From f90f81eb59d4d1c69eafd66693d91a3c65c1800a Mon Sep 17 00:00:00 2001 From: Nagkumar Arkalgud Date: Mon, 7 Oct 2024 17:54:25 -0700 Subject: [PATCH 1/9] Update evaluate-models-target.ipynb --- scenarios/evaluate-model-endpoints/evaluate-models-target.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scenarios/evaluate-model-endpoints/evaluate-models-target.ipynb b/scenarios/evaluate-model-endpoints/evaluate-models-target.ipynb index 486a5405..8f7eb770 100644 --- a/scenarios/evaluate-model-endpoints/evaluate-models-target.ipynb +++ b/scenarios/evaluate-model-endpoints/evaluate-models-target.ipynb @@ -112,7 +112,7 @@ "metadata": {}, "source": [ "\n", - "Please provide Azure AI Project details so that traces and eval results are pushing in the project in Azure AI Studio." + "Please provide Azure AI Project details so that traces and eval results are pushing in the project in Azure AI Studio. More information about the evaluators are found here - [evaluate-sdk](https://learn.microsoft.com/en-us/azure/ai-studio/how-to/develop/evaluate-sdk#risk-and-safety-evaluators)" ] }, { From 5ad991fcc03dd4ea9bf740e7bce2a8b28adb71d1 Mon Sep 17 00:00:00 2001 From: Nagkumar Arkalgud Date: Wed, 9 Oct 2024 14:59:24 -0700 Subject: [PATCH 2/9] Update sample to include instructions of getting data from search index and use model_config for the latest version of the simulator --- .../generate-data-query-response.ipynb | 301 +++++++++++++++--- .../user_override.prompty | 36 +++ 2 files changed, 296 insertions(+), 41 deletions(-) create mode 100644 scenarios/generate-synthetic-data/ai-generated-data-query-response/user_override.prompty diff --git a/scenarios/generate-synthetic-data/ai-generated-data-query-response/generate-data-query-response.ipynb b/scenarios/generate-synthetic-data/ai-generated-data-query-response/generate-data-query-response.ipynb index 1d6266c6..844abf0b 100644 --- a/scenarios/generate-synthetic-data/ai-generated-data-query-response/generate-data-query-response.ipynb +++ b/scenarios/generate-synthetic-data/ai-generated-data-query-response/generate-data-query-response.ipynb @@ -85,7 +85,7 @@ "source": [ "### Parameters\n", "\n", - "Lets initialize some variables. For `subscription_id`, `resource_group_name` and `project_name`, you can go to the Project Overview page in the AI Studio. Replace the items in <> with values for your project. " + "Lets initialize some variables. We need a way to connect to a LLM to use the notebook. This sample suggests a way to use `gpt-4o-mini` deployment in your Azure AI project. Replace the `azure_endpoint` with a link to your endpoint." ] }, { @@ -95,9 +95,8 @@ "outputs": [], "source": [ "# project details\n", - "subscription_id: str = \"\"\n", - "resource_group_name: str = \"\"\n", - "project_name: str = \"\"\n", + "azure_endpoint = \"https://.openai.azure.com\"\n", + "azure_deployment = \"gpt-4o-mini\" # replace with your deployment name, if different\n", "\n", "should_cleanup: bool = False" ] @@ -108,7 +107,7 @@ "source": [ "## Connect to your project\n", "\n", - "To start with let us create a config file with your project details. Replace the items in <> with values for your project" + "To start with let us create a config file with your project details. " ] }, { @@ -117,27 +116,23 @@ "metadata": {}, "outputs": [], "source": [ - "import json\n", + "import requests\n", "import os\n", + "from pathlib import Path\n", "\n", - "azure_ai_project = {\n", - " \"subscription_id\": subscription_id,\n", - " \"resource_group_name\": resource_group_name,\n", - " \"project_name\": project_name,\n", + "model_config = {\n", + " \"azure_endpoint\": azure_endpoint,\n", + " \"azure_deployment\": azure_deployment,\n", "}\n", "\n", - "os.environ[\"AZURE_OPENAI_API_KEY\"] = \"\"\n", - "os.environ[\"AZURE_OPENAI_ENDPOINT\"] = \"\"\n", - "# JSON mode supported model preferred to avoid errors ex. gpt-4o-mini, gpt-4o, gpt-4 (1106)\n", - "os.environ[\"AZURE_DEPLOYMENT\"] = \"\"\n", - "os.environ[\"AZURE_API_VERSION\"] = \"\"" + "# JSON mode supported model preferred to avoid errors ex. gpt-4o-mini, gpt-4o, gpt-4 (1106)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "Let us connect to the project" + "Let us connect to the project. DefaultAzureCredentails will be picked up by the SDK which runs the prompty files to authenticate your requests. If you want to use your AzureOpenAI key to authenticate, you can do so by setting the `api_key` in your `model_config`" ] }, { @@ -147,16 +142,95 @@ "outputs": [], "source": [ "from azure.ai.evaluation.simulator import Simulator\n", - "from azure.identity import DefaultAzureCredential\n", "\n", - "simulator = Simulator(azure_ai_project=azure_ai_project, credential=DefaultAzureCredential())" + "simulator = Simulator(model_config=model_config)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Data Source for the simulator" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Wikipedia article\n", + "In this example we use a wikipedia article as raw text generate Query Response pairs." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import wikipedia\n", + "\n", + "search_term = \"Leonardo da vinci\"\n", + "wiki_title = wikipedia.search(search_term)[0]\n", + "wiki_page = wikipedia.page(wiki_title)\n", + "text = wiki_page.summary[:5000]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Using data from an Azure AI Search Index\n", + "Use this step if you want to obtain text from your search index. This part assumes that you have a search index and have populated the index with necessary data that is relevant to your application." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "\n", + "search_term = \"camping stove\"\n", + "endpoint = \"https://.search.windows.net\"\n", + "index_name = \"\"\n", + "api_key = \"\"\n", + "url = f\"{endpoint}/indexes/{index_name}/docs/search?api-version=2024-07-01\"\n", + "\n", + "headers = {\n", + " \"api-key\": api_key,\n", + " \"Content-Type\": \"application/json\",\n", + "}\n", + "search_query = {\"search\": search_term, \"top\": 1}\n", + "\n", + "response = requests.post(url, headers=headers, data=json.dumps(search_query))\n", + "if response.status_code == 200:\n", + " results = response.json()\n", + " for result in results[\"value\"]:\n", + " text = result[\"content\"]\n", + "\n", + "text = text[:5000] # lets make sure the text isn't too long to exceed the model's token limit" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "Connecting the simulator to your application" + "### Connecting the simulator to your application\n", + "\n", + "This part assumes that you application is a call to `AzureOpenAI`'s chat completion endpoint. It also assumes that the following values are set in the environment. Feel free to change this method to call your application with its configuration." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "os.environ[\"AZURE_OPENAI_API_KEY\"] = \"\"\n", + "os.environ[\"AZURE_OPENAI_ENDPOINT\"] = \"\"\n", + "os.environ[\"AZURE_DEPLOYMENT\"] = \"\"\n", + "os.environ[\"AZURE_API_VERSION\"] = \"\"" ] }, { @@ -211,8 +285,11 @@ " latest_message = messages_list[-1]\n", " query = latest_message[\"content\"]\n", " context = None\n", - " # call your endpoint or ai application here\n", - " response = call_to_your_ai_application(query)\n", + " try:\n", + " # call your endpoint or ai application here\n", + " response = call_to_your_ai_application(query)\n", + " except Exception as e:\n", + " response = f\"An error occurred: {e!s}\"\n", " # we are formatting the response to follow the openAI chat protocol format\n", " formatted_response = {\n", " \"content\": response,\n", @@ -229,8 +306,11 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### Generate Query Responses from raw text\n", - "In this example we use a wikipedia article as raw text generate Query Response pairs." + "### Call to simulator\n", + "This call to the simulator generates 4 query response pairs in its first pass.\n", + "In the second pass, it picks up one task, pairs it with a query (generated in previous pass) and sends it to the configured llm to build the first user turn. This user turn is then passed to the `callback` method. The conversation continutes till the `max_conversation_turns` turns.\n", + "\n", + "The output of the simulator will have the original task, original query, the original query and the response generated from the first turn as expected response. You can find them in the `context` key of the conversation." ] }, { @@ -239,23 +319,29 @@ "metadata": {}, "outputs": [], "source": [ - "import wikipedia\n", - "\n", - "wiki_search_term = \"Leonardo da vinci\"\n", - "wiki_title = wikipedia.search(wiki_search_term)[0]\n", - "wiki_page = wikipedia.page(wiki_title)\n", - "text = wiki_page.summary[:5000]" + "outputs = await simulator(\n", + " target=callback,\n", + " text=text,\n", + " num_queries=4,\n", + " max_conversation_turns=1,\n", + " tasks=[\n", + " f\"I am a student and I want to learn more about {search_term}\",\n", + " f\"I am a teacher and I want to teach my students about {search_term}\",\n", + " f\"I am a researcher and I want to do a detailed research on {search_term}\",\n", + " f\"I am a statistician and I want to do a detailed table of factual data concerning {search_term}\",\n", + " ],\n", + ")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "### Call to simulator\n", - "This call to the simulator generates 4 query response pairs in its first pass.\n", - "In the second pass, it picks up one task, pairs it with a query (generated in previous pass) and sends it to the configured llm to build the first user turn. This user turn is then passed to the `callback` method. The conversation continutes till the `max_conversation_turns` turns.\n", + "### Overriding the user simulating behavior\n", "\n", - "The output of the simulator will have the original task, original query, the original query and the response generated from the first turn as expected response. You can find them in the `context` key of the conversation." + "Internally, the SDK has a `prompty` file that defines how the LLM which simulates the user should behave. Our SDK also offers an option for users to override the file, to support your own prompty files. Here's a brief overview of how to accomplish overriding the user behavior.\n", + "\n", + "Make sure you have `user_override.prompty` file in the same directory. The file in this repo takes an additional argument called `mood`. This is to show how you can add any additional keyword arguments to your prompty." ] }, { @@ -264,17 +350,23 @@ "metadata": {}, "outputs": [], "source": [ + "current_directory = Path.cwd()\n", + "user_override_prompty = Path(current_directory) / \"user_override.prompty\"\n", + "user_prompty_kwargs = {\"mood\": \"happy\"}\n", + "\n", "outputs = await simulator(\n", " target=callback,\n", " text=text,\n", " num_queries=4,\n", - " max_conversation_turns=3,\n", + " max_conversation_turns=1,\n", " tasks=[\n", - " f\"I am a student and I want to learn more about {wiki_search_term}\",\n", - " f\"I am a teacher and I want to teach my students about {wiki_search_term}\",\n", - " f\"I am a researcher and I want to do a detailed research on {wiki_search_term}\",\n", - " f\"I am a statistician and I want to do a detailed table of factual data concerning {wiki_search_term}\",\n", + " f\"I am a student and I want to learn more about {search_term}\",\n", + " f\"I am a teacher and I want to teach my students about {search_term}\",\n", + " f\"I am a researcher and I want to do a detailed research on {search_term}\",\n", + " f\"I am a statistician and I want to do a detailed table of factual data concerning {search_term}\",\n", " ],\n", + " user_simulator_prompty=user_override_prompty,\n", + " user_simulator_prompty_kwargs=user_prompty_kwargs,\n", ")" ] }, @@ -291,13 +383,140 @@ "metadata": {}, "outputs": [], "source": [ - "from pathlib import Path\n", - "\n", "output_file = Path(\"output.json\")\n", "with output_file.open(\"a\") as f:\n", " json.dump(outputs, f)" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Running evaluations on the simulated data\n", + "\n", + "Here we will try to run GroundednessEvaluator, RelevanceEvaluator, CoherenceEvaluator, FluencyEvaluator, SimilarityEvaluator, F1ScoreEvaluator on the output data from the simulator. \n", + "\n", + "From the [documentation](https://learn.microsoft.com/en-us/azure/ai-studio/how-to/develop/evaluate-sdk) we know that running those evaluators need the following data:\n", + "`query`, `response`, `context`, `ground_truth`\n", + "\n", + "For simplicity's sake, we can use our source document `text` as both `context` and `ground_truth`. This step only evaluates the first user message and first response from your AI Application for each of the simulated conversations. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "eval_input_data_json_lines = \"\"\n", + "for output in outputs:\n", + " query = None\n", + " response = None\n", + " context = text\n", + " ground_truth = text\n", + " for message in output[\"messages\"]:\n", + " if message[\"role\"] == \"user\":\n", + " query = message[\"content\"]\n", + " if message[\"role\"] == \"assistant\":\n", + " response = message[\"content\"]\n", + " if query and response:\n", + " eval_input_data_json_lines += (\n", + " json.dumps(\n", + " {\n", + " \"query\": query,\n", + " \"response\": response,\n", + " \"context\": context,\n", + " \"ground_truth\": ground_truth,\n", + " }\n", + " )\n", + " + \"\\n\"\n", + " )" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Store the output in a file" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "eval_input_data_file = Path(\"eval_input_data.jsonl\")\n", + "with eval_input_data_file.open(\"w\") as f:\n", + " f.write(eval_input_data_json_lines)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Run evaluation\n", + "\n", + "`QAEvaluator` is a composite evaluator which runs GroundednessEvaluator, RelevanceEvaluator, CoherenceEvaluator, FluencyEvaluator, SimilarityEvaluator, F1ScoreEvaluator\n", + "\n", + "Optionally set the `azure_ai_project` to upload the evaluation results to Azure AI Studio. To support that, we need to install `promptflow-azure`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%pip install promptflow-azure" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "azure_ai_project = {\n", + " \"subscription_id\": \"\",\n", + " \"resource_group\": \"\",\n", + " \"workspace_name\": \"\",\n", + "}\n", + "azure_ai_project = {\n", + " \"subscription_id\": \"fac34303-435d-4486-8c3f-7094d82a0b60\",\n", + " \"resource_group_name\": \"rg-naarkalgaihub\",\n", + " \"project_name\": \"naarkalg-rai-test\",\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from azure.ai.evaluation import evaluate, QAEvaluator\n", + "\n", + "qa_evaluator = QAEvaluator(model_config=model_config)\n", + "\n", + "eval_output = evaluate(\n", + " data=str(eval_input_data_file),\n", + " evaluators={\"QAEvaluator\": qa_evaluator},\n", + " evaluator_config={\n", + " \"QAEvaluator\": {\n", + " \"column_mapping\": {\n", + " \"query\": \"${data.query}\",\n", + " \"response\": \"${data.response}\",\n", + " \"context\": \"${data.context}\",\n", + " \"ground_truth\": \"${data.ground_truth}\",\n", + " }\n", + " }\n", + " },\n", + " azure_ai_project=azure_ai_project, # optional to store the evaluation results in Azure AI Studio\n", + " output_path=\"./myevalresults.json\", # optional to store the evaluation results in a file\n", + ")" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -317,7 +536,7 @@ "toc_visible": true }, "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": ".env", "language": "python", "name": "python3" }, diff --git a/scenarios/generate-synthetic-data/ai-generated-data-query-response/user_override.prompty b/scenarios/generate-synthetic-data/ai-generated-data-query-response/user_override.prompty new file mode 100644 index 00000000..0cffb969 --- /dev/null +++ b/scenarios/generate-synthetic-data/ai-generated-data-query-response/user_override.prompty @@ -0,0 +1,36 @@ +--- +name: TaskSimulatorWithPersona +description: Simulates a user to complete a conversation +model: + api: chat + parameters: + temperature: 0.0 + top_p: 1.0 + presence_penalty: 0 + frequency_penalty: 0 + response_format: + type: json_object + +inputs: + task: + type: string + conversation_history: + type: dict + mood: + type: string + default: neutral + +--- +system: +You must behave as a user who wants accomplish this task: {{ task }} and you continue to interact with a system that responds to your queries. If there is a message in the conversation history from the assistant, make sure you read the content of the message and include it your first response. Your mood is {{ mood }} +Make sure your conversation is engaging and interactive. +Output must be in JSON format +Here's a sample output: +{ + "content": "Here is my follow-up question.", + "user": "user" +} + +Output with a json object that continues the conversation, given the conversation history: +{{ conversation_history }} + From f75a4d17847eb377dc50e17a989e34c7cefb30f4 Mon Sep 17 00:00:00 2001 From: Nagkumar Arkalgud Date: Thu, 10 Oct 2024 12:02:00 -0700 Subject: [PATCH 3/9] remove azure_ai_project --- .../generate-data-query-response.ipynb | 5 ----- 1 file changed, 5 deletions(-) diff --git a/scenarios/generate-synthetic-data/ai-generated-data-query-response/generate-data-query-response.ipynb b/scenarios/generate-synthetic-data/ai-generated-data-query-response/generate-data-query-response.ipynb index 844abf0b..9e83b24d 100644 --- a/scenarios/generate-synthetic-data/ai-generated-data-query-response/generate-data-query-response.ipynb +++ b/scenarios/generate-synthetic-data/ai-generated-data-query-response/generate-data-query-response.ipynb @@ -481,11 +481,6 @@ " \"subscription_id\": \"\",\n", " \"resource_group\": \"\",\n", " \"workspace_name\": \"\",\n", - "}\n", - "azure_ai_project = {\n", - " \"subscription_id\": \"fac34303-435d-4486-8c3f-7094d82a0b60\",\n", - " \"resource_group_name\": \"rg-naarkalgaihub\",\n", - " \"project_name\": \"naarkalg-rai-test\",\n", "}" ] }, From 977c0be1bbfaa27a10f5c67d85708b1ff776c574 Mon Sep 17 00:00:00 2001 From: Nagkumar Arkalgud Date: Tue, 4 Feb 2025 09:30:14 -0800 Subject: [PATCH 4/9] Fix to param name --- .../Simulate_From_Input_Text/Simulate_From_Input_Text.ipynb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scenarios/evaluate/Simulators/Simulate_Context-Relevant_Data/Simulate_From_Input_Text/Simulate_From_Input_Text.ipynb b/scenarios/evaluate/Simulators/Simulate_Context-Relevant_Data/Simulate_From_Input_Text/Simulate_From_Input_Text.ipynb index 59f5d342..92448941 100644 --- a/scenarios/evaluate/Simulators/Simulate_Context-Relevant_Data/Simulate_From_Input_Text/Simulate_From_Input_Text.ipynb +++ b/scenarios/evaluate/Simulators/Simulate_Context-Relevant_Data/Simulate_From_Input_Text/Simulate_From_Input_Text.ipynb @@ -343,7 +343,7 @@ " f\"I am a statistician and I want to do a detailed table of factual data concerning {wiki_search_term}\",\n", " ],\n", " user_simulator_prompty=user_override_prompty,\n", - " user_simulator_prompty_kwargs=user_prompty_kwargs,\n", + " user_simulator_prompty_options=user_prompty_kwargs,\n", ")" ] }, @@ -489,7 +489,7 @@ "toc_visible": true }, "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": ".env", "language": "python", "name": "python3" }, From 836363715133b2c8addbfa8cfb97e256a279c3fe Mon Sep 17 00:00:00 2001 From: Nagkumar Arkalgud Date: Tue, 4 Feb 2025 09:31:48 -0800 Subject: [PATCH 5/9] Remove redundant file --- .../user_override.prompty | 36 ------------------- 1 file changed, 36 deletions(-) delete mode 100644 scenarios/generate-synthetic-data/ai-generated-data-query-response/user_override.prompty diff --git a/scenarios/generate-synthetic-data/ai-generated-data-query-response/user_override.prompty b/scenarios/generate-synthetic-data/ai-generated-data-query-response/user_override.prompty deleted file mode 100644 index 0cffb969..00000000 --- a/scenarios/generate-synthetic-data/ai-generated-data-query-response/user_override.prompty +++ /dev/null @@ -1,36 +0,0 @@ ---- -name: TaskSimulatorWithPersona -description: Simulates a user to complete a conversation -model: - api: chat - parameters: - temperature: 0.0 - top_p: 1.0 - presence_penalty: 0 - frequency_penalty: 0 - response_format: - type: json_object - -inputs: - task: - type: string - conversation_history: - type: dict - mood: - type: string - default: neutral - ---- -system: -You must behave as a user who wants accomplish this task: {{ task }} and you continue to interact with a system that responds to your queries. If there is a message in the conversation history from the assistant, make sure you read the content of the message and include it your first response. Your mood is {{ mood }} -Make sure your conversation is engaging and interactive. -Output must be in JSON format -Here's a sample output: -{ - "content": "Here is my follow-up question.", - "user": "user" -} - -Output with a json object that continues the conversation, given the conversation history: -{{ conversation_history }} - From 7e19ef63cd98464d6ce6ae48be1515377b9d7e26 Mon Sep 17 00:00:00 2001 From: Nagkumar Arkalgud Date: Fri, 11 Apr 2025 17:50:13 -0700 Subject: [PATCH 6/9] change the azure_ai_project --- .../Simulate_Evaluate_Groundedness.ipynb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scenarios/evaluate/Simulators/Simulate_Evaluate_Groundedness/Simulate_Evaluate_Groundedness.ipynb b/scenarios/evaluate/Simulators/Simulate_Evaluate_Groundedness/Simulate_Evaluate_Groundedness.ipynb index ccb85314..0c6b11e2 100644 --- a/scenarios/evaluate/Simulators/Simulate_Evaluate_Groundedness/Simulate_Evaluate_Groundedness.ipynb +++ b/scenarios/evaluate/Simulators/Simulate_Evaluate_Groundedness/Simulate_Evaluate_Groundedness.ipynb @@ -85,9 +85,9 @@ "outputs": [], "source": [ "azure_ai_project = {\n", - " \"subscription_id\": \"\",\n", - " \"resource_group\": \"\",\n", - " \"workspace_name\": \"\",\n", + " \"subscription_id\": \"AZURE_SUBSCRIPTION_ID\",\n", + " \"resource_group_name\": \"RESOURCE_GROUP\",\n", + " \"project_name\": \"PROJECT_NAME\",\n", "}\n", "\n", "azure_openai_endpoint = \"\"\n", From e853525c748d63d02c9f482a7612d78b6e4524bd Mon Sep 17 00:00:00 2001 From: Nagkumar Arkalgud Date: Fri, 26 Sep 2025 10:36:57 -0700 Subject: [PATCH 7/9] Samples for tracing --- scenarios/agent-tracing/langchain/README.md | 62 +++++ .../langchain/dev-requirements.txt | 9 + .../agent-tracing/langchain/requirements.txt | 7 + scenarios/agent-tracing/langchain/weather.py | 217 +++++++++++++++ scenarios/agent-tracing/langgraph/README.md | 62 +++++ .../langgraph/dev-requirements.txt | 10 + .../agent-tracing/langgraph/requirements.txt | 8 + scenarios/agent-tracing/langgraph/weather.py | 261 ++++++++++++++++++ .../agent-tracing/openai-agents/README.md | 61 ++++ .../openai-agents/dev-requirements.txt | 9 + .../openai-agents/requirements.txt | 8 + .../openai-agents/spanish_tutor.py | 204 ++++++++++++++ 12 files changed, 918 insertions(+) create mode 100644 scenarios/agent-tracing/langchain/README.md create mode 100644 scenarios/agent-tracing/langchain/dev-requirements.txt create mode 100644 scenarios/agent-tracing/langchain/requirements.txt create mode 100644 scenarios/agent-tracing/langchain/weather.py create mode 100644 scenarios/agent-tracing/langgraph/README.md create mode 100644 scenarios/agent-tracing/langgraph/dev-requirements.txt create mode 100644 scenarios/agent-tracing/langgraph/requirements.txt create mode 100644 scenarios/agent-tracing/langgraph/weather.py create mode 100644 scenarios/agent-tracing/openai-agents/README.md create mode 100644 scenarios/agent-tracing/openai-agents/dev-requirements.txt create mode 100644 scenarios/agent-tracing/openai-agents/requirements.txt create mode 100644 scenarios/agent-tracing/openai-agents/spanish_tutor.py diff --git a/scenarios/agent-tracing/langchain/README.md b/scenarios/agent-tracing/langchain/README.md new file mode 100644 index 00000000..9a526bd9 --- /dev/null +++ b/scenarios/agent-tracing/langchain/README.md @@ -0,0 +1,62 @@ +--- +page_type: sample +languages: +- python +products: +- ai-services +- azure-openai +description: Pure LangChain weather assistant with Azure tracing and manual tool-calling loop. +--- + +## Pure LangChain Weather (Tracing) + +### Overview + +This sample demonstrates a pure LangChain agent that uses a manual tool-calling loop, instrumented with Azure Application Insights via `langchain-azure-ai`. It calls Azure OpenAI chat models and a simple `get_weather` tool, and emits OpenTelemetry traces locally or to Azure Monitor. + +### Objective + +- Use `langchain` and `langchain-openai` with Azure OpenAI chat models. +- Add tracing via `langchain-azure-ai` and OpenTelemetry. +- Implement a manual tool-calling loop for clarity and control. + +### Programming Languages + +- Python + +### Estimated Runtime: 10 mins + +## Set up + +Create and activate a local virtual environment, then install dependencies: + +``` +python -m venv .venv +source .venv/bin/activate # Windows: .venv\\Scripts\\activate +pip install -r requirements.txt +``` + +Copy the environment template and set required variables: + +``` +cp .env.sample .env +``` + +Required: + +- `AZURE_OPENAI_API_KEY` +- `AZURE_OPENAI_ENDPOINT` +- `AZURE_OPENAI_DEPLOYMENT` +- `AZURE_OPENAI_API_VERSION` (default `2024-02-15-preview`) + +Optional for tracing: + +- `APPLICATION_INSIGHTS_CONNECTION_STRING` + +## Run + +``` +python weather.py +``` + +You can send traces to an OTLP-compatible backend by setting `OTEL_EXPORTER_OTLP_ENDPOINT`, or to Azure Monitor by setting `APPLICATION_INSIGHTS_CONNECTION_STRING`. diff --git a/scenarios/agent-tracing/langchain/dev-requirements.txt b/scenarios/agent-tracing/langchain/dev-requirements.txt new file mode 100644 index 00000000..3016f177 --- /dev/null +++ b/scenarios/agent-tracing/langchain/dev-requirements.txt @@ -0,0 +1,9 @@ +# Full dev setup (optional) +langchain +langchain-openai +langchain-azure-ai +opentelemetry-api +opentelemetry-sdk +opentelemetry-exporter-otlp +python-dotenv +azure-monitor-opentelemetry diff --git a/scenarios/agent-tracing/langchain/requirements.txt b/scenarios/agent-tracing/langchain/requirements.txt new file mode 100644 index 00000000..9fb42537 --- /dev/null +++ b/scenarios/agent-tracing/langchain/requirements.txt @@ -0,0 +1,7 @@ +langchain +langchain-openai +langchain-azure-ai +opentelemetry-api +opentelemetry-sdk +opentelemetry-exporter-otlp +python-dotenv diff --git a/scenarios/agent-tracing/langchain/weather.py b/scenarios/agent-tracing/langchain/weather.py new file mode 100644 index 00000000..5b9997a3 --- /dev/null +++ b/scenarios/agent-tracing/langchain/weather.py @@ -0,0 +1,217 @@ +""" +LangChain Weather Assistant with manual tool-calling loop + Azure tracing. + +Env vars required: + AZURE_OPENAI_API_KEY=... + AZURE_OPENAI_ENDPOINT=https://YOUR-RESOURCE.openai.azure.com + AZURE_OPENAI_DEPLOYMENT=yourDeploymentName + AZURE_OPENAI_API_VERSION=2024-02-15-preview (or compatible) + +Optional tracing: + APPLICATION_INSIGHTS_CONNECTION_STRING=InstrumentationKey=...;IngestionEndpoint=... + +Run: + python weather.py +""" + +import os +import json +import logging +from datetime import datetime +from typing import List, Any, Optional, Dict + +from langchain_core.tools import tool +from langchain_core.messages import ( + SystemMessage, + HumanMessage, + AIMessage, + ToolMessage, + BaseMessage, +) +from langchain_openai import AzureChatOpenAI + +try: + from langchain_azure_ai.callbacks.tracers import AzureAIInferenceTracer +except ImportError: + AzureAIInferenceTracer = None + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger("langchain_weather") + + +# ----------------------------------------------------------------------------- +# Tracing Setup (cached) +# ----------------------------------------------------------------------------- +_TRACERS: Optional[List[Any]] = None + + +def setup_tracing() -> List[Any]: + global _TRACERS + if _TRACERS is not None: + return _TRACERS + tracers: List[Any] = [] + conn = os.getenv("APPLICATION_INSIGHTS_CONNECTION_STRING") + if conn and AzureAIInferenceTracer: + try: + tracer = AzureAIInferenceTracer( + connection_string=conn, + enable_content_recording=True, + name="langchain_weather", + id="weather_agent", + endpoint="weather_single", + scope="Pure LangChain Weather", + ) + tracers.append(tracer) + logger.info("Azure tracing enabled.") + except Exception as e: + logger.warning(f"Failed to init tracer: {e}") + else: + logger.info("Tracing not enabled (missing APPLICATION_INSIGHTS_CONNECTION_STRING or dependency).") + _TRACERS = tracers + return tracers + + +def trace_config(agent_name: str, session_id: str) -> Dict[str, Any]: + tracers = setup_tracing() + return { + "callbacks": tracers, + "tags": [f"agent:{agent_name}", agent_name, "weather-langchain"], + "metadata": { + "agent_name": agent_name, + "agent_type": agent_name, + "langgraph_node": agent_name, # kept for parity + "session_id": session_id, + "thread_id": session_id, + "system": "langchain-weather", + }, + } + + +# ----------------------------------------------------------------------------- +# Tool +# ----------------------------------------------------------------------------- +@tool +def get_weather(location: str, date: Optional[str] = None) -> str: + """ + Return a mock weather forecast as JSON. + """ + if not date: + date = datetime.utcnow().strftime("%Y-%m-%d") + seed = sum(ord(c) for c in location.lower()) % 5 + conditions = ["Sunny", "Partly Cloudy", "Light Rain", "Overcast", "Showers"] + cond = conditions[seed] + forecast = { + "location": location, + "date": date, + "condition": cond, + "temp_high_c": 24 + seed, + "temp_low_c": 14 + seed, + "advice": "Great day outside!" if cond == "Sunny" else "Plan for changing conditions.", + } + return json.dumps(forecast, indent=2) + + +TOOLS = [get_weather] +TOOLS_BY_NAME = {t.name: t for t in TOOLS} + + +# ----------------------------------------------------------------------------- +# LLM Factory +# ----------------------------------------------------------------------------- +def build_llm(session_id: str) -> AzureChatOpenAI: + required = [ + "AZURE_OPENAI_API_KEY", + "AZURE_OPENAI_ENDPOINT", + "AZURE_OPENAI_DEPLOYMENT", + ] + missing = [v for v in required if not os.getenv(v)] + if missing: + raise RuntimeError(f"Missing Azure OpenAI env vars: {', '.join(missing)}") + return AzureChatOpenAI( + api_key=os.environ["AZURE_OPENAI_API_KEY"], + azure_endpoint=os.environ["AZURE_OPENAI_ENDPOINT"], + azure_deployment=os.environ["AZURE_OPENAI_DEPLOYMENT"], + api_version=os.getenv("AZURE_OPENAI_API_VERSION", "2024-02-15-preview"), + temperature=0.2, + callbacks=setup_tracing(), + tags=["weather_agent", "weather-langchain"], + metadata={ + "agent_type": "weather_agent", + "agent_name": "weather_agent", + "system": "langchain-weather", + "session_id": session_id, + "thread_id": session_id, + }, + ) + + +SYSTEM_PROMPT = """You are a weather assistant. +If user asks about weather, call the get_weather tool with (location, date if given). +If ambiguous date, assume tomorrow. +After tool output, summarize succinctly for the user. +""" + + +# ----------------------------------------------------------------------------- +# Agent Loop (manual) +# ----------------------------------------------------------------------------- +def run_weather_conversation(user_query: str, session_id: str) -> str: + llm = build_llm(session_id) + # Bind tools for tool-calling (function-calling) capability + tool_llm = llm.bind_tools(TOOLS) + + messages: List[BaseMessage] = [ + SystemMessage(content=SYSTEM_PROMPT), + HumanMessage(content=user_query), + ] + + # We allow up to N reasoning/tool steps (simple guard) + for step in range(5): + logger.info(f"LLM step {step + 1}") + response: AIMessage = tool_llm.invoke(messages, config=trace_config("weather_agent", session_id)) + messages.append(response) + + # If the model decided not to call any tools, we stop + tool_calls = getattr(response, "tool_calls", None) + if not tool_calls: + logger.info("No tool calls; finishing.") + break + + # Execute each tool call and append ToolMessage + for tc in tool_calls: + name = tc["name"] + args = tc.get("args", {}) + tool_obj = TOOLS_BY_NAME.get(name) + if not tool_obj: + tool_output = f"Tool '{name}' not found." + else: + try: + tool_output = tool_obj.invoke(args) + except Exception as e: + tool_output = f"Error executing tool '{name}': {e}" + messages.append( + ToolMessage( + content=tool_output, + name=name, + tool_call_id=tc["id"], + ) + ) + + # Final answer: last AI message with no tool calls OR last AI message overall + final_ai = next((m for m in reversed(messages) if isinstance(m, AIMessage)), None) + return final_ai.content if final_ai else "No AI response." + + +def main() -> None: + print("Pure LangChain Weather (Instrumented)") + q = input("Ask a weather question (e.g. 'Weather in Tokyo tomorrow'): ").strip() + if not q: + q = "Weather in Paris" + session_id = f"lc-session-{datetime.utcnow().strftime('%Y%m%d%H%M%S')}" + answer = run_weather_conversation(q, session_id) + print("\n--- Answer ---") + print(answer) + + +if __name__ == "__main__": + main() diff --git a/scenarios/agent-tracing/langgraph/README.md b/scenarios/agent-tracing/langgraph/README.md new file mode 100644 index 00000000..c0617b2c --- /dev/null +++ b/scenarios/agent-tracing/langgraph/README.md @@ -0,0 +1,62 @@ +--- +page_type: sample +languages: +- python +products: +- ai-services +- azure-openai +description: Pure LangGraph weather workflow with Azure tracing and single-node tool execution. +--- + +## Pure LangGraph Weather (Tracing) + +### Overview + +This sample demonstrates a LangGraph single-node workflow that calls Azure OpenAI chat models and a `get_weather` tool, with traces exported via `langchain-azure-ai` to local OTLP endpoints or Azure Monitor. + +### Objective + +- Use `langgraph` with `langchain` and Azure OpenAI. +- Instrument with `langchain-azure-ai` and OpenTelemetry for tracing. +- Stream steps and inspect final state in a simple weather flow. + +### Programming Languages + +- Python + +### Estimated Runtime: 10 mins + +## Set up + +Create and activate a local virtual environment, then install dependencies: + +``` +python -m venv .venv +source .venv/bin/activate # Windows: .venv\\Scripts\\activate +pip install -r requirements.txt +``` + +Copy the environment template and set required variables: + +``` +cp .env.sample .env +``` + +Required: + +- `AZURE_OPENAI_API_KEY` +- `AZURE_OPENAI_ENDPOINT` +- `AZURE_OPENAI_DEPLOYMENT` +- `AZURE_OPENAI_API_VERSION` (optional; defaults to `2024-02-15-preview`) + +Optional for tracing: + +- `APPLICATION_INSIGHTS_CONNECTION_STRING` + +## Run + +``` +python weather.py +``` + +Optionally set `OTEL_EXPORTER_OTLP_ENDPOINT` for local OTLP backends, or `APPLICATION_INSIGHTS_CONNECTION_STRING` to send traces to Azure Monitor. diff --git a/scenarios/agent-tracing/langgraph/dev-requirements.txt b/scenarios/agent-tracing/langgraph/dev-requirements.txt new file mode 100644 index 00000000..65e8c41b --- /dev/null +++ b/scenarios/agent-tracing/langgraph/dev-requirements.txt @@ -0,0 +1,10 @@ +# Full dev setup (optional) +langchain +langchain-openai +langchain-azure-ai +langgraph +opentelemetry-api +opentelemetry-sdk +opentelemetry-exporter-otlp +python-dotenv +azure-monitor-opentelemetry diff --git a/scenarios/agent-tracing/langgraph/requirements.txt b/scenarios/agent-tracing/langgraph/requirements.txt new file mode 100644 index 00000000..1317833c --- /dev/null +++ b/scenarios/agent-tracing/langgraph/requirements.txt @@ -0,0 +1,8 @@ +langchain +langchain-openai +langchain-azure-ai +langgraph +opentelemetry-api +opentelemetry-sdk +opentelemetry-exporter-otlp +python-dotenv diff --git a/scenarios/agent-tracing/langgraph/weather.py b/scenarios/agent-tracing/langgraph/weather.py new file mode 100644 index 00000000..ea51862c --- /dev/null +++ b/scenarios/agent-tracing/langgraph/weather.py @@ -0,0 +1,261 @@ +""" +LangGraph single-node weather workflow with Azure tracing. + +Env vars required (same as langchain/weather.py): + AZURE_OPENAI_API_KEY + AZURE_OPENAI_ENDPOINT + AZURE_OPENAI_DEPLOYMENT +Optional: + AZURE_OPENAI_API_VERSION + APPLICATION_INSIGHTS_CONNECTION_STRING + +Run: + python weather.py +""" + +import os +import json +import logging +from datetime import datetime +from typing import TypedDict, List, Annotated, Optional, Any, Dict + +from langchain_core.tools import tool +from langchain_core.messages import ( + HumanMessage, + SystemMessage, + AIMessage, + ToolMessage, + AnyMessage, +) +from langgraph.graph import StateGraph, START, END +from langgraph.graph.message import add_messages +from langgraph.checkpoint.memory import MemorySaver +from langchain_openai import AzureChatOpenAI + +try: + from langchain_azure_ai.callbacks.tracers import AzureAIInferenceTracer +except ImportError: + AzureAIInferenceTracer = None + +logging.basicConfig(level=logging.INFO) +log = logging.getLogger("langgraph_weather") + + +# ----------------------------------------------------------------------------- +# Tracing +# ----------------------------------------------------------------------------- +_TRACERS: Optional[list[Any]] = None + + +def setup_tracing() -> list[Any]: + global _TRACERS + if _TRACERS is not None: + return _TRACERS + tracers: list[Any] = [] + conn = os.getenv("APPLICATION_INSIGHTS_CONNECTION_STRING") + if conn and AzureAIInferenceTracer: + try: + tracers.append( + AzureAIInferenceTracer( + connection_string=conn, + enable_content_recording=True, + name="langgraph_weather", + id="weather_graph_agent", + endpoint="weather_graph", + scope="LangGraph Weather Flow", + ) + ) + log.info("Azure tracing enabled.") + except Exception as e: + log.warning(f"Tracing init failed: {e}") + else: + log.info("Tracing disabled (no APPLICATION_INSIGHTS_CONNECTION_STRING).") + _TRACERS = tracers + return tracers + + +def trace_config(agent_name: str, session_id: str) -> Dict[str, Any]: + tracers = setup_tracing() + return { + "callbacks": tracers, + "tags": [f"agent:{agent_name}", agent_name, "weather-langgraph"], + "metadata": { + "agent_name": agent_name, + "agent_type": agent_name, + "langgraph_node": agent_name, + "session_id": session_id, + "thread_id": session_id, + "system": "langgraph-weather", + }, + } + + +# ----------------------------------------------------------------------------- +# Tool +# ----------------------------------------------------------------------------- +@tool +def get_weather(location: str, date: Optional[str] = None) -> str: + if not date: + date = datetime.utcnow().strftime("%Y-%m-%d") + seed = sum(ord(c) for c in location.lower()) % 4 + conds = ["Sunny", "Windy", "Showers", "Cloudy"] + cond = conds[seed] + return json.dumps( + { + "location": location, + "date": date, + "condition": cond, + "high_c": 23 + seed, + "low_c": 13 + seed, + "advice": "Bring a jacket." if cond != "Sunny" else "Enjoy the sunshine!", + }, + indent=2, + ) + + +TOOLS = [get_weather] +TOOLS_BY_NAME = {t.name: t for t in TOOLS} + + +# ----------------------------------------------------------------------------- +# State +# ----------------------------------------------------------------------------- +class WeatherState(TypedDict): + messages: Annotated[List[AnyMessage], add_messages] + session_id: str + done: bool + + +SYSTEM_PROMPT = """You are a weather assistant. +Use get_weather tool exactly once if user asks about conditions. +Return a concise summary referencing the tool output. +""" + + +# ----------------------------------------------------------------------------- +# LLM +# ----------------------------------------------------------------------------- +def build_llm(session_id: str) -> AzureChatOpenAI: + required = [ + "AZURE_OPENAI_API_KEY", + "AZURE_OPENAI_ENDPOINT", + "AZURE_OPENAI_DEPLOYMENT", + ] + missing = [v for v in required if not os.getenv(v)] + if missing: + raise RuntimeError(f"Missing Azure OpenAI env vars: {', '.join(missing)}") + return AzureChatOpenAI( + api_key=os.environ["AZURE_OPENAI_API_KEY"], + azure_endpoint=os.environ["AZURE_OPENAI_ENDPOINT"], + azure_deployment=os.environ["AZURE_OPENAI_DEPLOYMENT"], + api_version=os.getenv("AZURE_OPENAI_API_VERSION", "2024-02-15-preview"), + temperature=0.2, + callbacks=setup_tracing(), + tags=["weather_agent", "weather-langgraph"], + metadata={ + "agent_type": "weather_agent", + "agent_name": "weather_agent", + "system": "langgraph-weather", + "session_id": session_id, + "thread_id": session_id, + }, + ) + + +# ----------------------------------------------------------------------------- +# Node +# ----------------------------------------------------------------------------- +def weather_node(state: WeatherState) -> WeatherState: + # If already done, just pass state (idempotency) + if state.get("done"): + return state + + llm = build_llm(state["session_id"]) + tool_llm = llm.bind_tools(TOOLS) + + # Step 1: ask the model + response: AIMessage = tool_llm.invoke( + state["messages"], + config=trace_config("weather_agent", state["session_id"]), + ) + state["messages"].append(response) + + tool_calls = getattr(response, "tool_calls", None) + if tool_calls: + # Execute tool calls + for tc in tool_calls: + name = tc["name"] + args = tc.get("args", {}) + tool_obj = TOOLS_BY_NAME.get(name) + if not tool_obj: + output = f"Tool '{name}' not found." + else: + try: + output = tool_obj.invoke(args) + except Exception as e: + output = f"Error executing tool '{name}': {e}" + state["messages"].append(ToolMessage(name=name, tool_call_id=tc["id"], content=output)) + # After tool outputs, ask model again to summarize + final_response: AIMessage = llm.invoke( + state["messages"], + config=trace_config("weather_agent", state["session_id"]), + ) + state["messages"].append(final_response) + + # Mark done + state["done"] = True + return state + + +# ----------------------------------------------------------------------------- +# Control Flow +# ----------------------------------------------------------------------------- +def route(state: WeatherState) -> str: + if not state.get("done"): + return "weather_agent" + return END + + +def build_app() -> StateGraph: + g = StateGraph(WeatherState) + g.add_node("weather_agent", weather_node) + g.add_conditional_edges(START, lambda _state: "weather_agent") + g.add_conditional_edges("weather_agent", route) + checkpointer = MemorySaver() + return g.compile(checkpointer=checkpointer) + + +# ----------------------------------------------------------------------------- +# CLI +# ----------------------------------------------------------------------------- +def main() -> None: + print("Pure LangGraph Weather (Instrumented)") + query = input("Ask a weather question: ").strip() + if not query: + query = "What's the weather in Lisbon tomorrow?" + session_id = f"graph-session-{datetime.utcnow().strftime('%Y%m%d%H%M%S')}" + + app = build_app() + initial: WeatherState = { + "messages": [ + SystemMessage(content=SYSTEM_PROMPT), + HumanMessage(content=query), + ], + "session_id": session_id, + "done": False, + } + + print("\n--- Streaming Steps ---") + for step in app.stream(initial, {"configurable": {"thread_id": session_id}}): + print(step) + + final_state = app.get_state({"configurable": {"thread_id": session_id}}) + all_msgs = final_state.values["messages"] + final_ai = [m for m in all_msgs if isinstance(m, AIMessage)] + if final_ai: + print("\n--- Final Answer ---") + print(final_ai[-1].content) + + +if __name__ == "__main__": + main() diff --git a/scenarios/agent-tracing/openai-agents/README.md b/scenarios/agent-tracing/openai-agents/README.md new file mode 100644 index 00000000..54d31519 --- /dev/null +++ b/scenarios/agent-tracing/openai-agents/README.md @@ -0,0 +1,61 @@ +--- +page_type: sample +languages: +- python +products: +- ai-services +- azure-openai +description: OpenAI Agents sample (Spanish Tutor) instrumented with OpenTelemetry for console or Azure Monitor. +--- + +## OpenAI Agents Spanish Tutor (Tracing) + +### Overview + +This sample uses the `openai-agents` Python SDK with Azure OpenAI (Chat Completions) and instruments agent runs via OpenTelemetry. It authenticates using `DefaultAzureCredential` (supports `az login` or service principal) and exports spans either to Azure Monitor or the console. + +### Objective + +- Configure Azure OpenAI with the `openai` SDK and `openai-agents`. +- Instrument the Agents framework with `opentelemetry-instrumentation-openai-agents`. +- Export traces to Azure Monitor or console. + +### Programming Languages + +- Python + +### Estimated Runtime: 10 mins + +## Set up + +Create a virtual environment and install dependencies: + +``` +python -m venv .venv +source .venv/bin/activate # Windows: .venv\\Scripts\\activate +pip install -r requirements.txt +``` + +Copy the environment template and set required variables: + +``` +cp .env.sample .env +``` + +Required: + +- `AZURE_OPENAI_ENDPOINT` +- `AZURE_OPENAI_VERSION` +- `AZURE_OPENAI_CHAT_DEPLOYMENT` + +Optional tracing: + +- `APPLICATION_INSIGHTS_CONNECTION_STRING` + +## Run + +``` +API_HOST=azure python spanish_tutor.py +``` + +Provide `APPLICATION_INSIGHTS_CONNECTION_STRING` to export traces to Azure Monitor, otherwise spans are printed to console. Ensure you have `az login` or a valid service principal configured for `DefaultAzureCredential`. diff --git a/scenarios/agent-tracing/openai-agents/dev-requirements.txt b/scenarios/agent-tracing/openai-agents/dev-requirements.txt new file mode 100644 index 00000000..6370a177 --- /dev/null +++ b/scenarios/agent-tracing/openai-agents/dev-requirements.txt @@ -0,0 +1,9 @@ +# Full dev setup (optional) +openai>=1.35.0 +openai-agents>=0.2.9 +azure-identity>=1.17.0 +opentelemetry-api>=1.27.0 +opentelemetry-sdk>=1.27.0 +opentelemetry-instrumentation-openai-agents[instruments] @ git+https://github.com/nagkumar91/opentelemetry-python-contrib@main#subdirectory=instrumentation-genai/opentelemetry-instrumentation-openai-agents +python-dotenv>=1.0.1 +azure-monitor-opentelemetry-exporter diff --git a/scenarios/agent-tracing/openai-agents/requirements.txt b/scenarios/agent-tracing/openai-agents/requirements.txt new file mode 100644 index 00000000..9ba04af7 --- /dev/null +++ b/scenarios/agent-tracing/openai-agents/requirements.txt @@ -0,0 +1,8 @@ +openai>=1.35.0 +openai-agents>=0.2.9 +azure-identity>=1.17.0 +opentelemetry-api>=1.27.0 +opentelemetry-sdk>=1.27.0 +opentelemetry-instrumentation-openai-agents[instruments] @ git+https://github.com/nagkumar91/opentelemetry-python-contrib@main#subdirectory=instrumentation-genai/opentelemetry-instrumentation-openai-agents +python-dotenv>=1.0.1 +azure-monitor-opentelemetry-exporter diff --git a/scenarios/agent-tracing/openai-agents/spanish_tutor.py b/scenarios/agent-tracing/openai-agents/spanish_tutor.py new file mode 100644 index 00000000..fdb7ebd0 --- /dev/null +++ b/scenarios/agent-tracing/openai-agents/spanish_tutor.py @@ -0,0 +1,204 @@ +""" +Spanish Tutor (Azure OpenAI) + OpenTelemetry (Console or Azure Monitor) + +This script: + * Uses Azure OpenAI (Chat Completions) via the `openai` Python SDK (>=1.x) + * Authenticates with DefaultAzureCredential (so you can use az login or a service principal) + * Instruments the OpenAI Agents framework with OpenTelemetry + * Captures rich GenAI semantic attributes (messages, system instructions, tool definitions) + * Exports spans either to: + - Azure Monitor (if APPLICATION_INSIGHTS_CONNECTION_STRING is set), or + - Console (fallback) + +Prerequisites: + pip install: + openai + openai-agents + azure-identity + opentelemetry-sdk + opentelemetry-api + opentelemetry-instrumentation-openai-agents + (optional) azure-monitor-opentelemetry-exporter + +Run: + API_HOST=azure python spanish_tutor.py +""" + +from __future__ import annotations + +import asyncio +import logging +import os +from dataclasses import dataclass +from typing import Callable +from urllib.parse import urlparse + +import azure.identity +import openai +from agents import Agent, OpenAIChatCompletionsModel, Runner # from openai-agents + +from opentelemetry import trace +from opentelemetry.instrumentation.openai_agents import OpenAIAgentsInstrumentor +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import ( + BatchSpanProcessor, + ConsoleSpanExporter, +) + +logging.basicConfig(level=logging.INFO) + + +@dataclass +class _ApiConfig: + build_client: Callable[[], openai.AsyncAzureOpenAI] + model_name: str + base_url: str + provider: str + + +def _set_capture_env(_provider: str, base_url: str) -> None: + """ + Set OpenTelemetry + GenAI semantic capture environment toggles if not already provided. + These default to 'true' to showcase the richest possible trace payload. + """ + capture_defaults = { + # Enable instrumentation features + "OTEL_INSTRUMENTATION_OPENAI_AGENTS_CAPTURE_CONTENT": "true", + "OTEL_INSTRUMENTATION_OPENAI_AGENTS_CAPTURE_METRICS": "true", + # Generic GenAI semantic attrs capture + "OTEL_GENAI_CAPTURE_MESSAGES": "true", + "OTEL_GENAI_CAPTURE_SYSTEM_INSTRUCTIONS": "true", + "OTEL_GENAI_CAPTURE_TOOL_DEFINITIONS": "true", + "OTEL_GENAI_EMIT_OPERATION_DETAILS": "true", + # Agent identity metadata + "OTEL_GENAI_AGENT_NAME": os.getenv("OTEL_GENAI_AGENT_NAME", "Spanish Tutor Agent"), + "OTEL_GENAI_AGENT_DESCRIPTION": os.getenv( + "OTEL_GENAI_AGENT_DESCRIPTION", + "Conversational tutor that always replies in Spanish", + ), + "OTEL_GENAI_AGENT_ID": os.getenv("OTEL_GENAI_AGENT_ID", "spanish-tutor"), + } + for k, v in capture_defaults.items(): + os.environ.setdefault(k, v) + + # 'provider' is used for identity metadata in env; parsing base_url sets server attrs + parsed = urlparse(base_url) + if parsed.hostname: + os.environ.setdefault("OTEL_GENAI_SERVER_ADDRESS", parsed.hostname) + if parsed.port: + os.environ.setdefault("OTEL_GENAI_SERVER_PORT", str(parsed.port)) + + +def _resolve_api_config() -> _ApiConfig: + """ + For this Azure-focused sample we expect API_HOST=azure. + """ + host = os.getenv("API_HOST", "azure").lower() + if host != "azure": + raise ValueError("This sample is locked to API_HOST=azure for clarity.") + + endpoint = os.environ["AZURE_OPENAI_ENDPOINT"].rstrip("/") + api_version = os.environ["AZURE_OPENAI_VERSION"] + deployment = os.environ["AZURE_OPENAI_CHAT_DEPLOYMENT"] + + credential = azure.identity.DefaultAzureCredential() + token_provider = azure.identity.get_bearer_token_provider( + credential, + "https://cognitiveservices.azure.com/.default", + ) + + def _build_client() -> openai.AsyncAzureOpenAI: + return openai.AsyncAzureOpenAI( + api_version=api_version, + azure_endpoint=endpoint, + azure_ad_token_provider=token_provider, + ) + + return _ApiConfig( + build_client=_build_client, + model_name=deployment, + base_url=endpoint, + provider="azure.ai.openai", + ) + + +def _configure_otel() -> None: + """ + Configure TracerProvider + exporter. + If APPLICATION_INSIGHTS_CONNECTION_STRING is set, export to Azure Monitor. + Otherwise, export spans to the console. + """ + conn = os.getenv("APPLICATION_INSIGHTS_CONNECTION_STRING") + resource = Resource.create( + { + "service.name": os.getenv("OTEL_SERVICE_NAME", "spanish-tutor-app"), + "service.namespace": "language-learning", + "service.version": os.getenv("SERVICE_VERSION", "1.0.0"), + } + ) + + tracer_provider = TracerProvider(resource=resource) + + if conn: + try: + from azure.monitor.opentelemetry.exporter import ( # type: ignore + AzureMonitorTraceExporter, + ) + except ImportError: + print( + "Azure Monitor exporter not installed. Falling back to console. " + "Install with: pip install azure-monitor-opentelemetry-exporter" + ) + tracer_provider.add_span_processor(BatchSpanProcessor(ConsoleSpanExporter())) + else: + tracer_provider.add_span_processor( + BatchSpanProcessor(AzureMonitorTraceExporter.from_connection_string(conn)) + ) + print("[otel] Azure Monitor trace exporter configured") + else: + tracer_provider.add_span_processor(BatchSpanProcessor(ConsoleSpanExporter())) + print("[otel] Console span exporter configured") + print("[otel] Provide APPLICATION_INSIGHTS_CONNECTION_STRING to export to Azure Monitor.") + + trace.set_tracer_provider(tracer_provider) + + +def _infer_span_name(provider: str) -> str: + return f"spanish_tutor_session[{provider}]" + + +async def main() -> None: + api_config = _resolve_api_config() + _set_capture_env(api_config.provider, api_config.base_url) + _configure_otel() + + # Instrument AFTER setting tracer provider + OpenAIAgentsInstrumentor().instrument(tracer_provider=trace.get_tracer_provider()) + + client = api_config.build_client() + + agent = Agent( + name="Spanish tutor", + instructions="You are a Spanish tutor. Help the user learn Spanish. ONLY respond in Spanish.", + model=OpenAIChatCompletionsModel( + model=api_config.model_name, + openai_client=client, + ), + ) + + tracer = trace.get_tracer(__name__) + # Create a session span + with tracer.start_as_current_span(_infer_span_name(api_config.provider)): + result = await Runner.run(agent, input="Hola, ¿cómo estás?") + print("\n=== Final Tutor Reply (Spanish) ===") + print(result.final_output) + print("==================================\n") + + +if __name__ == "__main__": + try: + asyncio.run(main()) + finally: + # Ensure processors flush + trace.get_tracer_provider().shutdown() From 7097e9eeec592125565221d6d96b49aaa04dbfa8 Mon Sep 17 00:00:00 2001 From: Nagkumar Arkalgud Date: Fri, 26 Sep 2025 15:37:43 -0700 Subject: [PATCH 8/9] Rename tracer and add env samples --- scenarios/agent-tracing/README.md | 31 +++++++++++++++++++ scenarios/agent-tracing/langchain/.env.sample | 9 ++++++ scenarios/agent-tracing/langchain/weather.py | 8 ++--- scenarios/agent-tracing/langgraph/.env.sample | 9 ++++++ scenarios/agent-tracing/langgraph/weather.py | 8 ++--- .../agent-tracing/openai-agents/.env.sample | 8 +++++ 6 files changed, 65 insertions(+), 8 deletions(-) create mode 100644 scenarios/agent-tracing/README.md create mode 100644 scenarios/agent-tracing/langchain/.env.sample create mode 100644 scenarios/agent-tracing/langgraph/.env.sample create mode 100644 scenarios/agent-tracing/openai-agents/.env.sample diff --git a/scenarios/agent-tracing/README.md b/scenarios/agent-tracing/README.md new file mode 100644 index 00000000..15ac22c0 --- /dev/null +++ b/scenarios/agent-tracing/README.md @@ -0,0 +1,31 @@ +# Agent Tracing + +Reasoning about agent executions is critical for troubleshooting and debugging. Complex agents can involve many nested steps, variable execution paths, and long inputs/outputs, which makes it hard to pinpoint issues. Tracing provides a clear, chronological view of the inputs and outputs for each primitive involved in a run. + +This scenario sets up a structure for agent tracing using OpenTelemetry. It supports: +- Local tracing via console or any OTLP-compatible backend (e.g., Aspire Dashboard). +- Cloud tracing via Azure Monitor when Application Insights is enabled for your Azure AI Studio project. + +## Structure +- `langchain/`: Tracing patterns for LangChain flows. +- `langgraph/`: Tracing patterns for LangGraph agents/graphs. +- `openai-agents/`: Tracing for OpenAI Agents with Azure OpenAI. + +Each subfolder contains its own `requirements.txt`, optional `dev-requirements.txt`, and `.env.sample` tailored for that sample. + +## Prerequisites +- Python 3.10+ recommended. +- An Azure AI Studio project (optional, for Azure Monitor tracing). +- If using Azure Monitor, enable the Tracing tab in your AI Studio project to provision Application Insights and retrieve the connection string. + +## Installation +Navigate to a subfolder and install its `requirements.txt`. Use `dev-requirements.txt` if you want Azure Monitor integrations. + +## Configuration +- Local OTLP exporter: + - Set `OTEL_EXPORTER_OTLP_ENDPOINT` to your backend (e.g., Aspire Dashboard default: `http://localhost:4317` for gRPC, or `http://localhost:4318` for HTTP). +- Azure Monitor: + - Copy the subfolder `.env.sample` to `.env` and set required values. + +## Notes +- The initial release of Azure AI Projects had a known issue where agent function tool call details might be included in traces even when content recording is disabled. Be cautious with sensitive data and review Tracing settings in AI Studio. diff --git a/scenarios/agent-tracing/langchain/.env.sample b/scenarios/agent-tracing/langchain/.env.sample new file mode 100644 index 00000000..bf05b82f --- /dev/null +++ b/scenarios/agent-tracing/langchain/.env.sample @@ -0,0 +1,9 @@ +# Azure OpenAI configuration (required) +AZURE_OPENAI_API_KEY= +AZURE_OPENAI_ENDPOINT= +AZURE_OPENAI_DEPLOYMENT= +AZURE_OPENAI_API_VERSION=2024-02-15-preview + +# Optional: Enable Azure Monitor tracing via Application Insights +APPLICATION_INSIGHTS_CONNECTION_STRING= + diff --git a/scenarios/agent-tracing/langchain/weather.py b/scenarios/agent-tracing/langchain/weather.py index 5b9997a3..51d46795 100644 --- a/scenarios/agent-tracing/langchain/weather.py +++ b/scenarios/agent-tracing/langchain/weather.py @@ -31,9 +31,9 @@ from langchain_openai import AzureChatOpenAI try: - from langchain_azure_ai.callbacks.tracers import AzureAIInferenceTracer + from langchain_azure_ai.callbacks.tracers import AzureAIOpenTelemetryTracer except ImportError: - AzureAIInferenceTracer = None + AzureAIOpenTelemetryTracer = None logging.basicConfig(level=logging.INFO) logger = logging.getLogger("langchain_weather") @@ -51,9 +51,9 @@ def setup_tracing() -> List[Any]: return _TRACERS tracers: List[Any] = [] conn = os.getenv("APPLICATION_INSIGHTS_CONNECTION_STRING") - if conn and AzureAIInferenceTracer: + if conn and AzureAIOpenTelemetryTracer: try: - tracer = AzureAIInferenceTracer( + tracer = AzureAIOpenTelemetryTracer( connection_string=conn, enable_content_recording=True, name="langchain_weather", diff --git a/scenarios/agent-tracing/langgraph/.env.sample b/scenarios/agent-tracing/langgraph/.env.sample new file mode 100644 index 00000000..bf05b82f --- /dev/null +++ b/scenarios/agent-tracing/langgraph/.env.sample @@ -0,0 +1,9 @@ +# Azure OpenAI configuration (required) +AZURE_OPENAI_API_KEY= +AZURE_OPENAI_ENDPOINT= +AZURE_OPENAI_DEPLOYMENT= +AZURE_OPENAI_API_VERSION=2024-02-15-preview + +# Optional: Enable Azure Monitor tracing via Application Insights +APPLICATION_INSIGHTS_CONNECTION_STRING= + diff --git a/scenarios/agent-tracing/langgraph/weather.py b/scenarios/agent-tracing/langgraph/weather.py index ea51862c..1bde88c5 100644 --- a/scenarios/agent-tracing/langgraph/weather.py +++ b/scenarios/agent-tracing/langgraph/weather.py @@ -33,9 +33,9 @@ from langchain_openai import AzureChatOpenAI try: - from langchain_azure_ai.callbacks.tracers import AzureAIInferenceTracer + from langchain_azure_ai.callbacks.tracers import AzureAIOpenTelemetryTracer except ImportError: - AzureAIInferenceTracer = None + AzureAIOpenTelemetryTracer = None logging.basicConfig(level=logging.INFO) log = logging.getLogger("langgraph_weather") @@ -53,10 +53,10 @@ def setup_tracing() -> list[Any]: return _TRACERS tracers: list[Any] = [] conn = os.getenv("APPLICATION_INSIGHTS_CONNECTION_STRING") - if conn and AzureAIInferenceTracer: + if conn and AzureAIOpenTelemetryTracer: try: tracers.append( - AzureAIInferenceTracer( + AzureAIOpenTelemetryTracer( connection_string=conn, enable_content_recording=True, name="langgraph_weather", diff --git a/scenarios/agent-tracing/openai-agents/.env.sample b/scenarios/agent-tracing/openai-agents/.env.sample new file mode 100644 index 00000000..20bd37e2 --- /dev/null +++ b/scenarios/agent-tracing/openai-agents/.env.sample @@ -0,0 +1,8 @@ +# Azure OpenAI Agents (required) +AZURE_OPENAI_ENDPOINT= +AZURE_OPENAI_VERSION= +AZURE_OPENAI_CHAT_DEPLOYMENT= + +# Optional tracing +APPLICATION_INSIGHTS_CONNECTION_STRING= + From 4a08bd50e3f54c2ea987df8f681a0d9621304a41 Mon Sep 17 00:00:00 2001 From: Nagkumar Arkalgud Date: Fri, 26 Sep 2025 15:55:31 -0700 Subject: [PATCH 9/9] remove scope and endpoint --- scenarios/agent-tracing/langchain/weather.py | 2 -- scenarios/agent-tracing/langgraph/weather.py | 2 -- scenarios/langchain/tracing-with-langchain.ipynb | 4 ++-- 3 files changed, 2 insertions(+), 6 deletions(-) diff --git a/scenarios/agent-tracing/langchain/weather.py b/scenarios/agent-tracing/langchain/weather.py index 51d46795..f2bc0926 100644 --- a/scenarios/agent-tracing/langchain/weather.py +++ b/scenarios/agent-tracing/langchain/weather.py @@ -58,8 +58,6 @@ def setup_tracing() -> List[Any]: enable_content_recording=True, name="langchain_weather", id="weather_agent", - endpoint="weather_single", - scope="Pure LangChain Weather", ) tracers.append(tracer) logger.info("Azure tracing enabled.") diff --git a/scenarios/agent-tracing/langgraph/weather.py b/scenarios/agent-tracing/langgraph/weather.py index 1bde88c5..956f8042 100644 --- a/scenarios/agent-tracing/langgraph/weather.py +++ b/scenarios/agent-tracing/langgraph/weather.py @@ -61,8 +61,6 @@ def setup_tracing() -> list[Any]: enable_content_recording=True, name="langgraph_weather", id="weather_graph_agent", - endpoint="weather_graph", - scope="LangGraph Weather Flow", ) ) log.info("Azure tracing enabled.") diff --git a/scenarios/langchain/tracing-with-langchain.ipynb b/scenarios/langchain/tracing-with-langchain.ipynb index 98163df8..752f3492 100644 --- a/scenarios/langchain/tracing-with-langchain.ipynb +++ b/scenarios/langchain/tracing-with-langchain.ipynb @@ -135,9 +135,9 @@ "metadata": {}, "outputs": [], "source": [ - "from langchain_azure_ai.callbacks.tracers import AzureAIInferenceTracer\n", + "from langchain_azure_ai.callbacks.tracers import AzureAIOpenTelemetryTracer\n", "\n", - "tracer = AzureAIInferenceTracer(\n", + "tracer = AzureAIOpenTelemetryTracer(\n", " connection_string=application_insights_connection_string,\n", " enable_content_recording=True,\n", ")"