mirror of
https://github.com/microsoft/agent-framework.git
synced 2026-06-16 21:04:09 +08:00
Python: Telemetry and observability follow-up (#833)
* updated telemetry work * updated telemetry * slight improvement * updated tests * fixes for telemetry * fixes for mypy * added settings setup to runner to avoid error * streamline usage * updated tests * updated tests * further refinement * fix dumped item for otel * removed enable_workflow_otel * final fixes * final fixes * updated samples * removed exporters * fix tests * fixed last import' * fixed devui
This commit is contained in:
Eduard van Valkenburg
committed by
GitHub
Unverified
GitHub
parent
f93f16a9ad
commit
2576e7a091
@@ -0,0 +1,12 @@
|
||||
# Connector environment variables
|
||||
# Foundry
|
||||
# see ../../../env.example for details
|
||||
# OpenAI
|
||||
# see ../../../env.example for details
|
||||
|
||||
# Otel specific variables
|
||||
APPLICATIONINSIGHTS_CONNECTION_STRING="..."
|
||||
APPLICATIONINSIGHTS_LIVE_METRICS=true
|
||||
OTLP_ENDPOINT="http://localhost:4317/"
|
||||
ENABLE_OTEL=true
|
||||
ENABLE_SENSITIVE_DATA=true
|
||||
@@ -0,0 +1,82 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
# type: ignore
|
||||
import asyncio
|
||||
from random import randint
|
||||
from typing import TYPE_CHECKING, Annotated
|
||||
|
||||
from agent_framework.openai import OpenAIResponsesClient
|
||||
from pydantic import Field
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from agent_framework import ChatClientProtocol
|
||||
|
||||
|
||||
"""
|
||||
This is the simplest sample of using the Agent Framework with telemetry.
|
||||
|
||||
This relies on the environment setting up the telemetry, you can test this with
|
||||
by navigating to this folder and running:
|
||||
uv run --env-file=zero_code.env opentelemetry-instrument python 01-zero_code.py
|
||||
|
||||
Check the zero_code.env file for the settings used in this example and to adapt it to your environment.
|
||||
"""
|
||||
|
||||
|
||||
async def get_weather(
|
||||
location: Annotated[str, Field(description="The location to get the weather for.")],
|
||||
) -> str:
|
||||
"""Get the weather for a given location."""
|
||||
await asyncio.sleep(randint(0, 10) / 10.0) # Simulate a network call
|
||||
conditions = ["sunny", "cloudy", "rainy", "stormy"]
|
||||
return f"The weather in {location} is {conditions[randint(0, 3)]} with a high of {randint(10, 30)}°C."
|
||||
|
||||
|
||||
async def run_chat_client(client: "ChatClientProtocol", stream: bool = False) -> None:
|
||||
"""Run an AI service.
|
||||
|
||||
This function runs an AI service and prints the output.
|
||||
Telemetry will be collected for the service execution behind the scenes,
|
||||
and the traces will be sent to the configured telemetry backend.
|
||||
|
||||
The telemetry will include information about the AI service execution.
|
||||
|
||||
Args:
|
||||
stream: Whether to use streaming for the plugin
|
||||
|
||||
Remarks:
|
||||
When function calling is outside the open telemetry loop
|
||||
each of the call to the model is handled as a seperate span,
|
||||
while when the open telemetry is put last, a single span
|
||||
is shown, which might include one or more rounds of function calling.
|
||||
|
||||
So for the scenario below, you should see the following:
|
||||
|
||||
2 spans with gen_ai.operation.name=chat
|
||||
The first has finish_reason "tool_calls"
|
||||
The second has finish_reason "stop"
|
||||
2 spans with gen_ai.operation.name=execute_tool
|
||||
|
||||
"""
|
||||
message = "What's the weather in Amsterdam and in Paris?"
|
||||
print(f"User: {message}")
|
||||
if stream:
|
||||
print("Assistant: ", end="")
|
||||
async for chunk in client.get_streaming_response(message, tools=get_weather):
|
||||
if str(chunk):
|
||||
print(str(chunk), end="")
|
||||
print("")
|
||||
else:
|
||||
response = await client.get_response(message, tools=get_weather)
|
||||
print(f"Assistant: {response}")
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
client = OpenAIResponsesClient()
|
||||
|
||||
# Scenarios where telemetry is collected in the SDK, from the most basic to the most complex.
|
||||
await run_chat_client(client, stream=True)
|
||||
await run_chat_client(client, stream=False)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,126 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
# type: ignore
|
||||
import argparse
|
||||
import asyncio
|
||||
from contextlib import suppress
|
||||
from random import randint
|
||||
from typing import TYPE_CHECKING, Annotated, Literal
|
||||
|
||||
from agent_framework import ai_function
|
||||
from agent_framework.observability import get_tracer, setup_observability
|
||||
from agent_framework.openai import OpenAIResponsesClient
|
||||
from opentelemetry import trace
|
||||
from opentelemetry.trace.span import format_trace_id
|
||||
from pydantic import Field
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from agent_framework import ChatClientProtocol
|
||||
|
||||
"""
|
||||
This sample, show how you can get telemetry from a chat client and tool.
|
||||
it uses the `tracer` that is configured by agent framework,
|
||||
which also sets up the traces with the configured environment.
|
||||
"""
|
||||
|
||||
|
||||
# Define the scenarios that can be run
|
||||
SCENARIOS = ["chat_client", "chat_client_stream", "ai_function", "all"]
|
||||
|
||||
|
||||
async def get_weather(
|
||||
location: Annotated[str, Field(description="The location to get the weather for.")],
|
||||
) -> str:
|
||||
"""Get the weather for a given location."""
|
||||
await asyncio.sleep(randint(0, 10) / 10.0) # Simulate a network call
|
||||
conditions = ["sunny", "cloudy", "rainy", "stormy"]
|
||||
return f"The weather in {location} is {conditions[randint(0, 3)]} with a high of {randint(10, 30)}°C."
|
||||
|
||||
|
||||
async def run_chat_client(client: "ChatClientProtocol", stream: bool = False) -> None:
|
||||
"""Run an AI service.
|
||||
|
||||
This function runs an AI service and prints the output.
|
||||
Telemetry will be collected for the service execution behind the scenes,
|
||||
and the traces will be sent to the configured telemetry backend.
|
||||
|
||||
The telemetry will include information about the AI service execution.
|
||||
|
||||
Args:
|
||||
client: The chat client to use.
|
||||
stream: Whether to use streaming for the response
|
||||
|
||||
Remarks:
|
||||
For the scenario below, you should see the following:
|
||||
1 Client span, with 4 children:
|
||||
2 Internal span with gen_ai.operation.name=chat
|
||||
The first has finish_reason "tool_calls"
|
||||
The second has finish_reason "stop"
|
||||
2 Internal span with gen_ai.operation.name=execute_tool
|
||||
|
||||
"""
|
||||
scenario_name = "Chat Client Stream" if stream else "Chat Client"
|
||||
with get_tracer().start_as_current_span(name=f"Scenario: {scenario_name}", kind=trace.SpanKind.CLIENT):
|
||||
print("Running scenario:", scenario_name)
|
||||
message = "What's the weather in Amsterdam and in Paris?"
|
||||
print(f"User: {message}")
|
||||
if stream:
|
||||
print("Assistant: ", end="")
|
||||
async for chunk in client.get_streaming_response(message, tools=get_weather):
|
||||
if str(chunk):
|
||||
print(str(chunk), end="")
|
||||
print("")
|
||||
else:
|
||||
response = await client.get_response(message, tools=get_weather)
|
||||
print(f"Assistant: {response}")
|
||||
|
||||
|
||||
async def run_ai_function() -> None:
|
||||
"""Run a AI function.
|
||||
|
||||
This function runs a AI function and prints the output.
|
||||
Telemetry will be collected for the function execution behind the scenes,
|
||||
and the traces will be sent to the configured telemetry backend.
|
||||
|
||||
The telemetry will include information about the AI function execution
|
||||
and the AI service execution.
|
||||
"""
|
||||
with get_tracer().start_as_current_span("Scenario: AI Function", kind=trace.SpanKind.CLIENT):
|
||||
print("Running scenario: AI Function")
|
||||
func = ai_function(get_weather)
|
||||
weather = await func.invoke(location="Amsterdam")
|
||||
print(f"Weather in Amsterdam:\n{weather}")
|
||||
|
||||
|
||||
async def main(scenario: Literal["chat_client", "chat_client_stream", "ai_function", "all"] = "all"):
|
||||
"""Run the selected scenario(s)."""
|
||||
setup_observability()
|
||||
with get_tracer().start_as_current_span("Sample Scenario's", kind=trace.SpanKind.CLIENT) as current_span:
|
||||
print(f"Trace ID: {format_trace_id(current_span.get_span_context().trace_id)}")
|
||||
|
||||
client = OpenAIResponsesClient()
|
||||
|
||||
# Scenarios where telemetry is collected in the SDK, from the most basic to the most complex.
|
||||
if scenario == "ai_function" or scenario == "all":
|
||||
with suppress(Exception):
|
||||
await run_ai_function()
|
||||
if scenario == "chat_client_stream" or scenario == "all":
|
||||
with suppress(Exception):
|
||||
await run_chat_client(client, stream=True)
|
||||
if scenario == "chat_client" or scenario == "all":
|
||||
with suppress(Exception):
|
||||
await run_chat_client(client, stream=False)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
arg_parser = argparse.ArgumentParser()
|
||||
|
||||
arg_parser.add_argument(
|
||||
"--scenario",
|
||||
type=str,
|
||||
choices=SCENARIOS,
|
||||
default="all",
|
||||
help="The scenario to run. Default is all.",
|
||||
)
|
||||
|
||||
args = arg_parser.parse_args()
|
||||
asyncio.run(main(args.scenario))
|
||||
@@ -0,0 +1,88 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
# type: ignore
|
||||
import asyncio
|
||||
import os
|
||||
from random import randint
|
||||
from typing import Annotated
|
||||
|
||||
from agent_framework import HostedCodeInterpreterTool
|
||||
from agent_framework.foundry import FoundryChatClient
|
||||
from agent_framework.observability import get_tracer, setup_observability
|
||||
from azure.ai.projects.aio import AIProjectClient
|
||||
from azure.identity.aio import AzureCliCredential
|
||||
from opentelemetry.trace import SpanKind
|
||||
from opentelemetry.trace.span import format_trace_id
|
||||
from pydantic import Field
|
||||
|
||||
"""
|
||||
This sample, shows you can leverage the built-in telemetry in Foundry.
|
||||
It uses the Foundry client to setup the telemetry, this calls
|
||||
out to Foundry for a telemetry connection strings,
|
||||
and then call the setup_observability function in the agent framework.
|
||||
If you want to compare with the trace sent to a generic OTLP endpoint,
|
||||
switch the `use_foundry_telemetry` variable to False.
|
||||
"""
|
||||
|
||||
|
||||
# ANSI color codes for printing in blue and resetting after each print
|
||||
BLUE = "\x1b[34m"
|
||||
RESET = "\x1b[0m"
|
||||
|
||||
|
||||
async def get_weather(
|
||||
location: Annotated[str, Field(description="The location to get the weather for.")],
|
||||
) -> str:
|
||||
"""Get the weather for a given location."""
|
||||
await asyncio.sleep(randint(0, 10) / 10.0) # Simulate a network call
|
||||
conditions = ["sunny", "cloudy", "rainy", "stormy"]
|
||||
return f"The weather in {location} is {conditions[randint(0, 3)]} with a high of {randint(10, 30)}°C."
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
"""Run an AI service.
|
||||
|
||||
This function runs an AI service and prints the output.
|
||||
Telemetry will be collected for the service execution behind the scenes,
|
||||
and the traces will be sent to the configured telemetry backend.
|
||||
|
||||
The telemetry will include information about the AI service execution.
|
||||
|
||||
In foundry you will also see specific operations happening that are called by the Foundry implementation,
|
||||
such as `create_agent`.
|
||||
"""
|
||||
use_foundry_obs = True
|
||||
questions = [
|
||||
"What's the weather in Amsterdam and in Paris?",
|
||||
"Why is the sky blue?",
|
||||
"Tell me about AI.",
|
||||
"Can you write a python function that adds two numbers? and use it to add 8483 and 5692?",
|
||||
]
|
||||
async with (
|
||||
AzureCliCredential() as credential,
|
||||
AIProjectClient(endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], credential=credential) as project,
|
||||
FoundryChatClient(client=project, setup_tracing=False) as client,
|
||||
):
|
||||
if use_foundry_obs:
|
||||
await client.setup_foundry_observability(enable_live_metrics=True)
|
||||
else:
|
||||
setup_observability()
|
||||
|
||||
with get_tracer().start_as_current_span(
|
||||
name="Foundry Telemetry from Agent Framework", kind=SpanKind.CLIENT
|
||||
) as span:
|
||||
for question in questions:
|
||||
print(f"{BLUE}User: {question}{RESET}")
|
||||
print(f"{BLUE}Assistant: {RESET}", end="")
|
||||
async for chunk in client.get_streaming_response(
|
||||
question, tools=[get_weather, HostedCodeInterpreterTool()]
|
||||
):
|
||||
if str(chunk):
|
||||
print(f"{BLUE}{str(chunk)}{RESET}", end="")
|
||||
print(f"{BLUE}{RESET}")
|
||||
|
||||
print(f"{BLUE}Done{RESET}")
|
||||
print(f"{BLUE}Operation ID: {format_trace_id(span.get_span_context().trace_id)}{RESET}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,56 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
# type: ignore
|
||||
import asyncio
|
||||
from random import randint
|
||||
from typing import Annotated
|
||||
|
||||
from agent_framework import ChatAgent
|
||||
from agent_framework.observability import get_tracer, setup_observability
|
||||
from agent_framework.openai import OpenAIChatClient
|
||||
from opentelemetry.trace import SpanKind
|
||||
from pydantic import Field
|
||||
|
||||
"""
|
||||
This sample shows you can can setup telemetry with a agent.
|
||||
The agent invoke is a additional Semantic Convention that now
|
||||
will wrap the calls made by the underlying chat client and tools.
|
||||
"""
|
||||
|
||||
|
||||
async def get_weather(
|
||||
location: Annotated[str, Field(description="The location to get the weather for.")],
|
||||
) -> str:
|
||||
"""Get the weather for a given location."""
|
||||
await asyncio.sleep(randint(0, 10) / 10.0) # Simulate a network call
|
||||
conditions = ["sunny", "cloudy", "rainy", "stormy"]
|
||||
return f"The weather in {location} is {conditions[randint(0, 3)]} with a high of {randint(10, 30)}°C."
|
||||
|
||||
|
||||
async def main():
|
||||
# Set up the telemetry
|
||||
|
||||
questions = ["What's the weather in Amsterdam?", "and in Paris, and which is better?", "Why is the sky blue?"]
|
||||
setup_observability()
|
||||
with get_tracer().start_as_current_span("Scenario: Agent Chat", kind=SpanKind.CLIENT):
|
||||
print("Running scenario: Agent Chat")
|
||||
print("Welcome to the chat, type 'exit' to quit.")
|
||||
agent = ChatAgent(
|
||||
chat_client=OpenAIChatClient(),
|
||||
tools=get_weather,
|
||||
name="WeatherAgent",
|
||||
instructions="You are a weather assistant.",
|
||||
)
|
||||
thread = agent.get_new_thread()
|
||||
for question in questions:
|
||||
print(f"User: {question}")
|
||||
print(f"{agent.display_name}: ", end="")
|
||||
async for update in agent.run_stream(
|
||||
question,
|
||||
thread=thread,
|
||||
):
|
||||
if update.text:
|
||||
print(update.text, end="")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,64 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
# type: ignore
|
||||
import asyncio
|
||||
import os
|
||||
from random import randint
|
||||
from typing import Annotated
|
||||
|
||||
from agent_framework import ChatAgent
|
||||
from agent_framework.observability import get_tracer
|
||||
from agent_framework_foundry import FoundryChatClient
|
||||
from azure.ai.projects.aio import AIProjectClient
|
||||
from azure.identity.aio import AzureCliCredential
|
||||
from opentelemetry.trace import SpanKind
|
||||
from pydantic import Field
|
||||
|
||||
"""
|
||||
This sample shows you can can setup telemetry with a agent from Foundry.
|
||||
We once again call the `setup_foundry_observability` method to set up telemetry in order to include the overall spans.
|
||||
"""
|
||||
|
||||
|
||||
async def get_weather(
|
||||
location: Annotated[str, Field(description="The location to get the weather for.")],
|
||||
) -> str:
|
||||
"""Get the weather for a given location."""
|
||||
await asyncio.sleep(randint(0, 10) / 10.0) # Simulate a network call
|
||||
conditions = ["sunny", "cloudy", "rainy", "stormy"]
|
||||
return f"The weather in {location} is {conditions[randint(0, 3)]} with a high of {randint(10, 30)}°C."
|
||||
|
||||
|
||||
async def main():
|
||||
# Set up the providers
|
||||
# This must be done before any other telemetry calls
|
||||
questions = ["What's the weather in Amsterdam?", "and in Paris, and which is better?", "Why is the sky blue?"]
|
||||
async with (
|
||||
AzureCliCredential() as credential,
|
||||
AIProjectClient(endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], credential=credential) as project,
|
||||
# this calls `setup_foundry_observability` through the context manager
|
||||
FoundryChatClient(client=project) as client,
|
||||
):
|
||||
await client.setup_foundry_observability(enable_live_metrics=True)
|
||||
with get_tracer().start_as_current_span("Single Agent Chat", kind=SpanKind.CLIENT):
|
||||
print("Running Single Agent Chat")
|
||||
print("Welcome to the chat, type 'exit' to quit.")
|
||||
agent = ChatAgent(
|
||||
chat_client=client,
|
||||
tools=get_weather,
|
||||
name="WeatherAgent",
|
||||
instructions="You are a weather assistant.",
|
||||
)
|
||||
thread = agent.get_new_thread()
|
||||
for question in questions:
|
||||
print(f"User: {question}")
|
||||
print(f"{agent.display_name}: ", end="")
|
||||
async for update in agent.run_stream(
|
||||
question,
|
||||
thread=thread,
|
||||
):
|
||||
if update.text:
|
||||
print(update.text, end="")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,115 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
# type: ignore
|
||||
import asyncio
|
||||
from typing import Any
|
||||
|
||||
from agent_framework import (
|
||||
Executor,
|
||||
WorkflowBuilder,
|
||||
WorkflowCompletedEvent,
|
||||
WorkflowContext,
|
||||
handler,
|
||||
)
|
||||
from agent_framework.observability import get_tracer, setup_observability
|
||||
from opentelemetry.trace import SpanKind
|
||||
from opentelemetry.trace.span import format_trace_id
|
||||
|
||||
"""Telemetry sample demonstrating OpenTelemetry integration with Agent Framework workflows.
|
||||
|
||||
This sample runs a simple sequential workflow with telemetry collection,
|
||||
showing telemetry collection for workflow execution, executor processing,
|
||||
and message publishing between executors.
|
||||
"""
|
||||
tracer = get_tracer("agent_framework.workflow")
|
||||
|
||||
|
||||
# Executors for sequential workflow
|
||||
class UpperCaseExecutor(Executor):
|
||||
"""An executor that converts text to uppercase."""
|
||||
|
||||
@handler
|
||||
async def to_upper_case(self, text: str, ctx: WorkflowContext[str]) -> None:
|
||||
"""Execute the task by converting the input string to uppercase."""
|
||||
print(f"UpperCaseExecutor: Processing '{text}'")
|
||||
result = text.upper()
|
||||
print(f"UpperCaseExecutor: Result '{result}'")
|
||||
|
||||
# Send the result to the next executor in the workflow.
|
||||
await ctx.send_message(result)
|
||||
|
||||
|
||||
class ReverseTextExecutor(Executor):
|
||||
"""An executor that reverses text."""
|
||||
|
||||
@handler
|
||||
async def reverse_text(self, text: str, ctx: WorkflowContext[Any]) -> None:
|
||||
"""Execute the task by reversing the input string."""
|
||||
print(f"ReverseTextExecutor: Processing '{text}'")
|
||||
result = text[::-1]
|
||||
print(f"ReverseTextExecutor: Result '{result}'")
|
||||
|
||||
# Send the result with a workflow completion event.
|
||||
await ctx.add_event(WorkflowCompletedEvent(result))
|
||||
|
||||
|
||||
async def run_sequential_workflow() -> None:
|
||||
"""Run a simple sequential workflow demonstrating telemetry collection.
|
||||
|
||||
This workflow processes a string through two executors in sequence:
|
||||
1. UpperCaseExecutor converts the input to uppercase
|
||||
2. ReverseTextExecutor reverses the string and completes the workflow
|
||||
|
||||
Telemetry data collected includes:
|
||||
- Overall workflow execution spans
|
||||
- Individual executor processing spans
|
||||
- Message publishing between executors
|
||||
- Workflow completion events
|
||||
"""
|
||||
with tracer.start_as_current_span("Scenario: Sequential Workflow", kind=SpanKind.CLIENT) as current_span:
|
||||
print("Running scenario: Sequential Workflow")
|
||||
try:
|
||||
# Step 1: Create the executors.
|
||||
upper_case_executor = UpperCaseExecutor(id="upper_case_executor")
|
||||
reverse_text_executor = ReverseTextExecutor(id="reverse_text_executor")
|
||||
|
||||
# Step 2: Build the workflow with the defined edges.
|
||||
workflow = (
|
||||
WorkflowBuilder()
|
||||
.add_edge(upper_case_executor, reverse_text_executor)
|
||||
.set_start_executor(upper_case_executor)
|
||||
.build()
|
||||
)
|
||||
|
||||
# Step 3: Run the workflow with an initial message.
|
||||
input_text = "hello world"
|
||||
print(f"Starting workflow with input: '{input_text}'")
|
||||
|
||||
completion_event = None
|
||||
async for event in workflow.run_stream(input_text):
|
||||
print(f"Event: {event}")
|
||||
if isinstance(event, WorkflowCompletedEvent):
|
||||
# The WorkflowCompletedEvent contains the final result.
|
||||
completion_event = event
|
||||
|
||||
if completion_event:
|
||||
print(f"Workflow completed with result: '{completion_event.data}'")
|
||||
else:
|
||||
print("Workflow completed without a completion event")
|
||||
|
||||
except Exception as e:
|
||||
current_span.record_exception(e)
|
||||
print(f"Error running workflow: {e}")
|
||||
|
||||
|
||||
async def main():
|
||||
"""Run the telemetry sample with a simple sequential workflow."""
|
||||
setup_observability()
|
||||
with tracer.start_as_current_span("Sequential Workflow Scenario", kind=SpanKind.CLIENT) as current_span:
|
||||
print(f"Trace ID: {format_trace_id(current_span.get_span_context().trace_id)}")
|
||||
|
||||
# Run the sequential workflow scenario
|
||||
await run_sequential_workflow()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,188 @@
|
||||
# Agent Framework Python Observability
|
||||
|
||||
This sample folder shows how a Python application can be configured to send Agent Framework observability data to the Application Performance Management (APM) vendor(s) of your choice based on the Open Telemetry standard.
|
||||
|
||||
In this sample, we provide options to send telemetry to [Application Insights](https://learn.microsoft.com/en-us/azure/azure-monitor/app/app-insights-overview), [Aspire Dashboard](https://learn.microsoft.com/en-us/dotnet/aspire/fundamentals/dashboard/overview?tabs=bash) and the console.
|
||||
|
||||
> **Quick Start**: For local development without Azure setup, you can use the [Aspire Dashboard](https://learn.microsoft.com/en-us/dotnet/aspire/fundamentals/dashboard/standalone) which runs locally via Docker and provides an excellent telemetry viewing experience for OpenTelemetry data.
|
||||
Or you can use the built-in tracing module of the [AI Toolkit for VS Code](https://marketplace.visualstudio.com/items?itemName=ms-windows-ai-studio.windows-ai-studio).
|
||||
|
||||
> Note that it is also possible to use other Application Performance Management (APM) vendors. An example is [Prometheus](https://prometheus.io/docs/introduction/overview/). Please refer to this [link](https://opentelemetry.io/docs/languages/python/exporters/) to learn more about exporters.
|
||||
|
||||
For more information, please refer to the following resources:
|
||||
|
||||
1. [Azure Monitor OpenTelemetry Exporter](https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/monitor/azure-monitor-opentelemetry-exporter)
|
||||
2. [Aspire Dashboard for Python Apps](https://learn.microsoft.com/en-us/dotnet/aspire/fundamentals/dashboard/standalone-for-python?tabs=flask%2Cwindows)
|
||||
2. [AI Toolkit for VS Code](https://marketplace.visualstudio.com/items?itemName=ms-windows-ai-studio.windows-ai-studio)
|
||||
3. [Python Logging](https://docs.python.org/3/library/logging.html)
|
||||
4. [Observability in Python](https://www.cncf.io/blog/2022/04/22/opentelemetry-and-python-a-complete-instrumentation-guide/)
|
||||
|
||||
## What to expect
|
||||
|
||||
The Agent Framework Python SDK is designed to efficiently generate comprehensive logs, traces, and metrics throughout the flow of function execution and model invocation. This allows you to effectively monitor your AI application's performance and accurately track token consumption. It does so based on the Semantic Conventions for GenAI defined by OpenTelemetry, and the workflows emit their own spans to provide end-to-end visibility.
|
||||
|
||||
## Configuration
|
||||
|
||||
### Required resources
|
||||
|
||||
2. OpenAI or [Azure OpenAI](https://learn.microsoft.com/en-us/azure/ai-services/openai/how-to/create-resource?pivots=web-portal)
|
||||
2. [Foundry project](https://ai.azure.com/doc/azure/ai-foundry/what-is-azure-ai-foundry)
|
||||
|
||||
### Optional resources
|
||||
|
||||
1. [Application Insights](https://learn.microsoft.com/en-us/azure/azure-monitor/app/create-workspace-resource)
|
||||
2. [Aspire Dashboard](https://learn.microsoft.com/en-us/dotnet/aspire/fundamentals/dashboard/standalone-for-python?tabs=flask%2Cwindows#start-the-aspire-dashboard)
|
||||
|
||||
### Dependencies
|
||||
No additional dependencies are required to enable telemetry. The necessary packages are included as part of the `agent-framework` package. Unless you want to use a different APM vendor, in which case you will need to install the appropriate OpenTelemetry exporter package.
|
||||
|
||||
### Environment variables
|
||||
The following environment variables can be set to configure telemetry, the first two set the basic configuration:
|
||||
|
||||
- ENABLE_OTEL=true
|
||||
- ENABLE_SENSITIVE_DATA=true
|
||||
|
||||
Next we need to know where to send the telemetry, for that you can use either a OTLP endpoint or a connection string for Application Insights:
|
||||
- OTLP_ENDPOINT="<url to OTLP endpoint>"
|
||||
or
|
||||
- APPLICATIONINSIGHTS_CONNECTION_STRING="<connection string>"
|
||||
Finally, you can enable live metrics streaming to Application Insights:
|
||||
- APPLICATIONINSIGHTS_LIVE_METRICS=true
|
||||
|
||||
## Samples
|
||||
This folder contains different samples demonstrating how to use telemetry in various scenarios.
|
||||
|
||||
### [01 - zero_code](./01-zero_code.py):
|
||||
A simple example showing how to enable telemetry in a zero-touch scenario. When the above environment variables are set, telemetry will be automatically enabled, however since you do not define any overarching tracer, you will only see the spans for the specific calls to the chat client and tools.
|
||||
|
||||
### [02a](./02a-generic_chat_client.py) and [02b](./02b-foundry_chat_client.py) Chat Clients:
|
||||
These two samples show how to first setup the telemetry by manually importing the `setup_observability` function from the `agent_framework.observability` module and calling it. After this is done, the trace that get's created will live in the same context as the chat client calls, allowing you to see the end-to-end flow of your application. For Foundry, there is a method in the Foundry project client to get the azure monitor connection string for your project, the `.setup_foundry_observability()` method in the `FoundryChatClient` class will use this url to configure telemetry and you then do not have to import and call `setup_observability()` manually.
|
||||
If you or some other process already configure global tracer_providers or metrics_providers, the `setup_observability()` function will not override them, but instead use the existing tracer_provider, if possible. Metrics cannot be setup this way, so if you want to use metrics, you will have to call `setup_observability()` manually, before another process.
|
||||
|
||||
### [03a](./03a-generic_agent.py) and [03b](./03b-foundry_agent.py) Agents:
|
||||
These two samples show how to setup telemetry when using the Agent Framework's agent abstraction layer. They are similar to the chat client samples, but also show how to create an agent and invoke it. The same rules apply for setting up telemetry, you can either call `setup_observability()` manually, or use the `setup_foundry_observability()` method in the `FoundryChatClient` class.
|
||||
|
||||
### [04 - workflow](./04-workflow.py) Workflow:
|
||||
This sample shows how to setup telemetry when using the Agent Framework's workflow execution engine. It demonstrates a simple workflow scenario with telemetry.
|
||||
|
||||
|
||||
## Running the samples
|
||||
|
||||
1. Open a terminal and navigate to this folder: `python/samples/getting_started/observability/`. This is necessary for the `.env` file to be read correctly.
|
||||
2. Create a `.env` file if one doesn't already exist in this folder. Please refer to the [example file](./.env.example).
|
||||
> Note that `APPLICATIONINSIGHTS_CONNECTION_STRING` and `OTLP_ENDPOINT` are optional. If you don't configure them, everything will get outputted to the console.
|
||||
> Set `ENABLE_OTEL=true` to enable telemetry and `ENABLE_SENSITIVE_DATA=true` to include sensitive information like prompts and responses.
|
||||
> Sensitive information should only be enabled in a development or test environment. It is not recommended to enable this in production environments as it may expose sensitive data.
|
||||
3. Activate your python virtual environment, and then run `python 01-zero_code.py` or others.
|
||||
> This will also print the Operation/Trace ID, which can be used later for filtering.
|
||||
|
||||
## Application Insights/Azure Monitor
|
||||
|
||||
### Authentication
|
||||
|
||||
You can connect to your Application Insights instance using a connection string. You can also authenticate using Entra ID by passing a [TokenCredential](https://learn.microsoft.com/en-us/python/api/azure-core/azure.core.credentials.tokencredential?view=azure-python) to the `setup_observability()` function used in the samples above.
|
||||
|
||||
```python
|
||||
from azure.identity import DefaultAzureCredential
|
||||
|
||||
setup_observability(credential=DefaultAzureCredential())
|
||||
```
|
||||
|
||||
It is recommended to use [DefaultAzureCredential](https://learn.microsoft.com/en-us/python/api/azure-identity/azure.identity.defaultazurecredential?view=azure-python) for local development and [ManagedIdentityCredential](https://learn.microsoft.com/en-us/python/api/azure-identity/azure.identity.managedidentitycredential?view=azure-python) for production environments.
|
||||
|
||||
### Logs and traces
|
||||
|
||||
Go to your Application Insights instance, click on _Transaction search_ on the left menu. Use the operation id printed by the program to search for the logs and traces associated with the operation. Click on any of the search result to view the end-to-end transaction details. Read more [here](https://learn.microsoft.com/en-us/azure/azure-monitor/app/transaction-search-and-diagnostics?tabs=transaction-search).
|
||||
|
||||
### Metrics
|
||||
|
||||
Running the application once will only generate one set of measurements (for each metrics). Run the application a couple times to generate more sets of measurements.
|
||||
|
||||
> Note: Make sure not to run the program too frequently. Otherwise, you may get throttled.
|
||||
|
||||
Please refer to here on how to analyze metrics in [Azure Monitor](https://learn.microsoft.com/en-us/azure/azure-monitor/essentials/analyze-metrics).
|
||||
|
||||
### Adding exporters
|
||||
|
||||
You can also create exporters directly and have those added to the tracer_providers, logger_providers and metrics_providers, this is useful if you want to add a different exporter on the fly, or if you want to customize the exporter. Here is an example of how to create an OTLP exporter and add it to the observability setup:
|
||||
|
||||
```python
|
||||
from grpc import Compression
|
||||
from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
|
||||
from agent_framework.observability import setup_observability
|
||||
|
||||
exporter = OTLPSpanExporter(endpoint="your-otlp-endpoint", compression=Compression.Gzip)
|
||||
setup_observability(exporters=[exporter])
|
||||
```
|
||||
|
||||
## Logs
|
||||
|
||||
When you are in Azure Monitor and want to have a overall view of the span, use this query in the logs section:
|
||||
|
||||
```kusto
|
||||
dependencies
|
||||
| where operation_Id in (dependencies
|
||||
| project operation_Id, timestamp
|
||||
| order by timestamp desc
|
||||
| summarize operations = make_set(operation_Id), timestamp = max(timestamp) by operation_Id
|
||||
| order by timestamp desc
|
||||
| project operation_Id
|
||||
| take 2)
|
||||
| evaluate bag_unpack(customDimensions)
|
||||
| extend tool_call_id = tostring(["gen_ai.tool.call.id"])
|
||||
| join kind=leftouter (customMetrics
|
||||
| extend tool_call_id = tostring(customDimensions['gen_ai.tool.call.id'])
|
||||
| where isnotempty(tool_call_id)
|
||||
| project tool_call_duration = value, tool_call_id)
|
||||
on tool_call_id
|
||||
| project-keep timestamp, target, operation_Id, tool_call_duration, duration, gen_ai*
|
||||
| order by timestamp asc
|
||||
```
|
||||
|
||||
## Aspire Dashboard
|
||||
|
||||
The [Aspire Dashboard](https://learn.microsoft.com/en-us/dotnet/aspire/fundamentals/dashboard/standalone) is a local telemetry viewing tool that provides an excellent experience for viewing OpenTelemetry data without requiring Azure setup.
|
||||
|
||||
### Setting up Aspire Dashboard with Docker
|
||||
|
||||
The easiest way to run the Aspire Dashboard locally is using Docker:
|
||||
|
||||
```bash
|
||||
# Pull and run the Aspire Dashboard container
|
||||
docker run --rm -it -d \
|
||||
-p 18888:18888 \
|
||||
-p 4317:18889 \
|
||||
--name aspire-dashboard \
|
||||
mcr.microsoft.com/dotnet/aspire-dashboard:latest
|
||||
```
|
||||
|
||||
This will start the dashboard with:
|
||||
|
||||
- **Web UI**: Available at <http://localhost:18888>
|
||||
- **OTLP endpoint**: Available at `http://localhost:4317` for your applications to send telemetry data
|
||||
|
||||
### Configuring your application
|
||||
|
||||
Make sure your `.env` file includes the OTLP endpoint:
|
||||
|
||||
```bash
|
||||
OTLP_ENDPOINT=http://localhost:4317
|
||||
```
|
||||
|
||||
Or set it as an environment variable when running your samples:
|
||||
|
||||
```bash
|
||||
ENABLE_OTEL=true OTLP_ENDPOINT=http://localhost:4317 python 01-zero_code.py
|
||||
```
|
||||
|
||||
### Viewing telemetry data
|
||||
|
||||
> Make sure you have the dashboard running to receive telemetry data.
|
||||
|
||||
Once your sample finishes running, navigate to <http://localhost:18888> in a web browser to see the telemetry data. Follow the [Aspire Dashboard exploration guide](https://learn.microsoft.com/en-us/dotnet/aspire/fundamentals/dashboard/explore) to authenticate to the dashboard and start exploring your traces, logs, and metrics!
|
||||
|
||||
## Console output
|
||||
|
||||
You won't have to deploy an Application Insights resource or install Docker to run Aspire Dashboard if you choose to inspect telemetry data in a console. However, it is difficult to navigate through all the spans and logs produced, so **this method is only recommended when you are just getting started**.
|
||||
|
||||
Use the guides from OpenTelemetry to setup exporters for [the console](https://opentelemetry.io/docs/languages/python/getting-started/), or use [manual_setup_console_output](./manual_setup_console_output.py) as a reference, just know that there are a lot of options you can setup and this is not a comprehensive example.
|
||||
@@ -0,0 +1,114 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
# type: ignore
|
||||
import asyncio
|
||||
import logging
|
||||
from random import randint
|
||||
from typing import Annotated
|
||||
|
||||
from agent_framework.openai import OpenAIChatClient
|
||||
from opentelemetry._logs import set_logger_provider
|
||||
from opentelemetry.metrics import set_meter_provider
|
||||
from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
|
||||
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor, ConsoleLogExporter
|
||||
from opentelemetry.sdk.metrics import MeterProvider
|
||||
from opentelemetry.sdk.metrics.export import ConsoleMetricExporter, PeriodicExportingMetricReader
|
||||
from opentelemetry.sdk.resources import Resource
|
||||
from opentelemetry.sdk.trace import TracerProvider
|
||||
from opentelemetry.sdk.trace.export import BatchSpanProcessor, ConsoleSpanExporter
|
||||
from opentelemetry.semconv.resource import ResourceAttributes
|
||||
from opentelemetry.trace import set_tracer_provider
|
||||
from pydantic import Field
|
||||
|
||||
"""
|
||||
This sample shows how to manually set up OpenTelemetry to log to the console.
|
||||
And this can also be used as a reference for more complex telemetry setups.
|
||||
"""
|
||||
|
||||
resource = Resource.create({ResourceAttributes.SERVICE_NAME: "ManualSetup"})
|
||||
|
||||
|
||||
def setup_console_telemetry():
|
||||
# Create and set a global logger provider for the application.
|
||||
logger_provider = LoggerProvider(resource=resource)
|
||||
# Log processors are initialized with an exporter which is responsible
|
||||
logger_provider.add_log_record_processor(BatchLogRecordProcessor(ConsoleLogExporter()))
|
||||
# Sets the global default logger provider
|
||||
set_logger_provider(logger_provider)
|
||||
|
||||
# Create a logging handler to write logging records, in OTLP format, to the exporter.
|
||||
handler = LoggingHandler()
|
||||
# Attach the handler to the root logger. `getLogger()` with no arguments returns the root logger.
|
||||
# Events from all child loggers will be processed by this handler.
|
||||
logger = logging.getLogger()
|
||||
logger.addHandler(handler)
|
||||
# Set the logging level to NOTSET to allow all records to be processed by the handler.
|
||||
logger.setLevel(logging.NOTSET)
|
||||
# Initialize a trace provider for the application. This is a factory for creating tracers.
|
||||
tracer_provider = TracerProvider(resource=resource)
|
||||
# Span processors are initialized with an exporter which is responsible
|
||||
# for sending the telemetry data to a particular backend.
|
||||
tracer_provider.add_span_processor(BatchSpanProcessor(ConsoleSpanExporter()))
|
||||
# Sets the global default tracer provider
|
||||
set_tracer_provider(tracer_provider)
|
||||
# Initialize a metric provider for the application. This is a factory for creating meters.
|
||||
meter_provider = MeterProvider(
|
||||
metric_readers=[PeriodicExportingMetricReader(ConsoleMetricExporter(), export_interval_millis=5000)],
|
||||
resource=resource,
|
||||
)
|
||||
# Sets the global default meter provider
|
||||
set_meter_provider(meter_provider)
|
||||
|
||||
|
||||
async def get_weather(
|
||||
location: Annotated[str, Field(description="The location to get the weather for.")],
|
||||
) -> str:
|
||||
"""Get the weather for a given location."""
|
||||
await asyncio.sleep(randint(0, 10) / 10.0) # Simulate a network call
|
||||
conditions = ["sunny", "cloudy", "rainy", "stormy"]
|
||||
return f"The weather in {location} is {conditions[randint(0, 3)]} with a high of {randint(10, 30)}°C."
|
||||
|
||||
|
||||
async def run_chat_client() -> None:
|
||||
"""Run an AI service.
|
||||
|
||||
This function runs an AI service and prints the output.
|
||||
Telemetry will be collected for the service execution behind the scenes,
|
||||
and the traces will be sent to the configured telemetry backend.
|
||||
|
||||
The telemetry will include information about the AI service execution.
|
||||
|
||||
Args:
|
||||
stream: Whether to use streaming for the plugin
|
||||
|
||||
Remarks:
|
||||
When function calling is outside the open telemetry loop
|
||||
each of the call to the model is handled as a seperate span,
|
||||
while when the open telemetry is put last, a single span
|
||||
is shown, which might include one or more rounds of function calling.
|
||||
|
||||
So for the scenario below, you should see the following:
|
||||
|
||||
2 spans with gen_ai.operation.name=chat
|
||||
The first has finish_reason "tool_calls"
|
||||
The second has finish_reason "stop"
|
||||
2 spans with gen_ai.operation.name=execute_tool
|
||||
|
||||
"""
|
||||
client = OpenAIChatClient()
|
||||
message = "What's the weather in Amsterdam and in Paris?"
|
||||
print(f"User: {message}")
|
||||
print("Assistant: ", end="")
|
||||
async for chunk in client.get_streaming_response(message, tools=get_weather):
|
||||
if str(chunk):
|
||||
print(str(chunk), end="")
|
||||
print("")
|
||||
|
||||
|
||||
async def main():
|
||||
"""Run the selected scenario(s)."""
|
||||
setup_console_telemetry()
|
||||
await run_chat_client()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,6 @@
|
||||
ENABLE_OTEL=true
|
||||
ENABLE_SENSITIVE_DATA=true
|
||||
OTEL_SERVICE_NAME=agent_framework
|
||||
OTEL_TRACES_EXPORTER=otlp
|
||||
OTEL_EXPORTER_OTLP_TRACES_ENDPOINT="http://localhost:4317/"
|
||||
# APPLICATIONINSIGHTS_CONNECTION_STRING=""
|
||||
Reference in New Issue
Block a user