mirror of
https://github.com/microsoft/agent-framework.git
synced 2026-06-16 21:04:09 +08:00
Enable instrumentation by default
This commit is contained in:
+1
-2
@@ -44,7 +44,6 @@ GEMINI_MODEL=""
|
||||
# Ollama
|
||||
OLLAMA_ENDPOINT=""
|
||||
OLLAMA_MODEL=""
|
||||
# Observability
|
||||
ENABLE_INSTRUMENTATION=true
|
||||
# Observability (instrumentation is enabled by default; set "ENABLE_INSTRUMENTATION" to "false" to opt out)
|
||||
ENABLE_SENSITIVE_DATA=true
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT="http://localhost:4317/"
|
||||
|
||||
+1
-1
@@ -72,7 +72,7 @@ def equal(arg1: str, arg2: str) -> bool:
|
||||
from agent_framework import Agent, Message, tool
|
||||
|
||||
# Components
|
||||
from agent_framework.observability import enable_instrumentation
|
||||
from agent_framework.observability import enable_sensitive_telemetry
|
||||
|
||||
# Connectors (lazy-loaded)
|
||||
from agent_framework.openai import OpenAIChatClient
|
||||
|
||||
@@ -186,7 +186,7 @@ The package follows a flat import structure:
|
||||
|
||||
- **Components**: Import from `agent_framework.<component>`
|
||||
```python
|
||||
from agent_framework.observability import enable_instrumentation, configure_otel_providers
|
||||
from agent_framework.observability import enable_sensitive_telemetry, configure_otel_providers
|
||||
```
|
||||
|
||||
- **Connectors**: Import from `agent_framework.<vendor/platform>`
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
"""Observability and OpenTelemetry helpers for Agent Framework.
|
||||
|
||||
Commonly used exports:
|
||||
- enable_instrumentation
|
||||
- enable_sensitive_telemetry
|
||||
- configure_otel_providers
|
||||
- AgentTelemetryLayer
|
||||
- ChatTelemetryLayer
|
||||
@@ -80,7 +80,7 @@ __all__ = [
|
||||
"configure_otel_providers",
|
||||
"create_metric_views",
|
||||
"create_resource",
|
||||
"enable_instrumentation",
|
||||
"enable_sensitive_telemetry",
|
||||
"get_meter",
|
||||
"get_tracer",
|
||||
]
|
||||
@@ -643,8 +643,8 @@ class ObservabilitySettings:
|
||||
Sensitive events should only be enabled on test and development environments.
|
||||
|
||||
Keyword Args:
|
||||
enable_instrumentation: Enable OpenTelemetry diagnostics. Default is False.
|
||||
Can be set via environment variable ENABLE_INSTRUMENTATION.
|
||||
enable_instrumentation: Enable OpenTelemetry diagnostics. Default is True.
|
||||
Can be disabled by setting environment variable ENABLE_INSTRUMENTATION=false.
|
||||
enable_sensitive_data: Enable OpenTelemetry sensitive events. Default is False.
|
||||
Can be set via environment variable ENABLE_SENSITIVE_DATA.
|
||||
enable_console_exporters: Enable console exporters for traces, logs, and metrics.
|
||||
@@ -659,12 +659,12 @@ class ObservabilitySettings:
|
||||
from agent_framework import ObservabilitySettings
|
||||
|
||||
# Using environment variables
|
||||
# Set ENABLE_INSTRUMENTATION=true
|
||||
# Instrumentation is enabled by default; set ENABLE_INSTRUMENTATION=false to disable.
|
||||
# Set ENABLE_CONSOLE_EXPORTERS=true
|
||||
settings = ObservabilitySettings()
|
||||
|
||||
# Or passing parameters directly
|
||||
settings = ObservabilitySettings(enable_instrumentation=True, enable_console_exporters=True)
|
||||
settings = ObservabilitySettings(enable_console_exporters=True)
|
||||
"""
|
||||
|
||||
def __init__(self, **kwargs: Any) -> None:
|
||||
@@ -677,7 +677,10 @@ class ObservabilitySettings:
|
||||
env_file_encoding=env_file_encoding,
|
||||
**kwargs,
|
||||
)
|
||||
self.enable_instrumentation: bool = data.get("enable_instrumentation") or False
|
||||
# `enable_instrumentation` is defaulted to True if not set
|
||||
instrumentation_value = data.get("enable_instrumentation")
|
||||
self.enable_instrumentation: bool = True if instrumentation_value is None else instrumentation_value
|
||||
|
||||
self.enable_sensitive_data: bool = data.get("enable_sensitive_data") or False
|
||||
self.enable_console_exporters: bool = data.get("enable_console_exporters") or False
|
||||
self.vs_code_extension_port: int | None = data.get("vs_code_extension_port")
|
||||
@@ -951,30 +954,22 @@ def _read_int_env(name: str, *, default: int | None = None) -> int | None:
|
||||
return default
|
||||
|
||||
|
||||
def enable_instrumentation(
|
||||
*,
|
||||
enable_sensitive_data: bool | None = None,
|
||||
) -> None:
|
||||
"""Enable instrumentation for your application.
|
||||
def enable_sensitive_telemetry() -> None:
|
||||
"""Enable capture of sensitive data in telemetry for your application.
|
||||
|
||||
Calling this method implies you want to enable observability in your application.
|
||||
Instrumentation is enabled by default; this method exists to opt-in to capturing
|
||||
sensitive event payloads (e.g., chat messages, tool arguments).
|
||||
|
||||
This method does not configure exporters or providers.
|
||||
It only updates the global variables that trigger the instrumentation code.
|
||||
If you have already set the environment variable ENABLE_INSTRUMENTATION=true,
|
||||
calling this method has no effect, unless you want to enable or disable sensitive data events.
|
||||
This method does not configure exporters or providers. It also ensures that
|
||||
instrumentation is enabled (in case it was explicitly disabled via the
|
||||
ENABLE_INSTRUMENTATION environment variable).
|
||||
|
||||
Keyword Args:
|
||||
enable_sensitive_data: Enable OpenTelemetry sensitive events. Overrides
|
||||
the environment variable ENABLE_SENSITIVE_DATA if set. Default is None.
|
||||
Warning:
|
||||
Sensitive events should only be enabled on test and development environments.
|
||||
"""
|
||||
global OBSERVABILITY_SETTINGS
|
||||
OBSERVABILITY_SETTINGS.enable_instrumentation = True
|
||||
if enable_sensitive_data is not None:
|
||||
OBSERVABILITY_SETTINGS.enable_sensitive_data = enable_sensitive_data
|
||||
else:
|
||||
# Re-read from current environment in case env vars were set after import (e.g. load_dotenv())
|
||||
OBSERVABILITY_SETTINGS.enable_sensitive_data = _read_bool_env("ENABLE_SENSITIVE_DATA")
|
||||
OBSERVABILITY_SETTINGS.enable_sensitive_data = True
|
||||
|
||||
|
||||
def configure_otel_providers(
|
||||
@@ -1008,7 +1003,7 @@ def configure_otel_providers(
|
||||
Since you can only setup one provider per signal type (logs, traces, metrics),
|
||||
you can choose to use this method and take the exporter and provider that we created.
|
||||
Alternatively, you can setup the providers yourself, or through another library
|
||||
(e.g., Azure Monitor) and just call `enable_instrumentation()` to enable instrumentation.
|
||||
(e.g., Azure Monitor) and just call `enable_sensitive_telemetry()` to opt-in to sensitive data capture.
|
||||
|
||||
Note:
|
||||
By default, the Agent Framework emits metrics with the prefixes `agent_framework`
|
||||
@@ -1042,7 +1037,6 @@ def configure_otel_providers(
|
||||
from agent_framework.observability import configure_otel_providers
|
||||
|
||||
# Using environment variables (recommended)
|
||||
# Set ENABLE_INSTRUMENTATION=true
|
||||
# Set OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317
|
||||
configure_otel_providers()
|
||||
|
||||
@@ -1087,12 +1081,13 @@ def configure_otel_providers(
|
||||
.. code-block:: python
|
||||
|
||||
# when azure monitor is installed
|
||||
from agent_framework.observability import enable_instrumentation
|
||||
from agent_framework.observability import enable_sensitive_telemetry
|
||||
from azure.monitor.opentelemetry import configure_azure_monitor
|
||||
|
||||
connection_string = "InstrumentationKey=your_instrumentation_key_here;..."
|
||||
configure_azure_monitor(connection_string=connection_string)
|
||||
enable_instrumentation()
|
||||
# Optional: opt into capturing sensitive data (dev/test only)
|
||||
enable_sensitive_telemetry()
|
||||
|
||||
References:
|
||||
- https://opentelemetry.io/docs/languages/sdk-configuration/general/
|
||||
|
||||
@@ -1015,11 +1015,8 @@ def test_observability_settings_is_setup_initial(monkeypatch):
|
||||
assert settings.is_setup is False
|
||||
|
||||
|
||||
# region Test enable_instrumentation function
|
||||
|
||||
|
||||
def test_enable_instrumentation_function(monkeypatch):
|
||||
"""Test enable_instrumentation function enables instrumentation."""
|
||||
def test_enable_sensitive_telemetry_function(monkeypatch):
|
||||
"""Test enable_sensitive_telemetry function enables instrumentation."""
|
||||
import importlib
|
||||
|
||||
monkeypatch.setenv("ENABLE_INSTRUMENTATION", "false")
|
||||
@@ -1030,41 +1027,7 @@ def test_enable_instrumentation_function(monkeypatch):
|
||||
|
||||
assert observability.OBSERVABILITY_SETTINGS.enable_instrumentation is False
|
||||
|
||||
observability.enable_instrumentation()
|
||||
assert observability.OBSERVABILITY_SETTINGS.enable_instrumentation is True
|
||||
|
||||
|
||||
def test_enable_instrumentation_with_sensitive_data(monkeypatch):
|
||||
"""Test enable_instrumentation function with sensitive_data parameter."""
|
||||
import importlib
|
||||
|
||||
monkeypatch.setenv("ENABLE_INSTRUMENTATION", "false")
|
||||
monkeypatch.setenv("ENABLE_SENSITIVE_DATA", "false")
|
||||
|
||||
observability = importlib.import_module("agent_framework.observability")
|
||||
importlib.reload(observability)
|
||||
|
||||
observability.enable_instrumentation(enable_sensitive_data=True)
|
||||
assert observability.OBSERVABILITY_SETTINGS.enable_instrumentation is True
|
||||
assert observability.OBSERVABILITY_SETTINGS.enable_sensitive_data is True
|
||||
|
||||
|
||||
def test_enable_instrumentation_reads_env_sensitive_data(monkeypatch):
|
||||
"""Test enable_instrumentation re-reads ENABLE_SENSITIVE_DATA from os.environ when not explicitly passed."""
|
||||
import importlib
|
||||
|
||||
monkeypatch.setenv("ENABLE_INSTRUMENTATION", "false")
|
||||
monkeypatch.setenv("ENABLE_SENSITIVE_DATA", "false")
|
||||
|
||||
observability = importlib.import_module("agent_framework.observability")
|
||||
importlib.reload(observability)
|
||||
|
||||
assert observability.OBSERVABILITY_SETTINGS.enable_sensitive_data is False
|
||||
|
||||
# Simulate load_dotenv() setting env var after import
|
||||
monkeypatch.setenv("ENABLE_SENSITIVE_DATA", "true")
|
||||
|
||||
observability.enable_instrumentation()
|
||||
observability.enable_sensitive_telemetry()
|
||||
assert observability.OBSERVABILITY_SETTINGS.enable_instrumentation is True
|
||||
assert observability.OBSERVABILITY_SETTINGS.enable_sensitive_data is True
|
||||
|
||||
@@ -1154,24 +1117,8 @@ def test_configure_otel_providers_explicit_param_overrides_env(monkeypatch):
|
||||
assert observability.OBSERVABILITY_SETTINGS.enable_sensitive_data is False
|
||||
|
||||
|
||||
def test_enable_instrumentation_explicit_param_overrides_env(monkeypatch):
|
||||
"""Test that explicit enable_sensitive_data parameter to enable_instrumentation overrides env var."""
|
||||
import importlib
|
||||
|
||||
monkeypatch.setenv("ENABLE_INSTRUMENTATION", "false")
|
||||
monkeypatch.setenv("ENABLE_SENSITIVE_DATA", "true")
|
||||
|
||||
observability = importlib.import_module("agent_framework.observability")
|
||||
importlib.reload(observability)
|
||||
|
||||
# Explicit False should override the env var True
|
||||
observability.enable_instrumentation(enable_sensitive_data=False)
|
||||
assert observability.OBSERVABILITY_SETTINGS.enable_instrumentation is True
|
||||
assert observability.OBSERVABILITY_SETTINGS.enable_sensitive_data is False
|
||||
|
||||
|
||||
def test_enable_instrumentation_does_not_touch_console_exporters(monkeypatch):
|
||||
"""Test enable_instrumentation does not modify enable_console_exporters (it is an exporter concern)."""
|
||||
def test_enable_sensitive_telemetry_does_not_touch_console_exporters(monkeypatch):
|
||||
"""Test enable_sensitive_telemetry does not modify enable_console_exporters (it is an exporter concern)."""
|
||||
import importlib
|
||||
|
||||
monkeypatch.setenv("ENABLE_INSTRUMENTATION", "false")
|
||||
@@ -1185,14 +1132,14 @@ def test_enable_instrumentation_does_not_touch_console_exporters(monkeypatch):
|
||||
# Simulate load_dotenv() setting env var after import
|
||||
monkeypatch.setenv("ENABLE_CONSOLE_EXPORTERS", "true")
|
||||
|
||||
observability.enable_instrumentation()
|
||||
# enable_console_exporters is not managed by enable_instrumentation;
|
||||
observability.enable_sensitive_telemetry()
|
||||
# enable_console_exporters is not managed by enable_sensitive_telemetry;
|
||||
# it is only read by configure_otel_providers.
|
||||
assert observability.OBSERVABILITY_SETTINGS.enable_console_exporters is False
|
||||
|
||||
|
||||
def test_enable_instrumentation_does_not_clobber_console_exporters(monkeypatch):
|
||||
"""Test enable_instrumentation does not reset enable_console_exporters set by prior configure call."""
|
||||
def test_enable_sensitive_telemetry_does_not_clobber_console_exporters(monkeypatch):
|
||||
"""Test enable_sensitive_telemetry does not reset enable_console_exporters set by prior configure call."""
|
||||
import importlib
|
||||
|
||||
monkeypatch.setenv("ENABLE_INSTRUMENTATION", "false")
|
||||
@@ -1215,42 +1162,13 @@ def test_enable_instrumentation_does_not_clobber_console_exporters(monkeypatch):
|
||||
observability.configure_otel_providers(enable_console_exporters=True)
|
||||
assert observability.OBSERVABILITY_SETTINGS.enable_console_exporters is True
|
||||
|
||||
# Calling enable_instrumentation should not clobber the value
|
||||
observability.enable_instrumentation()
|
||||
# Calling enable_sensitive_telemetry should not clobber the value
|
||||
observability.enable_sensitive_telemetry()
|
||||
assert observability.OBSERVABILITY_SETTINGS.enable_console_exporters is True
|
||||
|
||||
|
||||
def test_enable_instrumentation_with_sensitive_data_does_not_touch_console_exporters(monkeypatch):
|
||||
"""Test enable_console_exporters is untouched even when enable_sensitive_data is explicitly passed."""
|
||||
import importlib
|
||||
|
||||
monkeypatch.setenv("ENABLE_INSTRUMENTATION", "false")
|
||||
monkeypatch.delenv("ENABLE_CONSOLE_EXPORTERS", raising=False)
|
||||
monkeypatch.delenv("ENABLE_SENSITIVE_DATA", raising=False)
|
||||
monkeypatch.delenv("VS_CODE_EXTENSION_PORT", raising=False)
|
||||
for key in [
|
||||
"OTEL_EXPORTER_OTLP_ENDPOINT",
|
||||
"OTEL_EXPORTER_OTLP_TRACES_ENDPOINT",
|
||||
"OTEL_EXPORTER_OTLP_METRICS_ENDPOINT",
|
||||
"OTEL_EXPORTER_OTLP_LOGS_ENDPOINT",
|
||||
]:
|
||||
monkeypatch.delenv(key, raising=False)
|
||||
|
||||
observability = importlib.import_module("agent_framework.observability")
|
||||
importlib.reload(observability)
|
||||
|
||||
# Set console exporters via configure_otel_providers
|
||||
with patch.object(observability.OBSERVABILITY_SETTINGS, "_configure"):
|
||||
observability.configure_otel_providers(enable_console_exporters=True)
|
||||
assert observability.OBSERVABILITY_SETTINGS.enable_console_exporters is True
|
||||
|
||||
# Calling enable_instrumentation with explicit sensitive_data should not clobber console exporters
|
||||
observability.enable_instrumentation(enable_sensitive_data=True)
|
||||
assert observability.OBSERVABILITY_SETTINGS.enable_console_exporters is True
|
||||
|
||||
|
||||
def test_enable_instrumentation_preserves_console_exporters_after_env_removed(monkeypatch):
|
||||
"""Test enable_instrumentation preserves enable_console_exporters when env var is removed after reload."""
|
||||
def test_enable_sensitive_telemetry_preserves_console_exporters_after_env_removed(monkeypatch):
|
||||
"""Test enable_sensitive_telemetry preserves enable_console_exporters when env var is removed after reload."""
|
||||
import importlib
|
||||
|
||||
monkeypatch.setenv("ENABLE_INSTRUMENTATION", "false")
|
||||
@@ -1264,8 +1182,8 @@ def test_enable_instrumentation_preserves_console_exporters_after_env_removed(mo
|
||||
# Remove the env var after reload
|
||||
monkeypatch.delenv("ENABLE_CONSOLE_EXPORTERS", raising=False)
|
||||
|
||||
# enable_instrumentation should not reset the value
|
||||
observability.enable_instrumentation()
|
||||
# enable_sensitive_telemetry should not reset the value
|
||||
observability.enable_sensitive_telemetry()
|
||||
assert observability.OBSERVABILITY_SETTINGS.enable_console_exporters is True
|
||||
|
||||
|
||||
|
||||
@@ -94,7 +94,6 @@ def serve(
|
||||
auto_open: bool = False,
|
||||
cors_origins: list[str] | None = None,
|
||||
ui_enabled: bool = True,
|
||||
instrumentation_enabled: bool = False,
|
||||
mode: str = "developer",
|
||||
auth_enabled: bool = True,
|
||||
auth_token: str | None = None,
|
||||
@@ -109,7 +108,6 @@ def serve(
|
||||
auto_open: Whether to automatically open browser
|
||||
cors_origins: List of allowed CORS origins
|
||||
ui_enabled: Whether to enable the UI
|
||||
instrumentation_enabled: Whether to enable OpenTelemetry instrumentation
|
||||
mode: Server mode - 'developer' (full access, verbose errors) or 'user' (restricted APIs, generic errors)
|
||||
auth_enabled: Whether to enable Bearer token authentication
|
||||
auth_token: Custom authentication token (auto-generated if not provided with auth_enabled=True)
|
||||
@@ -126,13 +124,6 @@ def serve(
|
||||
if not isinstance(port, int) or not (1 <= port <= 65535):
|
||||
raise ValueError(f"Invalid port: {port}. Must be integer between 1 and 65535")
|
||||
|
||||
# Enable instrumentation if requested
|
||||
if instrumentation_enabled:
|
||||
from agent_framework.observability import enable_instrumentation
|
||||
|
||||
enable_instrumentation(enable_sensitive_data=True)
|
||||
logger.info("Enabled Agent Framework instrumentation with sensitive data")
|
||||
|
||||
# Create server with direct parameters
|
||||
server = DevServer(
|
||||
entities_dir=entities_dir,
|
||||
|
||||
@@ -98,7 +98,7 @@ class AgentFrameworkExecutor:
|
||||
try:
|
||||
from agent_framework.observability import OBSERVABILITY_SETTINGS, configure_otel_providers
|
||||
|
||||
# Configure if instrumentation is enabled (via enable_instrumentation() or env var)
|
||||
# Configure if instrumentation is enabled (on by default, via env var, or enable_sensitive_telemetry())
|
||||
if OBSERVABILITY_SETTINGS.ENABLED:
|
||||
# Call configure_otel_providers to set up exporters.
|
||||
# If OTEL_EXPORTER_OTLP_ENDPOINT is set, exporters will be created automatically.
|
||||
|
||||
@@ -518,7 +518,7 @@ class DevServer:
|
||||
framework="agent_framework",
|
||||
runtime="python", # Python DevUI backend
|
||||
capabilities={
|
||||
"instrumentation": os.getenv("ENABLE_INSTRUMENTATION") == "true",
|
||||
"instrumentation": os.getenv("ENABLE_INSTRUMENTATION", "true").lower() != "false",
|
||||
"openai_proxy": openai_executor.is_configured,
|
||||
"deployment": True, # Deployment feature is available
|
||||
},
|
||||
|
||||
@@ -248,8 +248,8 @@ services:
|
||||
- AZURE_OPENAI_API_KEY=\${AZURE_OPENAI_API_KEY}
|
||||
- AZURE_OPENAI_ENDPOINT=\${AZURE_OPENAI_ENDPOINT}
|
||||
- AZURE_OPENAI_MODEL=\${AZURE_OPENAI_MODEL}
|
||||
# Optional: Enable instrumentation
|
||||
- ENABLE_INSTRUMENTATION=\${ENABLE_INSTRUMENTATION:-false}
|
||||
# Optional: Disable instrumentation (enabled by default)
|
||||
- ENABLE_INSTRUMENTATION=\${ENABLE_INSTRUMENTATION:-true}
|
||||
ports:
|
||||
- "8080:8080"
|
||||
restart: unless-stopped
|
||||
|
||||
@@ -817,7 +817,7 @@ class RawFoundryAgent( # type: ignore[misc]
|
||||
"Install it with: pip install azure-monitor-opentelemetry"
|
||||
) from exc
|
||||
|
||||
from agent_framework.observability import create_metric_views, create_resource, enable_instrumentation
|
||||
from agent_framework.observability import create_metric_views, create_resource, enable_sensitive_telemetry
|
||||
|
||||
if "resource" not in kwargs:
|
||||
kwargs["resource"] = create_resource()
|
||||
@@ -828,7 +828,8 @@ class RawFoundryAgent( # type: ignore[misc]
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
enable_instrumentation(enable_sensitive_data=enable_sensitive_data)
|
||||
if enable_sensitive_data:
|
||||
enable_sensitive_telemetry()
|
||||
|
||||
|
||||
class FoundryAgent( # type: ignore[misc]
|
||||
|
||||
@@ -291,7 +291,7 @@ class RawFoundryChatClient( # type: ignore[misc]
|
||||
"Install it with: pip install azure-monitor-opentelemetry"
|
||||
) from exc
|
||||
|
||||
from agent_framework.observability import create_metric_views, create_resource, enable_instrumentation
|
||||
from agent_framework.observability import create_metric_views, create_resource, enable_sensitive_telemetry
|
||||
|
||||
if "resource" not in kwargs:
|
||||
kwargs["resource"] = create_resource()
|
||||
@@ -302,7 +302,8 @@ class RawFoundryChatClient( # type: ignore[misc]
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
enable_instrumentation(enable_sensitive_data=enable_sensitive_data)
|
||||
if enable_sensitive_data:
|
||||
enable_sensitive_telemetry()
|
||||
|
||||
# region Tool factory methods (override OpenAI defaults with Foundry versions)
|
||||
|
||||
|
||||
@@ -699,7 +699,7 @@ async def test_foundry_agent_configure_azure_monitor() -> None:
|
||||
),
|
||||
patch("agent_framework.observability.create_metric_views", mock_views),
|
||||
patch("agent_framework.observability.create_resource", return_value=mock_resource),
|
||||
patch("agent_framework.observability.enable_instrumentation", mock_enable),
|
||||
patch("agent_framework.observability.enable_sensitive_telemetry", mock_enable),
|
||||
):
|
||||
await agent.configure_azure_monitor(enable_sensitive_data=True)
|
||||
|
||||
@@ -708,7 +708,7 @@ async def test_foundry_agent_configure_azure_monitor() -> None:
|
||||
assert call_kwargs["connection_string"] == "InstrumentationKey=test-key;IngestionEndpoint=https://test.endpoint"
|
||||
assert call_kwargs["views"] == []
|
||||
assert call_kwargs["resource"] is mock_resource
|
||||
mock_enable.assert_called_once_with(enable_sensitive_data=True)
|
||||
mock_enable.assert_called_once()
|
||||
|
||||
|
||||
async def test_foundry_agent_configure_azure_monitor_resource_not_found() -> None:
|
||||
|
||||
@@ -249,7 +249,7 @@ async def test_configure_azure_monitor() -> None:
|
||||
),
|
||||
patch("agent_framework.observability.create_metric_views", mock_views),
|
||||
patch("agent_framework.observability.create_resource", return_value=mock_resource),
|
||||
patch("agent_framework.observability.enable_instrumentation", mock_enable),
|
||||
patch("agent_framework.observability.enable_sensitive_telemetry", mock_enable),
|
||||
):
|
||||
await client.configure_azure_monitor(enable_sensitive_data=True)
|
||||
|
||||
@@ -259,7 +259,7 @@ async def test_configure_azure_monitor() -> None:
|
||||
assert call_kwargs["connection_string"] == "InstrumentationKey=test-key;IngestionEndpoint=https://test.endpoint"
|
||||
assert call_kwargs["views"] == []
|
||||
assert call_kwargs["resource"] is mock_resource
|
||||
mock_enable.assert_called_once_with(enable_sensitive_data=True)
|
||||
mock_enable.assert_called_once()
|
||||
|
||||
|
||||
async def test_configure_azure_monitor_resource_not_found() -> None:
|
||||
@@ -324,7 +324,7 @@ async def test_configure_azure_monitor_with_custom_resource() -> None:
|
||||
),
|
||||
patch("agent_framework.observability.create_metric_views", return_value=[]),
|
||||
patch("agent_framework.observability.create_resource") as mock_create_resource,
|
||||
patch("agent_framework.observability.enable_instrumentation"),
|
||||
patch("agent_framework.observability.enable_sensitive_telemetry"),
|
||||
):
|
||||
await client.configure_azure_monitor(resource=custom_resource)
|
||||
|
||||
|
||||
@@ -6,7 +6,6 @@ from __future__ import annotations
|
||||
|
||||
import importlib.metadata
|
||||
|
||||
from agent_framework.observability import enable_instrumentation
|
||||
from agentlightning.tracer import ( # type: ignore[reportMissingImports]
|
||||
AgentOpsTracer, # type: ignore[reportMissingImports, import-not-found]
|
||||
)
|
||||
@@ -26,7 +25,6 @@ class AgentFrameworkTracer(AgentOpsTracer): # type: ignore
|
||||
|
||||
def init(self) -> None:
|
||||
"""Initialize the agent-framework-lab-lightning for training."""
|
||||
enable_instrumentation()
|
||||
super().init() # pyright: ignore[reportUnknownMemberType]
|
||||
|
||||
def teardown(self) -> None:
|
||||
|
||||
@@ -27,6 +27,9 @@ OTEL_EXPORTER_OTLP_ENDPOINT="http://localhost:4317"
|
||||
# Agent Framework specific settings
|
||||
# ==================================
|
||||
|
||||
# Observability is enabled by default. Set to "false" to opt out.
|
||||
# ENABLE_INSTRUMENTATION=false
|
||||
|
||||
# Enable sensitive data logging (prompts, responses, etc.)
|
||||
# WARNING: Only enable in dev/test environments
|
||||
ENABLE_SENSITIVE_DATA=true
|
||||
@@ -34,9 +37,6 @@ ENABLE_SENSITIVE_DATA=true
|
||||
# Optional: Enable console exporters for debugging
|
||||
# ENABLE_CONSOLE_EXPORTERS=true
|
||||
|
||||
# Optional: Enable observability (automatically enabled if env vars are set or configure_otel_providers() is called)
|
||||
# ENABLE_INSTRUMENTATION=true
|
||||
|
||||
# OpenAI specific variables
|
||||
# ==========================
|
||||
OPENAI_API_KEY="..."
|
||||
|
||||
@@ -73,11 +73,11 @@ configure_otel_providers(exporters=exporters, enable_sensitive_data=True)
|
||||
|
||||
**3. Third party setup**
|
||||
|
||||
A lot of third party specific otel package, have their own easy setup methods, for example Azure Monitor has `configure_azure_monitor()`. You can use those methods to setup the third party first, and then call `enable_instrumentation()` from the `agent_framework.observability` module to activate the Agent Framework telemetry code paths. In all these cases, if you already setup observability via environment variables, you don't need to call `enable_instrumentation()` as it will be enabled automatically.
|
||||
A lot of third party specific otel package, have their own easy setup methods, for example Azure Monitor has `configure_azure_monitor()`. You can use those methods to setup the third party first. Agent Framework instrumentation is enabled by default, so you don't need to call anything extra. If you want to also capture sensitive data (dev/test only), call `enable_sensitive_telemetry()` from the `agent_framework.observability` module.
|
||||
|
||||
```python
|
||||
from azure.monitor.opentelemetry import configure_azure_monitor
|
||||
from agent_framework.observability import create_resource, enable_instrumentation
|
||||
from agent_framework.observability import create_resource, enable_sensitive_telemetry
|
||||
|
||||
# Configure Azure Monitor first
|
||||
configure_azure_monitor(
|
||||
@@ -86,9 +86,8 @@ configure_azure_monitor(
|
||||
enable_live_metrics=True,
|
||||
)
|
||||
|
||||
# Then activate Agent Framework's telemetry code paths
|
||||
# This is optional if ENABLE_INSTRUMENTATION and or ENABLE_SENSITIVE_DATA are set in env vars
|
||||
enable_instrumentation(enable_sensitive_data=False)
|
||||
# Optional: opt in to capturing sensitive data (dev/test only)
|
||||
enable_sensitive_telemetry()
|
||||
```
|
||||
For Microsoft Foundry projects, use `client.configure_azure_monitor()` which retrieves the connection string from the project and configures everything:
|
||||
|
||||
@@ -110,7 +109,7 @@ Or with [Langfuse](https://langfuse.com/integrations/frameworks/microsoft-agent-
|
||||
|
||||
```python
|
||||
# environment should be setup correctly, with langfuse urls and keys
|
||||
from agent_framework.observability import enable_instrumentation
|
||||
from agent_framework.observability import enable_sensitive_telemetry
|
||||
from langfuse import get_client
|
||||
|
||||
langfuse = get_client()
|
||||
@@ -121,9 +120,9 @@ if langfuse.auth_check():
|
||||
else:
|
||||
print("Authentication failed. Please check your credentials and host.")
|
||||
|
||||
# Then activate Agent Framework's telemetry code paths
|
||||
# This is optional if ENABLE_INSTRUMENTATION and or ENABLE_SENSITIVE_DATA are set in env vars
|
||||
enable_instrumentation(enable_sensitive_data=False)
|
||||
# Agent Framework instrumentation is on by default.
|
||||
# Optional: opt in to capturing sensitive data (dev/test only)
|
||||
enable_sensitive_telemetry()
|
||||
```
|
||||
|
||||
Or with [Comet Opik](https://www.comet.com/docs/opik/integrations/microsoft-agent-framework):
|
||||
@@ -131,15 +130,15 @@ Or with [Comet Opik](https://www.comet.com/docs/opik/integrations/microsoft-agen
|
||||
```python
|
||||
import os
|
||||
|
||||
from agent_framework.observability import enable_instrumentation
|
||||
from agent_framework.observability import enable_sensitive_telemetry
|
||||
|
||||
# Use Opik OTLP settings from your project settings
|
||||
os.environ["OTEL_EXPORTER_OTLP_ENDPOINT"] = "<opik_otlp_endpoint>"
|
||||
os.environ["OTEL_EXPORTER_OTLP_HEADERS"] = "<opik_otlp_headers>"
|
||||
|
||||
# Then activate Agent Framework's telemetry code paths
|
||||
# This is optional if ENABLE_INSTRUMENTATION and or ENABLE_SENSITIVE_DATA are set in env vars
|
||||
enable_instrumentation(enable_sensitive_data=False)
|
||||
# Agent Framework instrumentation is on by default.
|
||||
# Optional: opt in to capturing sensitive data (dev/test only)
|
||||
enable_sensitive_telemetry()
|
||||
```
|
||||
|
||||
**4. Manual setup**
|
||||
@@ -169,11 +168,11 @@ The following environment variables are used to turn on/off observability of the
|
||||
- `ENABLE_SENSITIVE_DATA`
|
||||
- `ENABLE_CONSOLE_EXPORTERS`
|
||||
|
||||
All of these are booleans and default to `false`.
|
||||
`ENABLE_INSTRUMENTATION` defaults to `true` (set it to `false` to opt out). `ENABLE_SENSITIVE_DATA` and `ENABLE_CONSOLE_EXPORTERS` default to `false`.
|
||||
|
||||
Finally we have `VS_CODE_EXTENSION_PORT` which you can set to a port, which can be used to setup the AI Toolkit for VS Code tracing integration. See [here](https://marketplace.visualstudio.com/items?itemName=ms-windows-ai-studio.windows-ai-studio#tracing) for more details.
|
||||
|
||||
The framework will emit observability data when the `ENABLE_INSTRUMENTATION` environment variable is set to `true`. If both are `true` then it will also emit sensitive information. When these are not set, or set to false, you can use the `enable_instrumentation()` function from the `agent_framework.observability` module to turn on instrumentation programmatically. This is useful when you want to control this via code instead of environment variables.
|
||||
The framework emits observability data by default; set `ENABLE_INSTRUMENTATION=false` to disable. If `ENABLE_SENSITIVE_DATA=true` is set, the framework will also emit sensitive information. You can also call `enable_sensitive_telemetry()` from the `agent_framework.observability` module to opt-in to sensitive-data capture programmatically.
|
||||
|
||||
> **Note**: Sensitive information includes prompts, responses, and more, and should only be enabled in a development or test environment. It is not recommended to enable this in production environments as it may expose sensitive data.
|
||||
|
||||
@@ -233,14 +232,14 @@ This folder contains different samples demonstrating how to use telemetry in var
|
||||
| [agent_observability.py](./agent_observability.py) | Shows telemetry collection for an agentic application with tool calls using environment variables. |
|
||||
| [foundry_tracing.py](./foundry_tracing.py) | Shows Azure Monitor integration with Foundry for any chat client. |
|
||||
| [advanced_manual_setup_console_output.py](./advanced_manual_setup_console_output.py) | Advanced: Shows manual setup of exporters and providers with console output. Useful for understanding how observability works under the hood. |
|
||||
| [advanced_zero_code.py](./advanced_zero_code.py) | Advanced: Shows zero-code telemetry setup using the `opentelemetry-enable_instrumentation` CLI tool. |
|
||||
| [advanced_zero_code.py](./advanced_zero_code.py) | Advanced: Shows zero-code telemetry setup using the `opentelemetry-instrument` CLI tool. |
|
||||
| [workflow_observability.py](./workflow_observability.py) | Shows telemetry collection for a workflow with multiple executors and message passing. |
|
||||
|
||||
### Running the samples
|
||||
|
||||
1. Open a terminal and navigate to this folder: `python/samples/02-agents/observability/`. This is necessary for the `.env` file to be read correctly.
|
||||
2. Create a `.env` file if one doesn't already exist in this folder. Please refer to the [example file](./.env.example).
|
||||
> **Note**: You can start with just `ENABLE_INSTRUMENTATION=true` and add `OTEL_EXPORTER_OTLP_ENDPOINT` or other configuration as needed. If no exporters are configured, you can set `ENABLE_CONSOLE_EXPORTERS=true` for console output.
|
||||
> **Note**: Instrumentation is on by default. Set `OTEL_EXPORTER_OTLP_ENDPOINT` or other configuration as needed. If no exporters are configured, you can set `ENABLE_CONSOLE_EXPORTERS=true` for console output.
|
||||
3. Choose one environment-loading approach:
|
||||
- **A. Sample-managed loading (current samples):** run from this folder so the sample's `load_dotenv()` call can find `.env`.
|
||||
- **B. Shell/IDE-managed environment:** set/export environment variables directly, or use an IDE run configuration that injects env vars / `.env`.
|
||||
@@ -249,7 +248,7 @@ This folder contains different samples demonstrating how to use telemetry in var
|
||||
`uv run --env-file=.env python configure_otel_providers_with_env_var.py`
|
||||
4. Activate your python virtual environment, then run a sample (for example `python configure_otel_providers_with_env_var.py`).
|
||||
|
||||
> If you do manual provider setup (e.g., Azure Monitor), call `enable_instrumentation()` to turn on Agent Framework telemetry code paths; if you want Agent Framework to configure exporters/providers for you, call `configure_otel_providers(...)`.
|
||||
> If you do manual provider setup (e.g., Azure Monitor), Agent Framework instrumentation is on by default; call `enable_sensitive_telemetry()` if you also want to capture sensitive data. If you want Agent Framework to configure exporters/providers for you, call `configure_otel_providers(...)`.
|
||||
|
||||
> Each sample will print the Operation/Trace ID, which can be used later for filtering logs and traces in Application Insights or Aspire Dashboard.
|
||||
|
||||
@@ -352,7 +351,7 @@ setup_observability(
|
||||
|
||||
```python
|
||||
from agent_framework.foundry import FoundryChatClient
|
||||
from agent_framework.observability import create_resource, enable_instrumentation
|
||||
from agent_framework.observability import create_resource, enable_sensitive_telemetry
|
||||
from azure.identity import AzureCliCredential
|
||||
from azure.monitor.opentelemetry import configure_azure_monitor
|
||||
|
||||
@@ -371,7 +370,8 @@ async def main():
|
||||
resource=create_resource(),
|
||||
enable_live_metrics=True,
|
||||
)
|
||||
enable_instrumentation()
|
||||
# Optional: opt in to capturing sensitive data (dev/test only)
|
||||
enable_sensitive_telemetry()
|
||||
```
|
||||
|
||||
### Console Output
|
||||
@@ -437,7 +437,7 @@ OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317
|
||||
Or set it as an environment variable when running your samples:
|
||||
|
||||
```bash
|
||||
ENABLE_INSTRUMENTATION=true OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317 python configure_otel_providers_with_env_var.py
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317 python configure_otel_providers_with_env_var.py
|
||||
```
|
||||
|
||||
### Viewing telemetry data
|
||||
|
||||
@@ -7,7 +7,7 @@ from typing import Annotated
|
||||
|
||||
from agent_framework import Message, tool
|
||||
from agent_framework.foundry import FoundryChatClient
|
||||
from agent_framework.observability import enable_instrumentation
|
||||
from agent_framework.observability import enable_sensitive_telemetry
|
||||
from azure.identity import AzureCliCredential
|
||||
from dotenv import load_dotenv
|
||||
from opentelemetry._logs import set_logger_provider
|
||||
@@ -135,7 +135,8 @@ async def main():
|
||||
setup_logging()
|
||||
setup_tracing()
|
||||
setup_metrics()
|
||||
enable_instrumentation()
|
||||
# Instrumentation is enabled by default; call this to also capture sensitive data (dev/test only).
|
||||
enable_sensitive_telemetry()
|
||||
|
||||
await run_chat_client()
|
||||
|
||||
|
||||
@@ -22,7 +22,7 @@ What this example shows:
|
||||
- executor_completed events (type='executor_completed') contain the messages sent via ctx.send_message() in event.data
|
||||
- How to generically observe all executor I/O through workflow streaming events
|
||||
|
||||
This approach allows you to enable_instrumentation any workflow for observability without
|
||||
This approach allows you to instrument any workflow for observability without
|
||||
changing the executor implementations.
|
||||
|
||||
Prerequisites:
|
||||
|
||||
-1
@@ -1,4 +1,3 @@
|
||||
FOUNDRY_PROJECT_ENDPOINT="..."
|
||||
AZURE_AI_MODEL_DEPLOYMENT_NAME="..."
|
||||
ENABLE_INSTRUMENTATION=true
|
||||
ENABLE_SENSITIVE_DATA=true
|
||||
+1
-1
@@ -16,7 +16,7 @@ The agent is hosted using the [Agent Framework](https://github.com/microsoft/age
|
||||
|
||||
### Instrumentation
|
||||
|
||||
Agent Framework is [**natively instrumented**](https://learn.microsoft.com/en-us/agent-framework/agents/observability?pivots=programming-language-python) to capture diagnostics and telemetry for agent execution, but it's turned off by default. This sample demonstrates how to enable instrumentation via environment variables in `agent.manifest.yaml` and `agent.yaml`. The relevant environment variables are `ENABLE_INSTRUMENTATION` and `ENABLE_SENSITIVE_DATA`, which can be set to `true` to enable diagnostics and capture sensitive events respectively.
|
||||
Agent Framework is [**natively instrumented**](https://learn.microsoft.com/en-us/agent-framework/agents/observability?pivots=programming-language-python) to capture diagnostics and telemetry for agent execution. Instrumentation is enabled by default; set `ENABLE_INSTRUMENTATION=false` to opt out. To also capture sensitive event payloads (prompts, tool arguments, etc.) set `ENABLE_SENSITIVE_DATA=true`. This sample demonstrates how to manage these settings via environment variables in `agent.manifest.yaml` and `agent.yaml`.
|
||||
|
||||
Foundry Hosted Agent has built-in observability thus you don't need to set up exporters manually to capture telemetry from your code. The traces, metrics, and logs generated by the agent are automatically collected and made available through Foundry's observability stack via Azure Monitor/Application Insights. The `APPLICATIONINSIGHTS_CONNECTION_STRING` environment variable is injected when the agent is deployed to Foundry, however it is still required to be set in your environment if you want to run the agent host locally and have telemetry sent to Application Insights from your local environment.
|
||||
|
||||
|
||||
-2
@@ -17,8 +17,6 @@ template:
|
||||
environment_variables:
|
||||
- name: AZURE_AI_MODEL_DEPLOYMENT_NAME
|
||||
value: "{{AZURE_AI_MODEL_DEPLOYMENT_NAME}}"
|
||||
- name: ENABLE_INSTRUMENTATION
|
||||
value: true
|
||||
- name: ENABLE_SENSITIVE_DATA
|
||||
value: true
|
||||
resources:
|
||||
|
||||
+3
-5
@@ -5,12 +5,10 @@ protocols:
|
||||
- protocol: responses
|
||||
version: 1.0.0
|
||||
resources:
|
||||
cpu: '0.25'
|
||||
memory: '0.5Gi'
|
||||
cpu: "0.25"
|
||||
memory: "0.5Gi"
|
||||
environment_variables:
|
||||
- name: AZURE_AI_MODEL_DEPLOYMENT_NAME
|
||||
value: ${AZURE_AI_MODEL_DEPLOYMENT_NAME}
|
||||
- name: ENABLE_INSTRUMENTATION
|
||||
value: true
|
||||
- name: ENABLE_SENSITIVE_DATA
|
||||
value: true
|
||||
value: true
|
||||
|
||||
+15
-15
@@ -117,21 +117,21 @@ variable.
|
||||
| `agent-framework-copilotstudio` | `CopilotStudioAgent` | `COPILOTSTUDIOAGENT__SCHEMANAME` | `cr123_agentname` |
|
||||
| `agent-framework-copilotstudio` | `CopilotStudioAgent` | `COPILOTSTUDIOAGENT__TENANTID` | `11111111-1111-1111-1111-111111111111` |
|
||||
| `agent-framework-copilotstudio` | `CopilotStudioAgent` | `COPILOTSTUDIOAGENT__AGENTAPPID` | `22222222-2222-2222-2222-222222222222` |
|
||||
| `agent-framework-core` | `enable_instrumentation()` | `ENABLE_INSTRUMENTATION` | `true` |
|
||||
| `agent-framework-core` | `enable_instrumentation()` | `ENABLE_SENSITIVE_DATA` | `false` |
|
||||
| `agent-framework-core` | `enable_instrumentation()` | `ENABLE_CONSOLE_EXPORTERS` | `true` |
|
||||
| `agent-framework-core` | `enable_instrumentation()` | `OTEL_EXPORTER_OTLP_ENDPOINT` | `http://localhost:4317` |
|
||||
| `agent-framework-core` | `enable_instrumentation()` | `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` | `http://localhost:4318/v1/traces` |
|
||||
| `agent-framework-core` | `enable_instrumentation()` | `OTEL_EXPORTER_OTLP_METRICS_ENDPOINT` | `http://localhost:4318/v1/metrics` |
|
||||
| `agent-framework-core` | `enable_instrumentation()` | `OTEL_EXPORTER_OTLP_LOGS_ENDPOINT` | `http://localhost:4318/v1/logs` |
|
||||
| `agent-framework-core` | `enable_instrumentation()` | `OTEL_EXPORTER_OTLP_PROTOCOL` | `grpc` |
|
||||
| `agent-framework-core` | `enable_instrumentation()` | `OTEL_EXPORTER_OTLP_HEADERS` | `api-key=demo` |
|
||||
| `agent-framework-core` | `enable_instrumentation()` | `OTEL_EXPORTER_OTLP_TRACES_HEADERS` | `api-key=trace-demo` |
|
||||
| `agent-framework-core` | `enable_instrumentation()` | `OTEL_EXPORTER_OTLP_METRICS_HEADERS` | `api-key=metric-demo` |
|
||||
| `agent-framework-core` | `enable_instrumentation()` | `OTEL_EXPORTER_OTLP_LOGS_HEADERS` | `api-key=log-demo` |
|
||||
| `agent-framework-core` | `enable_instrumentation()` | `OTEL_SERVICE_NAME` | `sample-agent` |
|
||||
| `agent-framework-core` | `enable_instrumentation()` | `OTEL_SERVICE_VERSION` | `1.0.0` |
|
||||
| `agent-framework-core` | `enable_instrumentation()` | `OTEL_RESOURCE_ATTRIBUTES` | `deployment.environment=dev,service.namespace=agent-framework` |
|
||||
| `agent-framework-core` | `observability` | `ENABLE_INSTRUMENTATION` | `true` |
|
||||
| `agent-framework-core` | `observability` | `ENABLE_SENSITIVE_DATA` | `false` |
|
||||
| `agent-framework-core` | `observability` | `ENABLE_CONSOLE_EXPORTERS` | `true` |
|
||||
| `agent-framework-core` | `observability` | `OTEL_EXPORTER_OTLP_ENDPOINT` | `http://localhost:4317` |
|
||||
| `agent-framework-core` | `observability` | `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` | `http://localhost:4318/v1/traces` |
|
||||
| `agent-framework-core` | `observability` | `OTEL_EXPORTER_OTLP_METRICS_ENDPOINT` | `http://localhost:4318/v1/metrics` |
|
||||
| `agent-framework-core` | `observability` | `OTEL_EXPORTER_OTLP_LOGS_ENDPOINT` | `http://localhost:4318/v1/logs` |
|
||||
| `agent-framework-core` | `observability` | `OTEL_EXPORTER_OTLP_PROTOCOL` | `grpc` |
|
||||
| `agent-framework-core` | `observability` | `OTEL_EXPORTER_OTLP_HEADERS` | `api-key=demo` |
|
||||
| `agent-framework-core` | `observability` | `OTEL_EXPORTER_OTLP_TRACES_HEADERS` | `api-key=trace-demo` |
|
||||
| `agent-framework-core` | `observability` | `OTEL_EXPORTER_OTLP_METRICS_HEADERS` | `api-key=metric-demo` |
|
||||
| `agent-framework-core` | `observability` | `OTEL_EXPORTER_OTLP_LOGS_HEADERS` | `api-key=log-demo` |
|
||||
| `agent-framework-core` | `observability` | `OTEL_SERVICE_NAME` | `sample-agent` |
|
||||
| `agent-framework-core` | `observability` | `OTEL_SERVICE_VERSION` | `1.0.0` |
|
||||
| `agent-framework-core` | `observability` | `OTEL_RESOURCE_ATTRIBUTES` | `deployment.environment=dev,service.namespace=agent-framework` |
|
||||
| `agent-framework-devui` | `DevUI server` | `DEVUI_AUTH_TOKEN` | `my-devui-token` |
|
||||
| `agent-framework-foundry` | `FoundryChatClient` | `FOUNDRY_PROJECT_ENDPOINT` | `https://my-project.services.ai.azure.com/api/projects/my-project` |
|
||||
| `agent-framework-foundry` | `FoundryChatClient` | `FOUNDRY_MODEL` | `gpt-4o` |
|
||||
|
||||
Reference in New Issue
Block a user