agent/agent-framework
2
0
Fork 0
mirror of https://github.com/microsoft/agent-framework.git synced 2026-06-16 21:04:09 +08:00
Code Issues Packages Projects Releases Wiki Activity

[BREAKING] Python: Observability cleanup (#905)

Browse Source 
* Further observability cleanup and update telemetry samples

* Add VS Code Extension config

* Fix unit tests

* Fix unit tests

* Add more comments

* Remove live metric
This commit is contained in:
Tao Chen
2025-09-25 11:50:50 -07:00
committed by GitHub
Unverified
parent f527fbe6ce
commit a480ccfd16
24 changed files with 724 additions and 493 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/packages/azure-ai/agent_framework_azure_ai/_chat_client.py
+1 -3
View File
@@ -165,7 +165,6 @@ class AzureAIAgentClient(BaseChatClient):
model_deployment_name: The model deployment name to use for agent creation.
Can also be set via 'AZURE_AI_MODEL_DEPLOYMENT_NAME' environment variable.
async_credential: Azure async credential to use for authentication.
setup_tracing: Whether to setup tracing for the project_client. Defaults to True.
env_file_path: Path to environment file for loading settings.
env_file_encoding: Encoding of the environment file.
**kwargs: Additional keyword arguments passed to the parent class.
@@ -216,7 +215,7 @@ class AzureAIAgentClient(BaseChatClient):
)
self._should_close_client = should_close_client
async def setup_observability(self, enable_live_metrics: bool = False) -> None:
async def setup_observability(self) -> None:
"""Use this method to setup tracing in your Azure AI Project.
This will take the connection string from the project project_client.
@@ -227,7 +226,6 @@ class AzureAIAgentClient(BaseChatClient):
setup_observability(
applicationinsights_connection_string=await self.project_client.telemetry.get_application_insights_connection_string(), # noqa: E501
enable_live_metrics=enable_live_metrics,
)
async def __aenter__(self) -> "Self":
python/packages/devui/agent_framework_devui/_discovery.py
+3 -3
View File
@@ -40,7 +40,7 @@ class EntityDiscovery:
logger.info("No Agent Framework entities directory configured")
return []
entities_dir = Path(self.entities_dir).resolve()
entities_dir = Path(self.entities_dir).resolve() # noqa: ASYNC240
await self._scan_entities_directory(entities_dir)
logger.info(f"Discovered {len(self._entities)} Agent Framework entities")
@@ -152,7 +152,7 @@ class EntityDiscovery:
Args:
entities_dir: Directory to scan for entities
"""
if not entities_dir.exists():
if not entities_dir.exists(): # noqa: ASYNC240
logger.warning(f"Entities directory not found: {entities_dir}")
return
@@ -164,7 +164,7 @@ class EntityDiscovery:
sys.path.insert(0, entities_dir_str)
# Scan for directories and Python files
for item in entities_dir.iterdir():
for item in entities_dir.iterdir(): # noqa: ASYNC240
if item.name.startswith(".") or item.name == "__pycache__":
continue
python/packages/lab/gaia/agent_framework_lab_gaia/gaia.py
-4
View File
@@ -33,7 +33,6 @@ class GAIATelemetryConfig:
enable_tracing: bool = False,
otlp_endpoint: str | None = None,
applicationinsights_connection_string: str | None = None,
enable_live_metrics: bool = False,
trace_to_file: bool = False,
file_path: str | None = None,
):
@@ -44,14 +43,12 @@ class GAIATelemetryConfig:
enable_tracing: Whether to enable OpenTelemetry tracing
otlp_endpoint: OTLP endpoint for trace export
applicationinsights_connection_string: Azure Monitor connection string
enable_live_metrics: Enable Azure Monitor live metrics
trace_to_file: Whether to export traces to local file
file_path: Path for local file export (defaults to gaia_traces.json)
"""
self.enable_tracing = enable_tracing
self.otlp_endpoint = otlp_endpoint
self.applicationinsights_connection_string = applicationinsights_connection_string
self.enable_live_metrics = enable_live_metrics
self.trace_to_file = trace_to_file
self.file_path = file_path or "gaia_traces.json"
@@ -66,7 +63,6 @@ class GAIATelemetryConfig:
enable_sensitive_data=True, # Enable for detailed task traces
otlp_endpoint=self.otlp_endpoint,
applicationinsights_connection_string=self.applicationinsights_connection_string,
enable_live_metrics=self.enable_live_metrics,
)
# Set up local file export if requested
python/packages/lab/gaia/samples/gaia_sample.py
-1
View File
@@ -30,7 +30,6 @@ async def main() -> None:
# Optional: Configure external endpoints
# otlp_endpoint="http://localhost:4317", # For Aspire Dashboard or other OTLP endpoints
# applicationinsights_connection_string="your_connection_string", # For Azure Monitor
# enable_live_metrics=True, # Enable Azure Monitor live metrics
# Configure local file tracing
trace_to_file=True, # Export traces to local file
file_path="gaia_benchmark_traces.jsonl", # Custom file path for traces
python/packages/main/agent_framework/_tools.py
+6 -7
View File
@@ -404,18 +404,17 @@ class AIFunction(BaseTool, Generic[ArgsT, ReturnT]):
Args:
arguments: A Pydantic model instance containing the arguments for the function.
otel_settings: Optional model diagnostics settings to override the default settings.
kwargs: keyword arguments to pass to the function, will not be used if `arguments` is provided.
"""
global OTEL_SETTINGS
from .observability import OTEL_SETTINGS
global OBSERVABILITY_SETTINGS
from .observability import OBSERVABILITY_SETTINGS
tool_call_id = kwargs.pop("tool_call_id", None)
if arguments is not None:
if not isinstance(arguments, self.input_model):
raise TypeError(f"Expected {self.input_model.__name__}, got {type(arguments).__name__}")
kwargs = arguments.model_dump(exclude_none=True)
if not OTEL_SETTINGS.ENABLED: # type: ignore[name-defined]
if not OBSERVABILITY_SETTINGS.ENABLED: # type: ignore[name-defined]
logger.info(f"Function name: {self.name}")
logger.debug(f"Function arguments: {kwargs}")
res = self.__call__(**kwargs)
@@ -425,7 +424,7 @@ class AIFunction(BaseTool, Generic[ArgsT, ReturnT]):
return result # type: ignore[reportReturnType]
attributes = get_function_span_attributes(self, tool_call_id=tool_call_id)
if OTEL_SETTINGS.SENSITIVE_DATA_ENABLED: # type: ignore[name-defined]
if OBSERVABILITY_SETTINGS.SENSITIVE_DATA_ENABLED: # type: ignore[name-defined]
attributes.update({
OtelAttr.TOOL_ARGUMENTS: arguments.model_dump_json()
if arguments
@@ -436,7 +435,7 @@ class AIFunction(BaseTool, Generic[ArgsT, ReturnT]):
with get_function_span(attributes=attributes) as span:
attributes[OtelAttr.MEASUREMENT_FUNCTION_TAG_NAME] = self.name
logger.info(f"Function name: {self.name}")
if OTEL_SETTINGS.SENSITIVE_DATA_ENABLED: # type: ignore[name-defined]
if OBSERVABILITY_SETTINGS.SENSITIVE_DATA_ENABLED: # type: ignore[name-defined]
logger.debug(f"Function arguments: {kwargs}")
start_time_stamp = perf_counter()
end_time_stamp: float | None = None
@@ -452,7 +451,7 @@ class AIFunction(BaseTool, Generic[ArgsT, ReturnT]):
raise
else:
logger.info(f"Function {self.name} succeeded.")
if OTEL_SETTINGS.SENSITIVE_DATA_ENABLED: # type: ignore[name-defined]
if OBSERVABILITY_SETTINGS.SENSITIVE_DATA_ENABLED: # type: ignore[name-defined]
try:
json_result = json.dumps(result)
except (TypeError, OverflowError):
python/packages/main/agent_framework/_workflow/_workflow_context.py
+3 -3
View File
@@ -356,8 +356,8 @@ class WorkflowContext(Generic[T_Out, T_W_Out]):
target_id: The ID of the target executor to send the message to.
If None, the message will be sent to all target executors.
"""
global OTEL_SETTINGS
from ..observability import OTEL_SETTINGS
global OBSERVABILITY_SETTINGS
from ..observability import OBSERVABILITY_SETTINGS
# Create publishing span (inherits current trace context automatically)
attributes: dict[str, str] = {OtelAttr.MESSAGE_TYPE: type(message).__name__}
@@ -368,7 +368,7 @@ class WorkflowContext(Generic[T_Out, T_W_Out]):
msg = Message(data=message, source_id=self._executor_id, target_id=target_id)
# Inject current trace context if tracing enabled
if OTEL_SETTINGS.ENABLED and span and span.is_recording(): # type: ignore[name-defined]
if OBSERVABILITY_SETTINGS.ENABLED and span and span.is_recording(): # type: ignore[name-defined]
trace_context: dict[str, str] = {}
inject(trace_context) # Inject current trace context for message propagation
python/packages/main/agent_framework/observability.py
+193 -154
View File
@@ -20,13 +20,9 @@ from .exceptions import AgentInitializationError, ChatClientInitializationError
if TYPE_CHECKING: # pragma: no cover
from azure.core.credentials import TokenCredential
from opentelemetry.sdk._events import EventLoggerProvider
from opentelemetry.sdk._logs import LoggerProvider
from opentelemetry.sdk._logs._internal.export import LogExporter
from opentelemetry.sdk.metrics import MeterProvider
from opentelemetry.sdk.metrics.export import MetricExporter
from opentelemetry.sdk.resources import Resource
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import SpanExporter
from opentelemetry.trace import Tracer
from opentelemetry.util._decorator import _AgnosticContextManager # type: ignore[reportPrivateUsage]
@@ -328,11 +324,9 @@ def _create_resource() -> "Resource":
return Resource.create({service_attributes.SERVICE_NAME: service_name})
class OtelSettings(AFBaseSettings):
"""Settings for Open Telemetry.
class ObservabilitySettings(AFBaseSettings):
"""Settings for Agent Framework Observability.
The settings are first loaded from environment variables with
the prefix 'AGENT_FRAMEWORK_GENAI_'.
If the environment variables are not found, the settings can
be loaded from a .env file with the encoding 'utf-8'.
If the settings are not found in the .env file, the settings
@@ -349,10 +343,11 @@ class OtelSettings(AFBaseSettings):
(Env var ENABLE_SENSITIVE_DATA)
applicationinsights_connection_string: The Azure Monitor connection string. Default is None.
(Env var APPLICATIONINSIGHTS_CONNECTION_STRING)
applicationinsights_live_metrics: Enable Azure Monitor live metrics. Default is False.
(Env var APPLICATIONINSIGHTS_LIVE_METRICS)
otlp_endpoint: The OpenTelemetry Protocol (OTLP) endpoint. Default is None.
(Env var OTLP_ENDPOINT)
vs_code_extension_port: The port the AI Toolkit or AzureAI Foundry VS Code extensions are listening on.
Default is None.
(Env var VS_CODE_EXTENSION_PORT)
"""
env_prefix: ClassVar[str] = ""
@@ -360,14 +355,10 @@ class OtelSettings(AFBaseSettings):
enable_otel: bool = False
enable_sensitive_data: bool = False
applicationinsights_connection_string: str | list[str] | None = None
applicationinsights_live_metrics: bool = False
otlp_endpoint: str | list[str] | None = None
vs_code_extension_port: int | None = None
_resource: "Resource" = PrivateAttr(default_factory=_create_resource)
_executed_setup: bool = PrivateAttr(default=False)
_tracer_provider: "TracerProvider | None" = PrivateAttr(default=None)
_meter_provider: "MeterProvider | None" = PrivateAttr(default=None)
_logger_provider: "LoggerProvider | None" = PrivateAttr(default=None)
_event_logger_provider: "EventLoggerProvider | None" = PrivateAttr(default=None)
@property
def ENABLED(self) -> bool:
@@ -400,24 +391,24 @@ class OtelSettings(AFBaseSettings):
"""Set the resource."""
self._resource = value
def setup_observability(
def _configure(
self,
credential: "TokenCredential | None" = None,
additional_exporters: list["LogExporter | SpanExporter | MetricExporter"] | None = None,
force_setup: bool = False,
) -> None:
"""Setup telemetry based on the settings.
"""Configure application-wide observability based on the settings.
This method is a helper method to create the log, trace and metric providers.
This method is intended to be called once during the application startup. Calling it multiple times
will have no effect.
Args:
credential: The credential to use for Azure Monitor Entra ID authentication. Default is None.
additional_exporters: A list of additional exporters to add to the configuration. Default is None.
force_setup: Force the setup to be executed even if it has already been executed. Default is False.
"""
if (not self.ENABLED) or (self._executed_setup and not force_setup):
if not self.ENABLED or self._executed_setup:
return
global_logger = logging.getLogger()
global_logger.setLevel(logging.NOTSET)
exporters: list["LogExporter | SpanExporter | MetricExporter"] = additional_exporters or []
if self.otlp_endpoint:
exporters.extend(
@@ -438,26 +429,6 @@ class OtelSettings(AFBaseSettings):
)
self._configure_providers(exporters)
self._executed_setup = True
if self.applicationinsights_connection_string and self.applicationinsights_live_metrics:
from azure.monitor.opentelemetry import configure_azure_monitor
conn_strings = (
self.applicationinsights_connection_string
if isinstance(self.applicationinsights_connection_string, list)
else [self.applicationinsights_connection_string]
)
for con_str in conn_strings:
# only configure using this for live_metrics, ignore the rest.
configure_azure_monitor(
connection_string=con_str,
credential=credential,
logger_name="agent_framework",
resource=self.resource,
enable_live_metrics=self.applicationinsights_live_metrics,
disable_logging=True,
disable_metric=True,
disable_tracing=True,
)
def check_endpoint_already_configured(self, otlp_endpoint: str) -> bool:
"""Check if the endpoint is already configured.
@@ -467,9 +438,7 @@ class OtelSettings(AFBaseSettings):
"""
if not self.otlp_endpoint:
return False
return otlp_endpoint not in (
self.otlp_endpoint if isinstance(self.otlp_endpoint, list) else [self.otlp_endpoint]
)
return otlp_endpoint in (self.otlp_endpoint if isinstance(self.otlp_endpoint, list) else [self.otlp_endpoint])
def check_connection_string_already_configured(self, connection_string: str) -> bool:
"""Check if the connection string is already configured.
@@ -479,7 +448,7 @@ class OtelSettings(AFBaseSettings):
"""
if not self.applicationinsights_connection_string:
return False
return connection_string not in (
return connection_string in (
self.applicationinsights_connection_string
if isinstance(self.applicationinsights_connection_string, list)
else [self.applicationinsights_connection_string]
@@ -488,7 +457,6 @@ class OtelSettings(AFBaseSettings):
def _configure_providers(self, exporters: list["LogExporter | MetricExporter | SpanExporter"]) -> None:
"""Configure tracing, logging, events and metrics with the provided exporters."""
from opentelemetry._logs import set_logger_provider
from opentelemetry.sdk._events import EventLoggerProvider
from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
from opentelemetry.sdk._logs._internal.export import LogExporter
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
@@ -498,52 +466,49 @@ class OtelSettings(AFBaseSettings):
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor, SpanExporter
# Use SimpleSpanProcessor for in-memory exporter (tests) so spans are
# exported synchronously and immediately available via
# InMemorySpanExporter.get_finished_spans(). For all other exporters
# keep using the BatchSpanProcessor behavior.
# Tracing
if not self._executed_setup:
new_tracer_provider = TracerProvider(resource=self.resource)
# setting global tracer provider, other libaries can use this,
# but if another global tracer provider is already set this will not override it.
trace.set_tracer_provider(new_tracer_provider)
tracer_provider = trace.get_tracer_provider()
tracer_provider = TracerProvider(resource=self.resource)
trace.set_tracer_provider(tracer_provider)
should_add_console_exporter = True
for exporter in exporters:
if not isinstance(exporter, SpanExporter):
continue
if (add_span_processor := getattr(tracer_provider, "add_span_processor", None)) and callable(
add_span_processor
):
add_span_processor(BatchSpanProcessor(exporter))
if isinstance(exporter, SpanExporter):
tracer_provider.add_span_processor(BatchSpanProcessor(exporter))
should_add_console_exporter = False
if should_add_console_exporter:
from opentelemetry.sdk.trace.export import ConsoleSpanExporter
tracer_provider.add_span_processor(BatchSpanProcessor(ConsoleSpanExporter()))
# Logging
if not self._logger_provider:
self._logger_provider = LoggerProvider(resource=self.resource)
logger_provider = LoggerProvider(resource=self.resource)
should_add_console_exporter = True
for exporter in exporters:
if isinstance(exporter, LogExporter):
logger_provider.add_log_record_processor(BatchLogRecordProcessor(exporter))
should_add_console_exporter = False
if should_add_console_exporter:
from opentelemetry.sdk._logs._internal.export import ConsoleLogExporter
[
self._logger_provider.add_log_record_processor(BatchLogRecordProcessor(exporter))
for exporter in exporters
if isinstance(exporter, LogExporter)
]
logger = get_logger()
if not any(isinstance(handler, LoggingHandler) for handler in logger.handlers):
handler = LoggingHandler(logger_provider=self._logger_provider)
logger.addHandler(handler)
logger.setLevel(logging.NOTSET)
set_logger_provider(self._logger_provider)
# Events
if not self._event_logger_provider:
self._event_logger_provider = EventLoggerProvider(self._logger_provider)
logger_provider.add_log_record_processor(BatchLogRecordProcessor(ConsoleLogExporter()))
# Attach a handler with the provider to the root logger
logger = logging.getLogger()
handler = LoggingHandler(logger_provider=logger_provider)
logger.addHandler(handler)
set_logger_provider(logger_provider)
# metrics
metric_readers = [
PeriodicExportingMetricReader(exporter, export_interval_millis=5000)
for exporter in exporters
if isinstance(exporter, MetricExporter)
]
if not metric_readers:
from opentelemetry.sdk.metrics.export import ConsoleMetricExporter
metric_readers = [PeriodicExportingMetricReader(ConsoleMetricExporter(), export_interval_millis=5000)]
meter_provider = MeterProvider(
metric_readers=[
PeriodicExportingMetricReader(exporter, export_interval_millis=5000)
for exporter in exporters
if isinstance(exporter, MetricExporter)
],
metric_readers=metric_readers,
resource=self.resource,
views=[
# Dropping all instrument names except for those starting with "agent_framework"
@@ -555,10 +520,6 @@ class OtelSettings(AFBaseSettings):
metrics.set_meter_provider(meter_provider)
global OTEL_SETTINGS
OTEL_SETTINGS: OtelSettings = OtelSettings()
def get_tracer(
instrumenting_module_name: str = "agent_framework",
instrumenting_library_version: str = version_info,
@@ -606,74 +567,152 @@ def get_meter(
return metrics.get_meter(name=name, version=version, schema_url=schema_url, attributes=attributes)
global OBSERVABILITY_SETTINGS
OBSERVABILITY_SETTINGS: ObservabilitySettings = ObservabilitySettings()
def setup_observability(
enable_sensitive_data: bool | None = None,
otlp_endpoint: str | list[str] | None = None,
applicationinsights_connection_string: str | list[str] | None = None,
credential: "TokenCredential | None" = None,
enable_live_metrics: bool | None = None,
exporters: list["LogExporter | SpanExporter | MetricExporter"] | None = None,
vs_code_extension_port: int | None = None,
) -> None:
"""Setup telemetry with optionally provided settings, it is implied that you want to enable telemetry.
"""Convenient method to setup observability for the application.
All of these values can be set through environment variables or you can pass them here,
in the case where both are present, the provided value takes precedence.
This method will create the exporters and the providers for the application,
based on the provided values and the environment variables.
If you have both connection_string and otlp_endpoint, the connection_string will be used.
Call this method once during application startup, before any telemetry is captured.
DO NOT call this method multiple times, as it may lead to unexpected behavior.
If you have configured the providers manually, calling this method will not have any effect:
```python
# Some where in your application startup code
trace.set_tracer_provider(TracerProvider(...))
# After the above call, calling setup_observability will not have any effect
setup_observability()
```
The reverse is also true:
```python
# Some where in your application startup code
setup_observability()
# After the above call, calling trace.set_tracer_provider will not have any effect
trace.set_tracer_provider(TracerProvider(...))
```
The OTel endpoint and the Application Insights connection string can be set through
environment variables or you can pass additional ones here. In the case where both
are present, non-duplicate values will be added:
## With environment variables
This method will read the settings from the environment:
```python
setup_observability()
```
## Without environment variables and use parameters
It is also possible to pass the settings directly:
```python
setup_observability(
enable_sensitive_data=True,
otlp_endpoint=["http://localhost:7431"],
applicationinsights_connection_string=["..."],
exporters=[...], # your custom exporters
vs_code_extension_port=4317,
)
```
## Mixed
When both environment variables and parameters are used, the following settings will get overridden:
- enable_sensitive_data
- vs_code_extension_port
The endpoints and connection strings will be combined, excluding duplicates.
```env
OTEL_ENDPOINT="http://localhost:7431"
```
```python
setup_observability(
enable_sensitive_data=True,
otlp_endpoint=["http://localhost:4317"],
)
```
Exporters will be created for both endpoints.
Args:
enable_sensitive_data: Enable OpenTelemetry sensitive events. Default is False.
enable_sensitive_data: Enable OpenTelemetry sensitive events.
If set, this will override the value set through the environment variable.
Default is None.
otlp_endpoint: The OpenTelemetry Protocol (OTLP) endpoint. Default is None.
Will be used to create a `OTLPLogExporter`, `OTLPMetricExporter` and `OTLPSpanExporter`
applicationinsights_connection_string: The Azure Monitor connection string. Default is None.
Will be used to create AzureMonitorExporters.
credential: The credential to use for Azure Monitor Entra ID authentication.
Default is None.
enable_live_metrics: Enable Azure Monitor live metrics. Default is False.
exporters: a list of exporters, for logs, metrics or spans, or any combination,
these will be added directly, and allows you to customize the spans completely
exporters: A list of exporters, for logs, metrics or spans, or any combination.
These will be added directly, and allows you to customize the spans completely.
vs_code_extension_port: The port the AI Toolkit or AzureAI Foundry VS Code extensions are
listening on. When this is set, additional OTEL exporters will be created with endpoint
`http://localhost:{vs_code_extension_port}` unless this endpoint is already configured.
This will override the value set through the environment variable.
Default is None.
"""
global OTEL_SETTINGS
# Update the otel settings with the provided values
OTEL_SETTINGS.enable_otel = True
global OBSERVABILITY_SETTINGS
# Update the observability settings with the provided values
OBSERVABILITY_SETTINGS.enable_otel = True
if enable_sensitive_data is not None:
OTEL_SETTINGS.enable_sensitive_data = enable_sensitive_data
if enable_live_metrics is not None:
OTEL_SETTINGS.applicationinsights_live_metrics = enable_live_metrics
# Run the initial setup, which will create the providers, and add env setting exporters
new_exporters: list["LogExporter | SpanExporter | MetricExporter"] = []
if OTEL_SETTINGS.ENABLED and (otlp_endpoint or applicationinsights_connection_string or exporters):
# create the exporters, after checking if they are already configured through the env.
new_exporters = exporters or []
if otlp_endpoint:
if isinstance(otlp_endpoint, str):
otlp_endpoint = [otlp_endpoint]
new_exporters.extend(
_get_otlp_exporters(
endpoints=[
endpoint
for endpoint in otlp_endpoint
if not OTEL_SETTINGS.check_endpoint_already_configured(endpoint)
]
)
OBSERVABILITY_SETTINGS.enable_sensitive_data = enable_sensitive_data
if vs_code_extension_port is not None:
OBSERVABILITY_SETTINGS.vs_code_extension_port = vs_code_extension_port
# Create exporters, after checking if they are already configured through the env.
new_exporters: list["LogExporter | SpanExporter | MetricExporter"] = exporters or []
if otlp_endpoint:
if isinstance(otlp_endpoint, str):
otlp_endpoint = [otlp_endpoint]
new_exporters.extend(
_get_otlp_exporters(
endpoints=[
endpoint
for endpoint in otlp_endpoint
if not OBSERVABILITY_SETTINGS.check_endpoint_already_configured(endpoint)
]
)
if applicationinsights_connection_string:
if isinstance(applicationinsights_connection_string, str):
applicationinsights_connection_string = [applicationinsights_connection_string]
new_exporters.extend(
_get_azure_monitor_exporters(
connection_strings=[
conn_str
for conn_str in applicationinsights_connection_string
if not OTEL_SETTINGS.check_connection_string_already_configured(conn_str)
],
credential=credential,
)
)
if applicationinsights_connection_string:
if isinstance(applicationinsights_connection_string, str):
applicationinsights_connection_string = [applicationinsights_connection_string]
new_exporters.extend(
_get_azure_monitor_exporters(
connection_strings=[
conn_str
for conn_str in applicationinsights_connection_string
if not OBSERVABILITY_SETTINGS.check_connection_string_already_configured(conn_str)
],
credential=credential,
)
OTEL_SETTINGS.setup_observability(
credential=credential, additional_exporters=new_exporters, force_setup=bool(new_exporters)
)
)
if OBSERVABILITY_SETTINGS.vs_code_extension_port:
endpoint = f"http://localhost:{OBSERVABILITY_SETTINGS.vs_code_extension_port}"
if OBSERVABILITY_SETTINGS.check_endpoint_already_configured(endpoint):
new_exporters.extend(_get_otlp_exporters(endpoints=[endpoint]))
OBSERVABILITY_SETTINGS._configure(credential=credential, additional_exporters=new_exporters) # pyright: ignore[reportPrivateUsage]
# region Chat Client Telemetry
@@ -721,8 +760,8 @@ def _trace_get_response(
messages: "str | ChatMessage | list[str] | list[ChatMessage]",
**kwargs: Any,
) -> "ChatResponse":
global OTEL_SETTINGS
if not OTEL_SETTINGS.ENABLED:
global OBSERVABILITY_SETTINGS
if not OBSERVABILITY_SETTINGS.ENABLED:
# If model diagnostics are not enabled, just return the completion
return await func(
self,
@@ -747,7 +786,7 @@ def _trace_get_response(
**kwargs,
)
with _get_span(attributes=attributes, span_name_attribute=SpanAttributes.LLM_REQUEST_MODEL) as span:
if OTEL_SETTINGS.SENSITIVE_DATA_ENABLED and messages:
if OBSERVABILITY_SETTINGS.SENSITIVE_DATA_ENABLED and messages:
_capture_messages(span=span, provider_name=provider_name, messages=messages)
start_time_stamp = perf_counter()
end_time_stamp: float | None = None
@@ -767,7 +806,7 @@ def _trace_get_response(
token_usage_histogram=self.additional_properties["token_usage_histogram"],
operation_duration_histogram=self.additional_properties["operation_duration_histogram"],
)
if OTEL_SETTINGS.SENSITIVE_DATA_ENABLED and response.messages:
if OBSERVABILITY_SETTINGS.SENSITIVE_DATA_ENABLED and response.messages:
_capture_messages(
span=span,
provider_name=provider_name,
@@ -803,8 +842,8 @@ def _trace_get_streaming_response(
async def trace_get_streaming_response(
self: "ChatClientProtocol", messages: "str | ChatMessage | list[str] | list[ChatMessage]", **kwargs: Any
) -> AsyncIterable["ChatResponseUpdate"]:
global OTEL_SETTINGS
if not OTEL_SETTINGS.ENABLED:
global OBSERVABILITY_SETTINGS
if not OBSERVABILITY_SETTINGS.ENABLED:
# If model diagnostics are not enabled, just return the completion
async for update in func(self, messages=messages, **kwargs):
yield update
@@ -829,7 +868,7 @@ def _trace_get_streaming_response(
)
all_updates: list["ChatResponseUpdate"] = []
with _get_span(attributes=attributes, span_name_attribute=SpanAttributes.LLM_REQUEST_MODEL) as span:
if OTEL_SETTINGS.SENSITIVE_DATA_ENABLED and messages:
if OBSERVABILITY_SETTINGS.SENSITIVE_DATA_ENABLED and messages:
_capture_messages(
span=span,
provider_name=provider_name,
@@ -859,7 +898,7 @@ def _trace_get_streaming_response(
operation_duration_histogram=self.additional_properties["operation_duration_histogram"],
)
if OTEL_SETTINGS.SENSITIVE_DATA_ENABLED and response.messages:
if OBSERVABILITY_SETTINGS.SENSITIVE_DATA_ENABLED and response.messages:
_capture_messages(
span=span,
provider_name=provider_name,
@@ -937,9 +976,9 @@ def _trace_agent_run(
thread: "AgentThread | None" = None,
**kwargs: Any,
) -> "AgentRunResponse":
global OTEL_SETTINGS
global OBSERVABILITY_SETTINGS
if not OTEL_SETTINGS.ENABLED:
if not OBSERVABILITY_SETTINGS.ENABLED:
# If model diagnostics are not enabled, just return the completion
return await run_func(self, messages=messages, thread=thread, **kwargs)
@@ -953,7 +992,7 @@ def _trace_agent_run(
**kwargs,
)
with _get_span(attributes=attributes, span_name_attribute=OtelAttr.AGENT_NAME) as span:
if OTEL_SETTINGS.SENSITIVE_DATA_ENABLED and messages:
if OBSERVABILITY_SETTINGS.SENSITIVE_DATA_ENABLED and messages:
_capture_messages(
span=span,
provider_name=provider_name,
@@ -968,7 +1007,7 @@ def _trace_agent_run(
else:
attributes = _get_response_attributes(attributes, response)
_capture_response(span=span, attributes=attributes)
if OTEL_SETTINGS.SENSITIVE_DATA_ENABLED and response.messages:
if OBSERVABILITY_SETTINGS.SENSITIVE_DATA_ENABLED and response.messages:
_capture_messages(
span=span,
provider_name=provider_name,
@@ -1000,9 +1039,9 @@ def _trace_agent_run_stream(
thread: "AgentThread | None" = None,
**kwargs: Any,
) -> AsyncIterable["AgentRunResponseUpdate"]:
global OTEL_SETTINGS
global OBSERVABILITY_SETTINGS
if not OTEL_SETTINGS.ENABLED:
if not OBSERVABILITY_SETTINGS.ENABLED:
# If model diagnostics are not enabled, just return the completion
async for streaming_agent_response in run_streaming_func(self, messages=messages, thread=thread, **kwargs):
yield streaming_agent_response
@@ -1022,7 +1061,7 @@ def _trace_agent_run_stream(
**kwargs,
)
with _get_span(attributes=attributes, span_name_attribute=OtelAttr.AGENT_NAME) as span:
if OTEL_SETTINGS.SENSITIVE_DATA_ENABLED and messages:
if OBSERVABILITY_SETTINGS.SENSITIVE_DATA_ENABLED and messages:
_capture_messages(
span=span,
provider_name=provider_name,
@@ -1040,7 +1079,7 @@ def _trace_agent_run_stream(
response = AgentRunResponse.from_agent_run_response_updates(all_updates)
attributes = _get_response_attributes(attributes, response)
_capture_response(span=span, attributes=attributes)
if OTEL_SETTINGS.SENSITIVE_DATA_ENABLED and response.messages:
if OBSERVABILITY_SETTINGS.SENSITIVE_DATA_ENABLED and response.messages:
_capture_messages(
span=span,
provider_name=provider_name,
@@ -1342,8 +1381,8 @@ class EdgeGroupDeliveryStatus(Enum):
def workflow_tracer() -> "Tracer":
"""Get a workflow tracer or a no-op tracer if not enabled."""
global OTEL_SETTINGS
return get_tracer() if OTEL_SETTINGS.ENABLED else trace.NoOpTracer()
global OBSERVABILITY_SETTINGS
return get_tracer() if OBSERVABILITY_SETTINGS.ENABLED else trace.NoOpTracer()
def create_workflow_span(
python/packages/main/tests/conftest.py
+14 -12
View File
@@ -23,14 +23,13 @@ def enable_sensitive_data(request: Any) -> bool:
@fixture(autouse=True)
def span_exporter(monkeypatch, enable_otel: bool, enable_sensitive_data: bool) -> Generator[SpanExporter]:
"""Fixture to remove environment variables for OtelSettings."""
"""Fixture to remove environment variables for ObservabilitySettings."""
env_vars = [
"ENABLE_OTEL",
"ENABLE_SENSITIVE_DATA",
"OTLP_ENDPOINT",
"APPLICATIONINSIGHTS_CONNECTION_STRING",
"APPLICATIONINSIGHTS_LIVE_METRICS",
]
for key in env_vars:
@@ -47,22 +46,25 @@ def span_exporter(monkeypatch, enable_otel: bool, enable_sensitive_data: bool) -
import agent_framework.observability as observability
# Reload the module to ensure a clean state for tests, then create a
# fresh OtelSettings instance and patch the module attribute.
# fresh ObservabilitySettings instance and patch the module attribute.
importlib.reload(observability)
# recreate otel settings with values from above and no file.
otel = observability.OtelSettings(env_file_path="test.env")
otel.setup_observability()
monkeypatch.setattr(observability, "OTEL_SETTINGS", otel, raising=False) # type: ignore
exporter = InMemorySpanExporter()
# recreate observability settings with values from above and no file.
observability_settings = observability.ObservabilitySettings(env_file_path="test.env")
observability_settings._configure() # pyright: ignore[reportPrivateUsage]
monkeypatch.setattr(observability, "OBSERVABILITY_SETTINGS", observability_settings, raising=False) # type: ignore
with (
patch("agent_framework.observability.OTEL_SETTINGS", otel),
patch("agent_framework.observability.OBSERVABILITY_SETTINGS", observability_settings),
patch("agent_framework.observability.setup_observability"),
):
exporter = InMemorySpanExporter()
if enable_otel or enable_sensitive_data:
trace.get_tracer_provider().add_span_processor(
SimpleSpanProcessor(exporter) # type: ignore[func-returns-value]
)
tracer_provider = trace.get_tracer_provider()
if not hasattr(tracer_provider, "add_span_processor"):
raise RuntimeError("Tracer provider does not support adding span processors.")
tracer_provider.add_span_processor(SimpleSpanProcessor(exporter)) # type: ignore
yield exporter
# Clean up