Python: improve .env handling and observability samples (#4032)

* Python: improve .env precedence and observability samples

- Switch load_settings to explicit precedence: overrides -> explicit .env -> environment -> defaults\n- Raise when env_file_path is provided but missing\n- Update settings docs and tests for new behavior\n- Refresh observability samples and README guidance for env loading options\n\nCloses #3864\n\nCo-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

* fixed some imports

* Fix load_settings CI regressions

Allow explicit env_file_path values that exist but are not regular files (for example /dev/null) by checking path existence before dotenv parsing, and restore a dict accumulator with typed return cast to satisfy mypy.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

* Avoid implicit dotenv in observability

Only load dotenv in observability helpers when env_file_path is explicitly provided, and remove test os.devnull workarounds that are no longer necessary.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

---------

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

python/packages/core/agent_framework/__init__.py

@@ -56,6 +56,7 @@ from ._sessions import (
     SessionContext,
     register_state_type,
 )
+from ._settings import SecretString, load_settings
 from ._telemetry import (
     AGENT_FRAMEWORK_USER_AGENT,
     APP_INFO,
@@ -67,6 +68,7 @@ from ._tools import (
     FunctionInvocationConfiguration,
     FunctionInvocationLayer,
     FunctionTool,
+    ToolTypes,
     normalize_function_invocation_configuration,
     tool,
 )
@@ -234,6 +236,7 @@ __all__ = [
     "RoleLiteral",
     "Runner",
     "RunnerContext",
+    "SecretString",
     "SessionContext",
     "SingleEdgeGroup",
     "SubWorkflowRequestMessage",
@@ -250,6 +253,7 @@ __all__ = [
     "SwitchCaseEdgeGroupDefault",
     "TextSpanRegion",
     "ToolMode",
+    "ToolTypes",
     "TypeCompatibilityError",
     "UpdateT",
     "UsageDetails",
@@ -282,6 +286,7 @@ __all__ = [
     "executor",
     "function_middleware",
     "handler",
+    "load_settings",
     "map_chat_to_agent_update",
     "merge_chat_options",
     "normalize_function_invocation_configuration",

python/packages/core/agent_framework/_settings.py

@@ -36,7 +36,7 @@ from collections.abc import Callable, Sequence
 from contextlib import suppress
 from typing import Any, Union, get_args, get_origin, get_type_hints
 
-from dotenv import load_dotenv
+from dotenv import dotenv_values
 
 from .exceptions import SettingNotFoundError
 
@@ -172,14 +172,14 @@ def load_settings(
     required_fields: Sequence[str | tuple[str, ...]] | None = None,
     **overrides: Any,
 ) -> SettingsT:
-    """Load settings from environment variables, a ``.env`` file, and explicit overrides.
+    """Load settings from explicit overrides, an optional ``.env`` file, and environment variables.
 
     The *settings_type* must be a ``TypedDict`` subclass.  Values are resolved in
     this order (highest priority first):
 
     1. Explicit keyword *overrides* (``None`` values are filtered out).
-    2. Environment variables (``<env_prefix><FIELD_NAME>``).
-    3. A ``.env`` file (loaded via ``python-dotenv``; existing env vars take precedence).
+    2. A ``.env`` file (when *env_file_path* is explicitly provided).
+    3. Environment variables (``<env_prefix><FIELD_NAME>``).
     4. Default values — fields with class-level defaults on the TypedDict, or
        ``None`` for optional fields.
 
@@ -192,7 +192,8 @@ def load_settings(
     Args:
         settings_type: A ``TypedDict`` class describing the settings schema.
         env_prefix: Prefix for environment variable lookup (e.g. ``"OPENAI_"``).
-        env_file_path: Path to ``.env`` file.  Defaults to ``".env"`` when omitted.
+        env_file_path: Path to ``.env`` file. When provided, the file is required
+            and values are resolved before process environment variables.
         env_file_encoding: Encoding for reading the ``.env`` file.  Defaults to ``"utf-8"``.
         required_fields: Field names (``str``) that must resolve to a non-``None``
             value, or tuples of field names where exactly one must be set.
@@ -203,16 +204,22 @@ def load_settings(
         A populated dict matching *settings_type*.
 
     Raises:
+        FileNotFoundError: If *env_file_path* was provided but the file does not exist.
         SettingNotFoundError: If a required field could not be resolved from any
             source, or if a mutually exclusive constraint is violated.
         ServiceInitializationError: If an override value has an incompatible type.
     """
     encoding = env_file_encoding or "utf-8"
 
-    # Load .env file if it exists (existing env vars take precedence by default)
-    env_path = env_file_path or ".env"
-    if os.path.isfile(env_path):
-        load_dotenv(dotenv_path=env_path, encoding=encoding)
+    loaded_dotenv_values: dict[str, str] = {}
+    if env_file_path is not None:
+        if not os.path.exists(env_file_path):
+            raise FileNotFoundError(env_file_path)
+
+        raw_dotenv_values = dotenv_values(dotenv_path=env_file_path, encoding=encoding)
+        loaded_dotenv_values = {
+            key: value for key, value in raw_dotenv_values.items() if key is not None and value is not None
+        }
 
     # Filter out None overrides so defaults / env vars are preserved
     overrides = {k: v for k, v in overrides.items() if v is not None}
@@ -235,8 +242,19 @@ def load_settings(
             result[field_name] = override_value
             continue
 
-        # 2. Environment variable
         env_var_name = f"{env_prefix}{field_name.upper()}"
+
+        # 2. Optional .env value (only when env_file_path is explicitly provided)
+        if loaded_dotenv_values:
+            dotenv_value = loaded_dotenv_values.get(env_var_name)
+            if dotenv_value is not None:
+                try:
+                    result[field_name] = _coerce_value(dotenv_value, field_type)
+                except (ValueError, TypeError):
+                    result[field_name] = dotenv_value
+                continue
+
+        # 3. Environment variable
         env_value = os.getenv(env_var_name)
         if env_value is not None:
             try:
@@ -245,7 +263,7 @@ def load_settings(
                 result[field_name] = env_value
             continue
 
-        # 3. Default from TypedDict class-level defaults, or None for optional fields
+        # 4. Default from TypedDict class-level defaults, or None for optional fields
         if hasattr(settings_type, field_name):
             result[field_name] = getattr(settings_type, field_name)
         else:

python/packages/core/agent_framework/azure/_shared.py

@@ -33,10 +33,9 @@ DEFAULT_AZURE_TOKEN_ENDPOINT: Final[str] = "https://cognitiveservices.azure.com/
 class AzureOpenAISettings(TypedDict, total=False):
     """AzureOpenAI model settings.
 
-    The settings are first loaded from environment variables with the prefix 'AZURE_OPENAI_'.
-    If the environment variables are not found, the settings can be loaded from a .env file
-    with the encoding 'utf-8'. If the settings are not found in the .env file, the settings
-    are ignored; however, validation will fail alerting that the settings are missing.
+    Settings are resolved in this order: explicit keyword arguments, values from an
+    explicitly provided .env file, then environment variables with the prefix
+    'AZURE_OPENAI_'. If settings are missing after resolution, validation will fail.
 
     Keyword Args:
         endpoint: The endpoint of the Azure deployment. This value

python/packages/core/agent_framework/observability.py

@@ -440,7 +440,7 @@ def _get_exporters_from_env(
 
     Args:
         env_file_path: Path to a .env file to load environment variables from.
-            Default is None, which loads from '.env' if present.
+            Default is None, which does not load a .env file.
         env_file_encoding: Encoding to use when reading the .env file.
             Default is None, which uses the system default encoding.
 
@@ -451,8 +451,9 @@ def _get_exporters_from_env(
         - https://opentelemetry.io/docs/languages/sdk-configuration/general/
         - https://opentelemetry.io/docs/languages/sdk-configuration/otlp-exporter/
     """
-    # Load environment variables from .env file if present
-    load_dotenv(dotenv_path=env_file_path, encoding=env_file_encoding)
+    # Load environment variables from a .env file only when explicitly provided
+    if env_file_path is not None:
+        load_dotenv(dotenv_path=env_file_path, encoding=env_file_encoding)
 
     # Get base endpoint
     base_endpoint = os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT")
@@ -513,7 +514,7 @@ def create_resource(
         service_version: Override the service version. If not provided, reads from
             OTEL_SERVICE_VERSION environment variable or defaults to the package version.
         env_file_path: Path to a .env file to load environment variables from.
-            Default is None, which loads from '.env' if present.
+            Default is None, which does not load a .env file.
         env_file_encoding: Encoding to use when reading the .env file.
             Default is None, which uses the system default encoding.
         **attributes: Additional resource attributes to include. These will be merged
@@ -541,8 +542,9 @@ def create_resource(
             # Load from custom .env file
             resource = create_resource(env_file_path="config/.env")
     """
-    # Load environment variables from .env file if present
-    load_dotenv(dotenv_path=env_file_path, encoding=env_file_encoding)
+    # Load environment variables from a .env file only when explicitly provided
+    if env_file_path is not None:
+        load_dotenv(dotenv_path=env_file_path, encoding=env_file_encoding)
 
     # Start with provided attributes
     resource_attributes: dict[str, Any] = dict(attributes)

python/packages/core/agent_framework/openai/_shared.py

@@ -78,10 +78,9 @@ def _check_openai_version_for_callable_api_key() -> None:
 class OpenAISettings(TypedDict, total=False):
     """OpenAI environment settings.
 
-    The settings are first loaded from environment variables with the prefix 'OPENAI_'.
-    If the environment variables are not found, the settings can be loaded from a .env file with the
-    encoding 'utf-8'. If the settings are not found in the .env file, the settings are ignored;
-    however, validation will fail alerting that the settings are missing.
+    Settings are resolved in this order: explicit keyword arguments, values from an
+    explicitly provided .env file, then environment variables with the prefix
+    'OPENAI_'. If settings are missing after resolution, validation will fail.
 
     Keyword Args:
         api_key: OpenAI API key, see https://platform.openai.com/account/api-keys.

python/packages/core/tests/azure/test_azure_assistants_client.py

@@ -130,7 +130,7 @@ def test_azure_assistants_client_init_missing_deployment_name(azure_openai_unit_
     """Test AzureOpenAIAssistantsClient initialization with missing deployment name."""
     with pytest.raises(ServiceInitializationError):
         AzureOpenAIAssistantsClient(
-            api_key=azure_openai_unit_test_env.get("AZURE_OPENAI_API_KEY", "test-key"), env_file_path="nonexistent.env"
+            api_key=azure_openai_unit_test_env.get("AZURE_OPENAI_API_KEY", "test-key")
         )
 
 

python/packages/core/tests/azure/test_azure_chat_client.py

@@ -95,7 +95,6 @@ def test_init_endpoint(azure_openai_unit_test_env: dict[str, str]) -> None:
 def test_init_with_empty_deployment_name(azure_openai_unit_test_env: dict[str, str]) -> None:
     with pytest.raises(ServiceInitializationError):
         AzureOpenAIChatClient(
-            env_file_path="test.env",
         )
 
 
@@ -103,7 +102,6 @@ def test_init_with_empty_deployment_name(azure_openai_unit_test_env: dict[str, s
 def test_init_with_empty_endpoint_and_base_url(azure_openai_unit_test_env: dict[str, str]) -> None:
     with pytest.raises(ServiceInitializationError):
         AzureOpenAIChatClient(
-            env_file_path="test.env",
         )
 
 
@@ -126,7 +124,6 @@ def test_serialize(azure_openai_unit_test_env: dict[str, str]) -> None:
         "api_key": azure_openai_unit_test_env["AZURE_OPENAI_API_KEY"],
         "api_version": azure_openai_unit_test_env["AZURE_OPENAI_API_VERSION"],
         "default_headers": default_headers,
-        "env_file_path": "test.env",
     }
 
     azure_chat_client = AzureOpenAIChatClient.from_dict(settings)

python/packages/core/tests/azure/test_azure_responses_client.py

@@ -112,7 +112,6 @@ def test_init_with_default_header(azure_openai_unit_test_env: dict[str, str]) ->
 def test_init_with_empty_model_id(azure_openai_unit_test_env: dict[str, str]) -> None:
     with pytest.raises(ServiceInitializationError):
         AzureOpenAIResponsesClient(
-            env_file_path="test.env",
         )
 
 

python/packages/core/tests/conftest.py

@@ -61,7 +61,7 @@ def span_exporter(monkeypatch, enable_instrumentation: bool, enable_sensitive_da
     importlib.reload(observability)
 
     # recreate observability settings with values from above and no file.
-    observability_settings = observability.ObservabilitySettings(env_file_path="test.env")
+    observability_settings = observability.ObservabilitySettings()
 
     # Configure providers manually without calling _configure() to avoid OTLP imports
     if enable_instrumentation or enable_sensitive_data:

python/packages/core/tests/core/test_observability.py

@@ -901,7 +901,7 @@ def test_console_exporters_opt_in_false(monkeypatch):
     monkeypatch.setenv("ENABLE_CONSOLE_EXPORTERS", "false")
     monkeypatch.delenv("OTEL_EXPORTER_OTLP_ENDPOINT", raising=False)
 
-    settings = ObservabilitySettings(env_file_path="test.env")
+    settings = ObservabilitySettings()
     assert settings.enable_console_exporters is False
 
 
@@ -911,7 +911,7 @@ def test_console_exporters_opt_in_true(monkeypatch):
 
     monkeypatch.setenv("ENABLE_CONSOLE_EXPORTERS", "true")
 
-    settings = ObservabilitySettings(env_file_path="test.env")
+    settings = ObservabilitySettings()
     assert settings.enable_console_exporters is True
 
 
@@ -921,7 +921,7 @@ def test_console_exporters_default_false(monkeypatch):
 
     monkeypatch.delenv("ENABLE_CONSOLE_EXPORTERS", raising=False)
 
-    settings = ObservabilitySettings(env_file_path="test.env")
+    settings = ObservabilitySettings()
     assert settings.enable_console_exporters is False
 
 
@@ -996,7 +996,7 @@ def test_observability_settings_is_setup_initial(monkeypatch):
     from agent_framework.observability import ObservabilitySettings
 
     monkeypatch.delenv("ENABLE_INSTRUMENTATION", raising=False)
-    settings = ObservabilitySettings(env_file_path="test.env")
+    settings = ObservabilitySettings()
     assert settings.is_setup is False
 
 
@@ -1464,7 +1464,7 @@ def test_observability_settings_configure_not_enabled(monkeypatch):
     from agent_framework.observability import ObservabilitySettings
 
     monkeypatch.setenv("ENABLE_INSTRUMENTATION", "false")
-    settings = ObservabilitySettings(env_file_path="test.env")
+    settings = ObservabilitySettings()
 
     # Should not raise, should just return early
     settings._configure()
@@ -1485,7 +1485,7 @@ def test_observability_settings_configure_already_setup(monkeypatch):
     ]:
         monkeypatch.delenv(key, raising=False)
 
-    settings = ObservabilitySettings(env_file_path="test.env")
+    settings = ObservabilitySettings()
 
     # Manually mark as set up
     settings._executed_setup = True
@@ -2021,7 +2021,7 @@ def test_configure_providers_with_span_exporters(monkeypatch):
     ]:
         monkeypatch.delenv(key, raising=False)
 
-    settings = ObservabilitySettings(env_file_path="test.env")
+    settings = ObservabilitySettings()
 
     # Create mock span exporter
     mock_span_exporter = Mock(spec=SpanExporter)

python/packages/core/tests/core/test_settings.py

@@ -8,7 +8,7 @@ from typing import TypedDict
 
 import pytest
 
-from agent_framework._settings import SecretString, load_settings
+from agent_framework import SecretString, load_settings
 
 
 class SimpleSettings(TypedDict, total=False):
@@ -106,7 +106,7 @@ class TestDotenvFile:
         finally:
             os.unlink(env_path)
 
-    def test_env_vars_override_dotenv(self, monkeypatch: pytest.MonkeyPatch) -> None:
+    def test_dotenv_overrides_env_vars_when_env_file_path_is_set(self, monkeypatch: pytest.MonkeyPatch) -> None:
         monkeypatch.setenv("TEST_APP_API_KEY", "real-env-key")
 
         with tempfile.NamedTemporaryFile(mode="w", suffix=".env", delete=False) as f:
@@ -117,15 +117,34 @@ class TestDotenvFile:
         try:
             settings = load_settings(SimpleSettings, env_prefix="TEST_APP_", env_file_path=env_path)
 
-            assert settings["api_key"] == "real-env-key"
+            assert settings["api_key"] == "dotenv-key"
         finally:
             os.unlink(env_path)
 
-    def test_missing_dotenv_file(self, monkeypatch: pytest.MonkeyPatch) -> None:
-        monkeypatch.delenv("TEST_APP_API_KEY", raising=False)
-        settings = load_settings(SimpleSettings, env_prefix="TEST_APP_", env_file_path="/nonexistent/.env")
+    def test_env_vars_are_used_when_env_file_path_is_not_set(self, monkeypatch: pytest.MonkeyPatch) -> None:
+        monkeypatch.setenv("TEST_APP_API_KEY", "real-env-key")
+        settings = load_settings(SimpleSettings, env_prefix="TEST_APP_")
 
-        assert settings["api_key"] is None
+        assert settings["api_key"] == "real-env-key"
+
+    def test_overrides_beat_dotenv_and_env_vars(self, monkeypatch: pytest.MonkeyPatch) -> None:
+        monkeypatch.setenv("TEST_APP_TIMEOUT", "120")
+
+        with tempfile.NamedTemporaryFile(mode="w", suffix=".env", delete=False) as f:
+            f.write("TEST_APP_TIMEOUT=90\n")
+            f.flush()
+            env_path = f.name
+
+        try:
+            settings = load_settings(SimpleSettings, env_prefix="TEST_APP_", env_file_path=env_path, timeout=60)
+
+            assert settings["timeout"] == 60
+        finally:
+            os.unlink(env_path)
+
+    def test_missing_dotenv_file_raises(self) -> None:
+        with pytest.raises(FileNotFoundError):
+            load_settings(SimpleSettings, env_prefix="TEST_APP_", env_file_path="/nonexistent/.env")
 
 
 class TestSecretString:

python/packages/core/tests/openai/test_openai_assistants_client.py

@@ -155,7 +155,7 @@ def test_init_missing_model_id(openai_unit_test_env: dict[str, str]) -> None:
     """Test OpenAIAssistantsClient initialization with missing model ID."""
     with pytest.raises(ServiceInitializationError):
         OpenAIAssistantsClient(
-            api_key=openai_unit_test_env.get("OPENAI_API_KEY", "test-key"), env_file_path="nonexistent.env"
+            api_key=openai_unit_test_env.get("OPENAI_API_KEY", "test-key")
         )
 
 
@@ -163,7 +163,7 @@ def test_init_missing_model_id(openai_unit_test_env: dict[str, str]) -> None:
 def test_init_missing_api_key(openai_unit_test_env: dict[str, str]) -> None:
     """Test OpenAIAssistantsClient initialization with missing API key."""
     with pytest.raises(ServiceInitializationError):
-        OpenAIAssistantsClient(model_id="gpt-4", env_file_path="nonexistent.env")
+        OpenAIAssistantsClient(model_id="gpt-4")
 
 
 def test_init_with_default_headers(openai_unit_test_env: dict[str, str]) -> None:

python/packages/core/tests/openai/test_openai_chat_client.py

@@ -98,7 +98,6 @@ def test_init_base_url_from_settings_env() -> None:
 def test_init_with_empty_model_id(openai_unit_test_env: dict[str, str]) -> None:
     with pytest.raises(ServiceInitializationError):
         OpenAIChatClient(
-            env_file_path="test.env",
         )
 
 
@@ -109,7 +108,6 @@ def test_init_with_empty_api_key(openai_unit_test_env: dict[str, str]) -> None:
     with pytest.raises(ServiceInitializationError):
         OpenAIChatClient(
             model_id=model_id,
-            env_file_path="test.env",
         )
 
 

python/packages/core/tests/openai/test_openai_responses_client.py

@@ -140,7 +140,6 @@ def test_init_with_default_header(openai_unit_test_env: dict[str, str]) -> None:
 def test_init_with_empty_model_id(openai_unit_test_env: dict[str, str]) -> None:
     with pytest.raises(ServiceInitializationError):
         OpenAIResponsesClient(
-            env_file_path="test.env",
         )
 
 
@@ -151,7 +150,6 @@ def test_init_with_empty_api_key(openai_unit_test_env: dict[str, str]) -> None:
     with pytest.raises(ServiceInitializationError):
         OpenAIResponsesClient(
             model_id=model_id,
-            env_file_path="test.env",
         )