agent/agent-framework
2
0
Fork 0
mirror of https://github.com/microsoft/agent-framework.git synced 2026-06-16 21:04:09 +08:00
Code Issues Packages Projects Releases Wiki Activity

Python: updated integration tests and guidance (#4181)

Browse Source 
* updated integration tests and guidance

* fixed merge test

* updated integration tests

* fix: remove duplicate --dist loadfile flag from pytest-xdist config

Only one --dist mode can be active at a time; the second value silently
overrides the first. Keep --dist worksteal (dynamic load balancing) and
remove the redundant --dist loadfile from all workflow files and
pyproject.toml configs.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

* docs: add keep-in-sync notes for merge and integration test workflows

Both python-merge-tests.yml and python-integration-tests.yml share the
same parallel job structure. Added sync reminders in workflow file
comments, the python-testing SKILL.md, and CODING_STANDARD.md.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

* refactor: remove RUN_INTEGRATION_TESTS flag

Integration test gating now uses two mechanisms:
- `@pytest.mark.integration` for test selection via `-m` filtering
- `skip_if_*_disabled` for credential/service availability checks

The RUN_INTEGRATION_TESTS env var was redundant since the marker handles
selection and the skip decorators already check for actual credentials.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

* fix: sync missing env vars from merge-tests to integration-tests

Add OPENAI_EMBEDDINGS_MODEL_ID and AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME
to python-integration-tests.yml to match python-merge-tests.yml.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

* fix: remove remaining RUN_INTEGRATION_TESTS from embedding tests and docs

Missed test_openai_embedding_client.py and vector-stores README in the
earlier cleanup.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

* set functions tests to 3.10

---------

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
Eduard van Valkenburg
2026-02-24 10:35:46 +01:00
committed by GitHub
Unverified
parent 6305e3e092
commit acc49196c1
52 changed files with 731 additions and 212 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/packages/azure-ai/pyproject.toml
+1 -1
View File
@@ -85,7 +85,7 @@ test = "pytest --cov=agent_framework_azure_ai --cov-report=term-missing:skip-cov
[tool.poe.tasks.integration-tests]
cmd = """
pytest --import-mode=importlib
-n logical --dist loadfile --dist worksteal
-n logical --dist worksteal
tests
"""
python/packages/azure-ai/tests/test_agent_provider.py
+8 -5
View File
@@ -29,11 +29,8 @@ from agent_framework_azure_ai._shared import (
)
skip_if_azure_ai_integration_tests_disabled = pytest.mark.skipif(
os.getenv("RUN_INTEGRATION_TESTS", "false").lower() != "true"
or os.getenv("AZURE_AI_PROJECT_ENDPOINT", "") in ("", "https://test-project.cognitiveservices.azure.com/"),
reason="No real AZURE_AI_PROJECT_ENDPOINT provided; skipping integration tests."
if os.getenv("RUN_INTEGRATION_TESTS", "false").lower() == "true"
else "Integration tests are disabled.",
os.getenv("AZURE_AI_PROJECT_ENDPOINT", "") in ("", "https://test-project.cognitiveservices.azure.com/"),
reason="No real AZURE_AI_PROJECT_ENDPOINT provided; skipping integration tests.",
)
# region Provider Initialization Tests
@@ -779,6 +776,8 @@ def test_from_azure_ai_agent_tools_unknown_dict() -> None:
# region Integration Tests
@pytest.mark.flaky
@pytest.mark.integration
@skip_if_azure_ai_integration_tests_disabled
async def test_integration_create_agent() -> None:
"""Integration test: Create an agent using the provider."""
@@ -801,6 +800,8 @@ async def test_integration_create_agent() -> None:
await provider._agents_client.delete_agent(agent.id) # type: ignore
@pytest.mark.flaky
@pytest.mark.integration
@skip_if_azure_ai_integration_tests_disabled
async def test_integration_get_agent() -> None:
"""Integration test: Get an existing agent using the provider."""
@@ -825,6 +826,8 @@ async def test_integration_get_agent() -> None:
await provider._agents_client.delete_agent(created.id) # type: ignore
@pytest.mark.flaky
@pytest.mark.integration
@skip_if_azure_ai_integration_tests_disabled
async def test_integration_create_and_run() -> None:
"""Integration test: Create an agent and run a conversation."""
python/packages/azure-ai/tests/test_azure_ai_agent_client.py
+16 -5
View File
@@ -50,11 +50,8 @@ from pydantic import BaseModel, Field
from agent_framework_azure_ai import AzureAIAgentClient, AzureAISettings
skip_if_azure_ai_integration_tests_disabled = pytest.mark.skipif(
os.getenv("RUN_INTEGRATION_TESTS", "false").lower() != "true"
or os.getenv("AZURE_AI_PROJECT_ENDPOINT", "") in ("", "https://test-project.cognitiveservices.azure.com/"),
reason="No real AZURE_AI_PROJECT_ENDPOINT provided; skipping integration tests."
if os.getenv("RUN_INTEGRATION_TESTS", "false").lower() == "true"
else "Integration tests are disabled.",
os.getenv("AZURE_AI_PROJECT_ENDPOINT", "") in ("", "https://test-project.cognitiveservices.azure.com/"),
reason="No real AZURE_AI_PROJECT_ENDPOINT provided; skipping integration tests.",
)
@@ -1379,6 +1376,7 @@ def get_weather(
@pytest.mark.flaky
@pytest.mark.integration
@skip_if_azure_ai_integration_tests_disabled
async def test_azure_ai_chat_client_get_response() -> None:
"""Test Azure AI Chat Client response."""
@@ -1404,6 +1402,7 @@ async def test_azure_ai_chat_client_get_response() -> None:
@pytest.mark.flaky
@pytest.mark.integration
@skip_if_azure_ai_integration_tests_disabled
async def test_azure_ai_chat_client_get_response_tools() -> None:
"""Test Azure AI Chat Client response with tools."""
@@ -1425,6 +1424,7 @@ async def test_azure_ai_chat_client_get_response_tools() -> None:
@pytest.mark.flaky
@pytest.mark.integration
@skip_if_azure_ai_integration_tests_disabled
async def test_azure_ai_chat_client_streaming() -> None:
"""Test Azure AI Chat Client streaming response."""
@@ -1456,6 +1456,7 @@ async def test_azure_ai_chat_client_streaming() -> None:
@pytest.mark.flaky
@pytest.mark.integration
@skip_if_azure_ai_integration_tests_disabled
async def test_azure_ai_chat_client_streaming_tools() -> None:
"""Test Azure AI Chat Client streaming response with tools."""
@@ -1483,6 +1484,7 @@ async def test_azure_ai_chat_client_streaming_tools() -> None:
@pytest.mark.flaky
@pytest.mark.integration
@skip_if_azure_ai_integration_tests_disabled
async def test_azure_ai_chat_client_agent_basic_run() -> None:
"""Test Agent basic run functionality with AzureAIAgentClient."""
@@ -1500,6 +1502,7 @@ async def test_azure_ai_chat_client_agent_basic_run() -> None:
@pytest.mark.flaky
@pytest.mark.integration
@skip_if_azure_ai_integration_tests_disabled
async def test_azure_ai_chat_client_agent_basic_run_streaming() -> None:
"""Test Agent basic streaming functionality with AzureAIAgentClient."""
@@ -1520,6 +1523,7 @@ async def test_azure_ai_chat_client_agent_basic_run_streaming() -> None:
@pytest.mark.flaky
@pytest.mark.integration
@skip_if_azure_ai_integration_tests_disabled
async def test_azure_ai_chat_client_agent_thread_persistence() -> None:
"""Test Agent session persistence across runs with AzureAIAgentClient."""
@@ -1546,6 +1550,7 @@ async def test_azure_ai_chat_client_agent_thread_persistence() -> None:
@pytest.mark.flaky
@pytest.mark.integration
@skip_if_azure_ai_integration_tests_disabled
async def test_azure_ai_chat_client_agent_existing_thread_id() -> None:
"""Test Agent existing thread ID functionality with AzureAIAgentClient."""
@@ -1584,6 +1589,7 @@ async def test_azure_ai_chat_client_agent_existing_thread_id() -> None:
@pytest.mark.flaky
@pytest.mark.integration
@skip_if_azure_ai_integration_tests_disabled
async def test_azure_ai_chat_client_agent_code_interpreter():
"""Test Agent with code interpreter through AzureAIAgentClient."""
@@ -1604,6 +1610,7 @@ async def test_azure_ai_chat_client_agent_code_interpreter():
@pytest.mark.flaky
@pytest.mark.integration
@skip_if_azure_ai_integration_tests_disabled
async def test_azure_ai_chat_client_agent_file_search():
"""Test Agent with file search through AzureAIAgentClient."""
@@ -1651,6 +1658,7 @@ async def test_azure_ai_chat_client_agent_file_search():
await client.close()
@pytest.mark.integration
@skip_if_azure_ai_integration_tests_disabled
async def test_azure_ai_chat_client_agent_hosted_mcp_tool() -> None:
"""Integration test for MCP tool with Azure AI Agent using Microsoft Learn MCP."""
@@ -1686,6 +1694,7 @@ async def test_azure_ai_chat_client_agent_hosted_mcp_tool() -> None:
@pytest.mark.flaky
@pytest.mark.integration
@skip_if_azure_ai_integration_tests_disabled
async def test_azure_ai_chat_client_agent_level_tool_persistence():
"""Test that agent-level tools persist across multiple runs with AzureAIAgentClient."""
@@ -1711,6 +1720,7 @@ async def test_azure_ai_chat_client_agent_level_tool_persistence():
assert any(term in second_response.text.lower() for term in ["miami", "sunny", "25"])
@pytest.mark.integration
@skip_if_azure_ai_integration_tests_disabled
async def test_azure_ai_chat_client_agent_chat_options_run_level() -> None:
"""Test ChatOptions parameter coverage at run level."""
@@ -1735,6 +1745,7 @@ async def test_azure_ai_chat_client_agent_chat_options_run_level() -> None:
assert len(response.text) > 0
@pytest.mark.integration
@skip_if_azure_ai_integration_tests_disabled
async def test_azure_ai_chat_client_agent_chat_options_agent_level() -> None:
"""Test ChatOptions parameter coverage agent level."""
python/packages/azure-ai/tests/test_azure_ai_client.py
+8 -7
View File
@@ -47,14 +47,9 @@ from agent_framework_azure_ai import AzureAIClient, AzureAISettings
from agent_framework_azure_ai._shared import from_azure_ai_tools
skip_if_azure_ai_integration_tests_disabled = pytest.mark.skipif(
os.getenv("RUN_INTEGRATION_TESTS", "false").lower() != "true"
or os.getenv("AZURE_AI_PROJECT_ENDPOINT", "") in ("", "https://test-project.cognitiveservices.azure.com/")
os.getenv("AZURE_AI_PROJECT_ENDPOINT", "") in ("", "https://test-project.cognitiveservices.azure.com/")
or os.getenv("AZURE_AI_MODEL_DEPLOYMENT_NAME", "") == "",
reason=(
"No real AZURE_AI_PROJECT_ENDPOINT or AZURE_AI_MODEL_DEPLOYMENT_NAME provided; skipping integration tests."
if os.getenv("RUN_INTEGRATION_TESTS", "false").lower() == "true"
else "Integration tests are disabled."
),
reason="No real AZURE_AI_PROJECT_ENDPOINT or AZURE_AI_MODEL_DEPLOYMENT_NAME provided; skipping integration tests.",
)
@@ -1329,6 +1324,7 @@ async def client() -> AsyncGenerator[AzureAIClient, None]:
@pytest.mark.flaky
@pytest.mark.integration
@skip_if_azure_ai_integration_tests_disabled
@pytest.mark.parametrize(
"option_name,option_value,needs_validation",
@@ -1443,6 +1439,7 @@ async def test_integration_options(
@pytest.mark.flaky
@pytest.mark.integration
@skip_if_azure_ai_integration_tests_disabled
@pytest.mark.parametrize(
"option_name,option_value,needs_validation",
@@ -1541,6 +1538,7 @@ async def test_integration_agent_options(
@pytest.mark.flaky
@pytest.mark.integration
@skip_if_azure_ai_integration_tests_disabled
async def test_integration_web_search() -> None:
async with temporary_chat_client(agent_name="af-int-test-web-search") as client:
@@ -1586,6 +1584,7 @@ async def test_integration_web_search() -> None:
@pytest.mark.flaky
@pytest.mark.integration
@skip_if_azure_ai_integration_tests_disabled
async def test_integration_agent_hosted_mcp_tool() -> None:
"""Integration test for MCP tool with Azure Response Agent using Microsoft Learn MCP."""
@@ -1610,6 +1609,7 @@ async def test_integration_agent_hosted_mcp_tool() -> None:
@pytest.mark.flaky
@pytest.mark.integration
@skip_if_azure_ai_integration_tests_disabled
async def test_integration_agent_hosted_code_interpreter_tool():
"""Test Azure Responses Client agent with code interpreter tool through AzureAIClient."""
@@ -1628,6 +1628,7 @@ async def test_integration_agent_hosted_code_interpreter_tool():
@pytest.mark.flaky
@pytest.mark.integration
@skip_if_azure_ai_integration_tests_disabled
async def test_integration_agent_existing_session():
"""Test Azure Responses Client agent with existing session to continue conversations across agent instances."""
python/packages/azure-ai/tests/test_provider.py
+3 -7
View File
@@ -20,14 +20,9 @@ from azure.identity.aio import AzureCliCredential
from agent_framework_azure_ai import AzureAIProjectAgentProvider
skip_if_azure_ai_integration_tests_disabled = pytest.mark.skipif(
os.getenv("RUN_INTEGRATION_TESTS", "false").lower() != "true"
or os.getenv("AZURE_AI_PROJECT_ENDPOINT", "") in ("", "https://test-project.cognitiveservices.azure.com/")
os.getenv("AZURE_AI_PROJECT_ENDPOINT", "") in ("", "https://test-project.cognitiveservices.azure.com/")
or os.getenv("AZURE_AI_MODEL_DEPLOYMENT_NAME", "") == "",
reason=(
"No real AZURE_AI_PROJECT_ENDPOINT or AZURE_AI_MODEL_DEPLOYMENT_NAME provided; skipping integration tests."
if os.getenv("RUN_INTEGRATION_TESTS", "false").lower() == "true"
else "Integration tests are disabled."
),
reason="No real AZURE_AI_PROJECT_ENDPOINT or AZURE_AI_MODEL_DEPLOYMENT_NAME provided; skipping integration tests.",
)
@@ -698,6 +693,7 @@ async def test_provider_create_agent_with_mcp_and_regular_tools(
@pytest.mark.flaky
@pytest.mark.integration
@skip_if_azure_ai_integration_tests_disabled
async def test_provider_create_and_get_agent_integration() -> None:
"""Integration test for provider create_agent and get_agent."""