agent/agent-framework
2
0
Fork 0
mirror of https://github.com/microsoft/agent-framework.git synced 2026-06-16 21:04:09 +08:00
Code Issues Packages Projects Releases Wiki Activity

Python: updated integration tests and guidance (#4181)

Browse Source 
* updated integration tests and guidance

* fixed merge test

* updated integration tests

* fix: remove duplicate --dist loadfile flag from pytest-xdist config

Only one --dist mode can be active at a time; the second value silently
overrides the first. Keep --dist worksteal (dynamic load balancing) and
remove the redundant --dist loadfile from all workflow files and
pyproject.toml configs.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

* docs: add keep-in-sync notes for merge and integration test workflows

Both python-merge-tests.yml and python-integration-tests.yml share the
same parallel job structure. Added sync reminders in workflow file
comments, the python-testing SKILL.md, and CODING_STANDARD.md.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

* refactor: remove RUN_INTEGRATION_TESTS flag

Integration test gating now uses two mechanisms:
- `@pytest.mark.integration` for test selection via `-m` filtering
- `skip_if_*_disabled` for credential/service availability checks

The RUN_INTEGRATION_TESTS env var was redundant since the marker handles
selection and the skip decorators already check for actual credentials.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

* fix: sync missing env vars from merge-tests to integration-tests

Add OPENAI_EMBEDDINGS_MODEL_ID and AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME
to python-integration-tests.yml to match python-merge-tests.yml.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

* fix: remove remaining RUN_INTEGRATION_TESTS from embedding tests and docs

Missed test_openai_embedding_client.py and vector-stores README in the
earlier cleanup.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

* set functions tests to 3.10

---------

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
Eduard van Valkenburg
2026-02-24 10:35:46 +01:00
committed by GitHub
Unverified
parent 6305e3e092
commit acc49196c1
52 changed files with 731 additions and 212 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/packages/core/tests/openai/test_assistant_provider.py
+4 -5
View File
@@ -755,14 +755,13 @@ class TestToolMerging:
# region Integration Tests
skip_if_openai_integration_tests_disabled = pytest.mark.skipif(
os.getenv("RUN_INTEGRATION_TESTS", "false").lower() != "true"
or os.getenv("OPENAI_API_KEY", "") in ("", "test-dummy-key"),
reason="No real OPENAI_API_KEY provided; skipping integration tests."
if os.getenv("RUN_INTEGRATION_TESTS", "false").lower() == "true"
else "Integration tests are disabled.",
os.getenv("OPENAI_API_KEY", "") in ("", "test-dummy-key"),
reason="No real OPENAI_API_KEY provided; skipping integration tests.",
)
@pytest.mark.flaky
@pytest.mark.integration
@skip_if_openai_integration_tests_disabled
class TestOpenAIAssistantProviderIntegration:
"""Integration tests requiring real OpenAI API."""
python/packages/core/tests/openai/test_openai_assistants_client.py
+15 -5
View File
@@ -25,11 +25,8 @@ from agent_framework import (
from agent_framework.openai import OpenAIAssistantsClient
skip_if_openai_integration_tests_disabled = pytest.mark.skipif(
os.getenv("RUN_INTEGRATION_TESTS", "false").lower() != "true"
or os.getenv("OPENAI_API_KEY", "") in ("", "test-dummy-key"),
reason="No real OPENAI_API_KEY provided; skipping integration tests."
if os.getenv("RUN_INTEGRATION_TESTS", "false").lower() == "true"
else "Integration tests are disabled.",
os.getenv("OPENAI_API_KEY", "") in ("", "test-dummy-key"),
reason="No real OPENAI_API_KEY provided; skipping integration tests.",
)
INTEGRATION_TEST_MODEL = "gpt-4.1-nano"
@@ -1075,6 +1072,7 @@ def get_weather(
@pytest.mark.flaky
@pytest.mark.integration
@skip_if_openai_integration_tests_disabled
async def test_get_response() -> None:
"""Test OpenAI Assistants Client response."""
@@ -1100,6 +1098,7 @@ async def test_get_response() -> None:
@pytest.mark.flaky
@pytest.mark.integration
@skip_if_openai_integration_tests_disabled
async def test_get_response_tools() -> None:
"""Test OpenAI Assistants Client response with tools."""
@@ -1121,6 +1120,7 @@ async def test_get_response_tools() -> None:
@pytest.mark.flaky
@pytest.mark.integration
@skip_if_openai_integration_tests_disabled
async def test_streaming() -> None:
"""Test OpenAI Assistants Client streaming response."""
@@ -1152,6 +1152,7 @@ async def test_streaming() -> None:
@pytest.mark.flaky
@pytest.mark.integration
@skip_if_openai_integration_tests_disabled
async def test_streaming_tools() -> None:
"""Test OpenAI Assistants Client streaming response with tools."""
@@ -1182,6 +1183,7 @@ async def test_streaming_tools() -> None:
@pytest.mark.flaky
@pytest.mark.integration
@skip_if_openai_integration_tests_disabled
async def test_with_existing_assistant() -> None:
"""Test OpenAI Assistants Client with existing assistant ID."""
@@ -1210,6 +1212,7 @@ async def test_with_existing_assistant() -> None:
@pytest.mark.flaky
@pytest.mark.integration
@skip_if_openai_integration_tests_disabled
@pytest.mark.skip(reason="OpenAI file search functionality is currently broken - tracked in GitHub issue")
async def test_file_search() -> None:
@@ -1236,6 +1239,7 @@ async def test_file_search() -> None:
@pytest.mark.flaky
@pytest.mark.integration
@skip_if_openai_integration_tests_disabled
@pytest.mark.skip(reason="OpenAI file search functionality is currently broken - tracked in GitHub issue")
async def test_file_search_streaming() -> None:
@@ -1270,6 +1274,7 @@ async def test_file_search_streaming() -> None:
@pytest.mark.flaky
@pytest.mark.integration
@skip_if_openai_integration_tests_disabled
async def test_openai_assistants_agent_basic_run():
"""Test Agent basic run functionality with OpenAIAssistantsClient."""
@@ -1287,6 +1292,7 @@ async def test_openai_assistants_agent_basic_run():
@pytest.mark.flaky
@pytest.mark.integration
@skip_if_openai_integration_tests_disabled
async def test_openai_assistants_agent_basic_run_streaming():
"""Test Agent basic streaming functionality with OpenAIAssistantsClient."""
@@ -1307,6 +1313,7 @@ async def test_openai_assistants_agent_basic_run_streaming():
@pytest.mark.flaky
@pytest.mark.integration
@skip_if_openai_integration_tests_disabled
async def test_openai_assistants_agent_session_persistence():
"""Test Agent session persistence across runs with OpenAIAssistantsClient."""
@@ -1336,6 +1343,7 @@ async def test_openai_assistants_agent_session_persistence():
@pytest.mark.flaky
@pytest.mark.integration
@skip_if_openai_integration_tests_disabled
async def test_openai_assistants_agent_existing_session_id():
"""Test Agent with existing session ID to continue conversations across agent instances."""
@@ -1381,6 +1389,7 @@ async def test_openai_assistants_agent_existing_session_id():
@pytest.mark.flaky
@pytest.mark.integration
@skip_if_openai_integration_tests_disabled
async def test_openai_assistants_agent_code_interpreter():
"""Test Agent with code interpreter through OpenAIAssistantsClient."""
@@ -1401,6 +1410,7 @@ async def test_openai_assistants_agent_code_interpreter():
@pytest.mark.flaky
@pytest.mark.integration
@skip_if_openai_integration_tests_disabled
async def test_agent_level_tool_persistence():
"""Test that agent-level tools persist across multiple runs with OpenAI Assistants Client."""
python/packages/core/tests/openai/test_openai_chat_client.py
+4 -5
View File
@@ -24,11 +24,8 @@ from agent_framework.openai import OpenAIChatClient
from agent_framework.openai._exceptions import OpenAIContentFilterException
skip_if_openai_integration_tests_disabled = pytest.mark.skipif(
os.getenv("RUN_INTEGRATION_TESTS", "false").lower() != "true"
or os.getenv("OPENAI_API_KEY", "") in ("", "test-dummy-key"),
reason="No real OPENAI_API_KEY provided; skipping integration tests."
if os.getenv("RUN_INTEGRATION_TESTS", "false").lower() == "true"
else "Integration tests are disabled.",
os.getenv("OPENAI_API_KEY", "") in ("", "test-dummy-key"),
reason="No real OPENAI_API_KEY provided; skipping integration tests.",
)
@@ -1087,6 +1084,7 @@ class OutputStruct(BaseModel):
@pytest.mark.flaky
@pytest.mark.integration
@skip_if_openai_integration_tests_disabled
@pytest.mark.parametrize(
"option_name,option_value,needs_validation",
@@ -1225,6 +1223,7 @@ async def test_integration_options(
@pytest.mark.flaky
@pytest.mark.integration
@skip_if_openai_integration_tests_disabled
async def test_integration_web_search() -> None:
client = OpenAIChatClient(model_id="gpt-4o-search-preview")
python/packages/core/tests/openai/test_openai_embedding_client.py
+4 -10
View File
@@ -259,20 +259,14 @@ def test_azure_otel_provider_name() -> None:
# --- Integration tests ---
skip_if_openai_integration_tests_disabled = pytest.mark.skipif(
os.getenv("RUN_INTEGRATION_TESTS", "false").lower() != "true"
or os.getenv("OPENAI_API_KEY", "") in ("", "test-dummy-key"),
reason="No real OPENAI_API_KEY provided; skipping integration tests."
if os.getenv("RUN_INTEGRATION_TESTS", "false").lower() == "true"
else "Integration tests are disabled.",
os.getenv("OPENAI_API_KEY", "") in ("", "test-dummy-key"),
reason="No real OPENAI_API_KEY provided; skipping integration tests.",
)
skip_if_azure_openai_integration_tests_disabled = pytest.mark.skipif(
os.getenv("RUN_INTEGRATION_TESTS", "false").lower() != "true"
or not os.getenv("AZURE_OPENAI_ENDPOINT")
not os.getenv("AZURE_OPENAI_ENDPOINT")
or (not os.getenv("AZURE_OPENAI_API_KEY") and not os.getenv("AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME")),
reason="No Azure OpenAI credentials provided; skipping integration tests."
if os.getenv("RUN_INTEGRATION_TESTS", "false").lower() == "true"
else "Integration tests are disabled.",
reason="No Azure OpenAI credentials provided; skipping integration tests.",
)
python/packages/core/tests/openai/test_openai_responses_client.py
+6 -5
View File
@@ -40,11 +40,8 @@ from agent_framework.openai import OpenAIResponsesClient
from agent_framework.openai._exceptions import OpenAIContentFilterException
skip_if_openai_integration_tests_disabled = pytest.mark.skipif(
os.getenv("RUN_INTEGRATION_TESTS", "false").lower() != "true"
or os.getenv("OPENAI_API_KEY", "") in ("", "test-dummy-key"),
reason="No real OPENAI_API_KEY provided; skipping integration tests."
if os.getenv("RUN_INTEGRATION_TESTS", "false").lower() == "true"
else "Integration tests are disabled.",
os.getenv("OPENAI_API_KEY", "") in ("", "test-dummy-key"),
reason="No real OPENAI_API_KEY provided; skipping integration tests.",
)
@@ -2362,6 +2359,7 @@ def test_with_callable_api_key() -> None:
@pytest.mark.flaky
@pytest.mark.integration
@skip_if_openai_integration_tests_disabled
@pytest.mark.parametrize(
"option_name,option_value,needs_validation",
@@ -2500,6 +2498,7 @@ async def test_integration_options(
@pytest.mark.timeout(300)
@pytest.mark.flaky
@pytest.mark.integration
@skip_if_openai_integration_tests_disabled
async def test_integration_web_search() -> None:
client = OpenAIResponsesClient(model_id="gpt-5")
@@ -2553,6 +2552,7 @@ async def test_integration_web_search() -> None:
"race condition. See https://github.com/microsoft/agent-framework/issues/1669"
)
@pytest.mark.flaky
@pytest.mark.integration
@skip_if_openai_integration_tests_disabled
async def test_integration_file_search() -> None:
openai_responses_client = OpenAIResponsesClient()
@@ -2586,6 +2586,7 @@ async def test_integration_file_search() -> None:
"potential race condition. See https://github.com/microsoft/agent-framework/issues/1669"
)
@pytest.mark.flaky
@pytest.mark.integration
@skip_if_openai_integration_tests_disabled
async def test_integration_streaming_file_search() -> None:
openai_responses_client = OpenAIResponsesClient()