diff --git a/.github/workflows/python-merge-tests.yml b/.github/workflows/python-merge-tests.yml index 6378e4520e..819822ae27 100644 --- a/.github/workflows/python-merge-tests.yml +++ b/.github/workflows/python-merge-tests.yml @@ -17,6 +17,7 @@ env: # Configure a constant location for the uv cache UV_CACHE_DIR: /tmp/.uv-cache RUN_INTEGRATION_TESTS: "true" + RUN_SAMPLES_TESTS: ${{ vars.RUN_SAMPLES_TESTS }} jobs: paths-filter: @@ -79,6 +80,10 @@ jobs: - name: Test with pytest run: uv run poe --directory ./packages/${{ env.PACKAGE_NAME }} test --junitxml=coverage.xml working-directory: ./python + - name: Test openai samples + if: env.RUN_SAMPLES_TESTS == 'true' + run: uv run pytest tests/samples/ -m "openai" --junitxml=coverage_samples_main.xml + working-directory: ./python - name: Move coverage file run: | mv ./packages/${{ env.PACKAGE_NAME }}/coverage.xml coverage_${{ env.PACKAGE_NAME }}.xml @@ -141,6 +146,10 @@ jobs: - name: Test with pytest run: uv run poe --directory ./packages/${{ env.PACKAGE_NAME }} test --junitxml=coverage.xml working-directory: ./python + - name: Test azure samples + if: env.RUN_SAMPLES_TESTS == 'true' + run: uv run pytest tests/samples/ -m "azure" --junitxml=coverage_samples_azure.xml + working-directory: ./python - name: Move coverage file run: | mv ./packages/${{ env.PACKAGE_NAME }}/coverage.xml coverage_${{ env.PACKAGE_NAME }}.xml @@ -202,6 +211,10 @@ jobs: - name: Test with pytest run: uv run poe --directory ./packages/${{ env.PACKAGE_NAME }} test --junitxml=coverage.xml working-directory: ./python + - name: Test foundry samples + if: env.RUN_SAMPLES_TESTS == 'true' + run: uv run pytest tests/samples/ -m "foundry" --junitxml=coverage_samples_foundry.xml + working-directory: ./python - name: Move coverage file run: | mv ./packages/${{ env.PACKAGE_NAME }}/coverage.xml coverage_${{ env.PACKAGE_NAME }}.xml @@ -220,6 +233,7 @@ jobs: display-options: fEX fail-on-empty: true title: Test results + python-integration-tests-check: if: always() runs-on: ubuntu-latest diff --git a/python/packages/main/tests/openai/test_openai_assistants_client.py b/python/packages/main/tests/openai/test_openai_assistants_client.py index 7e3b455bc9..c78a45d1fb 100644 --- a/python/packages/main/tests/openai/test_openai_assistants_client.py +++ b/python/packages/main/tests/openai/test_openai_assistants_client.py @@ -1049,6 +1049,7 @@ async def test_openai_assistants_client_with_existing_assistant() -> None: @skip_if_openai_integration_tests_disabled +@pytest.mark.skip(reason="OpenAI file search functionality is currently broken - tracked in GitHub issue") async def test_openai_assistants_client_file_search() -> None: """Test OpenAI Assistants Client response.""" async with OpenAIAssistantsClient() as openai_assistants_client: @@ -1071,6 +1072,7 @@ async def test_openai_assistants_client_file_search() -> None: @skip_if_openai_integration_tests_disabled +@pytest.mark.skip(reason="OpenAI file search functionality is currently broken - tracked in GitHub issue") async def test_openai_assistants_client_file_search_streaming() -> None: """Test OpenAI Assistants Client response.""" async with OpenAIAssistantsClient() as openai_assistants_client: diff --git a/python/packages/main/tests/openai/test_openai_responses_client.py b/python/packages/main/tests/openai/test_openai_responses_client.py index 51d8633b79..f9e7c18ab6 100644 --- a/python/packages/main/tests/openai/test_openai_responses_client.py +++ b/python/packages/main/tests/openai/test_openai_responses_client.py @@ -1012,6 +1012,7 @@ async def test_openai_responses_client_web_search_streaming() -> None: @skip_if_openai_integration_tests_disabled +@pytest.mark.skip(reason="OpenAI file search functionality is currently broken - tracked in GitHub issue") async def test_openai_responses_client_file_search() -> None: openai_responses_client = OpenAIResponsesClient() @@ -1036,6 +1037,7 @@ async def test_openai_responses_client_file_search() -> None: @skip_if_openai_integration_tests_disabled +@pytest.mark.skip(reason="OpenAI file search functionality is currently broken - tracked in GitHub issue") async def test_openai_responses_client_streaming_file_search() -> None: openai_responses_client = OpenAIResponsesClient() diff --git a/python/pyproject.toml b/python/pyproject.toml index 0b04f3dd21..503de388d4 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -131,6 +131,11 @@ asyncio_mode = "auto" asyncio_default_fixture_loop_scope = "function" filterwarnings = [] timeout = 120 +markers = [ + "azure: marks tests as Azure provider specific", + "foundry: marks tests as Foundry provider specific", + "openai: marks tests as OpenAI provider specific", +] [tool.coverage.run] omit = [ diff --git a/python/samples/getting_started/threads/suspend_resume_thread.py b/python/samples/getting_started/threads/suspend_resume_thread.py index 28f445816e..88065aa987 100644 --- a/python/samples/getting_started/threads/suspend_resume_thread.py +++ b/python/samples/getting_started/threads/suspend_resume_thread.py @@ -2,46 +2,38 @@ import asyncio -from agent_framework.foundry import FoundryChatClient from agent_framework.openai import OpenAIChatClient -from azure.identity.aio import AzureCliCredential async def suspend_resume_service_managed_thread() -> None: """Demonstrates how to suspend and resume a service-managed thread.""" print("=== Suspend-Resume Service-Managed Thread ===") - # Foundry Chat Client is used as an example here, + # OpenAI Chat Client is used as an example here, # other chat clients can be used as well. - # For authentication, run `az login` command in terminal or replace AzureCliCredential with preferred - # authentication option. - async with ( - AzureCliCredential() as credential, - FoundryChatClient(async_credential=credential).create_agent( - name="Joker", instructions="You are good at telling jokes." - ) as agent, - ): - # Start a new thread for the agent conversation. - thread = agent.get_new_thread() + agent = OpenAIChatClient().create_agent(name="Joker", instructions="You are good at telling jokes.") - # Respond to user input. - query = "Tell me a joke about a pirate." - print(f"User: {query}") - print(f"Agent: {await agent.run(query, thread=thread)}\n") + # Start a new thread for the agent conversation. + thread = agent.get_new_thread() - # Serialize the thread state, so it can be stored for later use. - serialized_thread = await thread.serialize() + # Respond to user input. + query = "Tell me a joke about a pirate." + print(f"User: {query}") + print(f"Agent: {await agent.run(query, thread=thread)}\n") - # The thread can now be saved to a database, file, or any other storage mechanism and loaded again later. - print(f"Serialized thread: {serialized_thread}\n") + # Serialize the thread state, so it can be stored for later use. + serialized_thread = await thread.serialize() - # Deserialize the thread state after loading from storage. - resumed_thread = await agent.deserialize_thread(serialized_thread) + # The thread can now be saved to a database, file, or any other storage mechanism and loaded again later. + print(f"Serialized thread: {serialized_thread}\n") - # Respond to user input. - query = "Now tell the same joke in the voice of a pirate, and add some emojis to the joke." - print(f"User: {query}") - print(f"Agent: {await agent.run(query, thread=resumed_thread)}\n") + # Deserialize the thread state after loading from storage. + resumed_thread = await agent.deserialize_thread(serialized_thread) + + # Respond to user input. + query = "Now tell the same joke in the voice of a pirate, and add some emojis to the joke." + print(f"User: {query}") + print(f"Agent: {await agent.run(query, thread=resumed_thread)}\n") async def suspend_resume_in_memory_thread() -> None: diff --git a/python/tests/__init__.py b/python/tests/__init__.py new file mode 100644 index 0000000000..58b29ef7f3 --- /dev/null +++ b/python/tests/__init__.py @@ -0,0 +1,3 @@ +# Copyright (c) Microsoft. All rights reserved. + +"""Test utilities for sample testing.""" diff --git a/python/tests/sample_utils.py b/python/tests/sample_utils.py new file mode 100644 index 0000000000..878566e223 --- /dev/null +++ b/python/tests/sample_utils.py @@ -0,0 +1,43 @@ +# Copyright (c) Microsoft. All rights reserved. + +import asyncio +import logging +from collections.abc import Awaitable, Callable +from typing import Any + +logger = logging.getLogger(__name__) + + +async def retry( + func: Callable[[], Awaitable[Any]], + retries: int = 3, + reset: Callable[[], None] | None = None, + name: str | None = None, +) -> None: + """Retry function with reset capability and proper logging. + + Args: + func: The function to retry. + retries: Number of retries. + reset: Function to reset the state of any variables used in the function. + name: Optional name for logging purposes. + """ + func_name = name or func.__module__ + logger.info(f"Running {retries} retries with func: {func_name}") + + for i in range(retries): + logger.info(f" Try {i + 1} for {func_name}") + try: + if reset: + reset() + await func() + return + except Exception as e: + logger.warning(f" On try {i + 1} got this error: {e}") + if i == retries - 1: # Last retry + raise + + # Binary exponential backoff like Semantic Kernel + backoff = 2**i + logger.info(f" Sleeping for {backoff} seconds before retrying") + await asyncio.sleep(backoff) diff --git a/python/tests/samples/__init__.py b/python/tests/samples/__init__.py new file mode 100644 index 0000000000..1857030e0e --- /dev/null +++ b/python/tests/samples/__init__.py @@ -0,0 +1,3 @@ +# Copyright (c) Microsoft. All rights reserved. + +"""Sample tests package.""" diff --git a/python/tests/samples/getting_started/__init__.py b/python/tests/samples/getting_started/__init__.py new file mode 100644 index 0000000000..9c75ed9a9b --- /dev/null +++ b/python/tests/samples/getting_started/__init__.py @@ -0,0 +1,3 @@ +# Copyright (c) Microsoft. All rights reserved. + +"""Getting started sample tests.""" diff --git a/python/tests/samples/getting_started/test_agents.py b/python/tests/samples/getting_started/test_agents.py new file mode 100644 index 0000000000..5e04367061 --- /dev/null +++ b/python/tests/samples/getting_started/test_agents.py @@ -0,0 +1,559 @@ +# Copyright (c) Microsoft. All rights reserved. + +import copy +import os +from collections.abc import Awaitable, Callable +from typing import Any + +import pytest +from pytest import MonkeyPatch, mark, param + +from samples.getting_started.agents.azure_assistants_client.azure_assistants_basic import ( + main as azure_assistants_basic, +) +from samples.getting_started.agents.azure_assistants_client.azure_assistants_with_code_interpreter import ( + main as azure_assistants_with_code_interpreter, +) +from samples.getting_started.agents.azure_assistants_client.azure_assistants_with_existing_assistant import ( + main as azure_assistants_with_existing_assistant, +) +from samples.getting_started.agents.azure_assistants_client.azure_assistants_with_explicit_settings import ( + main as azure_assistants_with_explicit_settings, +) +from samples.getting_started.agents.azure_assistants_client.azure_assistants_with_function_tools import ( + main as azure_assistants_with_function_tools, +) +from samples.getting_started.agents.azure_assistants_client.azure_assistants_with_thread import ( + main as azure_assistants_with_thread, +) +from samples.getting_started.agents.azure_chat_client.azure_chat_client_basic import ( + main as azure_chat_client_basic, +) +from samples.getting_started.agents.azure_chat_client.azure_chat_client_with_explicit_settings import ( + main as azure_chat_client_with_explicit_settings, +) +from samples.getting_started.agents.azure_chat_client.azure_chat_client_with_function_tools import ( + main as azure_chat_client_with_function_tools, +) +from samples.getting_started.agents.azure_chat_client.azure_chat_client_with_thread import ( + main as azure_chat_client_with_thread, +) +from samples.getting_started.agents.azure_responses_client.azure_responses_client_basic import ( + main as azure_responses_client_basic, +) +from samples.getting_started.agents.azure_responses_client.azure_responses_client_with_code_interpreter import ( + main as azure_responses_client_with_code_interpreter, +) +from samples.getting_started.agents.azure_responses_client.azure_responses_client_with_explicit_settings import ( + main as azure_responses_client_with_explicit_settings, +) +from samples.getting_started.agents.azure_responses_client.azure_responses_client_with_function_tools import ( + main as azure_responses_client_with_function_tools, +) +from samples.getting_started.agents.azure_responses_client.azure_responses_client_with_thread import ( + main as azure_responses_client_with_thread, +) +from samples.getting_started.agents.foundry.foundry_basic import ( + main as foundry_basic, +) +from samples.getting_started.agents.foundry.foundry_with_code_interpreter import ( + main as foundry_with_code_interpreter, +) +from samples.getting_started.agents.foundry.foundry_with_existing_agent import ( + main as foundry_with_existing_agent, +) +from samples.getting_started.agents.foundry.foundry_with_explicit_settings import ( + main as foundry_with_explicit_settings, +) +from samples.getting_started.agents.foundry.foundry_with_function_tools import ( + main as foundry_with_function_tools, +) +from samples.getting_started.agents.foundry.foundry_with_thread import ( + main as foundry_with_thread, +) +from samples.getting_started.agents.openai_assistants_client.openai_assistants_basic import ( + main as openai_assistants_basic, +) +from samples.getting_started.agents.openai_assistants_client.openai_assistants_with_code_interpreter import ( + main as openai_assistants_with_code_interpreter, +) +from samples.getting_started.agents.openai_assistants_client.openai_assistants_with_existing_assistant import ( + main as openai_assistants_with_existing_assistant, +) +from samples.getting_started.agents.openai_assistants_client.openai_assistants_with_explicit_settings import ( + main as openai_assistants_with_explicit_settings, +) +from samples.getting_started.agents.openai_assistants_client.openai_assistants_with_file_search import ( + main as openai_assistants_with_file_search, +) +from samples.getting_started.agents.openai_assistants_client.openai_assistants_with_function_tools import ( + main as openai_assistants_with_function_tools, +) +from samples.getting_started.agents.openai_assistants_client.openai_assistants_with_thread import ( + main as openai_assistants_with_thread, +) +from samples.getting_started.agents.openai_chat_client.openai_chat_client_basic import ( + main as openai_chat_client_basic, +) +from samples.getting_started.agents.openai_chat_client.openai_chat_client_with_explicit_settings import ( + main as openai_chat_client_with_explicit_settings, +) +from samples.getting_started.agents.openai_chat_client.openai_chat_client_with_function_tools import ( + main as openai_chat_client_with_function_tools, +) +from samples.getting_started.agents.openai_chat_client.openai_chat_client_with_local_mcp import ( + main as openai_chat_client_with_local_mcp, +) +from samples.getting_started.agents.openai_chat_client.openai_chat_client_with_thread import ( + main as openai_chat_client_with_thread, +) +from samples.getting_started.agents.openai_chat_client.openai_chat_client_with_web_search import ( + main as openai_chat_client_with_web_search, +) +from samples.getting_started.agents.openai_responses_client.openai_responses_client_basic import ( + main as openai_responses_client_basic, +) +from samples.getting_started.agents.openai_responses_client.openai_responses_client_reasoning import ( + main as openai_responses_client_reasoning, +) +from samples.getting_started.agents.openai_responses_client.openai_responses_client_with_code_interpreter import ( + main as openai_responses_client_with_code_interpreter, +) +from samples.getting_started.agents.openai_responses_client.openai_responses_client_with_explicit_settings import ( + main as openai_responses_client_with_explicit_settings, +) +from samples.getting_started.agents.openai_responses_client.openai_responses_client_with_file_search import ( + main as openai_responses_client_with_file_search, +) +from samples.getting_started.agents.openai_responses_client.openai_responses_client_with_function_tools import ( + main as openai_responses_client_with_function_tools, +) +from samples.getting_started.agents.openai_responses_client.openai_responses_client_with_local_mcp import ( + main as openai_responses_client_with_local_mcp, +) +from samples.getting_started.agents.openai_responses_client.openai_responses_client_with_thread import ( + main as openai_responses_client_with_thread, +) +from samples.getting_started.agents.openai_responses_client.openai_responses_client_with_web_search import ( + main as openai_responses_client_with_web_search, +) +from tests.sample_utils import retry + +# Environment variable for controlling sample tests +RUN_SAMPLES_TESTS = "RUN_SAMPLES_TESTS" + +# All agent samples across providers +agent_samples = [ + # Azure Assistants Agent samples + param( + azure_assistants_basic, + [], # Non-interactive sample + id="azure_assistants_basic", + marks=[ + pytest.mark.azure, + pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."), + ], + ), + param( + azure_assistants_with_code_interpreter, + [], # Non-interactive sample + id="azure_assistants_with_code_interpreter", + marks=[ + pytest.mark.azure, + pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."), + ], + ), + param( + azure_assistants_with_function_tools, + [], # Non-interactive sample + id="azure_assistants_with_function_tools", + marks=[ + pytest.mark.azure, + pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."), + ], + ), + param( + azure_assistants_with_existing_assistant, + [], # Non-interactive sample + id="azure_assistants_with_existing_assistant", + marks=[ + pytest.mark.azure, + pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."), + ], + ), + param( + azure_assistants_with_explicit_settings, + [], # Non-interactive sample + id="azure_assistants_with_explicit_settings", + marks=[ + pytest.mark.azure, + pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."), + ], + ), + param( + azure_assistants_with_thread, + [], # Non-interactive sample + id="azure_assistants_with_thread", + marks=[ + pytest.mark.azure, + pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."), + ], + ), + # Azure Chat Client Agent samples + param( + azure_chat_client_basic, + [], # Non-interactive sample + id="azure_chat_client_basic", + marks=[ + pytest.mark.azure, + pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."), + ], + ), + param( + azure_chat_client_with_explicit_settings, + [], # Non-interactive sample + id="azure_chat_client_with_explicit_settings", + marks=[ + pytest.mark.azure, + pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."), + ], + ), + param( + azure_chat_client_with_function_tools, + [], # Non-interactive sample + id="azure_chat_client_with_function_tools", + marks=[ + pytest.mark.azure, + pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."), + ], + ), + param( + azure_chat_client_with_thread, + [], # Non-interactive sample + id="azure_chat_client_with_thread", + marks=[ + pytest.mark.azure, + pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."), + ], + ), + # Azure Responses Client Agent samples + param( + azure_responses_client_basic, + [], # Non-interactive sample + id="azure_responses_client_basic", + marks=[ + pytest.mark.azure, + pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."), + ], + ), + param( + azure_responses_client_with_code_interpreter, + [], # Non-interactive sample + id="azure_responses_client_with_code_interpreter", + marks=[ + pytest.mark.azure, + pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."), + ], + ), + param( + azure_responses_client_with_explicit_settings, + [], # Non-interactive sample + id="azure_responses_client_with_explicit_settings", + marks=[ + pytest.mark.azure, + pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."), + ], + ), + param( + azure_responses_client_with_function_tools, + [], # Non-interactive sample + id="azure_responses_client_with_function_tools", + marks=[ + pytest.mark.azure, + pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."), + ], + ), + param( + azure_responses_client_with_thread, + [], # Non-interactive sample + id="azure_responses_client_with_thread", + marks=[ + pytest.mark.azure, + pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."), + ], + ), + # Foundry Agent samples + param( + foundry_basic, + [], # Non-interactive sample + id="foundry_basic", + marks=[ + pytest.mark.foundry, + pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."), + ], + ), + param( + foundry_with_code_interpreter, + [], # Non-interactive sample + id="foundry_with_code_interpreter", + marks=[ + pytest.mark.foundry, + pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."), + ], + ), + param( + foundry_with_existing_agent, + [], # Non-interactive sample + id="foundry_with_existing_agent", + marks=[ + pytest.mark.foundry, + pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."), + ], + ), + param( + foundry_with_explicit_settings, + [], # Non-interactive sample + id="foundry_with_explicit_settings", + marks=[ + pytest.mark.foundry, + pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."), + ], + ), + param( + foundry_with_function_tools, + [], # Non-interactive sample + id="foundry_with_function_tools", + marks=[ + pytest.mark.foundry, + pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."), + ], + ), + param( + foundry_with_thread, + [], # Non-interactive sample + id="foundry_with_thread", + marks=[ + pytest.mark.foundry, + pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."), + ], + ), + # OpenAI Assistants Agent samples + param( + openai_assistants_basic, + [], # Non-interactive sample + id="openai_assistants_basic", + marks=[ + pytest.mark.openai, + pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."), + ], + ), + param( + openai_assistants_with_code_interpreter, + [], # Non-interactive sample + id="openai_assistants_with_code_interpreter", + marks=[ + pytest.mark.openai, + pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."), + ], + ), + param( + openai_assistants_with_existing_assistant, + [], # Non-interactive sample + id="openai_assistants_with_existing_assistant", + marks=[ + pytest.mark.openai, + pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."), + ], + ), + param( + openai_assistants_with_explicit_settings, + [], # Non-interactive sample + id="openai_assistants_with_explicit_settings", + marks=[ + pytest.mark.openai, + pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."), + ], + ), + param( + openai_assistants_with_file_search, + [], # Non-interactive sample + id="openai_assistants_with_file_search", + marks=[ + pytest.mark.openai, + pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."), + pytest.mark.skip(reason="OpenAI file search functionality is currently broken - tracked in GitHub issue"), + ], + ), + param( + openai_assistants_with_function_tools, + [], # Non-interactive sample + id="openai_assistants_with_function_tools", + marks=[ + pytest.mark.openai, + pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."), + ], + ), + param( + openai_assistants_with_thread, + [], # Non-interactive sample + id="openai_assistants_with_thread", + marks=[ + pytest.mark.openai, + pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."), + ], + ), + # OpenAI Chat Client Agent samples + param( + openai_chat_client_basic, + [], # Non-interactive sample + id="openai_chat_client_basic", + marks=[ + pytest.mark.openai, + pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."), + ], + ), + param( + openai_chat_client_with_explicit_settings, + [], # Non-interactive sample + id="openai_chat_client_with_explicit_settings", + marks=[ + pytest.mark.openai, + pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."), + ], + ), + param( + openai_chat_client_with_function_tools, + [], # Non-interactive sample + id="openai_chat_client_with_function_tools", + marks=[ + pytest.mark.openai, + pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."), + ], + ), + param( + openai_chat_client_with_local_mcp, + [], # Non-interactive sample + id="openai_chat_client_with_local_mcp", + marks=[ + pytest.mark.openai, + pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."), + ], + ), + param( + openai_chat_client_with_thread, + [], # Non-interactive sample + id="openai_chat_client_with_thread", + marks=[ + pytest.mark.openai, + pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."), + ], + ), + param( + openai_chat_client_with_web_search, + [], # Non-interactive sample + id="openai_chat_client_with_web_search", + marks=[ + pytest.mark.openai, + pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."), + ], + ), + # OpenAI Responses Client Agent samples + param( + openai_responses_client_basic, + [], # Non-interactive sample + id="openai_responses_client_basic", + marks=[ + pytest.mark.openai, + pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."), + ], + ), + param( + openai_responses_client_reasoning, + [], # Non-interactive sample + id="openai_responses_client_reasoning", + marks=[ + pytest.mark.openai, + pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."), + ], + ), + param( + openai_responses_client_with_code_interpreter, + [], # Non-interactive sample + id="openai_responses_client_with_code_interpreter", + marks=[ + pytest.mark.openai, + pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."), + ], + ), + param( + openai_responses_client_with_explicit_settings, + [], # Non-interactive sample + id="openai_responses_client_with_explicit_settings", + marks=[ + pytest.mark.openai, + pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."), + ], + ), + param( + openai_responses_client_with_file_search, + [], # Non-interactive sample + id="openai_responses_client_with_file_search", + marks=[ + pytest.mark.openai, + pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."), + pytest.mark.skip(reason="OpenAI file search functionality is currently broken - tracked in GitHub issue"), + ], + ), + param( + openai_responses_client_with_function_tools, + [], # Non-interactive sample + id="openai_responses_client_with_function_tools", + marks=[ + pytest.mark.openai, + pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."), + ], + ), + param( + openai_responses_client_with_local_mcp, + [], # Non-interactive sample + id="openai_responses_client_with_local_mcp", + marks=[ + pytest.mark.openai, + pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."), + ], + ), + param( + openai_responses_client_with_thread, + [], # Non-interactive sample + id="openai_responses_client_with_thread", + marks=[ + pytest.mark.openai, + pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."), + ], + ), + param( + openai_responses_client_with_web_search, + [], # Non-interactive sample + id="openai_responses_client_with_web_search", + marks=[ + pytest.mark.openai, + pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."), + ], + ), +] + + +@mark.parametrize("sample, responses", agent_samples) +async def test_agent_samples(sample: Callable[..., Awaitable[Any]], responses: list[str], monkeypatch: MonkeyPatch): + """Test agent samples with input mocking and retry logic.""" + saved_responses = copy.deepcopy(responses) + + def reset(): + responses.clear() + responses.extend(saved_responses) + + def mock_input(prompt: str = "") -> str: + return responses.pop(0) if responses else "exit" + + monkeypatch.setattr("builtins.input", mock_input) + await retry(sample, retries=3, reset=reset) diff --git a/python/tests/samples/getting_started/test_chat_client.py b/python/tests/samples/getting_started/test_chat_client.py new file mode 100644 index 0000000000..2c93ee0804 --- /dev/null +++ b/python/tests/samples/getting_started/test_chat_client.py @@ -0,0 +1,135 @@ +# Copyright (c) Microsoft. All rights reserved. + +import copy +import os +from collections.abc import Awaitable, Callable +from typing import Any + +import pytest +from pytest import MonkeyPatch, mark, param + +from samples.getting_started.chat_client.azure_assistants_client import ( + main as azure_assistants_client, +) +from samples.getting_started.chat_client.azure_chat_client import ( + main as azure_chat_client, +) +from samples.getting_started.chat_client.azure_responses_client import ( + main as azure_responses_client, +) +from samples.getting_started.chat_client.chat_response_cancellation import ( + main as chat_response_cancellation, +) +from samples.getting_started.chat_client.foundry_chat_client import ( + main as foundry_chat_client, +) +from samples.getting_started.chat_client.openai_assistants_client import ( + main as openai_assistants_client, +) +from samples.getting_started.chat_client.openai_chat_client import ( + main as openai_chat_client, +) +from samples.getting_started.chat_client.openai_responses_client import ( + main as openai_responses_client, +) +from tests.sample_utils import retry + +# Environment variable for controlling sample tests +RUN_SAMPLES_TESTS = "RUN_SAMPLES_TESTS" + +# All chat client samples across providers +chat_client_samples = [ + # Azure Chat Client samples + param( + azure_assistants_client, + [], # Non-interactive sample + id="azure_assistants_client", + marks=[ + pytest.mark.azure, + pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."), + ], + ), + param( + azure_chat_client, + [], # Non-interactive sample + id="azure_chat_client", + marks=[ + pytest.mark.azure, + pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."), + ], + ), + param( + azure_responses_client, + [], # Non-interactive sample + id="azure_responses_client", + marks=[ + pytest.mark.azure, + pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."), + ], + ), + # Foundry Chat Client samples + param( + foundry_chat_client, + [], # Non-interactive sample + id="foundry_chat_client", + marks=[ + pytest.mark.foundry, + pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."), + ], + ), + # OpenAI Chat Client samples + param( + openai_assistants_client, + [], # Non-interactive sample + id="openai_assistants_client", + marks=[ + pytest.mark.openai, + pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."), + ], + ), + param( + openai_chat_client, + [], # Non-interactive sample + id="openai_chat_client", + marks=[ + pytest.mark.openai, + pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."), + ], + ), + param( + openai_responses_client, + [], # Non-interactive sample + id="openai_responses_client", + marks=[ + pytest.mark.openai, + pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."), + ], + ), + # General Chat Client samples (no provider-specific environment variable) + param( + chat_response_cancellation, + [], # Non-interactive sample + id="chat_response_cancellation", + marks=pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."), + ), +] + + +@mark.parametrize("sample, responses", chat_client_samples) +async def test_chat_client_samples( + sample: Callable[..., Awaitable[Any]], + responses: list[str], + monkeypatch: MonkeyPatch, +): + """Test chat client samples with input mocking and retry logic.""" + saved_responses = copy.deepcopy(responses) + + def reset(): + responses.clear() + responses.extend(saved_responses) + + def mock_input(prompt: str = "") -> str: + return responses.pop(0) if responses else "exit" + + monkeypatch.setattr("builtins.input", mock_input) + await retry(sample, retries=3, reset=reset) diff --git a/python/tests/samples/getting_started/test_telemetry.py b/python/tests/samples/getting_started/test_telemetry.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/python/tests/samples/getting_started/test_threads.py b/python/tests/samples/getting_started/test_threads.py new file mode 100644 index 0000000000..ec530efa6e --- /dev/null +++ b/python/tests/samples/getting_started/test_threads.py @@ -0,0 +1,54 @@ +# Copyright (c) Microsoft. All rights reserved. + +import copy +import os +from collections.abc import Awaitable, Callable +from typing import Any + +import pytest +from pytest import MonkeyPatch, mark, param + +from samples.getting_started.threads.custom_chat_message_store_thread import main as threads_custom_store +from samples.getting_started.threads.suspend_resume_thread import main as threads_suspend_resume +from tests.sample_utils import retry + +# Environment variable for controlling sample tests +RUN_SAMPLES_TESTS = "RUN_SAMPLES_TESTS" + +# All thread samples +thread_samples = [ + param( + threads_custom_store, + [], # Non-interactive sample + id="threads_custom_store", + marks=[ + pytest.mark.openai, + pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."), + ], + ), + param( + threads_suspend_resume, + [], # Non-interactive sample + id="threads_suspend_resume", + marks=[ + pytest.mark.openai, + pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."), + ], + ), +] + + +@mark.parametrize("sample, responses", thread_samples) +async def test_thread_samples(sample: Callable[..., Awaitable[Any]], responses: list[str], monkeypatch: MonkeyPatch): + """Test thread samples with input mocking and retry logic.""" + saved_responses = copy.deepcopy(responses) + + def reset(): + responses.clear() + responses.extend(saved_responses) + + def mock_input(prompt: str = "") -> str: + return responses.pop(0) if responses else "exit" + + monkeypatch.setattr("builtins.input", mock_input) + await retry(sample, retries=3, reset=reset)