mirror of
https://github.com/microsoft/agent-framework.git
synced 2026-06-16 21:04:09 +08:00
Python: Samples Integration Tests (#615)
* Samples Tests * small fixes * job fix * telemetry dependency fix * job error fix * sorting provider specific tests * telemetry fixes * openai file search fix --------- Co-authored-by: Giles Odigwe <gilesodigwe@microsoft.com>
This commit is contained in:
Giles Odigwe
committed by
GitHub
Unverified
GitHub
parent
240edb00cd
commit
ee56314a26
@@ -0,0 +1,3 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
"""Test utilities for sample testing."""
|
||||
@@ -0,0 +1,43 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from collections.abc import Awaitable, Callable
|
||||
from typing import Any
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def retry(
|
||||
func: Callable[[], Awaitable[Any]],
|
||||
retries: int = 3,
|
||||
reset: Callable[[], None] | None = None,
|
||||
name: str | None = None,
|
||||
) -> None:
|
||||
"""Retry function with reset capability and proper logging.
|
||||
|
||||
Args:
|
||||
func: The function to retry.
|
||||
retries: Number of retries.
|
||||
reset: Function to reset the state of any variables used in the function.
|
||||
name: Optional name for logging purposes.
|
||||
"""
|
||||
func_name = name or func.__module__
|
||||
logger.info(f"Running {retries} retries with func: {func_name}")
|
||||
|
||||
for i in range(retries):
|
||||
logger.info(f" Try {i + 1} for {func_name}")
|
||||
try:
|
||||
if reset:
|
||||
reset()
|
||||
await func()
|
||||
return
|
||||
except Exception as e:
|
||||
logger.warning(f" On try {i + 1} got this error: {e}")
|
||||
if i == retries - 1: # Last retry
|
||||
raise
|
||||
|
||||
# Binary exponential backoff like Semantic Kernel
|
||||
backoff = 2**i
|
||||
logger.info(f" Sleeping for {backoff} seconds before retrying")
|
||||
await asyncio.sleep(backoff)
|
||||
@@ -0,0 +1,3 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
"""Sample tests package."""
|
||||
@@ -0,0 +1,3 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
"""Getting started sample tests."""
|
||||
@@ -0,0 +1,559 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
import copy
|
||||
import os
|
||||
from collections.abc import Awaitable, Callable
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from pytest import MonkeyPatch, mark, param
|
||||
|
||||
from samples.getting_started.agents.azure_assistants_client.azure_assistants_basic import (
|
||||
main as azure_assistants_basic,
|
||||
)
|
||||
from samples.getting_started.agents.azure_assistants_client.azure_assistants_with_code_interpreter import (
|
||||
main as azure_assistants_with_code_interpreter,
|
||||
)
|
||||
from samples.getting_started.agents.azure_assistants_client.azure_assistants_with_existing_assistant import (
|
||||
main as azure_assistants_with_existing_assistant,
|
||||
)
|
||||
from samples.getting_started.agents.azure_assistants_client.azure_assistants_with_explicit_settings import (
|
||||
main as azure_assistants_with_explicit_settings,
|
||||
)
|
||||
from samples.getting_started.agents.azure_assistants_client.azure_assistants_with_function_tools import (
|
||||
main as azure_assistants_with_function_tools,
|
||||
)
|
||||
from samples.getting_started.agents.azure_assistants_client.azure_assistants_with_thread import (
|
||||
main as azure_assistants_with_thread,
|
||||
)
|
||||
from samples.getting_started.agents.azure_chat_client.azure_chat_client_basic import (
|
||||
main as azure_chat_client_basic,
|
||||
)
|
||||
from samples.getting_started.agents.azure_chat_client.azure_chat_client_with_explicit_settings import (
|
||||
main as azure_chat_client_with_explicit_settings,
|
||||
)
|
||||
from samples.getting_started.agents.azure_chat_client.azure_chat_client_with_function_tools import (
|
||||
main as azure_chat_client_with_function_tools,
|
||||
)
|
||||
from samples.getting_started.agents.azure_chat_client.azure_chat_client_with_thread import (
|
||||
main as azure_chat_client_with_thread,
|
||||
)
|
||||
from samples.getting_started.agents.azure_responses_client.azure_responses_client_basic import (
|
||||
main as azure_responses_client_basic,
|
||||
)
|
||||
from samples.getting_started.agents.azure_responses_client.azure_responses_client_with_code_interpreter import (
|
||||
main as azure_responses_client_with_code_interpreter,
|
||||
)
|
||||
from samples.getting_started.agents.azure_responses_client.azure_responses_client_with_explicit_settings import (
|
||||
main as azure_responses_client_with_explicit_settings,
|
||||
)
|
||||
from samples.getting_started.agents.azure_responses_client.azure_responses_client_with_function_tools import (
|
||||
main as azure_responses_client_with_function_tools,
|
||||
)
|
||||
from samples.getting_started.agents.azure_responses_client.azure_responses_client_with_thread import (
|
||||
main as azure_responses_client_with_thread,
|
||||
)
|
||||
from samples.getting_started.agents.foundry.foundry_basic import (
|
||||
main as foundry_basic,
|
||||
)
|
||||
from samples.getting_started.agents.foundry.foundry_with_code_interpreter import (
|
||||
main as foundry_with_code_interpreter,
|
||||
)
|
||||
from samples.getting_started.agents.foundry.foundry_with_existing_agent import (
|
||||
main as foundry_with_existing_agent,
|
||||
)
|
||||
from samples.getting_started.agents.foundry.foundry_with_explicit_settings import (
|
||||
main as foundry_with_explicit_settings,
|
||||
)
|
||||
from samples.getting_started.agents.foundry.foundry_with_function_tools import (
|
||||
main as foundry_with_function_tools,
|
||||
)
|
||||
from samples.getting_started.agents.foundry.foundry_with_thread import (
|
||||
main as foundry_with_thread,
|
||||
)
|
||||
from samples.getting_started.agents.openai_assistants_client.openai_assistants_basic import (
|
||||
main as openai_assistants_basic,
|
||||
)
|
||||
from samples.getting_started.agents.openai_assistants_client.openai_assistants_with_code_interpreter import (
|
||||
main as openai_assistants_with_code_interpreter,
|
||||
)
|
||||
from samples.getting_started.agents.openai_assistants_client.openai_assistants_with_existing_assistant import (
|
||||
main as openai_assistants_with_existing_assistant,
|
||||
)
|
||||
from samples.getting_started.agents.openai_assistants_client.openai_assistants_with_explicit_settings import (
|
||||
main as openai_assistants_with_explicit_settings,
|
||||
)
|
||||
from samples.getting_started.agents.openai_assistants_client.openai_assistants_with_file_search import (
|
||||
main as openai_assistants_with_file_search,
|
||||
)
|
||||
from samples.getting_started.agents.openai_assistants_client.openai_assistants_with_function_tools import (
|
||||
main as openai_assistants_with_function_tools,
|
||||
)
|
||||
from samples.getting_started.agents.openai_assistants_client.openai_assistants_with_thread import (
|
||||
main as openai_assistants_with_thread,
|
||||
)
|
||||
from samples.getting_started.agents.openai_chat_client.openai_chat_client_basic import (
|
||||
main as openai_chat_client_basic,
|
||||
)
|
||||
from samples.getting_started.agents.openai_chat_client.openai_chat_client_with_explicit_settings import (
|
||||
main as openai_chat_client_with_explicit_settings,
|
||||
)
|
||||
from samples.getting_started.agents.openai_chat_client.openai_chat_client_with_function_tools import (
|
||||
main as openai_chat_client_with_function_tools,
|
||||
)
|
||||
from samples.getting_started.agents.openai_chat_client.openai_chat_client_with_local_mcp import (
|
||||
main as openai_chat_client_with_local_mcp,
|
||||
)
|
||||
from samples.getting_started.agents.openai_chat_client.openai_chat_client_with_thread import (
|
||||
main as openai_chat_client_with_thread,
|
||||
)
|
||||
from samples.getting_started.agents.openai_chat_client.openai_chat_client_with_web_search import (
|
||||
main as openai_chat_client_with_web_search,
|
||||
)
|
||||
from samples.getting_started.agents.openai_responses_client.openai_responses_client_basic import (
|
||||
main as openai_responses_client_basic,
|
||||
)
|
||||
from samples.getting_started.agents.openai_responses_client.openai_responses_client_reasoning import (
|
||||
main as openai_responses_client_reasoning,
|
||||
)
|
||||
from samples.getting_started.agents.openai_responses_client.openai_responses_client_with_code_interpreter import (
|
||||
main as openai_responses_client_with_code_interpreter,
|
||||
)
|
||||
from samples.getting_started.agents.openai_responses_client.openai_responses_client_with_explicit_settings import (
|
||||
main as openai_responses_client_with_explicit_settings,
|
||||
)
|
||||
from samples.getting_started.agents.openai_responses_client.openai_responses_client_with_file_search import (
|
||||
main as openai_responses_client_with_file_search,
|
||||
)
|
||||
from samples.getting_started.agents.openai_responses_client.openai_responses_client_with_function_tools import (
|
||||
main as openai_responses_client_with_function_tools,
|
||||
)
|
||||
from samples.getting_started.agents.openai_responses_client.openai_responses_client_with_local_mcp import (
|
||||
main as openai_responses_client_with_local_mcp,
|
||||
)
|
||||
from samples.getting_started.agents.openai_responses_client.openai_responses_client_with_thread import (
|
||||
main as openai_responses_client_with_thread,
|
||||
)
|
||||
from samples.getting_started.agents.openai_responses_client.openai_responses_client_with_web_search import (
|
||||
main as openai_responses_client_with_web_search,
|
||||
)
|
||||
from tests.sample_utils import retry
|
||||
|
||||
# Environment variable for controlling sample tests
|
||||
RUN_SAMPLES_TESTS = "RUN_SAMPLES_TESTS"
|
||||
|
||||
# All agent samples across providers
|
||||
agent_samples = [
|
||||
# Azure Assistants Agent samples
|
||||
param(
|
||||
azure_assistants_basic,
|
||||
[], # Non-interactive sample
|
||||
id="azure_assistants_basic",
|
||||
marks=[
|
||||
pytest.mark.azure,
|
||||
pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."),
|
||||
],
|
||||
),
|
||||
param(
|
||||
azure_assistants_with_code_interpreter,
|
||||
[], # Non-interactive sample
|
||||
id="azure_assistants_with_code_interpreter",
|
||||
marks=[
|
||||
pytest.mark.azure,
|
||||
pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."),
|
||||
],
|
||||
),
|
||||
param(
|
||||
azure_assistants_with_function_tools,
|
||||
[], # Non-interactive sample
|
||||
id="azure_assistants_with_function_tools",
|
||||
marks=[
|
||||
pytest.mark.azure,
|
||||
pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."),
|
||||
],
|
||||
),
|
||||
param(
|
||||
azure_assistants_with_existing_assistant,
|
||||
[], # Non-interactive sample
|
||||
id="azure_assistants_with_existing_assistant",
|
||||
marks=[
|
||||
pytest.mark.azure,
|
||||
pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."),
|
||||
],
|
||||
),
|
||||
param(
|
||||
azure_assistants_with_explicit_settings,
|
||||
[], # Non-interactive sample
|
||||
id="azure_assistants_with_explicit_settings",
|
||||
marks=[
|
||||
pytest.mark.azure,
|
||||
pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."),
|
||||
],
|
||||
),
|
||||
param(
|
||||
azure_assistants_with_thread,
|
||||
[], # Non-interactive sample
|
||||
id="azure_assistants_with_thread",
|
||||
marks=[
|
||||
pytest.mark.azure,
|
||||
pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."),
|
||||
],
|
||||
),
|
||||
# Azure Chat Client Agent samples
|
||||
param(
|
||||
azure_chat_client_basic,
|
||||
[], # Non-interactive sample
|
||||
id="azure_chat_client_basic",
|
||||
marks=[
|
||||
pytest.mark.azure,
|
||||
pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."),
|
||||
],
|
||||
),
|
||||
param(
|
||||
azure_chat_client_with_explicit_settings,
|
||||
[], # Non-interactive sample
|
||||
id="azure_chat_client_with_explicit_settings",
|
||||
marks=[
|
||||
pytest.mark.azure,
|
||||
pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."),
|
||||
],
|
||||
),
|
||||
param(
|
||||
azure_chat_client_with_function_tools,
|
||||
[], # Non-interactive sample
|
||||
id="azure_chat_client_with_function_tools",
|
||||
marks=[
|
||||
pytest.mark.azure,
|
||||
pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."),
|
||||
],
|
||||
),
|
||||
param(
|
||||
azure_chat_client_with_thread,
|
||||
[], # Non-interactive sample
|
||||
id="azure_chat_client_with_thread",
|
||||
marks=[
|
||||
pytest.mark.azure,
|
||||
pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."),
|
||||
],
|
||||
),
|
||||
# Azure Responses Client Agent samples
|
||||
param(
|
||||
azure_responses_client_basic,
|
||||
[], # Non-interactive sample
|
||||
id="azure_responses_client_basic",
|
||||
marks=[
|
||||
pytest.mark.azure,
|
||||
pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."),
|
||||
],
|
||||
),
|
||||
param(
|
||||
azure_responses_client_with_code_interpreter,
|
||||
[], # Non-interactive sample
|
||||
id="azure_responses_client_with_code_interpreter",
|
||||
marks=[
|
||||
pytest.mark.azure,
|
||||
pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."),
|
||||
],
|
||||
),
|
||||
param(
|
||||
azure_responses_client_with_explicit_settings,
|
||||
[], # Non-interactive sample
|
||||
id="azure_responses_client_with_explicit_settings",
|
||||
marks=[
|
||||
pytest.mark.azure,
|
||||
pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."),
|
||||
],
|
||||
),
|
||||
param(
|
||||
azure_responses_client_with_function_tools,
|
||||
[], # Non-interactive sample
|
||||
id="azure_responses_client_with_function_tools",
|
||||
marks=[
|
||||
pytest.mark.azure,
|
||||
pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."),
|
||||
],
|
||||
),
|
||||
param(
|
||||
azure_responses_client_with_thread,
|
||||
[], # Non-interactive sample
|
||||
id="azure_responses_client_with_thread",
|
||||
marks=[
|
||||
pytest.mark.azure,
|
||||
pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."),
|
||||
],
|
||||
),
|
||||
# Foundry Agent samples
|
||||
param(
|
||||
foundry_basic,
|
||||
[], # Non-interactive sample
|
||||
id="foundry_basic",
|
||||
marks=[
|
||||
pytest.mark.foundry,
|
||||
pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."),
|
||||
],
|
||||
),
|
||||
param(
|
||||
foundry_with_code_interpreter,
|
||||
[], # Non-interactive sample
|
||||
id="foundry_with_code_interpreter",
|
||||
marks=[
|
||||
pytest.mark.foundry,
|
||||
pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."),
|
||||
],
|
||||
),
|
||||
param(
|
||||
foundry_with_existing_agent,
|
||||
[], # Non-interactive sample
|
||||
id="foundry_with_existing_agent",
|
||||
marks=[
|
||||
pytest.mark.foundry,
|
||||
pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."),
|
||||
],
|
||||
),
|
||||
param(
|
||||
foundry_with_explicit_settings,
|
||||
[], # Non-interactive sample
|
||||
id="foundry_with_explicit_settings",
|
||||
marks=[
|
||||
pytest.mark.foundry,
|
||||
pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."),
|
||||
],
|
||||
),
|
||||
param(
|
||||
foundry_with_function_tools,
|
||||
[], # Non-interactive sample
|
||||
id="foundry_with_function_tools",
|
||||
marks=[
|
||||
pytest.mark.foundry,
|
||||
pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."),
|
||||
],
|
||||
),
|
||||
param(
|
||||
foundry_with_thread,
|
||||
[], # Non-interactive sample
|
||||
id="foundry_with_thread",
|
||||
marks=[
|
||||
pytest.mark.foundry,
|
||||
pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."),
|
||||
],
|
||||
),
|
||||
# OpenAI Assistants Agent samples
|
||||
param(
|
||||
openai_assistants_basic,
|
||||
[], # Non-interactive sample
|
||||
id="openai_assistants_basic",
|
||||
marks=[
|
||||
pytest.mark.openai,
|
||||
pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."),
|
||||
],
|
||||
),
|
||||
param(
|
||||
openai_assistants_with_code_interpreter,
|
||||
[], # Non-interactive sample
|
||||
id="openai_assistants_with_code_interpreter",
|
||||
marks=[
|
||||
pytest.mark.openai,
|
||||
pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."),
|
||||
],
|
||||
),
|
||||
param(
|
||||
openai_assistants_with_existing_assistant,
|
||||
[], # Non-interactive sample
|
||||
id="openai_assistants_with_existing_assistant",
|
||||
marks=[
|
||||
pytest.mark.openai,
|
||||
pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."),
|
||||
],
|
||||
),
|
||||
param(
|
||||
openai_assistants_with_explicit_settings,
|
||||
[], # Non-interactive sample
|
||||
id="openai_assistants_with_explicit_settings",
|
||||
marks=[
|
||||
pytest.mark.openai,
|
||||
pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."),
|
||||
],
|
||||
),
|
||||
param(
|
||||
openai_assistants_with_file_search,
|
||||
[], # Non-interactive sample
|
||||
id="openai_assistants_with_file_search",
|
||||
marks=[
|
||||
pytest.mark.openai,
|
||||
pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."),
|
||||
pytest.mark.skip(reason="OpenAI file search functionality is currently broken - tracked in GitHub issue"),
|
||||
],
|
||||
),
|
||||
param(
|
||||
openai_assistants_with_function_tools,
|
||||
[], # Non-interactive sample
|
||||
id="openai_assistants_with_function_tools",
|
||||
marks=[
|
||||
pytest.mark.openai,
|
||||
pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."),
|
||||
],
|
||||
),
|
||||
param(
|
||||
openai_assistants_with_thread,
|
||||
[], # Non-interactive sample
|
||||
id="openai_assistants_with_thread",
|
||||
marks=[
|
||||
pytest.mark.openai,
|
||||
pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."),
|
||||
],
|
||||
),
|
||||
# OpenAI Chat Client Agent samples
|
||||
param(
|
||||
openai_chat_client_basic,
|
||||
[], # Non-interactive sample
|
||||
id="openai_chat_client_basic",
|
||||
marks=[
|
||||
pytest.mark.openai,
|
||||
pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."),
|
||||
],
|
||||
),
|
||||
param(
|
||||
openai_chat_client_with_explicit_settings,
|
||||
[], # Non-interactive sample
|
||||
id="openai_chat_client_with_explicit_settings",
|
||||
marks=[
|
||||
pytest.mark.openai,
|
||||
pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."),
|
||||
],
|
||||
),
|
||||
param(
|
||||
openai_chat_client_with_function_tools,
|
||||
[], # Non-interactive sample
|
||||
id="openai_chat_client_with_function_tools",
|
||||
marks=[
|
||||
pytest.mark.openai,
|
||||
pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."),
|
||||
],
|
||||
),
|
||||
param(
|
||||
openai_chat_client_with_local_mcp,
|
||||
[], # Non-interactive sample
|
||||
id="openai_chat_client_with_local_mcp",
|
||||
marks=[
|
||||
pytest.mark.openai,
|
||||
pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."),
|
||||
],
|
||||
),
|
||||
param(
|
||||
openai_chat_client_with_thread,
|
||||
[], # Non-interactive sample
|
||||
id="openai_chat_client_with_thread",
|
||||
marks=[
|
||||
pytest.mark.openai,
|
||||
pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."),
|
||||
],
|
||||
),
|
||||
param(
|
||||
openai_chat_client_with_web_search,
|
||||
[], # Non-interactive sample
|
||||
id="openai_chat_client_with_web_search",
|
||||
marks=[
|
||||
pytest.mark.openai,
|
||||
pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."),
|
||||
],
|
||||
),
|
||||
# OpenAI Responses Client Agent samples
|
||||
param(
|
||||
openai_responses_client_basic,
|
||||
[], # Non-interactive sample
|
||||
id="openai_responses_client_basic",
|
||||
marks=[
|
||||
pytest.mark.openai,
|
||||
pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."),
|
||||
],
|
||||
),
|
||||
param(
|
||||
openai_responses_client_reasoning,
|
||||
[], # Non-interactive sample
|
||||
id="openai_responses_client_reasoning",
|
||||
marks=[
|
||||
pytest.mark.openai,
|
||||
pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."),
|
||||
],
|
||||
),
|
||||
param(
|
||||
openai_responses_client_with_code_interpreter,
|
||||
[], # Non-interactive sample
|
||||
id="openai_responses_client_with_code_interpreter",
|
||||
marks=[
|
||||
pytest.mark.openai,
|
||||
pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."),
|
||||
],
|
||||
),
|
||||
param(
|
||||
openai_responses_client_with_explicit_settings,
|
||||
[], # Non-interactive sample
|
||||
id="openai_responses_client_with_explicit_settings",
|
||||
marks=[
|
||||
pytest.mark.openai,
|
||||
pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."),
|
||||
],
|
||||
),
|
||||
param(
|
||||
openai_responses_client_with_file_search,
|
||||
[], # Non-interactive sample
|
||||
id="openai_responses_client_with_file_search",
|
||||
marks=[
|
||||
pytest.mark.openai,
|
||||
pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."),
|
||||
pytest.mark.skip(reason="OpenAI file search functionality is currently broken - tracked in GitHub issue"),
|
||||
],
|
||||
),
|
||||
param(
|
||||
openai_responses_client_with_function_tools,
|
||||
[], # Non-interactive sample
|
||||
id="openai_responses_client_with_function_tools",
|
||||
marks=[
|
||||
pytest.mark.openai,
|
||||
pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."),
|
||||
],
|
||||
),
|
||||
param(
|
||||
openai_responses_client_with_local_mcp,
|
||||
[], # Non-interactive sample
|
||||
id="openai_responses_client_with_local_mcp",
|
||||
marks=[
|
||||
pytest.mark.openai,
|
||||
pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."),
|
||||
],
|
||||
),
|
||||
param(
|
||||
openai_responses_client_with_thread,
|
||||
[], # Non-interactive sample
|
||||
id="openai_responses_client_with_thread",
|
||||
marks=[
|
||||
pytest.mark.openai,
|
||||
pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."),
|
||||
],
|
||||
),
|
||||
param(
|
||||
openai_responses_client_with_web_search,
|
||||
[], # Non-interactive sample
|
||||
id="openai_responses_client_with_web_search",
|
||||
marks=[
|
||||
pytest.mark.openai,
|
||||
pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."),
|
||||
],
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@mark.parametrize("sample, responses", agent_samples)
|
||||
async def test_agent_samples(sample: Callable[..., Awaitable[Any]], responses: list[str], monkeypatch: MonkeyPatch):
|
||||
"""Test agent samples with input mocking and retry logic."""
|
||||
saved_responses = copy.deepcopy(responses)
|
||||
|
||||
def reset():
|
||||
responses.clear()
|
||||
responses.extend(saved_responses)
|
||||
|
||||
def mock_input(prompt: str = "") -> str:
|
||||
return responses.pop(0) if responses else "exit"
|
||||
|
||||
monkeypatch.setattr("builtins.input", mock_input)
|
||||
await retry(sample, retries=3, reset=reset)
|
||||
@@ -0,0 +1,135 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
import copy
|
||||
import os
|
||||
from collections.abc import Awaitable, Callable
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from pytest import MonkeyPatch, mark, param
|
||||
|
||||
from samples.getting_started.chat_client.azure_assistants_client import (
|
||||
main as azure_assistants_client,
|
||||
)
|
||||
from samples.getting_started.chat_client.azure_chat_client import (
|
||||
main as azure_chat_client,
|
||||
)
|
||||
from samples.getting_started.chat_client.azure_responses_client import (
|
||||
main as azure_responses_client,
|
||||
)
|
||||
from samples.getting_started.chat_client.chat_response_cancellation import (
|
||||
main as chat_response_cancellation,
|
||||
)
|
||||
from samples.getting_started.chat_client.foundry_chat_client import (
|
||||
main as foundry_chat_client,
|
||||
)
|
||||
from samples.getting_started.chat_client.openai_assistants_client import (
|
||||
main as openai_assistants_client,
|
||||
)
|
||||
from samples.getting_started.chat_client.openai_chat_client import (
|
||||
main as openai_chat_client,
|
||||
)
|
||||
from samples.getting_started.chat_client.openai_responses_client import (
|
||||
main as openai_responses_client,
|
||||
)
|
||||
from tests.sample_utils import retry
|
||||
|
||||
# Environment variable for controlling sample tests
|
||||
RUN_SAMPLES_TESTS = "RUN_SAMPLES_TESTS"
|
||||
|
||||
# All chat client samples across providers
|
||||
chat_client_samples = [
|
||||
# Azure Chat Client samples
|
||||
param(
|
||||
azure_assistants_client,
|
||||
[], # Non-interactive sample
|
||||
id="azure_assistants_client",
|
||||
marks=[
|
||||
pytest.mark.azure,
|
||||
pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."),
|
||||
],
|
||||
),
|
||||
param(
|
||||
azure_chat_client,
|
||||
[], # Non-interactive sample
|
||||
id="azure_chat_client",
|
||||
marks=[
|
||||
pytest.mark.azure,
|
||||
pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."),
|
||||
],
|
||||
),
|
||||
param(
|
||||
azure_responses_client,
|
||||
[], # Non-interactive sample
|
||||
id="azure_responses_client",
|
||||
marks=[
|
||||
pytest.mark.azure,
|
||||
pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."),
|
||||
],
|
||||
),
|
||||
# Foundry Chat Client samples
|
||||
param(
|
||||
foundry_chat_client,
|
||||
[], # Non-interactive sample
|
||||
id="foundry_chat_client",
|
||||
marks=[
|
||||
pytest.mark.foundry,
|
||||
pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."),
|
||||
],
|
||||
),
|
||||
# OpenAI Chat Client samples
|
||||
param(
|
||||
openai_assistants_client,
|
||||
[], # Non-interactive sample
|
||||
id="openai_assistants_client",
|
||||
marks=[
|
||||
pytest.mark.openai,
|
||||
pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."),
|
||||
],
|
||||
),
|
||||
param(
|
||||
openai_chat_client,
|
||||
[], # Non-interactive sample
|
||||
id="openai_chat_client",
|
||||
marks=[
|
||||
pytest.mark.openai,
|
||||
pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."),
|
||||
],
|
||||
),
|
||||
param(
|
||||
openai_responses_client,
|
||||
[], # Non-interactive sample
|
||||
id="openai_responses_client",
|
||||
marks=[
|
||||
pytest.mark.openai,
|
||||
pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."),
|
||||
],
|
||||
),
|
||||
# General Chat Client samples (no provider-specific environment variable)
|
||||
param(
|
||||
chat_response_cancellation,
|
||||
[], # Non-interactive sample
|
||||
id="chat_response_cancellation",
|
||||
marks=pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."),
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@mark.parametrize("sample, responses", chat_client_samples)
|
||||
async def test_chat_client_samples(
|
||||
sample: Callable[..., Awaitable[Any]],
|
||||
responses: list[str],
|
||||
monkeypatch: MonkeyPatch,
|
||||
):
|
||||
"""Test chat client samples with input mocking and retry logic."""
|
||||
saved_responses = copy.deepcopy(responses)
|
||||
|
||||
def reset():
|
||||
responses.clear()
|
||||
responses.extend(saved_responses)
|
||||
|
||||
def mock_input(prompt: str = "") -> str:
|
||||
return responses.pop(0) if responses else "exit"
|
||||
|
||||
monkeypatch.setattr("builtins.input", mock_input)
|
||||
await retry(sample, retries=3, reset=reset)
|
||||
@@ -0,0 +1,54 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
import copy
|
||||
import os
|
||||
from collections.abc import Awaitable, Callable
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from pytest import MonkeyPatch, mark, param
|
||||
|
||||
from samples.getting_started.threads.custom_chat_message_store_thread import main as threads_custom_store
|
||||
from samples.getting_started.threads.suspend_resume_thread import main as threads_suspend_resume
|
||||
from tests.sample_utils import retry
|
||||
|
||||
# Environment variable for controlling sample tests
|
||||
RUN_SAMPLES_TESTS = "RUN_SAMPLES_TESTS"
|
||||
|
||||
# All thread samples
|
||||
thread_samples = [
|
||||
param(
|
||||
threads_custom_store,
|
||||
[], # Non-interactive sample
|
||||
id="threads_custom_store",
|
||||
marks=[
|
||||
pytest.mark.openai,
|
||||
pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."),
|
||||
],
|
||||
),
|
||||
param(
|
||||
threads_suspend_resume,
|
||||
[], # Non-interactive sample
|
||||
id="threads_suspend_resume",
|
||||
marks=[
|
||||
pytest.mark.openai,
|
||||
pytest.mark.skipif(os.getenv(RUN_SAMPLES_TESTS, None) is None, reason="Not running sample tests."),
|
||||
],
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@mark.parametrize("sample, responses", thread_samples)
|
||||
async def test_thread_samples(sample: Callable[..., Awaitable[Any]], responses: list[str], monkeypatch: MonkeyPatch):
|
||||
"""Test thread samples with input mocking and retry logic."""
|
||||
saved_responses = copy.deepcopy(responses)
|
||||
|
||||
def reset():
|
||||
responses.clear()
|
||||
responses.extend(saved_responses)
|
||||
|
||||
def mock_input(prompt: str = "") -> str:
|
||||
return responses.pop(0) if responses else "exit"
|
||||
|
||||
monkeypatch.setattr("builtins.input", mock_input)
|
||||
await retry(sample, retries=3, reset=reset)
|
||||
Reference in New Issue
Block a user