Python: updated integration tests and guidance (#4181)

* updated integration tests and guidance

* fixed merge test

* updated integration tests

* fix: remove duplicate --dist loadfile flag from pytest-xdist config

Only one --dist mode can be active at a time; the second value silently
overrides the first. Keep --dist worksteal (dynamic load balancing) and
remove the redundant --dist loadfile from all workflow files and
pyproject.toml configs.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

* docs: add keep-in-sync notes for merge and integration test workflows

Both python-merge-tests.yml and python-integration-tests.yml share the
same parallel job structure. Added sync reminders in workflow file
comments, the python-testing SKILL.md, and CODING_STANDARD.md.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

* refactor: remove RUN_INTEGRATION_TESTS flag

Integration test gating now uses two mechanisms:
- `@pytest.mark.integration` for test selection via `-m` filtering
- `skip_if_*_disabled` for credential/service availability checks

The RUN_INTEGRATION_TESTS env var was redundant since the marker handles
selection and the skip decorators already check for actual credentials.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

* fix: sync missing env vars from merge-tests to integration-tests

Add OPENAI_EMBEDDINGS_MODEL_ID and AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME
to python-integration-tests.yml to match python-merge-tests.yml.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

* fix: remove remaining RUN_INTEGRATION_TESTS from embedding tests and docs

Missed test_openai_embedding_client.py and vector-stores README in the
earlier cleanup.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

* set functions tests to 3.10

---------

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

python/packages/azurefunctions/tests/integration_tests/.env.example

@@ -2,7 +2,6 @@
 AZURE_OPENAI_ENDPOINT=https://your-resource.openai.azure.com/
 AZURE_OPENAI_CHAT_DEPLOYMENT_NAME=your-deployment-name
 FUNCTIONS_WORKER_RUNTIME=python
-RUN_INTEGRATION_TESTS=true
 
 # Azure Functions Configuration
 AzureWebJobsStorage=UseDevelopmentStorage=true

python/packages/azurefunctions/tests/integration_tests/conftest.py

@@ -90,13 +90,6 @@ def _should_skip_azure_functions_integration_tests() -> tuple[bool, str]:
     """Determine whether Azure Functions integration tests should be skipped."""
     _load_env_file_if_present()
 
-    run_integration_tests = os.getenv("RUN_INTEGRATION_TESTS", "false").lower() == "true"
-    if not run_integration_tests:
-        return (
-            True,
-            "Integration tests are disabled. Set RUN_INTEGRATION_TESTS=true to enable Azure Functions sample tests.",
-        )
-
     # Check for Azure Functions Core Tools
     if not _check_func_cli_available():
         return (

python/packages/azurefunctions/tests/integration_tests/test_01_single_agent.py

@@ -19,6 +19,8 @@ from agent_framework_durabletask import THREAD_ID_HEADER
 
 # Module-level markers - applied to all tests in this file
 pytestmark = [
+    pytest.mark.flaky,
+    pytest.mark.integration,
     pytest.mark.sample("01_single_agent"),
     pytest.mark.usefixtures("function_app_for_test"),
 ]

python/packages/azurefunctions/tests/integration_tests/test_02_multi_agent.py

@@ -18,6 +18,8 @@ import pytest
 
 # Module-level markers - applied to all tests in this file
 pytestmark = [
+    pytest.mark.flaky,
+    pytest.mark.integration,
     pytest.mark.sample("02_multi_agent"),
     pytest.mark.usefixtures("function_app_for_test"),
 ]

python/packages/azurefunctions/tests/integration_tests/test_03_reliable_streaming.py

@@ -22,6 +22,8 @@ import requests
 
 # Module-level markers - applied to all tests in this file
 pytestmark = [
+    pytest.mark.flaky,
+    pytest.mark.integration,
     pytest.mark.sample("03_reliable_streaming"),
     pytest.mark.usefixtures("function_app_for_test"),
     pytest.mark.skip(reason="Temp disabled to fix test instability - needs investigation into root cause"),

python/packages/azurefunctions/tests/integration_tests/test_04_single_agent_orchestration_chaining.py

@@ -22,6 +22,8 @@ import pytest
 
 # Module-level markers - applied to all tests in this file
 pytestmark = [
+    pytest.mark.flaky,
+    pytest.mark.integration,
     pytest.mark.sample("04_single_agent_orchestration_chaining"),
     pytest.mark.usefixtures("function_app_for_test"),
 ]

python/packages/azurefunctions/tests/integration_tests/test_05_multi_agent_orchestration_concurrency.py

@@ -22,6 +22,8 @@ import pytest
 
 # Module-level markers - applied to all tests in this file
 pytestmark = [
+    pytest.mark.flaky,
+    pytest.mark.integration,
     pytest.mark.orchestration,
     pytest.mark.sample("05_multi_agent_orchestration_concurrency"),
     pytest.mark.usefixtures("function_app_for_test"),

python/packages/azurefunctions/tests/integration_tests/test_06_multi_agent_orchestration_conditionals.py

@@ -22,6 +22,8 @@ import pytest
 
 # Module-level markers - applied to all tests in this file
 pytestmark = [
+    pytest.mark.flaky,
+    pytest.mark.integration,
     pytest.mark.orchestration,
     pytest.mark.sample("06_multi_agent_orchestration_conditionals"),
     pytest.mark.usefixtures("function_app_for_test"),

python/packages/azurefunctions/tests/integration_tests/test_07_single_agent_orchestration_hitl.py

@@ -24,6 +24,8 @@ import pytest
 
 # Module-level markers - applied to all tests in this file
 pytestmark = [
+    pytest.mark.flaky,
+    pytest.mark.integration,
     pytest.mark.sample("07_single_agent_orchestration_hitl"),
     pytest.mark.usefixtures("function_app_for_test"),
 ]

python/packages/azurefunctions/tests/integration_tests/test_09_workflow_shared_state.py

@@ -23,6 +23,8 @@ import pytest
 
 # Module-level markers - applied to all tests in this file
 pytestmark = [
+    pytest.mark.flaky,
+    pytest.mark.integration,
     pytest.mark.sample("09_workflow_shared_state"),
     pytest.mark.usefixtures("function_app_for_test"),
 ]

python/packages/azurefunctions/tests/integration_tests/test_10_workflow_no_shared_state.py

@@ -23,6 +23,8 @@ import pytest
 
 # Module-level markers - applied to all tests in this file
 pytestmark = [
+    pytest.mark.flaky,
+    pytest.mark.integration,
     pytest.mark.sample("10_workflow_no_shared_state"),
     pytest.mark.usefixtures("function_app_for_test"),
 ]

python/packages/azurefunctions/tests/integration_tests/test_11_workflow_parallel.py

@@ -25,6 +25,8 @@ import pytest
 
 # Module-level markers - applied to all tests in this file
 pytestmark = [
+    pytest.mark.flaky,
+    pytest.mark.integration,
     pytest.mark.sample("11_workflow_parallel"),
     pytest.mark.usefixtures("function_app_for_test"),
 ]

python/packages/azurefunctions/tests/integration_tests/test_12_workflow_hitl.py

@@ -25,6 +25,8 @@ import pytest
 
 # Module-level markers - applied to all tests in this file
 pytestmark = [
+    pytest.mark.flaky,
+    pytest.mark.integration,
     pytest.mark.sample("12_workflow_hitl"),
     pytest.mark.usefixtures("function_app_for_test"),
 ]