mirror of
https://github.com/microsoft/agent-framework.git
synced 2026-06-16 21:04:09 +08:00
Eduard van Valkenburg
7e9c043c4c
* Improve PR template and breaking-change label automation - Add a structured "Related Issue" section using GitHub closing keywords - Add a Review Guide prompt (major changes, impact, reviewer focus) with a note that the focus item is for human reviewers only - Add checklist items for issue linkage / no duplicate PRs and invert the breaking-change item (checked = not breaking) - Extend label-title-prefix to prepend [BREAKING] when the "breaking change" label is added - Add label-breaking-change workflow to apply the "breaking change" label when a PR title contains [BREAKING] Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Add pull-requests agent skill with dotnet/python links - Add root .github/skills/pull-requests/SKILL.md covering PR description authoring (following the PR template) and the review-comment workflow (review -> plan -> user review -> implement -> reply to all -> resolve) - Symlink the skill from python/.github/skills and dotnet/.github/skills - Reference the skill from python/AGENTS.md and dotnet/AGENTS.md Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Fold breaking-change labeling into label-pr workflow Move the title -> 'breaking change' label logic into the existing label-pr workflow (which already applies the python/.NET labels) and drop the separate label-breaking-change workflow. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Address PR title prefix review feedback Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Pin patched MessagePack for .NET restore Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Revert MessagePack central pin Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Move title prefix tests out of tracked GitHub tests Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Exclude skill docs from CI path filters Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Match skill symlinks in CI path exclusions Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Exclude AGENTS docs from CI path filters Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Scope title-prefix normalization to a real prefix The normalization branch in addTitlePrefix matched ^Python (no colon), so titles like "Python samples improvements" or "Pythonic refactor" were treated as already-prefixed and only re-cased, never receiving the "Python: " prefix. Scope the match to ^<prefix>:\s* so only an actual existing prefix is normalized; otherwise the prefix is prepended. Same fix applies to the .NET prefix (e.g. ".NETStandard bump"). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
814 lines
31 KiB
YAML
814 lines
31 KiB
YAML
name: Python - Merge - Tests
|
|
#
|
|
# NOTE: This workflow and python-integration-tests.yml share the same set of
|
|
# parallel test jobs. Keep them in sync — when adding, removing, or modifying a
|
|
# job here, apply the same change to python-integration-tests.yml.
|
|
#
|
|
|
|
on:
|
|
workflow_dispatch:
|
|
pull_request:
|
|
branches: ["main"]
|
|
merge_group:
|
|
branches: ["main"]
|
|
schedule:
|
|
- cron: "0 0 * * *" # Run at midnight UTC daily
|
|
|
|
permissions:
|
|
contents: read
|
|
id-token: write
|
|
|
|
env:
|
|
# Configure a constant location for the uv cache
|
|
UV_CACHE_DIR: /tmp/.uv-cache
|
|
UV_PYTHON: "3.13"
|
|
RUN_SAMPLES_TESTS: ${{ vars.RUN_SAMPLES_TESTS }}
|
|
|
|
jobs:
|
|
paths-filter:
|
|
runs-on: ubuntu-latest
|
|
permissions:
|
|
contents: read
|
|
pull-requests: read
|
|
outputs:
|
|
pythonChanges: ${{ steps.filter.outputs.python }}
|
|
coreChanged: ${{ steps.filter.outputs.core }}
|
|
openaiChanged: ${{ steps.filter.outputs.openai }}
|
|
azureChanged: ${{ steps.filter.outputs.azure }}
|
|
miscChanged: ${{ steps.filter.outputs.misc }}
|
|
functionsChanged: ${{ steps.filter.outputs.functions }}
|
|
foundryChanged: ${{ steps.filter.outputs.foundry }}
|
|
foundryHostingChanged: ${{ steps.filter.outputs.foundry_hosting }}
|
|
cosmosChanged: ${{ steps.filter.outputs.cosmos }}
|
|
githubCopilotChanged: ${{ steps.filter.outputs.github_copilot }}
|
|
steps:
|
|
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
|
- uses: dorny/paths-filter@d1c1ffe0248fe513906c8e24db8ea791d46f8590 # v3
|
|
id: filter
|
|
with:
|
|
filters: |
|
|
python:
|
|
- 'python/**'
|
|
- '!python/AGENTS.md'
|
|
- '!python/**/AGENTS.md'
|
|
- '!python/.github/skills/*'
|
|
- '!python/.github/skills/**'
|
|
- '.github/actions/setup-local-mcp-server/**'
|
|
- '.github/workflows/python-merge-tests.yml'
|
|
- '.github/workflows/python-integration-tests.yml'
|
|
core:
|
|
- 'python/packages/core/agent_framework/_*.py'
|
|
- 'python/packages/core/agent_framework/_workflows/**'
|
|
- 'python/packages/core/agent_framework/exceptions.py'
|
|
- 'python/packages/core/agent_framework/observability.py'
|
|
openai:
|
|
- 'python/packages/core/agent_framework/openai/**'
|
|
- 'python/packages/openai/**'
|
|
- 'python/samples/**/providers/openai/**'
|
|
azure:
|
|
- 'python/packages/openai/**'
|
|
- 'python/packages/core/agent_framework/azure/**'
|
|
- 'python/samples/**/providers/azure/**'
|
|
misc:
|
|
- 'python/packages/anthropic/**'
|
|
- 'python/packages/hyperlight/**'
|
|
- 'python/packages/ollama/**'
|
|
- 'python/packages/core/agent_framework/_mcp.py'
|
|
- 'python/packages/core/tests/core/test_mcp.py'
|
|
- 'python/scripts/local_mcp_streamable_http_server.py'
|
|
- '.github/actions/setup-local-mcp-server/**'
|
|
- '.github/workflows/python-merge-tests.yml'
|
|
- '.github/workflows/python-integration-tests.yml'
|
|
functions:
|
|
- 'python/packages/azurefunctions/**'
|
|
- 'python/packages/durabletask/**'
|
|
foundry:
|
|
- 'python/packages/foundry/**'
|
|
- 'python/samples/**/providers/foundry/**'
|
|
- 'python/samples/02-agents/embeddings/foundry_embeddings.py'
|
|
foundry_hosting:
|
|
- 'python/packages/foundry_hosting/**'
|
|
cosmos:
|
|
- 'python/packages/azure-cosmos/**'
|
|
github_copilot:
|
|
- 'python/packages/github_copilot/**'
|
|
# run only if 'python' files were changed
|
|
- name: python tests
|
|
if: steps.filter.outputs.python == 'true'
|
|
run: echo "Python file"
|
|
# run only if not 'python' files were changed
|
|
- name: not python tests
|
|
if: steps.filter.outputs.python != 'true'
|
|
run: echo "NOT python file"
|
|
# Unit tests: always run all non-integration tests across all packages
|
|
python-tests-unit:
|
|
name: Python Tests - Unit
|
|
needs: paths-filter
|
|
if: >
|
|
github.event_name != 'pull_request' &&
|
|
needs.paths-filter.outputs.pythonChanges == 'true'
|
|
runs-on: ubuntu-latest
|
|
environment: integration
|
|
defaults:
|
|
run:
|
|
working-directory: python
|
|
steps:
|
|
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
|
- name: Set up python and install the project
|
|
id: python-setup
|
|
uses: ./.github/actions/python-setup
|
|
with:
|
|
python-version: ${{ env.UV_PYTHON }}
|
|
os: ${{ runner.os }}
|
|
- name: Test with pytest (unit tests only)
|
|
run: >
|
|
uv run poe test -A
|
|
-m "not integration"
|
|
--timeout=120 --session-timeout=900 --timeout_method thread
|
|
--retries 2 --retry-delay 5
|
|
--junitxml=pytest.xml
|
|
working-directory: ./python
|
|
- name: Surface failing tests
|
|
if: always()
|
|
uses: pmeier/pytest-results-action@20b595761ba9bf89e115e875f8bc863f913bc8ad # v0.7.2
|
|
with:
|
|
path: ./python/pytest.xml
|
|
summary: true
|
|
display-options: fEX
|
|
fail-on-empty: false
|
|
title: Unit test results
|
|
|
|
# OpenAI integration tests
|
|
python-tests-openai:
|
|
name: Python Tests - OpenAI Integration
|
|
needs: paths-filter
|
|
if: >
|
|
github.event_name != 'pull_request' &&
|
|
needs.paths-filter.outputs.pythonChanges == 'true' &&
|
|
(github.event_name != 'merge_group' ||
|
|
needs.paths-filter.outputs.openaiChanged == 'true' ||
|
|
needs.paths-filter.outputs.coreChanged == 'true')
|
|
runs-on: ubuntu-latest
|
|
environment: integration
|
|
env:
|
|
OPENAI_CHAT_COMPLETION_MODEL: ${{ vars.OPENAI__CHATMODELID }}
|
|
OPENAI_CHAT_MODEL: ${{ vars.OPENAI__RESPONSESMODELID }}
|
|
OPENAI_MODEL: ${{ vars.OPENAI__RESPONSESMODELID }}
|
|
OPENAI_EMBEDDING_MODEL: ${{ vars.OPENAI_EMBEDDING_MODEL_ID }}
|
|
OPENAI_API_KEY: ${{ secrets.OPENAI__APIKEY }}
|
|
defaults:
|
|
run:
|
|
working-directory: python
|
|
steps:
|
|
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
|
- name: Set up python and install the project
|
|
id: python-setup
|
|
uses: ./.github/actions/python-setup
|
|
with:
|
|
python-version: ${{ env.UV_PYTHON }}
|
|
os: ${{ runner.os }}
|
|
- name: Test with pytest (OpenAI integration)
|
|
run: >
|
|
uv run pytest --import-mode=importlib
|
|
packages/openai/tests
|
|
-m "integration and not azure"
|
|
-n logical --dist worksteal
|
|
--timeout=120 --session-timeout=900 --timeout_method thread
|
|
--retries 2 --retry-delay 5
|
|
--junitxml=pytest.xml
|
|
working-directory: ./python
|
|
- name: Test OpenAI samples
|
|
timeout-minutes: 10
|
|
if: env.RUN_SAMPLES_TESTS == 'true'
|
|
run: uv run pytest tests/samples/ -m "openai"
|
|
working-directory: ./python
|
|
- name: Surface failing tests
|
|
if: always()
|
|
uses: pmeier/pytest-results-action@20b595761ba9bf89e115e875f8bc863f913bc8ad # v0.7.2
|
|
with:
|
|
path: ./python/pytest.xml
|
|
summary: true
|
|
display-options: fEX
|
|
fail-on-empty: false
|
|
title: OpenAI integration test results
|
|
- name: Upload test results
|
|
if: always()
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
|
with:
|
|
name: test-results-openai
|
|
path: ./python/pytest.xml
|
|
if-no-files-found: ignore
|
|
|
|
# Azure OpenAI integration tests
|
|
python-tests-azure-openai:
|
|
name: Python Tests - Azure OpenAI Integration
|
|
needs: paths-filter
|
|
if: >
|
|
github.event_name != 'pull_request' &&
|
|
needs.paths-filter.outputs.pythonChanges == 'true' &&
|
|
(github.event_name != 'merge_group' ||
|
|
needs.paths-filter.outputs.azureChanged == 'true' ||
|
|
needs.paths-filter.outputs.coreChanged == 'true')
|
|
runs-on: ubuntu-latest
|
|
environment: integration
|
|
env:
|
|
AZURE_OPENAI_CHAT_COMPLETION_MODEL: ${{ vars.AZUREOPENAI__CHATDEPLOYMENTNAME }}
|
|
AZURE_OPENAI_CHAT_MODEL: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
|
|
AZURE_OPENAI_MODEL: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
|
|
AZURE_OPENAI_EMBEDDING_MODEL: ${{ vars.AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME }}
|
|
AZURE_OPENAI_ENDPOINT: ${{ vars.AZUREOPENAI__ENDPOINT }}
|
|
defaults:
|
|
run:
|
|
working-directory: python
|
|
steps:
|
|
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
|
- name: Set up python and install the project
|
|
id: python-setup
|
|
uses: ./.github/actions/python-setup
|
|
with:
|
|
python-version: ${{ env.UV_PYTHON }}
|
|
os: ${{ runner.os }}
|
|
- name: Azure CLI Login
|
|
if: github.event_name != 'pull_request'
|
|
uses: azure/login@a457da9ea143d694b1b9c7c869ebb04ebe844ef5 # v2
|
|
with:
|
|
client-id: ${{ secrets.AZURE_CLIENT_ID }}
|
|
tenant-id: ${{ secrets.AZURE_TENANT_ID }}
|
|
subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
|
|
- name: Test with pytest (Azure OpenAI integration)
|
|
run: >
|
|
uv run pytest --import-mode=importlib
|
|
packages/openai/tests/openai/test_openai_chat_completion_client_azure.py
|
|
packages/openai/tests/openai/test_openai_chat_client_azure.py
|
|
packages/openai/tests/openai/test_openai_embedding_client_azure.py
|
|
-m integration
|
|
-n logical --dist worksteal
|
|
--timeout=120 --session-timeout=900 --timeout_method thread
|
|
--retries 2 --retry-delay 5
|
|
--junitxml=pytest.xml
|
|
working-directory: ./python
|
|
- name: Test Azure samples
|
|
timeout-minutes: 10
|
|
if: env.RUN_SAMPLES_TESTS == 'true'
|
|
run: uv run pytest tests/samples/ -m "azure"
|
|
working-directory: ./python
|
|
- name: Surface failing tests
|
|
if: always()
|
|
uses: pmeier/pytest-results-action@20b595761ba9bf89e115e875f8bc863f913bc8ad # v0.7.2
|
|
with:
|
|
path: ./python/pytest.xml
|
|
summary: true
|
|
display-options: fEX
|
|
fail-on-empty: false
|
|
title: Azure OpenAI integration test results
|
|
- name: Upload test results
|
|
if: always()
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
|
with:
|
|
name: test-results-azure-openai
|
|
path: ./python/pytest.xml
|
|
if-no-files-found: ignore
|
|
|
|
# Misc integration tests (Anthropic, Ollama, MCP)
|
|
python-tests-misc-integration:
|
|
name: Python Tests - Misc Integration
|
|
needs: paths-filter
|
|
if: >
|
|
github.event_name != 'pull_request' &&
|
|
needs.paths-filter.outputs.pythonChanges == 'true' &&
|
|
(github.event_name != 'merge_group' ||
|
|
needs.paths-filter.outputs.miscChanged == 'true' ||
|
|
needs.paths-filter.outputs.coreChanged == 'true')
|
|
runs-on: ubuntu-latest
|
|
environment: integration
|
|
env:
|
|
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
|
|
ANTHROPIC_CHAT_MODEL: ${{ vars.ANTHROPIC_CHAT_MODEL_ID }}
|
|
LOCAL_MCP_URL: ${{ vars.LOCAL_MCP__URL }}
|
|
OLLAMA_MODEL: qwen2.5:1.5b
|
|
OLLAMA_EMBEDDING_MODEL: nomic-embed-text
|
|
defaults:
|
|
run:
|
|
working-directory: python
|
|
steps:
|
|
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
|
- name: Set up python and install the project
|
|
id: python-setup
|
|
uses: ./.github/actions/python-setup
|
|
with:
|
|
python-version: ${{ env.UV_PYTHON }}
|
|
os: ${{ runner.os }}
|
|
- name: Install Ollama
|
|
run: curl -fsSL https://ollama.com/install.sh | sh
|
|
working-directory: .
|
|
- name: Cache Ollama models
|
|
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
|
|
with:
|
|
path: ~/.ollama/models
|
|
key: ollama-models-qwen2.5-1.5b-nomic-embed-text-v1
|
|
- name: Start Ollama and pull models
|
|
run: |
|
|
# Stop any Ollama instance auto-started by the install script
|
|
pkill ollama || true
|
|
sleep 2
|
|
ollama serve &
|
|
for i in $(seq 1 30); do
|
|
if curl -sf http://localhost:11434/api/tags > /dev/null 2>&1; then
|
|
break
|
|
fi
|
|
sleep 1
|
|
done
|
|
# Pull models with retry for transient 429 rate limits
|
|
for model in qwen2.5:1.5b nomic-embed-text; do
|
|
pulled=false
|
|
for attempt in 1 2 3; do
|
|
if ollama pull "$model"; then
|
|
pulled=true
|
|
break
|
|
fi
|
|
echo "Retry $attempt for $model (waiting 15s)..."
|
|
sleep 15
|
|
done
|
|
if [ "$pulled" != "true" ]; then
|
|
echo "ERROR: Failed to pull $model after 3 attempts"
|
|
exit 1
|
|
fi
|
|
done
|
|
working-directory: .
|
|
- name: Start local MCP server
|
|
id: local-mcp
|
|
uses: ./.github/actions/setup-local-mcp-server
|
|
with:
|
|
fallback_url: ${{ env.LOCAL_MCP_URL }}
|
|
- name: Prefer local MCP URL when available
|
|
run: echo "LOCAL_MCP_URL=${{ steps.local-mcp.outputs.effective_url }}" >> "$GITHUB_ENV"
|
|
- name: Test with pytest (Anthropic, Hyperlight, Ollama, MCP integration)
|
|
run: >
|
|
uv run pytest --import-mode=importlib
|
|
packages/anthropic/tests
|
|
packages/hyperlight/tests
|
|
packages/ollama/tests
|
|
packages/core/tests/core/test_mcp.py
|
|
-m integration
|
|
-n logical --dist worksteal
|
|
--timeout=120 --session-timeout=900 --timeout_method thread
|
|
--retries 2 --retry-delay 30
|
|
--junitxml=pytest.xml
|
|
working-directory: ./python
|
|
- name: Stop local MCP server
|
|
if: always()
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
server_pid="${{ steps.local-mcp.outputs.pid }}"
|
|
if [[ -z "$server_pid" ]]; then
|
|
exit 0
|
|
fi
|
|
if ! kill -0 "$server_pid" 2>/dev/null; then
|
|
exit 0
|
|
fi
|
|
kill -TERM -- "-$server_pid" 2>/dev/null || kill -TERM "$server_pid" 2>/dev/null || true
|
|
for _ in $(seq 1 10); do
|
|
if ! kill -0 "$server_pid" 2>/dev/null; then
|
|
exit 0
|
|
fi
|
|
sleep 1
|
|
done
|
|
kill -KILL -- "-$server_pid" 2>/dev/null || kill -KILL "$server_pid" 2>/dev/null || true
|
|
- name: Surface failing tests
|
|
if: always()
|
|
uses: pmeier/pytest-results-action@20b595761ba9bf89e115e875f8bc863f913bc8ad # v0.7.2
|
|
with:
|
|
path: ./python/pytest.xml
|
|
summary: true
|
|
display-options: fEX
|
|
fail-on-empty: false
|
|
title: Misc integration test results
|
|
- name: Upload test results
|
|
if: always()
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
|
with:
|
|
name: test-results-misc
|
|
path: ./python/pytest.xml
|
|
if-no-files-found: ignore
|
|
|
|
# Azure Functions + Durable Task integration tests
|
|
python-tests-functions:
|
|
name: Python Tests - Functions Integration
|
|
needs: paths-filter
|
|
if: >
|
|
github.event_name != 'pull_request' &&
|
|
needs.paths-filter.outputs.pythonChanges == 'true' &&
|
|
(github.event_name != 'merge_group' ||
|
|
needs.paths-filter.outputs.functionsChanged == 'true' ||
|
|
needs.paths-filter.outputs.coreChanged == 'true')
|
|
runs-on: ubuntu-latest
|
|
environment: integration
|
|
env:
|
|
UV_PYTHON: "3.11"
|
|
OPENAI_CHAT_COMPLETION_MODEL: ${{ vars.OPENAI__CHATMODELID }}
|
|
OPENAI_CHAT_MODEL: ${{ vars.OPENAI__RESPONSESMODELID }}
|
|
OPENAI_MODEL: ${{ vars.OPENAI__RESPONSESMODELID }}
|
|
OPENAI_EMBEDDING_MODEL: ${{ vars.OPENAI_EMBEDDING_MODEL_ID }}
|
|
OPENAI_API_KEY: ${{ secrets.OPENAI__APIKEY }}
|
|
AZURE_OPENAI_ENDPOINT: ${{ vars.AZUREOPENAI__ENDPOINT }}
|
|
AZURE_OPENAI_MODEL: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
|
|
AZURE_OPENAI_CHAT_MODEL: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
|
|
AZURE_OPENAI_CHAT_COMPLETION_MODEL: ${{ vars.AZUREOPENAI__CHATDEPLOYMENTNAME }}
|
|
FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT }}
|
|
FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL }}
|
|
FUNCTIONS_WORKER_RUNTIME: "python"
|
|
DURABLE_TASK_SCHEDULER_CONNECTION_STRING: "Endpoint=http://localhost:8080;TaskHub=default;Authentication=None"
|
|
AzureWebJobsStorage: "UseDevelopmentStorage=true"
|
|
defaults:
|
|
run:
|
|
working-directory: python
|
|
steps:
|
|
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
|
- name: Set up python and install the project
|
|
id: python-setup
|
|
uses: ./.github/actions/python-setup
|
|
with:
|
|
python-version: ${{ env.UV_PYTHON }}
|
|
os: ${{ runner.os }}
|
|
- name: Azure CLI Login
|
|
if: github.event_name != 'pull_request'
|
|
uses: azure/login@a457da9ea143d694b1b9c7c869ebb04ebe844ef5 # v2
|
|
with:
|
|
client-id: ${{ secrets.AZURE_CLIENT_ID }}
|
|
tenant-id: ${{ secrets.AZURE_TENANT_ID }}
|
|
subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
|
|
- name: Set up Azure Functions Integration Test Emulators
|
|
uses: ./.github/actions/azure-functions-integration-setup
|
|
id: azure-functions-setup
|
|
- name: Test with pytest (Functions + Durable Task integration)
|
|
run: >
|
|
uv run pytest --import-mode=importlib
|
|
packages/azurefunctions/tests/integration_tests
|
|
packages/durabletask/tests/integration_tests
|
|
-m integration
|
|
-n logical --dist worksteal
|
|
-x
|
|
--timeout=480 --session-timeout=900 --timeout_method thread
|
|
--retries 2 --retry-delay 5
|
|
--junitxml=pytest.xml
|
|
working-directory: ./python
|
|
- name: Surface failing tests
|
|
if: always()
|
|
uses: pmeier/pytest-results-action@20b595761ba9bf89e115e875f8bc863f913bc8ad # v0.7.2
|
|
with:
|
|
path: ./python/pytest.xml
|
|
summary: true
|
|
display-options: fEX
|
|
fail-on-empty: false
|
|
title: Functions integration test results
|
|
- name: Upload test results
|
|
if: always()
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
|
with:
|
|
name: test-results-functions
|
|
path: ./python/pytest.xml
|
|
if-no-files-found: ignore
|
|
|
|
python-tests-foundry:
|
|
name: Python Integration Tests - Foundry
|
|
needs: paths-filter
|
|
if: >
|
|
github.event_name != 'pull_request' &&
|
|
needs.paths-filter.outputs.pythonChanges == 'true' &&
|
|
(github.event_name != 'merge_group' ||
|
|
needs.paths-filter.outputs.foundryChanged == 'true' ||
|
|
needs.paths-filter.outputs.coreChanged == 'true')
|
|
runs-on: ubuntu-latest
|
|
environment: integration
|
|
env:
|
|
FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT }}
|
|
FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL }}
|
|
FOUNDRY_AGENT_NAME: ${{ vars.FOUNDRY_AGENT_NAME }}
|
|
FOUNDRY_AGENT_VERSION: ${{ vars.FOUNDRY_AGENT_VERSION }}
|
|
FOUNDRY_MODELS_ENDPOINT: ${{ vars.FOUNDRY_MODELS_ENDPOINT || '' }}
|
|
FOUNDRY_MODELS_API_KEY: ${{ secrets.FOUNDRY_MODELS_API_KEY || '' }}
|
|
FOUNDRY_EMBEDDING_MODEL: ${{ vars.FOUNDRY_EMBEDDING_MODEL || '' }}
|
|
FOUNDRY_IMAGE_EMBEDDING_MODEL: ${{ vars.FOUNDRY_IMAGE_EMBEDDING_MODEL || '' }}
|
|
LOCAL_MCP_URL: ${{ vars.LOCAL_MCP__URL }}
|
|
defaults:
|
|
run:
|
|
working-directory: python
|
|
steps:
|
|
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
|
- name: Set up python and install the project
|
|
id: python-setup
|
|
uses: ./.github/actions/python-setup
|
|
with:
|
|
python-version: ${{ env.UV_PYTHON }}
|
|
os: ${{ runner.os }}
|
|
- name: Azure CLI Login
|
|
if: github.event_name != 'pull_request'
|
|
uses: azure/login@a457da9ea143d694b1b9c7c869ebb04ebe844ef5 # v2
|
|
with:
|
|
client-id: ${{ secrets.AZURE_CLIENT_ID }}
|
|
tenant-id: ${{ secrets.AZURE_TENANT_ID }}
|
|
subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
|
|
- name: Test with pytest
|
|
timeout-minutes: 15
|
|
run: >
|
|
uv run pytest --import-mode=importlib
|
|
packages/foundry/tests
|
|
-m integration
|
|
-n logical --dist worksteal
|
|
--timeout=120 --session-timeout=900 --timeout_method thread
|
|
--retries 2 --retry-delay 5
|
|
--junitxml=pytest.xml
|
|
working-directory: ./python
|
|
- name: Surface failing tests
|
|
if: always()
|
|
uses: pmeier/pytest-results-action@20b595761ba9bf89e115e875f8bc863f913bc8ad # v0.7.2
|
|
with:
|
|
path: ./python/pytest.xml
|
|
summary: true
|
|
display-options: fEX
|
|
fail-on-empty: false
|
|
title: Test results
|
|
- name: Upload test results
|
|
if: always()
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
|
with:
|
|
name: test-results-foundry
|
|
path: ./python/pytest.xml
|
|
if-no-files-found: ignore
|
|
|
|
# Foundry Hosting integration tests
|
|
python-tests-foundry-hosting:
|
|
name: Python Tests - Foundry Hosting Integration
|
|
needs: paths-filter
|
|
if: >
|
|
github.event_name != 'pull_request' &&
|
|
needs.paths-filter.outputs.pythonChanges == 'true' &&
|
|
(github.event_name != 'merge_group' ||
|
|
needs.paths-filter.outputs.foundryHostingChanged == 'true' ||
|
|
needs.paths-filter.outputs.coreChanged == 'true')
|
|
runs-on: ubuntu-latest
|
|
environment: integration
|
|
env:
|
|
FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT }}
|
|
FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL }}
|
|
defaults:
|
|
run:
|
|
working-directory: python
|
|
steps:
|
|
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
|
- name: Set up python and install the project
|
|
id: python-setup
|
|
uses: ./.github/actions/python-setup
|
|
with:
|
|
python-version: ${{ env.UV_PYTHON }}
|
|
os: ${{ runner.os }}
|
|
- name: Azure CLI Login
|
|
if: github.event_name != 'pull_request'
|
|
uses: azure/login@a457da9ea143d694b1b9c7c869ebb04ebe844ef5 # v2
|
|
with:
|
|
client-id: ${{ secrets.AZURE_CLIENT_ID }}
|
|
tenant-id: ${{ secrets.AZURE_TENANT_ID }}
|
|
subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
|
|
- name: Test with pytest (Foundry Hosting integration)
|
|
timeout-minutes: 15
|
|
run: >
|
|
uv run pytest --import-mode=importlib
|
|
packages/foundry_hosting/tests
|
|
-m integration
|
|
-n logical --dist worksteal
|
|
--timeout=120 --session-timeout=900 --timeout_method thread
|
|
--retries 2 --retry-delay 5
|
|
--junitxml=pytest.xml
|
|
working-directory: ./python
|
|
- name: Surface failing tests
|
|
if: always()
|
|
uses: pmeier/pytest-results-action@20b595761ba9bf89e115e875f8bc863f913bc8ad # v0.7.2
|
|
with:
|
|
path: ./python/pytest.xml
|
|
summary: true
|
|
display-options: fEX
|
|
fail-on-empty: false
|
|
title: Foundry Hosting integration test results
|
|
- name: Upload test results
|
|
if: always()
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
|
with:
|
|
name: test-results-foundry-hosting
|
|
path: ./python/pytest.xml
|
|
if-no-files-found: ignore
|
|
|
|
# TODO: Add python-tests-lab
|
|
|
|
# Azure Cosmos integration tests
|
|
python-tests-cosmos:
|
|
name: Python Tests - Cosmos Integration
|
|
needs: paths-filter
|
|
if: >
|
|
github.event_name != 'pull_request' &&
|
|
needs.paths-filter.outputs.pythonChanges == 'true' &&
|
|
(github.event_name != 'merge_group' ||
|
|
needs.paths-filter.outputs.cosmosChanged == 'true' ||
|
|
needs.paths-filter.outputs.coreChanged == 'true')
|
|
runs-on: ubuntu-latest
|
|
environment: integration
|
|
services:
|
|
cosmosdb:
|
|
image: mcr.microsoft.com/cosmosdb/linux/azure-cosmos-emulator:vnext-preview
|
|
ports:
|
|
- 8081:8081
|
|
env:
|
|
AZURE_COSMOS_ENDPOINT: "http://localhost:8081/"
|
|
# Static Azure Cosmos DB emulator key (documented): https://learn.microsoft.com/en-us/azure/cosmos-db/emulator
|
|
AZURE_COSMOS_KEY: "C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw=="
|
|
AZURE_COSMOS_DATABASE_NAME: "agent-framework-cosmos-it-db"
|
|
AZURE_COSMOS_CONTAINER_NAME: "agent-framework-cosmos-it-container"
|
|
defaults:
|
|
run:
|
|
working-directory: python
|
|
steps:
|
|
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
|
- name: Set up python and install the project
|
|
id: python-setup
|
|
uses: ./.github/actions/python-setup
|
|
with:
|
|
python-version: ${{ env.UV_PYTHON }}
|
|
os: ${{ runner.os }}
|
|
- name: Wait for Cosmos DB emulator
|
|
run: |
|
|
for i in {1..60}; do
|
|
if curl --silent --show-error http://localhost:8081/ > /dev/null; then
|
|
echo "Cosmos DB emulator is ready."
|
|
exit 0
|
|
fi
|
|
sleep 2
|
|
done
|
|
echo "Cosmos DB emulator did not become ready in time." >&2
|
|
exit 1
|
|
- name: Test with pytest (Cosmos integration)
|
|
run: uv run --directory packages/azure-cosmos poe integration-tests -n logical --dist worksteal --timeout=120 --session-timeout=900 --timeout_method thread --retries 2 --retry-delay 5 --junitxml=${{ github.workspace }}/python/pytest.xml
|
|
working-directory: ./python
|
|
- name: Surface failing tests
|
|
if: always()
|
|
uses: pmeier/pytest-results-action@20b595761ba9bf89e115e875f8bc863f913bc8ad # v0.7.2
|
|
with:
|
|
path: ./python/pytest.xml
|
|
summary: true
|
|
display-options: fEX
|
|
fail-on-empty: false
|
|
title: Cosmos integration test results
|
|
- name: Upload test results
|
|
if: always()
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
|
with:
|
|
name: test-results-cosmos
|
|
path: ./python/pytest.xml
|
|
if-no-files-found: ignore
|
|
|
|
# GitHub Copilot integration tests
|
|
python-tests-github-copilot:
|
|
name: Python Tests - GitHub Copilot Integration
|
|
needs: paths-filter
|
|
if: >
|
|
github.event_name != 'pull_request' &&
|
|
needs.paths-filter.outputs.pythonChanges == 'true' &&
|
|
(github.event_name != 'merge_group' ||
|
|
needs.paths-filter.outputs.githubCopilotChanged == 'true' ||
|
|
needs.paths-filter.outputs.coreChanged == 'true')
|
|
runs-on: ubuntu-latest
|
|
environment: integration
|
|
timeout-minutes: 60
|
|
env:
|
|
COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }}
|
|
GITHUB_COPILOT_TIMEOUT: "120"
|
|
defaults:
|
|
run:
|
|
working-directory: python
|
|
steps:
|
|
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
|
- name: Set up python and install the project
|
|
id: python-setup
|
|
uses: ./.github/actions/python-setup
|
|
with:
|
|
python-version: ${{ env.UV_PYTHON }}
|
|
os: ${{ runner.os }}
|
|
- name: Test with pytest (GitHub Copilot integration)
|
|
run: >
|
|
uv run pytest --import-mode=importlib
|
|
packages/github_copilot/tests
|
|
-m integration
|
|
--timeout=120 --session-timeout=900 --timeout_method thread
|
|
--retries 2 --retry-delay 5
|
|
--junitxml=pytest.xml
|
|
- name: Surface failing tests
|
|
if: always()
|
|
uses: pmeier/pytest-results-action@20b595761ba9bf89e115e875f8bc863f913bc8ad # v0.7.2
|
|
with:
|
|
path: ./python/pytest.xml
|
|
summary: true
|
|
display-options: fEX
|
|
fail-on-empty: false
|
|
title: GitHub Copilot integration test results
|
|
- name: Upload test results
|
|
if: always()
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
|
with:
|
|
name: test-results-github-copilot
|
|
path: ./python/pytest.xml
|
|
if-no-files-found: ignore
|
|
|
|
# Integration test trend report (aggregates per-job JUnit XML results)
|
|
python-integration-test-report:
|
|
name: Integration Test Report
|
|
if: >
|
|
always() &&
|
|
(contains(join(needs.*.result, ','), 'success') ||
|
|
contains(join(needs.*.result, ','), 'failure'))
|
|
needs:
|
|
[
|
|
python-tests-openai,
|
|
python-tests-azure-openai,
|
|
python-tests-misc-integration,
|
|
python-tests-functions,
|
|
python-tests-foundry,
|
|
python-tests-foundry-hosting,
|
|
python-tests-cosmos,
|
|
python-tests-github-copilot,
|
|
]
|
|
runs-on: ubuntu-latest
|
|
defaults:
|
|
run:
|
|
working-directory: python
|
|
steps:
|
|
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
|
- name: Set up python and install the project
|
|
uses: ./.github/actions/python-setup
|
|
with:
|
|
python-version: ${{ env.UV_PYTHON }}
|
|
os: ${{ runner.os }}
|
|
- name: Download all test results from current run
|
|
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
|
|
with:
|
|
pattern: test-results-*
|
|
path: test-results/
|
|
- name: Restore report history cache
|
|
uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
|
|
with:
|
|
path: python/integration-report-history.json
|
|
key: integration-report-history-merge-${{ github.run_id }}
|
|
restore-keys: |
|
|
integration-report-history-merge-
|
|
- name: Generate trend report
|
|
run: >
|
|
uv run python scripts/integration_test_report/aggregate.py
|
|
../test-results/
|
|
integration-report-history.json
|
|
integration-test-report.md
|
|
- name: Post to Job Summary
|
|
if: always()
|
|
run: cat integration-test-report.md >> $GITHUB_STEP_SUMMARY
|
|
- name: Save report history cache
|
|
if: always()
|
|
uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
|
|
with:
|
|
path: python/integration-report-history.json
|
|
key: integration-report-history-merge-${{ github.run_id }}
|
|
- name: Upload unified trend report
|
|
if: always()
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
|
with:
|
|
name: integration-test-report
|
|
path: |
|
|
python/integration-test-report.md
|
|
python/integration-report-history.json
|
|
|
|
python-integration-tests-check:
|
|
if: always()
|
|
runs-on: ubuntu-latest
|
|
needs:
|
|
[
|
|
python-tests-unit,
|
|
python-tests-openai,
|
|
python-tests-azure-openai,
|
|
python-tests-misc-integration,
|
|
python-tests-functions,
|
|
python-tests-foundry,
|
|
python-tests-foundry-hosting,
|
|
python-tests-cosmos,
|
|
python-tests-github-copilot,
|
|
]
|
|
steps:
|
|
- name: Fail workflow if tests failed
|
|
id: check_tests_failed
|
|
if: contains(join(needs.*.result, ','), 'failure')
|
|
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
|
|
with:
|
|
script: core.setFailed('Integration Tests Failed!')
|
|
|
|
- name: Fail workflow if tests cancelled
|
|
id: check_tests_cancelled
|
|
if: contains(join(needs.*.result, ','), 'cancelled')
|
|
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
|
|
with:
|
|
script: core.setFailed('Integration Tests Cancelled!')
|