Compare commits

..
Author SHA1 Message Date
Tao ChenandGitHub 4c0f0ec99a Merge branch 'main' into taochen/python-update-sample-validation-scripts 2026-03-24 18:16:22 -07:00
Tao Chen 6185ba2125 force node22 2026-03-24 17:10:25 -07:00
Tao Chen dcc1eeac36 force node24 2026-03-24 17:06:41 -07:00
Tao Chen 3f2096595f force node24 2026-03-24 16:58:18 -07:00
Tao Chen 45a9da5523 Comments 2026-03-24 16:51:44 -07:00
Tao Chen 6364c05efc Add more env vars 2026-03-24 16:19:36 -07:00
Tao Chen bbb871e4cd Add timestamp 2026-03-24 12:47:55 -07:00
Tao Chen 7d7b8dd1a4 Create trend report 2026-03-24 11:33:20 -07:00
Tao Chen 2f51a5ca78 Add .env 2026-03-24 08:58:33 -07:00
Tao Chen ad5749c92a Split jobs 2026-03-23 17:16:19 -07:00
Tao Chen ed6b290457 Add fix suggestion 2026-03-23 16:15:47 -07:00
Tao Chen 63039cb748 Update autogen-migration samples 2026-03-23 15:51:17 -07:00
Tao Chen 3e7c94699f Adjust prompt 2026-03-23 15:17:45 -07:00
Tao Chen 6320443969 Update sample validation scripts 2026-03-23 14:40:37 -07:00
47 changed files with 967 additions and 1213 deletions
@@ -24,7 +24,9 @@ runs:
using: "composite"
steps:
- name: Set up Node.js environment
uses: actions/setup-node@v4
uses: actions/setup-node@v6
with:
node-version: 22
- name: Install Copilot CLI
shell: bash
+520 -8
View File
@@ -41,6 +41,13 @@ jobs:
azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
os: ${{ runner.os }}
- name: Create .env for samples
run: |
echo "AZURE_AI_PROJECT_ENDPOINT=$AZURE_AI_PROJECT_ENDPOINT" >> .env
echo "AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME=$AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME" >> .env
echo "AZURE_OPENAI_ENDPOINT=$AZURE_OPENAI_ENDPOINT" >> .env
echo "AZURE_OPENAI_CHAT_DEPLOYMENT_NAME=$AZURE_OPENAI_CHAT_DEPLOYMENT_NAME" >> .env
- name: Run sample validation
run: |
cd scripts && uv run python -m sample_validation --subdir 01-get-started --save-report --report-name 01-get-started
@@ -50,7 +57,7 @@ jobs:
if: always()
with:
name: validation-report-01-get-started
path: python/scripts/sample_validation/reports/
path: python/samples/sample_validation/reports/
validate-02-agents:
name: Validate 02-agents
@@ -64,10 +71,13 @@ jobs:
AZURE_OPENAI_ENDPOINT: ${{ vars.AZUREOPENAI__ENDPOINT }}
AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__CHATDEPLOYMENTNAME }}
AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__EMBEDDINGDEPLOYMENTNAME }}
# OpenAI configuration
OPENAI_API_KEY: ${{ secrets.OPENAI__APIKEY }}
OPENAI_CHAT_MODEL_ID: ${{ vars.OPENAI__CHATMODELID }}
OPENAI_RESPONSES_MODEL_ID: ${{ vars.OPENAI__RESPONSESMODELID }}
# GitHub MCP
GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
# Observability
ENABLE_INSTRUMENTATION: "true"
defaults:
@@ -84,16 +94,420 @@ jobs:
azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
os: ${{ runner.os }}
- name: Create .env for samples
run: |
echo "AZURE_AI_PROJECT_ENDPOINT=$AZURE_AI_PROJECT_ENDPOINT" >> .env
echo "AZURE_AI_MODEL_DEPLOYMENT_NAME=$AZURE_AI_MODEL_DEPLOYMENT_NAME" >> .env
echo "AZURE_OPENAI_ENDPOINT=$AZURE_OPENAI_ENDPOINT" >> .env
echo "AZURE_OPENAI_CHAT_DEPLOYMENT_NAME=$AZURE_OPENAI_CHAT_DEPLOYMENT_NAME" >> .env
echo "AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME=$AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME" >> .env
echo "AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME=$AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME" >> .env
echo "OPENAI_API_KEY=$OPENAI_API_KEY" >> .env
echo "OPENAI_CHAT_MODEL_ID=$OPENAI_CHAT_MODEL_ID" >> .env
echo "OPENAI_RESPONSES_MODEL_ID=$OPENAI_RESPONSES_MODEL_ID" >> .env
echo "GITHUB_PAT=$GITHUB_PAT" >> .env
- name: Run sample validation
run: |
cd scripts && uv run python -m sample_validation --subdir 02-agents --save-report --report-name 02-agents
cd scripts && uv run python -m sample_validation --subdir 02-agents --exclude providers --save-report --report-name 02-agents
- name: Upload validation report
uses: actions/upload-artifact@v7
if: always()
with:
name: validation-report-02-agents
path: python/scripts/sample_validation/reports/
path: python/samples/sample_validation/reports/
validate-02-agents-openai:
name: Validate 02-agents/providers/openai
runs-on: ubuntu-latest
environment: integration
env:
OPENAI_API_KEY: ${{ secrets.OPENAI__APIKEY }}
OPENAI_CHAT_MODEL_ID: ${{ vars.OPENAI__CHATMODELID }}
OPENAI_RESPONSES_MODEL_ID: ${{ vars.OPENAI__RESPONSESMODELID }}
defaults:
run:
working-directory: python
steps:
- uses: actions/checkout@v6
- name: Setup environment
uses: ./.github/actions/sample-validation-setup
with:
azure-client-id: ${{ secrets.AZURE_CLIENT_ID }}
azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }}
azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
os: ${{ runner.os }}
- name: Create .env for samples
run: |
echo "OPENAI_API_KEY=$OPENAI_API_KEY" >> .env
echo "OPENAI_CHAT_MODEL_ID=$OPENAI_CHAT_MODEL_ID" >> .env
echo "OPENAI_RESPONSES_MODEL_ID=$OPENAI_RESPONSES_MODEL_ID" >> .env
- name: Run sample validation
run: |
cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/openai --save-report --report-name 02-agents-openai
- name: Upload validation report
uses: actions/upload-artifact@v7
if: always()
with:
name: validation-report-02-agents-openai
path: python/samples/sample_validation/reports/
validate-02-agents-azure-openai:
name: Validate 02-agents/providers/azure_openai
runs-on: ubuntu-latest
environment: integration
env:
AZURE_AI_PROJECT_ENDPOINT: ${{ vars.AZURE_AI_PROJECT_ENDPOINT }}
AZURE_OPENAI_ENDPOINT: ${{ vars.AZUREOPENAI__ENDPOINT }}
AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__CHATDEPLOYMENTNAME }}
AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
defaults:
run:
working-directory: python
steps:
- uses: actions/checkout@v6
- name: Setup environment
uses: ./.github/actions/sample-validation-setup
with:
azure-client-id: ${{ secrets.AZURE_CLIENT_ID }}
azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }}
azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
os: ${{ runner.os }}
- name: Create .env for samples
run: |
echo "AZURE_AI_PROJECT_ENDPOINT=$AZURE_AI_PROJECT_ENDPOINT" >> .env
echo "AZURE_OPENAI_ENDPOINT=$AZURE_OPENAI_ENDPOINT" >> .env
echo "AZURE_OPENAI_CHAT_DEPLOYMENT_NAME=$AZURE_OPENAI_CHAT_DEPLOYMENT_NAME" >> .env
echo "AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME=$AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME" >> .env
- name: Run sample validation
run: |
cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/azure_openai --save-report --report-name 02-agents-azure-openai
- name: Upload validation report
uses: actions/upload-artifact@v7
if: always()
with:
name: validation-report-02-agents-azure-openai
path: python/samples/sample_validation/reports/
validate-02-agents-azure-ai:
name: Validate 02-agents/providers/azure_ai
runs-on: ubuntu-latest
environment: integration
env:
AZURE_AI_PROJECT_ENDPOINT: ${{ vars.AZURE_AI_PROJECT_ENDPOINT }}
AZURE_AI_MODEL_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
AZURE_AI_CHAT_MODEL_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__CHATDEPLOYMENTNAME }}
AZURE_AI_EMBEDDING_MODEL_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__EMBEDDINGDEPLOYMENTNAME }}
BING_CONNECTION_ID: ${{ secrets.BING_CONNECTION_ID }}
defaults:
run:
working-directory: python
steps:
- uses: actions/checkout@v6
- name: Setup environment
uses: ./.github/actions/sample-validation-setup
with:
azure-client-id: ${{ secrets.AZURE_CLIENT_ID }}
azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }}
azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
os: ${{ runner.os }}
- name: Create .env for samples
run: |
echo "AZURE_AI_PROJECT_ENDPOINT=$AZURE_AI_PROJECT_ENDPOINT" >> .env
echo "AZURE_AI_MODEL_DEPLOYMENT_NAME=$AZURE_AI_MODEL_DEPLOYMENT_NAME" >> .env
echo "AZURE_AI_CHAT_MODEL_DEPLOYMENT_NAME=$AZURE_AI_CHAT_MODEL_DEPLOYMENT_NAME" >> .env
echo "AZURE_AI_EMBEDDING_MODEL_DEPLOYMENT_NAME=$AZURE_AI_EMBEDDING_MODEL_DEPLOYMENT_NAME" >> .env
echo "BING_CONNECTION_ID=$BING_CONNECTION_ID" >> .env
- name: Run sample validation
run: |
cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/azure_ai --save-report --report-name 02-agents-azure-ai
- name: Upload validation report
uses: actions/upload-artifact@v7
if: always()
with:
name: validation-report-02-agents-azure-ai
path: python/samples/sample_validation/reports/
validate-02-agents-azure-ai-agent:
name: Validate 02-agents/providers/azure_ai_agent
runs-on: ubuntu-latest
environment: integration
env:
AZURE_AI_PROJECT_ENDPOINT: ${{ vars.AZURE_AI_PROJECT_ENDPOINT }}
AZURE_AI_MODEL_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
defaults:
run:
working-directory: python
steps:
- uses: actions/checkout@v6
- name: Setup environment
uses: ./.github/actions/sample-validation-setup
with:
azure-client-id: ${{ secrets.AZURE_CLIENT_ID }}
azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }}
azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
os: ${{ runner.os }}
- name: Create .env for samples
run: |
echo "AZURE_AI_PROJECT_ENDPOINT=$AZURE_AI_PROJECT_ENDPOINT" >> .env
echo "AZURE_AI_MODEL_DEPLOYMENT_NAME=$AZURE_AI_MODEL_DEPLOYMENT_NAME" >> .env
- name: Run sample validation
run: |
cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/azure_ai_agent --save-report --report-name 02-agents-azure-ai-agent
- name: Upload validation report
uses: actions/upload-artifact@v7
if: always()
with:
name: validation-report-02-agents-azure-ai-agent
path: python/samples/sample_validation/reports/
validate-02-agents-anthropic:
name: Validate 02-agents/providers/anthropic
runs-on: ubuntu-latest
environment: integration
env:
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
ANTHROPIC_CHAT_MODEL_ID: ${{ vars.ANTHROPIC_CHAT_MODEL_ID }}
defaults:
run:
working-directory: python
steps:
- uses: actions/checkout@v6
- name: Setup environment
uses: ./.github/actions/sample-validation-setup
with:
azure-client-id: ${{ secrets.AZURE_CLIENT_ID }}
azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }}
azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
os: ${{ runner.os }}
- name: Create .env for samples
run: |
echo "ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY" >> .env
echo "ANTHROPIC_CHAT_MODEL_ID=$ANTHROPIC_CHAT_MODEL_ID" >> .env
- name: Run sample validation
run: |
cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/anthropic --save-report --report-name 02-agents-anthropic
- name: Upload validation report
uses: actions/upload-artifact@v7
if: always()
with:
name: validation-report-02-agents-anthropic
path: python/samples/sample_validation/reports/
validate-02-agents-github-copilot:
name: Validate 02-agents/providers/github_copilot
runs-on: ubuntu-latest
environment: integration
defaults:
run:
working-directory: python
steps:
- uses: actions/checkout@v6
- name: Setup environment
uses: ./.github/actions/sample-validation-setup
with:
azure-client-id: ${{ secrets.AZURE_CLIENT_ID }}
azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }}
azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
os: ${{ runner.os }}
- name: Run sample validation
run: |
cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/github_copilot --save-report --report-name 02-agents-github-copilot
- name: Upload validation report
uses: actions/upload-artifact@v7
if: always()
with:
name: validation-report-02-agents-github-copilot
path: python/samples/sample_validation/reports/
validate-02-agents-amazon:
name: Validate 02-agents/providers/amazon
if: false # Temporarily disabled - requires AWS credentials
runs-on: ubuntu-latest
environment: integration
env:
BEDROCK_CHAT_MODEL_ID: ${{ vars.BEDROCK__CHATMODELID }}
defaults:
run:
working-directory: python
steps:
- uses: actions/checkout@v6
- name: Setup environment
uses: ./.github/actions/sample-validation-setup
with:
azure-client-id: ${{ secrets.AZURE_CLIENT_ID }}
azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }}
azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
os: ${{ runner.os }}
- name: Run sample validation
run: |
cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/amazon --save-report --report-name 02-agents-amazon
- name: Upload validation report
uses: actions/upload-artifact@v7
if: always()
with:
name: validation-report-02-agents-amazon
path: python/samples/sample_validation/reports/
validate-02-agents-ollama:
name: Validate 02-agents/providers/ollama
if: false # Temporarily disabled - requires local Ollama server
runs-on: ubuntu-latest
environment: integration
env:
OLLAMA_MODEL: ${{ vars.OLLAMA__MODEL }}
defaults:
run:
working-directory: python
steps:
- uses: actions/checkout@v6
- name: Setup environment
uses: ./.github/actions/sample-validation-setup
with:
azure-client-id: ${{ secrets.AZURE_CLIENT_ID }}
azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }}
azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
os: ${{ runner.os }}
- name: Run sample validation
run: |
cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/ollama --save-report --report-name 02-agents-ollama
- name: Upload validation report
uses: actions/upload-artifact@v7
if: always()
with:
name: validation-report-02-agents-ollama
path: python/samples/sample_validation/reports/
validate-02-agents-foundry-local:
name: Validate 02-agents/providers/foundry_local
if: false # Temporarily disabled - requires local Foundry setup
runs-on: ubuntu-latest
environment: integration
defaults:
run:
working-directory: python
steps:
- uses: actions/checkout@v6
- name: Setup environment
uses: ./.github/actions/sample-validation-setup
with:
azure-client-id: ${{ secrets.AZURE_CLIENT_ID }}
azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }}
azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
os: ${{ runner.os }}
- name: Run sample validation
run: |
cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/foundry_local --save-report --report-name 02-agents-foundry-local
- name: Upload validation report
uses: actions/upload-artifact@v7
if: always()
with:
name: validation-report-02-agents-foundry-local
path: python/samples/sample_validation/reports/
validate-02-agents-copilotstudio:
name: Validate 02-agents/providers/copilotstudio
if: false # Temporarily disabled - requires Copilot Studio setup
runs-on: ubuntu-latest
environment: integration
env:
COPILOTSTUDIOAGENT__ENVIRONMENTID: ${{ secrets.COPILOTSTUDIOAGENT__ENVIRONMENTID }}
COPILOTSTUDIOAGENT__SCHEMANAME: ${{ secrets.COPILOTSTUDIOAGENT__SCHEMANAME }}
COPILOTSTUDIOAGENT__TENANTID: ${{ secrets.COPILOTSTUDIOAGENT__TENANTID }}
COPILOTSTUDIOAGENT__AGENTAPPID: ${{ secrets.COPILOTSTUDIOAGENT__AGENTAPPID }}
defaults:
run:
working-directory: python
steps:
- uses: actions/checkout@v6
- name: Setup environment
uses: ./.github/actions/sample-validation-setup
with:
azure-client-id: ${{ secrets.AZURE_CLIENT_ID }}
azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }}
azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
os: ${{ runner.os }}
- name: Create .env for samples
run: |
echo "COPILOTSTUDIOAGENT__ENVIRONMENTID=$COPILOTSTUDIOAGENT__ENVIRONMENTID" >> .env
echo "COPILOTSTUDIOAGENT__SCHEMANAME=$COPILOTSTUDIOAGENT__SCHEMANAME" >> .env
echo "COPILOTSTUDIOAGENT__TENANTID=$COPILOTSTUDIOAGENT__TENANTID" >> .env
echo "COPILOTSTUDIOAGENT__AGENTAPPID=$COPILOTSTUDIOAGENT__AGENTAPPID" >> .env
- name: Run sample validation
run: |
cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/copilotstudio --save-report --report-name 02-agents-copilotstudio
- name: Upload validation report
uses: actions/upload-artifact@v7
if: always()
with:
name: validation-report-02-agents-copilotstudio
path: python/samples/sample_validation/reports/
validate-02-agents-custom:
name: Validate 02-agents/providers/custom
runs-on: ubuntu-latest
environment: integration
defaults:
run:
working-directory: python
steps:
- uses: actions/checkout@v6
- name: Setup environment
uses: ./.github/actions/sample-validation-setup
with:
azure-client-id: ${{ secrets.AZURE_CLIENT_ID }}
azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }}
azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
os: ${{ runner.os }}
- name: Run sample validation
run: |
cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/custom --save-report --report-name 02-agents-custom
- name: Upload validation report
uses: actions/upload-artifact@v7
if: always()
with:
name: validation-report-02-agents-custom
path: python/samples/sample_validation/reports/
validate-03-workflows:
name: Validate 03-workflows
@@ -121,6 +535,14 @@ jobs:
azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
os: ${{ runner.os }}
- name: Create .env for samples
run: |
echo "AZURE_AI_PROJECT_ENDPOINT=$AZURE_AI_PROJECT_ENDPOINT" >> .env
echo "AZURE_AI_MODEL_DEPLOYMENT_NAME=$AZURE_AI_MODEL_DEPLOYMENT_NAME" >> .env
echo "AZURE_OPENAI_ENDPOINT=$AZURE_OPENAI_ENDPOINT" >> .env
echo "AZURE_OPENAI_CHAT_DEPLOYMENT_NAME=$AZURE_OPENAI_CHAT_DEPLOYMENT_NAME" >> .env
echo "AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME=$AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME" >> .env
- name: Run sample validation
run: |
cd scripts && uv run python -m sample_validation --subdir 03-workflows --save-report --report-name 03-workflows
@@ -130,7 +552,7 @@ jobs:
if: always()
with:
name: validation-report-03-workflows
path: python/scripts/sample_validation/reports/
path: python/samples/sample_validation/reports/
validate-04-hosting:
name: Validate 04-hosting
@@ -169,7 +591,7 @@ jobs:
if: always()
with:
name: validation-report-04-hosting
path: python/scripts/sample_validation/reports/
path: python/samples/sample_validation/reports/
validate-05-end-to-end:
name: Validate 05-end-to-end
@@ -213,7 +635,7 @@ jobs:
if: always()
with:
name: validation-report-05-end-to-end
path: python/scripts/sample_validation/reports/
path: python/samples/sample_validation/reports/
validate-autogen-migration:
name: Validate autogen-migration
@@ -244,6 +666,16 @@ jobs:
azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
os: ${{ runner.os }}
- name: Create .env for samples
run: |
echo "AZURE_AI_PROJECT_ENDPOINT=$AZURE_AI_PROJECT_ENDPOINT" >> .env
echo "AZURE_AI_MODEL_DEPLOYMENT_NAME=$AZURE_AI_MODEL_DEPLOYMENT_NAME" >> .env
echo "AZURE_OPENAI_ENDPOINT=$AZURE_OPENAI_ENDPOINT" >> .env
echo "AZURE_OPENAI_CHAT_DEPLOYMENT_NAME=$AZURE_OPENAI_CHAT_DEPLOYMENT_NAME" >> .env
echo "OPENAI_API_KEY=$OPENAI_API_KEY" >> .env
echo "OPENAI_CHAT_MODEL_ID=$OPENAI_CHAT_MODEL_ID" >> .env
echo "OPENAI_RESPONSES_MODEL_ID=$OPENAI_RESPONSES_MODEL_ID" >> .env
- name: Run sample validation
run: |
cd scripts && uv run python -m sample_validation --subdir autogen-migration --save-report --report-name autogen-migration
@@ -253,7 +685,7 @@ jobs:
if: always()
with:
name: validation-report-autogen-migration
path: python/scripts/sample_validation/reports/
path: python/samples/sample_validation/reports/
validate-semantic-kernel-migration:
name: Validate semantic-kernel-migration
@@ -290,6 +722,21 @@ jobs:
azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
os: ${{ runner.os }}
- name: Create .env for samples
run: |
echo "AZURE_AI_PROJECT_ENDPOINT=$AZURE_AI_PROJECT_ENDPOINT" >> .env
echo "AZURE_AI_MODEL_DEPLOYMENT_NAME=$AZURE_AI_MODEL_DEPLOYMENT_NAME" >> .env
echo "AZURE_OPENAI_ENDPOINT=$AZURE_OPENAI_ENDPOINT" >> .env
echo "AZURE_OPENAI_CHAT_DEPLOYMENT_NAME=$AZURE_OPENAI_CHAT_DEPLOYMENT_NAME" >> .env
echo "AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME=$AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME" >> .env
echo "OPENAI_API_KEY=$OPENAI_API_KEY" >> .env
echo "OPENAI_CHAT_MODEL_ID=$OPENAI_CHAT_MODEL_ID" >> .env
echo "OPENAI_RESPONSES_MODEL_ID=$OPENAI_RESPONSES_MODEL_ID" >> .env
echo "COPILOTSTUDIOAGENT__ENVIRONMENTID=$COPILOTSTUDIOAGENT__ENVIRONMENTID" >> .env
echo "COPILOTSTUDIOAGENT__SCHEMANAME=$COPILOTSTUDIOAGENT__SCHEMANAME" >> .env
echo "COPILOTSTUDIOAGENT__TENANTID=$COPILOTSTUDIOAGENT__TENANTID" >> .env
echo "COPILOTSTUDIOAGENT__AGENTAPPID=$COPILOTSTUDIOAGENT__AGENTAPPID" >> .env
- name: Run sample validation
run: |
cd scripts && uv run python -m sample_validation --subdir semantic-kernel-migration --save-report --report-name semantic-kernel-migration
@@ -299,4 +746,69 @@ jobs:
if: always()
with:
name: validation-report-semantic-kernel-migration
path: python/scripts/sample_validation/reports/
path: python/samples/sample_validation/reports/
aggregate-results:
name: Aggregate Results
runs-on: ubuntu-latest
if: always()
needs:
- validate-01-get-started
- validate-02-agents
- validate-02-agents-openai
- validate-02-agents-azure-openai
- validate-02-agents-azure-ai
- validate-02-agents-azure-ai-agent
- validate-02-agents-anthropic
- validate-02-agents-github-copilot
- validate-02-agents-amazon
- validate-02-agents-ollama
- validate-02-agents-foundry-local
- validate-02-agents-copilotstudio
- validate-02-agents-custom
- validate-03-workflows
- validate-04-hosting
- validate-05-end-to-end
- validate-autogen-migration
- validate-semantic-kernel-migration
steps:
- uses: actions/checkout@v6
- name: Download all validation reports
uses: actions/download-artifact@v7
with:
pattern: validation-report-*
path: reports/
merge-multiple: true
- name: Restore validation history
id: cache-restore
uses: actions/cache/restore@v4
with:
path: validation-history/
key: validation-history-${{ github.run_id }}
restore-keys: |
validation-history-
- name: Aggregate results and generate trend report
run: |
python3 python/scripts/sample_validation/aggregate.py \
reports/ \
validation-history/history.json \
trend-report.md
- name: Write trend report to job summary
run: cat trend-report.md >> "$GITHUB_STEP_SUMMARY"
- name: Save validation history
uses: actions/cache/save@v4
with:
path: validation-history/
key: validation-history-${{ github.run_id }}
- name: Upload trend report
uses: actions/upload-artifact@v7
if: always()
with:
name: validation-trend-report
path: trend-report.md
-2
View File
@@ -76,8 +76,6 @@
<Project Path="samples/04-hosting/DurableWorkflows/AzureFunctions/01_SequentialWorkflow/01_SequentialWorkflow.csproj" />
<Project Path="samples/04-hosting/DurableWorkflows/AzureFunctions/02_ConcurrentWorkflow/02_ConcurrentWorkflow.csproj" />
<Project Path="samples/04-hosting/DurableWorkflows/AzureFunctions/03_WorkflowHITL/03_WorkflowHITL.csproj" />
<Project Path="samples/04-hosting/DurableWorkflows/AzureFunctions/04_WorkflowMcpTool/04_WorkflowMcpTool.csproj" />
<Project Path="samples/04-hosting/DurableWorkflows/AzureFunctions/05_WorkflowAndAgents/05_WorkflowAndAgents.csproj" />
</Folder>
<Folder Name="/Samples/GettingStarted/">
<File Path="samples/GettingStarted/README.md" />
@@ -1,35 +0,0 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFrameworks>net10.0</TargetFrameworks>
<AzureFunctionsVersion>v4</AzureFunctionsVersion>
<OutputType>Exe</OutputType>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<!-- The Functions build tools don't like namespaces that start with a number -->
<AssemblyName>WorkflowMcpTool</AssemblyName>
<RootNamespace>WorkflowMcpTool</RootNamespace>
</PropertyGroup>
<ItemGroup>
<FrameworkReference Include="Microsoft.AspNetCore.App" />
</ItemGroup>
<!-- Azure Functions packages -->
<ItemGroup>
<PackageReference Include="Microsoft.Azure.Functions.Worker" />
<PackageReference Include="Microsoft.Azure.Functions.Worker.Extensions.DurableTask" />
<PackageReference Include="Microsoft.Azure.Functions.Worker.Extensions.DurableTask.AzureManaged" />
<PackageReference Include="Microsoft.Azure.Functions.Worker.Extensions.Http.AspNetCore" />
<PackageReference Include="Microsoft.Azure.Functions.Worker.Sdk" />
</ItemGroup>
<!-- Local projects that should be switched to package references when using the sample outside of this MAF repo -->
<!--
<ItemGroup>
<PackageReference Include="Microsoft.Agents.AI.Hosting.AzureFunctions" />
</ItemGroup>
-->
<ItemGroup>
<ProjectReference Include="..\..\..\..\..\src\Microsoft.Agents.AI.Hosting.AzureFunctions\Microsoft.Agents.AI.Hosting.AzureFunctions.csproj" />
</ItemGroup>
</Project>
@@ -1,59 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
using Microsoft.Agents.AI.Workflows;
namespace WorkflowMcpTool;
internal sealed class TranslateText() : Executor<string, TranslationResult>("TranslateText")
{
public override ValueTask<TranslationResult> HandleAsync(
string message,
IWorkflowContext context,
CancellationToken cancellationToken = default)
{
Console.WriteLine($"[Activity] TranslateText: '{message}'");
return ValueTask.FromResult(new TranslationResult(message, message.ToUpperInvariant()));
}
}
internal sealed class FormatOutput() : Executor<TranslationResult, string>("FormatOutput")
{
public override ValueTask<string> HandleAsync(
TranslationResult message,
IWorkflowContext context,
CancellationToken cancellationToken = default)
{
Console.WriteLine("[Activity] FormatOutput: Formatting result");
return ValueTask.FromResult($"Original: {message.Original} => Translated: {message.Translated}");
}
}
internal sealed class LookupOrder() : Executor<string, OrderInfo>("LookupOrder")
{
public override ValueTask<OrderInfo> HandleAsync(
string message,
IWorkflowContext context,
CancellationToken cancellationToken = default)
{
Console.WriteLine($"[Activity] LookupOrder: '{message}'");
return ValueTask.FromResult(new OrderInfo(message, "Alice Johnson", "Wireless Headphones", Quantity: 2, UnitPrice: 49.99m));
}
}
internal sealed class EnrichOrder() : Executor<OrderInfo, OrderSummary>("EnrichOrder")
{
public override ValueTask<OrderSummary> HandleAsync(
OrderInfo message,
IWorkflowContext context,
CancellationToken cancellationToken = default)
{
Console.WriteLine($"[Activity] EnrichOrder: '{message.OrderId}'");
return ValueTask.FromResult(new OrderSummary(message, TotalPrice: message.Quantity * message.UnitPrice, Status: "Confirmed"));
}
}
internal sealed record TranslationResult(string Original, string Translated);
internal sealed record OrderInfo(string OrderId, string CustomerName, string Product, int Quantity, decimal UnitPrice);
internal sealed record OrderSummary(OrderInfo Order, decimal TotalPrice, string Status);
@@ -1,44 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
// This sample demonstrates how to expose a durable workflow as an MCP (Model Context Protocol) tool.
// When using AddWorkflow with exposeMcpToolTrigger: true, the Functions host will automatically
// generate a remote MCP endpoint for the app at /runtime/webhooks/mcp with a workflow-specific
// tool name. MCP-compatible clients can then invoke the workflow as a tool.
using Microsoft.Agents.AI.Hosting.AzureFunctions;
using Microsoft.Agents.AI.Workflows;
using Microsoft.Azure.Functions.Worker.Builder;
using Microsoft.Extensions.Hosting;
using WorkflowMcpTool;
// Define executors
TranslateText translateText = new();
FormatOutput formatOutput = new();
LookupOrder lookupOrder = new();
EnrichOrder enrichOrder = new();
// Build a simple workflow: TranslateText -> FormatOutput
Workflow translateWorkflow = new WorkflowBuilder(translateText)
.WithName("Translate")
.WithDescription("Translate text to uppercase and format the result")
.AddEdge(translateText, formatOutput)
.Build();
// Build a workflow that returns a POCO: LookupOrder -> EnrichOrder
Workflow orderLookupWorkflow = new WorkflowBuilder(lookupOrder)
.WithName("OrderLookup")
.WithDescription("Look up an order by ID and return enriched order details")
.AddEdge(lookupOrder, enrichOrder)
.Build();
using IHost app = FunctionsApplication
.CreateBuilder(args)
.ConfigureFunctionsWebApplication()
.ConfigureDurableWorkflows(workflows =>
{
// Expose both workflows as MCP tool triggers.
workflows.AddWorkflow(translateWorkflow, exposeStatusEndpoint: false, exposeMcpToolTrigger: true);
workflows.AddWorkflow(orderLookupWorkflow, exposeStatusEndpoint: false, exposeMcpToolTrigger: true);
})
.Build();
app.Run();
@@ -1,81 +0,0 @@
# Workflow as MCP Tool Sample
This sample demonstrates how to expose durable workflows as [MCP (Model Context Protocol)](https://modelcontextprotocol.io/) tools, enabling MCP-compatible clients to invoke workflows directly.
## Key Concepts Demonstrated
- **Workflow as MCP Tool**: Expose workflows as callable MCP tools using `exposeMcpToolTrigger: true`
- **MCP Server Hosting**: The Azure Functions host automatically generates a remote MCP endpoint at `/runtime/webhooks/mcp`
- **String and POCO Results**: Shows workflows returning both plain strings and structured JSON objects
## Sample Architecture
The sample creates two workflows exposed as MCP tools:
### Translate Workflow (returns a string)
| Executor | Input | Output | Description |
|----------|-------|--------|-------------|
| **TranslateText** | `string` | `TranslationResult` | Converts input text to uppercase |
| **FormatOutput** | `TranslationResult` | `string` | Formats the result into a readable string |
### OrderLookup Workflow (returns a POCO)
| Executor | Input | Output | Description |
|----------|-------|--------|-------------|
| **LookupOrder** | `string` | `OrderInfo` | Looks up an order by ID |
| **EnrichOrder** | `OrderInfo` | `OrderSummary` | Adds computed fields (total price, status) |
## Environment Setup
See the [README.md](../../README.md) file in the parent directory for complete setup instructions, including:
- Prerequisites installation
- Durable Task Scheduler setup
- Storage emulator configuration
For this sample, you'll also need [Node.js](https://nodejs.org/en/download) to use the [MCP Inspector](https://modelcontextprotocol.io/docs/tools/inspector).
## Running the Sample
1. **Start the Function App**:
```bash
cd dotnet/samples/04-hosting/DurableWorkflows/AzureFunctions/04_WorkflowMcpTool
func start
```
2. **Note the MCP Server Endpoint**: When the app starts, you'll see the MCP server endpoint in the terminal output:
```text
MCP server endpoint: http://localhost:7071/runtime/webhooks/mcp
```
## Invoking Workflows via MCP Inspector
1. Install and run the [MCP Inspector](https://modelcontextprotocol.io/docs/tools/inspector):
```bash
npx @modelcontextprotocol/inspector
```
2. Connect to the MCP server endpoint:
- For **Transport Type**, select **"Streamable HTTP"**
- For **URL**, enter `http://localhost:7071/runtime/webhooks/mcp`
- Click the **Connect** button
3. Click the **List Tools** button. You should see two tools: `Translate` and `OrderLookup`.
4. Test the **Translate** tool (returns a plain string):
- Select the `Translate` tool
- Set `hello world` as the `input` parameter
- Click **Run Tool**
- Expected result: `Original: hello world => Translated: HELLO WORLD`
5. Test the **OrderLookup** tool (returns a JSON object):
- Select the `OrderLookup` tool
- Set `ORD-2025-42` as the `input` parameter
- Click **Run Tool**
- Expected result: A JSON object containing order details such as `OrderId`, `CustomerName`, `Product`, `TotalPrice`, and `Status`
You'll see the workflow executor activities logged in the terminal where you ran `func start`.
@@ -1,20 +0,0 @@
{
"version": "2.0",
"logging": {
"logLevel": {
"Microsoft.Agents.AI.DurableTask": "Information",
"Microsoft.Agents.AI.Hosting.AzureFunctions": "Information",
"DurableTask": "Information",
"Microsoft.DurableTask": "Information"
}
},
"extensions": {
"durableTask": {
"hubName": "default",
"storageProvider": {
"type": "AzureManaged",
"connectionStringName": "DURABLE_TASK_SCHEDULER_CONNECTION_STRING"
}
}
}
}
@@ -1,8 +0,0 @@
{
"IsEncrypted": false,
"Values": {
"FUNCTIONS_WORKER_RUNTIME": "dotnet-isolated",
"AzureWebJobsStorage": "UseDevelopmentStorage=true",
"DURABLE_TASK_SCHEDULER_CONNECTION_STRING": "Endpoint=http://localhost:8080;TaskHub=default;Authentication=None"
}
}
@@ -1,42 +0,0 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFrameworks>net10.0</TargetFrameworks>
<AzureFunctionsVersion>v4</AzureFunctionsVersion>
<OutputType>Exe</OutputType>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<!-- The Functions build tools don't like namespaces that start with a number -->
<AssemblyName>WorkflowAndAgents</AssemblyName>
<RootNamespace>WorkflowAndAgents</RootNamespace>
</PropertyGroup>
<ItemGroup>
<FrameworkReference Include="Microsoft.AspNetCore.App" />
</ItemGroup>
<!-- Azure Functions packages -->
<ItemGroup>
<PackageReference Include="Microsoft.Azure.Functions.Worker" />
<PackageReference Include="Microsoft.Azure.Functions.Worker.Extensions.DurableTask" />
<PackageReference Include="Microsoft.Azure.Functions.Worker.Extensions.DurableTask.AzureManaged" />
<PackageReference Include="Microsoft.Azure.Functions.Worker.Extensions.Http.AspNetCore" />
<PackageReference Include="Microsoft.Azure.Functions.Worker.Sdk" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Azure.AI.OpenAI" />
<PackageReference Include="Azure.Identity" />
</ItemGroup>
<!-- Local projects that should be switched to package references when using the sample outside of this MAF repo -->
<!--
<ItemGroup>
<PackageReference Include="Microsoft.Agents.AI.Hosting.AzureFunctions" />
<PackageReference Include="Microsoft.Agents.AI.OpenAI" />
</ItemGroup>
-->
<ItemGroup>
<ProjectReference Include="..\..\..\..\..\src\Microsoft.Agents.AI.Hosting.AzureFunctions\Microsoft.Agents.AI.Hosting.AzureFunctions.csproj" />
<ProjectReference Include="..\..\..\..\..\src\Microsoft.Agents.AI.OpenAI\Microsoft.Agents.AI.OpenAI.csproj" />
</ItemGroup>
</Project>
@@ -1,31 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
using Microsoft.Agents.AI.Workflows;
namespace WorkflowAndAgents;
internal sealed class TranslateText() : Executor<string, TranslationResult>("TranslateText")
{
public override ValueTask<TranslationResult> HandleAsync(
string message,
IWorkflowContext context,
CancellationToken cancellationToken = default)
{
Console.WriteLine($"[Activity] TranslateText: '{message}'");
return ValueTask.FromResult(new TranslationResult(message, message.ToUpperInvariant()));
}
}
internal sealed class FormatOutput() : Executor<TranslationResult, string>("FormatOutput")
{
public override ValueTask<string> HandleAsync(
TranslationResult message,
IWorkflowContext context,
CancellationToken cancellationToken = default)
{
Console.WriteLine("[Activity] FormatOutput: Formatting result");
return ValueTask.FromResult($"Original: {message.Original} => Translated: {message.Translated}");
}
}
internal sealed record TranslationResult(string Original, string Translated);
@@ -1,64 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
// This sample demonstrates using ConfigureDurableOptions to register BOTH agents AND workflows
// in a single Azure Functions app. It uses a workflow to translate text and a standalone AI agent
// accessible via HTTP and MCP tool triggers.
#pragma warning disable IDE0002 // Simplify Member Access
using Azure;
using Azure.AI.OpenAI;
using Azure.Identity;
using Microsoft.Agents.AI;
using Microsoft.Agents.AI.Hosting.AzureFunctions;
using Microsoft.Agents.AI.Workflows;
using Microsoft.Azure.Functions.Worker.Builder;
using Microsoft.Extensions.Hosting;
using OpenAI.Chat;
using WorkflowAndAgents;
// Get the Azure OpenAI endpoint and deployment name from environment variables.
string endpoint = Environment.GetEnvironmentVariable("AZURE_OPENAI_ENDPOINT")
?? throw new InvalidOperationException("AZURE_OPENAI_ENDPOINT is not set.");
string deploymentName = Environment.GetEnvironmentVariable("AZURE_OPENAI_DEPLOYMENT_NAME")
?? throw new InvalidOperationException("AZURE_OPENAI_DEPLOYMENT_NAME is not set.");
// Use Azure Key Credential if provided, otherwise use Azure CLI Credential.
string? azureOpenAiKey = Environment.GetEnvironmentVariable("AZURE_OPENAI_API_KEY");
AzureOpenAIClient client = !string.IsNullOrEmpty(azureOpenAiKey)
? new AzureOpenAIClient(new Uri(endpoint), new AzureKeyCredential(azureOpenAiKey))
: new AzureOpenAIClient(new Uri(endpoint), new DefaultAzureCredential());
ChatClient chatClient = client.GetChatClient(deploymentName);
// Define a standalone AI agent
AIAgent assistant = chatClient.AsAIAgent(
"You are a helpful assistant. Answer questions clearly and concisely.",
"Assistant",
description: "A general-purpose helpful assistant.");
// Define workflow executors
TranslateText translateText = new();
FormatOutput formatOutput = new();
// Build a workflow: TranslateText -> FormatOutput
Workflow translateWorkflow = new WorkflowBuilder(translateText)
.WithName("Translate")
.WithDescription("Translate text to uppercase and format the result")
.AddEdge(translateText, formatOutput)
.Build();
// Use ConfigureDurableOptions to register both agents and workflows together
using IHost app = FunctionsApplication
.CreateBuilder(args)
.ConfigureFunctionsWebApplication()
.ConfigureDurableOptions(options =>
{
// Register the standalone agent with HTTP and MCP tool triggers
options.Agents.AddAIAgent(assistant, enableHttpTrigger: true, enableMcpToolTrigger: true);
// Register the workflow with an HTTP endpoint and MCP tool trigger
options.Workflows.AddWorkflow(translateWorkflow, exposeStatusEndpoint: false, exposeMcpToolTrigger: true);
})
.Build();
app.Run();
@@ -1,76 +0,0 @@
# Workflow and Agents Sample
This sample demonstrates how to use `ConfigureDurableOptions` to register **both** AI agents **and** workflows in a single Azure Functions app. This is the recommended approach when your application needs both standalone agents and orchestrated workflows.
## Key Concepts Demonstrated
- **Unified Configuration**: Use `ConfigureDurableOptions` to register agents and workflows together
- **Standalone Agent**: An AI agent accessible via HTTP and MCP tool triggers
- **Workflow**: A simple text translation workflow also exposed as an MCP tool
- **Mixed Triggers**: Both agents and workflows coexist in the same Functions host
## Sample Architecture
### Standalone Agent
| Agent | Description |
|-------|-------------|
| **Assistant** | A general-purpose AI assistant accessible via HTTP (`/agents/Assistant/run`) and as an MCP tool |
### Translate Workflow
| Executor | Input | Output | Description |
|----------|-------|--------|-------------|
| **TranslateText** | `string` | `TranslationResult` | Converts input text to uppercase |
| **FormatOutput** | `TranslationResult` | `string` | Formats the result into a readable string |
## Environment Setup
See the [README.md](../../README.md) file in the parent directory for complete setup instructions, including:
- Prerequisites installation
- Durable Task Scheduler setup
- Storage emulator configuration
This sample also requires Azure OpenAI credentials. Set the following in `local.settings.json`:
- `AZURE_OPENAI_ENDPOINT`: Your Azure OpenAI endpoint URL
- `AZURE_OPENAI_DEPLOYMENT_NAME`: Your chat model deployment name
- `AZURE_OPENAI_API_KEY` (optional): If not set, Azure CLI credential is used
## Running the Sample
1. **Start the Function App**:
```bash
cd dotnet/samples/04-hosting/DurableWorkflows/AzureFunctions/05_WorkflowAndAgents
func start
```
2. **Expected Functions**: When the app starts, you should see functions for both the agent and the workflow:
- `dafx-Assistant` (entity trigger for the agent)
- `http-Assistant` (HTTP trigger for the agent)
- `mcptool-Assistant` (MCP tool trigger for the agent)
- `wf-Translate` (orchestration trigger for the workflow)
- `mcptool-wf-Translate` (MCP tool trigger for the workflow)
## Invoking the Agent via HTTP
```bash
curl -X POST http://localhost:7071/agents/Assistant/run \
-H "Content-Type: application/json" \
-d '{"query": "What is the capital of France?"}'
```
## Invoking via MCP Inspector
1. Install and run the [MCP Inspector](https://modelcontextprotocol.io/docs/tools/inspector):
```bash
npx @modelcontextprotocol/inspector
```
2. Connect to `http://localhost:7071/runtime/webhooks/mcp` using **Streamable HTTP** transport.
3. Click **List Tools** to see both the `Assistant` agent tool and the `Translate` workflow tool.
@@ -1,20 +0,0 @@
{
"version": "2.0",
"logging": {
"logLevel": {
"Microsoft.Agents.AI.DurableTask": "Information",
"Microsoft.Agents.AI.Hosting.AzureFunctions": "Information",
"DurableTask": "Information",
"Microsoft.DurableTask": "Information"
}
},
"extensions": {
"durableTask": {
"hubName": "default",
"storageProvider": {
"type": "AzureManaged",
"connectionStringName": "DURABLE_TASK_SCHEDULER_CONNECTION_STRING"
}
}
}
}
@@ -1,10 +0,0 @@
{
"IsEncrypted": false,
"Values": {
"FUNCTIONS_WORKER_RUNTIME": "dotnet-isolated",
"AzureWebJobsStorage": "UseDevelopmentStorage=true",
"DURABLE_TASK_SCHEDULER_CONNECTION_STRING": "Endpoint=http://localhost:8080;TaskHub=default;Authentication=None",
"AZURE_OPENAI_ENDPOINT": "<AZURE_OPENAI_ENDPOINT>",
"AZURE_OPENAI_DEPLOYMENT_NAME": "<AZURE_OPENAI_DEPLOYMENT_NAME>"
}
}
@@ -48,4 +48,3 @@ $env:DURABLE_TASK_SCHEDULER_CONNECTION_STRING = "AccountEndpoint=http://localhos
| [01_SequentialWorkflow](AzureFunctions/01_SequentialWorkflow/) | Sequential workflow hosted in Azure Functions |
| [02_ConcurrentWorkflow](AzureFunctions/02_ConcurrentWorkflow/) | Concurrent workflow hosted in Azure Functions |
| [03_WorkflowHITL](AzureFunctions/03_WorkflowHITL/) | Human-in-the-loop workflow hosted in Azure Functions |
| [04_WorkflowMcpTool](AzureFunctions/04_WorkflowMcpTool/) | Workflow exposed as an MCP tool |
@@ -167,20 +167,6 @@ internal sealed class BuiltInFunctionExecutor : IFunctionExecutor
return;
}
if (context.FunctionDefinition.EntryPoint == BuiltInFunctions.RunWorkflowMcpToolFunctionEntryPoint)
{
if (mcpToolInvocationContext is null)
{
throw new InvalidOperationException($"MCP tool invocation context binding is missing for the invocation {context.InvocationId}.");
}
context.GetInvocationResult().Value = await BuiltInFunctions.RunWorkflowMcpToolAsync(
mcpToolInvocationContext,
durableTaskClient,
context);
return;
}
throw new InvalidOperationException($"Unsupported function entry point '{context.FunctionDefinition.EntryPoint}' for invocation {context.InvocationId}.");
}
@@ -29,7 +29,6 @@ internal static class BuiltInFunctions
internal static readonly string InvokeWorkflowActivityFunctionEntryPoint = $"{typeof(BuiltInFunctions).FullName!}.{nameof(InvokeWorkflowActivityAsync)}";
internal static readonly string GetWorkflowStatusHttpFunctionEntryPoint = $"{typeof(BuiltInFunctions).FullName!}.{nameof(GetWorkflowStatusAsync)}";
internal static readonly string RespondToWorkflowHttpFunctionEntryPoint = $"{typeof(BuiltInFunctions).FullName!}.{nameof(RespondToWorkflowAsync)}";
internal static readonly string RunWorkflowMcpToolFunctionEntryPoint = $"{typeof(BuiltInFunctions).FullName!}.{nameof(RunWorkflowMcpToolAsync)}";
#pragma warning disable IL3000 // Avoid accessing Assembly file path when publishing as a single file - Azure Functions does not use single-file publishing
internal static readonly string ScriptFile = Path.GetFileName(typeof(BuiltInFunctions).Assembly.Location);
@@ -379,55 +378,6 @@ internal static class BuiltInFunctions
return agentResponse.Text;
}
/// <summary>
/// Runs a workflow via MCP tool trigger.
/// Extracts the <c>input</c> argument, schedules a new orchestration, waits for completion, and returns the output.
/// </summary>
public static async Task<string?> RunWorkflowMcpToolAsync(
[McpToolTrigger("BuiltInWorkflowMcpTool")] ToolInvocationContext context,
[DurableClient] DurableTaskClient client,
FunctionContext functionContext)
{
if (context.Arguments is null)
{
throw new ArgumentException("MCP Tool invocation is missing required arguments.");
}
if (!context.Arguments.TryGetValue("input", out object? inputObj) || inputObj is not string input)
{
throw new ArgumentException("MCP Tool invocation is missing required 'input' argument of type string.");
}
string workflowName = context.Name;
string orchestrationFunctionName = WorkflowNamingHelper.ToOrchestrationFunctionName(workflowName);
DurableWorkflowInput<string> orchestrationInput = new() { Input = input };
string instanceId = await client.ScheduleNewOrchestrationInstanceAsync(orchestrationFunctionName, orchestrationInput);
OrchestrationMetadata? metadata = await client.WaitForInstanceCompletionAsync(
instanceId,
getInputsAndOutputs: true,
cancellation: functionContext.CancellationToken);
if (metadata is null)
{
throw new InvalidOperationException($"Workflow orchestration '{instanceId}' returned no metadata.");
}
if (metadata.RuntimeStatus is OrchestrationRuntimeStatus.Failed)
{
string errorMessage = metadata.FailureDetails?.ErrorMessage ?? "Unknown error";
throw new InvalidOperationException($"Workflow orchestration '{instanceId}' failed: {errorMessage}");
}
if (metadata.RuntimeStatus is not OrchestrationRuntimeStatus.Completed)
{
throw new InvalidOperationException($"Workflow orchestration '{instanceId}' ended with unexpected status '{metadata.RuntimeStatus}'.");
}
return metadata.ReadOutputAs<DurableWorkflowResult>()?.Result;
}
/// <summary>
/// Creates an error response with the specified status code and error message.
/// </summary>
@@ -2,7 +2,6 @@
## [Unreleased]
- Added MCP tool trigger support for durable workflows ([#4768](https://github.com/microsoft/agent-framework/pull/4768))
- Added Azure Functions hosting support for durable workflows ([#4436](https://github.com/microsoft/agent-framework/pull/4436))
## v1.0.0-preview.251219.1
@@ -6,8 +6,7 @@ namespace Microsoft.Agents.AI.Hosting.AzureFunctions;
/// <summary>
/// Provides access to agent-specific options for functions agents by name.
/// Returns <see langword="false"/> when no explicit options have been configured for an agent,
/// which distinguishes standalone agents from those auto-registered by workflows.
/// Returns default options (HTTP trigger enabled, MCP tool disabled) when no explicit options were configured.
/// </summary>
internal sealed class DefaultFunctionsAgentOptionsProvider(IReadOnlyDictionary<string, FunctionsAgentOptions> functionsAgentOptions)
: IFunctionsAgentOptionsProvider
@@ -15,19 +14,32 @@ internal sealed class DefaultFunctionsAgentOptionsProvider(IReadOnlyDictionary<s
private readonly IReadOnlyDictionary<string, FunctionsAgentOptions> _functionsAgentOptions =
functionsAgentOptions ?? throw new ArgumentNullException(nameof(functionsAgentOptions));
// Default options. HTTP trigger enabled, MCP tool disabled.
private static readonly FunctionsAgentOptions s_defaultOptions = new()
{
HttpTrigger = { IsEnabled = true },
McpToolTrigger = { IsEnabled = false }
};
/// <summary>
/// Attempts to retrieve the options associated with the specified agent name.
/// Returns <see langword="false"/> when no options have been explicitly configured for the agent.
/// If not found, a default options instance (with HTTP trigger enabled) is returned.
/// </summary>
/// <param name="agentName">The name of the agent whose options are to be retrieved. Cannot be null or empty.</param>
/// <param name="options">
/// When this method returns <see langword="true"/>, contains the options for the specified agent;
/// otherwise, <see langword="null"/>.
/// </param>
/// <returns><see langword="true"/> if options were found for the agent; otherwise, <see langword="false"/>.</returns>
/// <param name="options">The options for the specified agent. Will never be null.</param>
/// <returns>Always true. Returns configured options if present; otherwise default fallback options.</returns>
public bool TryGet(string agentName, [NotNullWhen(true)] out FunctionsAgentOptions? options)
{
ArgumentException.ThrowIfNullOrEmpty(agentName);
return this._functionsAgentOptions.TryGetValue(agentName, out options);
if (this._functionsAgentOptions.TryGetValue(agentName, out FunctionsAgentOptions? existing))
{
options = existing;
return true;
}
// If not defined, return default options.
options = s_defaultOptions;
return true;
}
}
@@ -6,13 +6,9 @@ using Microsoft.Extensions.Logging;
namespace Microsoft.Agents.AI.Hosting.AzureFunctions;
/// <summary>
/// Transforms function metadata by registering durable agent functions for each explicitly configured agent.
/// Transforms function metadata by registering durable agent functions for each configured agent.
/// </summary>
/// <remarks>
/// This transformer adds entity, HTTP, and MCP tool trigger functions for agents that have
/// explicit <see cref="FunctionsAgentOptions"/>. Agents auto-registered by workflows
/// (which lack explicit options) are handled by <see cref="DurableWorkflowsFunctionMetadataTransformer"/>.
/// </remarks>
/// <remarks>This transformer adds both entity trigger and HTTP trigger functions for every agent registered in the application.</remarks>
internal sealed class DurableAgentFunctionMetadataTransformer : IFunctionMetadataTransformer
{
private readonly ILogger<DurableAgentFunctionMetadataTransformer> _logger;
@@ -42,27 +38,24 @@ internal sealed class DurableAgentFunctionMetadataTransformer : IFunctionMetadat
{
string agentName = kvp.Key;
// Only generate triggers for agents with explicit Functions agent options.
// Agents auto-registered by workflows are handled by DurableWorkflowsFunctionMetadataTransformer.
if (!this._functionsAgentOptionsProvider.TryGet(agentName, out FunctionsAgentOptions? agentTriggerOptions))
{
continue;
}
this._logger.LogRegisteringTriggerForAgent(agentName, "entity");
original.Add(FunctionMetadataFactory.CreateEntityTrigger(agentName));
if (agentTriggerOptions.HttpTrigger.IsEnabled)
if (this._functionsAgentOptionsProvider.TryGet(agentName, out FunctionsAgentOptions? agentTriggerOptions))
{
this._logger.LogRegisteringTriggerForAgent(agentName, "http");
original.Add(FunctionMetadataFactory.CreateHttpTrigger(agentName, $"agents/{agentName}/run", BuiltInFunctions.RunAgentHttpFunctionEntryPoint));
}
if (agentTriggerOptions.HttpTrigger.IsEnabled)
{
this._logger.LogRegisteringTriggerForAgent(agentName, "http");
original.Add(FunctionMetadataFactory.CreateHttpTrigger(agentName, $"agents/{agentName}/run", BuiltInFunctions.RunAgentHttpFunctionEntryPoint));
}
if (agentTriggerOptions.McpToolTrigger.IsEnabled)
{
AIAgent agent = kvp.Value(this._serviceProvider);
this._logger.LogRegisteringTriggerForAgent(agentName, "mcpTool");
original.Add(CreateMcpToolTrigger(agentName, agent.Description));
if (agentTriggerOptions.McpToolTrigger.IsEnabled)
{
AIAgent agent = kvp.Value(this._serviceProvider);
this._logger.LogRegisteringTriggerForAgent(agentName, "mcpTool");
original.Add(CreateMcpToolTrigger(agentName, agent.Description));
}
}
}
}
@@ -134,17 +134,4 @@ public static class DurableAgentsOptionsExtensions
{
return new Dictionary<string, FunctionsAgentOptions>(s_agentOptions, StringComparer.OrdinalIgnoreCase);
}
/// <summary>
/// Ensures every agent in <paramref name="agentNames"/> has an entry in the
/// options registry. Agents that already have explicit options are left untouched.
/// New entries receive the default configuration (HTTP trigger enabled, MCP tool disabled).
/// </summary>
internal static void EnsureDefaultOptionsForAll(IEnumerable<string> agentNames)
{
foreach (string name in agentNames)
{
s_agentOptions.TryAdd(name, new FunctionsAgentOptions { HttpTrigger = { IsEnabled = true } });
}
}
}
@@ -1,6 +1,5 @@
// Copyright (c) Microsoft. All rights reserved.
using System.Text.Json.Nodes;
using Microsoft.Agents.AI.DurableTask;
using Microsoft.Azure.Functions.Worker.Core.FunctionMetadata;
@@ -99,65 +98,4 @@ internal static class FunctionMetadataFactory
ScriptFile = BuiltInFunctions.ScriptFile,
};
}
/// <summary>
/// Creates function metadata for an MCP tool trigger function that starts a workflow.
/// </summary>
/// <param name="workflowName">The name of the workflow to expose as an MCP tool.</param>
/// <param name="description">An optional description for the MCP tool. If null, a default description is generated.</param>
/// <returns>A <see cref="DefaultFunctionMetadata"/> configured for an MCP tool trigger.</returns>
internal static DefaultFunctionMetadata CreateWorkflowMcpToolTrigger(
string workflowName,
string? description)
{
var functionName = $"{BuiltInFunctions.McpToolPrefix}{workflowName}";
var toolDescription = description ?? $"Run the {workflowName} workflow";
var toolProperties = new JsonArray(new JsonObject
{
["propertyName"] = "input",
["propertyType"] = "string",
["description"] = "The input to the workflow.",
["isRequired"] = true,
["isArray"] = false,
});
var triggerBinding = new JsonObject
{
["name"] = "context",
["type"] = "mcpToolTrigger",
["direction"] = "In",
["toolName"] = workflowName,
["description"] = toolDescription,
["toolProperties"] = toolProperties.ToJsonString(),
};
var inputBinding = new JsonObject
{
["name"] = "input",
["type"] = "mcpToolProperty",
["direction"] = "In",
["propertyName"] = "input",
["description"] = "The input to the workflow",
["isRequired"] = true,
["dataType"] = "String",
["propertyType"] = "string",
};
var clientBinding = new JsonObject
{
["name"] = "client",
["type"] = "durableClient",
["direction"] = "In",
};
return new DefaultFunctionMetadata
{
Name = functionName,
Language = "dotnet-isolated",
RawBindings = [triggerBinding.ToJsonString(), inputBinding.ToJsonString(), clientBinding.ToJsonString()],
EntryPoint = BuiltInFunctions.RunWorkflowMcpToolFunctionEntryPoint,
ScriptFile = BuiltInFunctions.ScriptFile,
};
}
}
@@ -27,16 +27,9 @@ public static class FunctionsApplicationBuilderExtensions
{
ArgumentNullException.ThrowIfNull(configure);
// Create/get shared options BEFORE the DurableTask library call so it can find them.
FunctionsDurableOptions sharedOptions = GetOrCreateSharedOptions(builder.Services);
// The main agent services registration is done in Microsoft.DurableTask.Agents.
builder.Services.ConfigureDurableAgents(configure);
// Ensure all agents registered through this path have default FunctionsAgentOptions.
// This distinguishes them from agents auto-registered by workflows.
DurableAgentsOptionsExtensions.EnsureDefaultOptionsForAll(sharedOptions.Agents.GetAgentFactories().Keys);
builder.Services.TryAddSingleton<IFunctionsAgentOptionsProvider>(_ =>
new DefaultFunctionsAgentOptionsProvider(DurableAgentsOptionsExtensions.GetAgentOptionsSnapshot()));
@@ -74,13 +67,6 @@ public static class FunctionsApplicationBuilderExtensions
builder.Services.ConfigureDurableOptions(configure);
if (DurableAgentsOptionsExtensions.GetAgentOptionsSnapshot().Count > 0)
{
builder.Services.TryAddSingleton<IFunctionsAgentOptionsProvider>(_ =>
new DefaultFunctionsAgentOptionsProvider(DurableAgentsOptionsExtensions.GetAgentOptionsSnapshot()));
builder.Services.TryAddEnumerable(ServiceDescriptor.Singleton<IFunctionMetadataTransformer, DurableAgentFunctionMetadataTransformer>());
}
if (sharedOptions.Workflows.Workflows.Count > 0)
{
builder.Services.TryAddEnumerable(ServiceDescriptor.Singleton<IFunctionMetadataTransformer, DurableWorkflowsFunctionMetadataTransformer>());
@@ -116,14 +102,12 @@ public static class FunctionsApplicationBuilderExtensions
builder.UseWhen<BuiltInFunctionExecutionMiddleware>(static context =>
string.Equals(context.FunctionDefinition.EntryPoint, BuiltInFunctions.RunAgentHttpFunctionEntryPoint, StringComparison.Ordinal) ||
string.Equals(context.FunctionDefinition.EntryPoint, BuiltInFunctions.RunAgentMcpToolFunctionEntryPoint, StringComparison.Ordinal) ||
string.Equals(context.FunctionDefinition.EntryPoint, BuiltInFunctions.RunAgentEntityFunctionEntryPoint, StringComparison.Ordinal) ||
string.Equals(context.FunctionDefinition.EntryPoint, BuiltInFunctions.RunWorkflowOrchestrationHttpFunctionEntryPoint, StringComparison.Ordinal) ||
string.Equals(context.FunctionDefinition.EntryPoint, BuiltInFunctions.RunWorkflowOrchestrationFunctionEntryPoint, StringComparison.Ordinal) ||
string.Equals(context.FunctionDefinition.EntryPoint, BuiltInFunctions.InvokeWorkflowActivityFunctionEntryPoint, StringComparison.Ordinal) ||
string.Equals(context.FunctionDefinition.EntryPoint, BuiltInFunctions.GetWorkflowStatusHttpFunctionEntryPoint, StringComparison.Ordinal) ||
string.Equals(context.FunctionDefinition.EntryPoint, BuiltInFunctions.RespondToWorkflowHttpFunctionEntryPoint, StringComparison.Ordinal) ||
string.Equals(context.FunctionDefinition.EntryPoint, BuiltInFunctions.RunWorkflowMcpToolFunctionEntryPoint, StringComparison.Ordinal)
string.Equals(context.FunctionDefinition.EntryPoint, BuiltInFunctions.RespondToWorkflowHttpFunctionEntryPoint, StringComparison.Ordinal)
);
builder.Services.TryAddSingleton<BuiltInFunctionExecutor>();
}
@@ -10,7 +10,6 @@ namespace Microsoft.Agents.AI.Hosting.AzureFunctions;
internal sealed class FunctionsDurableOptions : DurableOptions
{
private readonly HashSet<string> _statusEndpointWorkflows = new(StringComparer.OrdinalIgnoreCase);
private readonly HashSet<string> _mcpToolTriggerWorkflows = new(StringComparer.OrdinalIgnoreCase);
/// <summary>
/// Enables the status HTTP endpoint for the specified workflow.
@@ -27,20 +26,4 @@ internal sealed class FunctionsDurableOptions : DurableOptions
{
return this._statusEndpointWorkflows.Contains(workflowName);
}
/// <summary>
/// Enables the MCP tool trigger for the specified workflow.
/// </summary>
internal void EnableMcpToolTrigger(string workflowName)
{
this._mcpToolTriggerWorkflows.Add(workflowName);
}
/// <summary>
/// Returns whether the MCP tool trigger is enabled for the specified workflow.
/// </summary>
internal bool IsMcpToolTriggerEnabled(string workflowName)
{
return this._mcpToolTriggerWorkflows.Contains(workflowName);
}
}
@@ -27,31 +27,4 @@ public static class DurableWorkflowOptionsExtensions
functionsOptions.EnableStatusEndpoint(workflow.Name!);
}
}
/// <summary>
/// Adds a workflow and configures whether to expose a status HTTP endpoint and/or an MCP tool trigger.
/// </summary>
/// <param name="options">The workflow options to add the workflow to.</param>
/// <param name="workflow">The workflow instance to add.</param>
/// <param name="exposeStatusEndpoint">If <see langword="true"/>, a GET endpoint is generated at <c>workflows/{name}/status/{runId}</c>.</param>
/// <param name="exposeMcpToolTrigger">If <see langword="true"/>, an MCP tool trigger is generated for the workflow.</param>
public static void AddWorkflow(this DurableWorkflowOptions options, Workflow workflow, bool exposeStatusEndpoint, bool exposeMcpToolTrigger)
{
ArgumentNullException.ThrowIfNull(options);
options.AddWorkflow(workflow);
if (options.ParentOptions is FunctionsDurableOptions functionsOptions)
{
if (exposeStatusEndpoint)
{
functionsOptions.EnableStatusEndpoint(workflow.Name!);
}
if (exposeMcpToolTrigger)
{
functionsOptions.EnableMcpToolTrigger(workflow.Name!);
}
}
}
}
@@ -50,11 +50,8 @@ internal sealed class DurableWorkflowsFunctionMetadataTransformer : IFunctionMet
int initialCount = original.Count;
this._logger.LogTransformingFunctionMetadata(initialCount);
// Seed with existing function names to avoid duplicates across transformers
// (e.g., when DurableAgentFunctionMetadataTransformer already registered entity triggers).
HashSet<string> registeredFunctions = new(
original.Select(f => f.Name!),
StringComparer.OrdinalIgnoreCase);
// Track registered function names to avoid duplicates when workflows share executors.
HashSet<string> registeredFunctions = [];
DurableWorkflowOptions workflowOptions = this._options.Workflows;
foreach (var workflow in workflowOptions.Workflows)
@@ -116,17 +113,6 @@ internal sealed class DurableWorkflowsFunctionMetadataTransformer : IFunctionMet
}
}
// Register an MCP tool trigger if opted in via AddWorkflow(exposeMcpToolTrigger: true).
if (this._options.IsMcpToolTriggerEnabled(workflow.Key))
{
string mcpToolFunctionName = $"{BuiltInFunctions.McpToolPrefix}{workflow.Key}";
if (registeredFunctions.Add(mcpToolFunctionName))
{
this._logger.LogRegisteringWorkflowTrigger(workflow.Key, mcpToolFunctionName, "mcpTool");
original.Add(FunctionMetadataFactory.CreateWorkflowMcpToolTrigger(workflow.Key, workflow.Value.Description));
}
}
// Register activity or entity functions for each executor in the workflow.
// ReflectExecutors() returns all executors across the graph; no need to manually traverse edges.
foreach (KeyValuePair<string, ExecutorBinding> entry in workflow.Value.ReflectExecutors())
@@ -5,8 +5,6 @@ using System.Reflection;
using System.Text;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Logging;
using ModelContextProtocol.Client;
using ModelContextProtocol.Protocol;
namespace Microsoft.Agents.AI.Hosting.AzureFunctions.IntegrationTests;
/// <summary>
@@ -237,114 +235,6 @@ public sealed class WorkflowSamplesValidation(ITestOutputHelper outputHelper) :
});
}
[Fact]
public async Task WorkflowMcpToolSampleValidationAsync()
{
string samplePath = Path.Combine(s_samplesPath, "04_WorkflowMcpTool");
await this.RunSampleTestAsync(samplePath, requiresOpenAI: false, async (logs) =>
{
// Connect to the MCP endpoint exposed by the Azure Functions host
IClientTransport clientTransport = new HttpClientTransport(new()
{
Endpoint = new Uri($"http://localhost:{AzureFunctionsPort}/runtime/webhooks/mcp")
});
await using McpClient mcpClient = await McpClient.CreateAsync(clientTransport);
// Verify both workflow tools are listed
IList<McpClientTool> tools = await mcpClient.ListToolsAsync();
this._outputHelper.WriteLine($"MCP tools found: {string.Join(", ", tools.Select(t => t.Name))}");
Assert.Single(tools, t => t.Name == "Translate");
Assert.Single(tools, t => t.Name == "OrderLookup");
// Invoke the Translate workflow via MCP tool (returns a string result)
this._outputHelper.WriteLine("Invoking MCP tool 'Translate'...");
CallToolResult translateResult = await mcpClient.CallToolAsync(
"Translate",
arguments: new Dictionary<string, object?> { { "input", "hello world" } });
Assert.NotEmpty(translateResult.Content);
string translateResponse = Assert.IsType<TextContentBlock>(translateResult.Content[0]).Text;
this._outputHelper.WriteLine($"Translate MCP tool response: {translateResponse}");
Assert.NotEmpty(translateResponse);
Assert.Contains("HELLO WORLD", translateResponse);
// Invoke the OrderLookup workflow via MCP tool (returns a POCO serialized as JSON)
this._outputHelper.WriteLine("Invoking MCP tool 'OrderLookup'...");
CallToolResult orderResult = await mcpClient.CallToolAsync(
"OrderLookup",
arguments: new Dictionary<string, object?> { { "input", "ORD-2025-42" } });
Assert.NotEmpty(orderResult.Content);
string orderResponse = Assert.IsType<TextContentBlock>(orderResult.Content[0]).Text;
this._outputHelper.WriteLine($"OrderLookup MCP tool response: {orderResponse}");
Assert.NotEmpty(orderResponse);
Assert.Contains("ORD-2025-42", orderResponse);
// Verify executor activities ran in the logs
lock (logs)
{
Assert.True(logs.Any(log => log.Message.Contains("[Activity] TranslateText:")), "TranslateText activity not found in logs.");
Assert.True(logs.Any(log => log.Message.Contains("[Activity] FormatOutput:")), "FormatOutput activity not found in logs.");
Assert.True(logs.Any(log => log.Message.Contains("[Activity] LookupOrder:")), "LookupOrder activity not found in logs.");
Assert.True(logs.Any(log => log.Message.Contains("[Activity] EnrichOrder:")), "EnrichOrder activity not found in logs.");
}
});
}
[Fact]
public async Task WorkflowAndAgentsSampleValidationAsync()
{
string samplePath = Path.Combine(s_samplesPath, "05_WorkflowAndAgents");
await this.RunSampleTestAsync(samplePath, requiresOpenAI: true, async (logs) =>
{
// Connect to the MCP endpoint exposed by the Azure Functions host
IClientTransport clientTransport = new HttpClientTransport(new()
{
Endpoint = new Uri($"http://localhost:{AzureFunctionsPort}/runtime/webhooks/mcp")
});
await using McpClient mcpClient = await McpClient.CreateAsync(clientTransport);
// Verify both the agent and workflow tools are listed
IList<McpClientTool> tools = await mcpClient.ListToolsAsync();
this._outputHelper.WriteLine($"MCP tools found: {string.Join(", ", tools.Select(t => t.Name))}");
Assert.Single(tools, t => t.Name == "Assistant");
Assert.Single(tools, t => t.Name == "Translate");
// Invoke the Translate workflow via MCP tool
this._outputHelper.WriteLine("Invoking MCP tool 'Translate'...");
CallToolResult translateResult = await mcpClient.CallToolAsync(
"Translate",
arguments: new Dictionary<string, object?> { { "input", "hello world" } });
Assert.NotEmpty(translateResult.Content);
string translateResponse = Assert.IsType<TextContentBlock>(translateResult.Content[0]).Text;
this._outputHelper.WriteLine($"Translate MCP tool response: {translateResponse}");
Assert.Contains("HELLO WORLD", translateResponse);
// Invoke the Assistant agent via MCP tool
this._outputHelper.WriteLine("Invoking MCP tool 'Assistant'...");
CallToolResult assistantResult = await mcpClient.CallToolAsync(
"Assistant",
arguments: new Dictionary<string, object?> { { "query", "What is 2 + 2?" } });
Assert.NotEmpty(assistantResult.Content);
string assistantResponse = Assert.IsType<TextContentBlock>(assistantResult.Content[0]).Text;
this._outputHelper.WriteLine($"Assistant MCP tool response: {assistantResponse}");
Assert.NotEmpty(assistantResponse);
// Verify workflow executor activities ran in the logs
lock (logs)
{
Assert.True(logs.Any(log => log.Message.Contains("[Activity] TranslateText:")), "TranslateText activity not found in logs.");
Assert.True(logs.Any(log => log.Message.Contains("[Activity] FormatOutput:")), "FormatOutput activity not found in logs.");
}
});
}
[Fact]
public async Task ConcurrentWorkflowSampleValidationAsync()
{
@@ -148,45 +148,6 @@ public sealed class DurableAgentFunctionMetadataTransformerTests
}
}
[Fact]
public void Transform_SkipsAgents_WithoutExplicitOptions()
{
// Arrange: two agents in the dictionary, but only one has explicit FunctionsAgentOptions.
// This simulates a workflow-auto-registered agent (workflowAgent) alongside a standalone agent.
Dictionary<string, Func<IServiceProvider, AIAgent>> agents = new()
{
{ "standaloneAgent", _ => new TestAgent("standaloneAgent", "Standalone agent") },
{ "workflowAgent", _ => new TestAgent("workflowAgent", "Auto-registered by workflow") }
};
FunctionsAgentOptions standaloneOptions = new();
standaloneOptions.HttpTrigger.IsEnabled = true;
// Only standaloneAgent has explicit options; workflowAgent does not.
IFunctionsAgentOptionsProvider agentOptionsProvider = new FakeOptionsProvider(new Dictionary<string, FunctionsAgentOptions>
{
{ "standaloneAgent", standaloneOptions }
});
List<IFunctionMetadata> metadataList = [];
DurableAgentFunctionMetadataTransformer transformer = new(
agents,
NullLogger<DurableAgentFunctionMetadataTransformer>.Instance,
new FakeServiceProvider(),
agentOptionsProvider);
// Act
transformer.Transform(metadataList);
// Assert: only standaloneAgent should have triggers (entity + http = 2).
// workflowAgent should be skipped entirely.
Assert.Equal(2, metadataList.Count);
Assert.Contains(metadataList, m => m.Name == "dafx-standaloneAgent");
Assert.Contains(metadataList, m => m.Name == "http-standaloneAgent");
Assert.DoesNotContain(metadataList, m => m.Name!.Contains("workflowAgent"));
}
private static List<IFunctionMetadata> BuildFunctionMetadataList(int numberOfFunctions)
{
List<IFunctionMetadata> list = [];
@@ -1,121 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
using System.Text.Json;
using Microsoft.Azure.Functions.Worker.Core.FunctionMetadata;
namespace Microsoft.Agents.AI.Hosting.AzureFunctions.UnitTests;
public sealed class FunctionMetadataFactoryTests
{
[Fact]
public void CreateEntityTrigger_SetsCorrectNameAndBindings()
{
DefaultFunctionMetadata metadata = FunctionMetadataFactory.CreateEntityTrigger("myAgent");
Assert.Equal("dafx-myAgent", metadata.Name);
Assert.Equal("dotnet-isolated", metadata.Language);
Assert.Equal(BuiltInFunctions.RunAgentEntityFunctionEntryPoint, metadata.EntryPoint);
Assert.NotNull(metadata.RawBindings);
Assert.Equal(2, metadata.RawBindings.Count);
Assert.Contains("entityTrigger", metadata.RawBindings[0]);
Assert.Contains("durableClient", metadata.RawBindings[1]);
}
[Fact]
public void CreateHttpTrigger_SetsCorrectNameRouteAndDefaults()
{
DefaultFunctionMetadata metadata = FunctionMetadataFactory.CreateHttpTrigger(
"myWorkflow", "workflows/myWorkflow/run", BuiltInFunctions.RunWorkflowOrchestrationHttpFunctionEntryPoint);
Assert.Equal("http-myWorkflow", metadata.Name);
Assert.Equal("dotnet-isolated", metadata.Language);
Assert.Equal(BuiltInFunctions.RunWorkflowOrchestrationHttpFunctionEntryPoint, metadata.EntryPoint);
Assert.NotNull(metadata.RawBindings);
Assert.Equal(3, metadata.RawBindings.Count);
Assert.Contains("httpTrigger", metadata.RawBindings[0]);
Assert.Contains("workflows/myWorkflow/run", metadata.RawBindings[0]);
Assert.Contains("\"post\"", metadata.RawBindings[0]);
Assert.Contains("http", metadata.RawBindings[1]);
Assert.Contains("durableClient", metadata.RawBindings[2]);
}
[Fact]
public void CreateHttpTrigger_RespectsCustomMethods()
{
DefaultFunctionMetadata metadata = FunctionMetadataFactory.CreateHttpTrigger(
"status", "workflows/status/{runId}", BuiltInFunctions.GetWorkflowStatusHttpFunctionEntryPoint, methods: "\"get\"");
Assert.NotNull(metadata.RawBindings);
Assert.Contains("\"get\"", metadata.RawBindings[0]);
Assert.DoesNotContain("\"post\"", metadata.RawBindings[0]);
}
[Fact]
public void CreateActivityTrigger_SetsCorrectNameAndBindings()
{
DefaultFunctionMetadata metadata = FunctionMetadataFactory.CreateActivityTrigger("dafx-MyExecutor");
Assert.Equal("dafx-MyExecutor", metadata.Name);
Assert.Equal("dotnet-isolated", metadata.Language);
Assert.Equal(BuiltInFunctions.InvokeWorkflowActivityFunctionEntryPoint, metadata.EntryPoint);
Assert.NotNull(metadata.RawBindings);
Assert.Equal(2, metadata.RawBindings.Count);
Assert.Contains("activityTrigger", metadata.RawBindings[0]);
Assert.Contains("durableClient", metadata.RawBindings[1]);
}
[Fact]
public void CreateOrchestrationTrigger_SetsCorrectNameAndBindings()
{
DefaultFunctionMetadata metadata = FunctionMetadataFactory.CreateOrchestrationTrigger(
"dafx-MyWorkflow", BuiltInFunctions.RunWorkflowOrchestrationFunctionEntryPoint);
Assert.Equal("dafx-MyWorkflow", metadata.Name);
Assert.Equal("dotnet-isolated", metadata.Language);
Assert.Equal(BuiltInFunctions.RunWorkflowOrchestrationFunctionEntryPoint, metadata.EntryPoint);
Assert.NotNull(metadata.RawBindings);
Assert.Single(metadata.RawBindings);
Assert.Contains("orchestrationTrigger", metadata.RawBindings[0]);
}
[Fact]
public void CreateWorkflowMcpToolTrigger_SetsCorrectNameAndBindings()
{
DefaultFunctionMetadata metadata = FunctionMetadataFactory.CreateWorkflowMcpToolTrigger("Translate", "Translate text");
Assert.Equal("mcptool-Translate", metadata.Name);
Assert.Equal("dotnet-isolated", metadata.Language);
Assert.Equal(BuiltInFunctions.RunWorkflowMcpToolFunctionEntryPoint, metadata.EntryPoint);
Assert.NotNull(metadata.RawBindings);
Assert.Equal(3, metadata.RawBindings.Count);
// Verify all bindings are valid JSON
foreach (string binding in metadata.RawBindings)
{
JsonDocument.Parse(binding);
}
// mcpToolTrigger binding
Assert.Contains("mcpToolTrigger", metadata.RawBindings[0]);
Assert.Contains("\"toolName\":\"Translate\"", metadata.RawBindings[0]);
Assert.Contains("\"description\":\"Translate text\"", metadata.RawBindings[0]);
Assert.Contains("toolProperties", metadata.RawBindings[0]);
// mcpToolProperty binding for input
Assert.Contains("mcpToolProperty", metadata.RawBindings[1]);
Assert.Contains("\"propertyName\":\"input\"", metadata.RawBindings[1]);
Assert.Contains("\"isRequired\":true", metadata.RawBindings[1]);
// durableClient binding
Assert.Contains("durableClient", metadata.RawBindings[2]);
}
[Fact]
public void CreateWorkflowMcpToolTrigger_UsesDefaultDescription_WhenNull()
{
DefaultFunctionMetadata metadata = FunctionMetadataFactory.CreateWorkflowMcpToolTrigger("MyWorkflow", description: null);
Assert.NotNull(metadata.RawBindings);
Assert.Contains("Run the MyWorkflow workflow", metadata.RawBindings[0]);
}
}
@@ -5,7 +5,7 @@ import os
from random import randint
from typing import Annotated, Any, Literal
from agent_framework import SupportsChatGetResponse, tool
from agent_framework import Message, SupportsChatGetResponse, tool
from agent_framework.azure import (
AzureAIAgentClient,
AzureOpenAIAssistantsClient,
@@ -117,35 +117,37 @@ async def main(client_name: ClientName = "openai_chat") -> None:
client = get_client(client_name)
# 1. Configure prompt and streaming mode.
message = "What's the weather in Amsterdam and in Paris?"
message = Message("user", text="What's the weather in Amsterdam and in Paris?")
stream = os.getenv("STREAM", "false").lower() == "true"
print(f"Client: {client_name}")
print(f"User: {message}")
print(f"User: {message.text}")
# 2. Run with context-managed clients.
if isinstance(client, OpenAIAssistantsClient | AzureOpenAIAssistantsClient | AzureAIAgentClient):
async with client:
if stream:
response_stream = client.get_response(message, stream=True, options={"tools": get_weather})
response_stream = client.get_response([message], stream=True, options={"tools": get_weather})
print("Assistant: ", end="")
async for chunk in response_stream:
if chunk.text:
print(chunk.text, end="")
print("")
else:
print(f"Assistant: {await client.get_response(message, stream=False, options={'tools': get_weather})}")
print(
f"Assistant: {await client.get_response([message], stream=False, options={'tools': get_weather})}"
)
return
# 3. Run with non-context-managed clients.
if stream:
response_stream = client.get_response(message, stream=True, options={"tools": get_weather})
response_stream = client.get_response([message], stream=True, options={"tools": get_weather})
print("Assistant: ", end="")
async for chunk in response_stream:
if chunk.text:
print(chunk.text, end="")
print("")
else:
print(f"Assistant: {await client.get_response(message, stream=False, options={'tools': get_weather})}")
print(f"Assistant: {await client.get_response([message], stream=False, options={'tools': get_weather})}")
if __name__ == "__main__":
@@ -1,25 +1,17 @@
# /// script
# requires-python = ">=3.10"
# dependencies = [
# "autogen-agentchat",
# "autogen-ext[openai]",
# ]
# ///
# Run with any PEP 723 compatible runner, e.g.:
# uv run samples/autogen-migration/orchestrations/01_round_robin_group_chat.py
# Copyright (c) Microsoft. All rights reserved.
"""AutoGen RoundRobinGroupChat vs Agent Framework GroupChatBuilder/SequentialBuilder.
Demonstrates sequential agent orchestration where agents take turns processing
the task in a round-robin fashion.
"""
import asyncio
from agent_framework import Message
from dotenv import load_dotenv
"""AutoGen RoundRobinGroupChat vs Agent Framework GroupChatBuilder/SequentialBuilder.
Demonstrates sequential agent orchestration where agents take turns processing
the task in a round-robin fashion.
"""
# Load environment variables from .env file
load_dotenv()
@@ -98,7 +90,7 @@ async def run_agent_framework() -> None:
print("[Agent Framework] Sequential conversation:")
async for event in workflow.run("Create a brief summary about electric vehicles", stream=True):
if event.type == "output" and isinstance(event.data, list):
for message in event.data:
for message in event.data: # type: ignore
if isinstance(message, Message) and message.role == "assistant" and message.text:
print(f"---------- {message.author_name} ----------")
print(message.text)
@@ -144,9 +136,7 @@ async def run_agent_framework_with_cycle() -> None:
if last_message and "APPROVED" in last_message.text:
await context.yield_output("Content approved.")
else:
await context.send_message(
AgentExecutorRequest(messages=response.full_conversation, should_respond=True)
)
await context.send_message(AgentExecutorRequest(messages=response.full_conversation, should_respond=True))
workflow = (
WorkflowBuilder(start_executor=researcher)
@@ -1,25 +1,17 @@
# /// script
# requires-python = ">=3.10"
# dependencies = [
# "autogen-agentchat",
# "autogen-ext[openai]",
# ]
# ///
# Run with any PEP 723 compatible runner, e.g.:
# uv run samples/autogen-migration/orchestrations/02_selector_group_chat.py
# Copyright (c) Microsoft. All rights reserved.
"""AutoGen SelectorGroupChat vs Agent Framework GroupChatBuilder.
Demonstrates LLM-based speaker selection where an orchestrator decides
which agent should speak next based on the conversation context.
"""
import asyncio
from agent_framework import Message
from dotenv import load_dotenv
"""AutoGen SelectorGroupChat vs Agent Framework GroupChatBuilder.
Demonstrates LLM-based speaker selection where an orchestrator decides
which agent should speak next based on the conversation context.
"""
# Load environment variables from .env file
load_dotenv()
@@ -113,7 +105,7 @@ async def run_agent_framework() -> None:
print("[Agent Framework] Group chat conversation:")
async for event in workflow.run("How do I connect to a PostgreSQL database using Python?", stream=True):
if event.type == "output" and isinstance(event.data, list):
for message in event.data:
for message in event.data: # type: ignore
if isinstance(message, Message) and message.role == "assistant" and message.text:
print(f"---------- {message.author_name} ----------")
print(message.text)
@@ -1,19 +1,4 @@
# /// script
# requires-python = ">=3.10"
# dependencies = [
# "autogen-agentchat",
# "autogen-ext[openai]",
# ]
# ///
# Run with any PEP 723 compatible runner, e.g.:
# uv run samples/autogen-migration/orchestrations/03_swarm.py
# Copyright (c) Microsoft. All rights reserved.
"""AutoGen Swarm pattern vs Agent Framework HandoffBuilder.
Demonstrates agent handoff coordination where agents can transfer control
to other specialized agents based on the task requirements.
"""
import asyncio
from typing import Any
@@ -21,6 +6,12 @@ from typing import Any
from agent_framework import AgentResponseUpdate, WorkflowEvent
from dotenv import load_dotenv
"""AutoGen Swarm pattern vs Agent Framework HandoffBuilder.
Demonstrates agent handoff coordination where agents can transfer control
to other specialized agents based on the task requirements.
"""
# Load environment variables from .env file
load_dotenv()
@@ -1,19 +1,4 @@
# /// script
# requires-python = ">=3.10"
# dependencies = [
# "autogen-agentchat",
# "autogen-ext[openai]",
# ]
# ///
# Run with any PEP 723 compatible runner, e.g.:
# uv run samples/autogen-migration/orchestrations/04_magentic_one.py
# Copyright (c) Microsoft. All rights reserved.
"""AutoGen MagenticOneGroupChat vs Agent Framework MagenticBuilder.
Demonstrates orchestrated multi-agent workflows with a central coordinator
managing specialized agents for complex tasks.
"""
import asyncio
import json
@@ -27,6 +12,12 @@ from agent_framework import (
from agent_framework.orchestrations import MagenticProgressLedger
from dotenv import load_dotenv
"""AutoGen MagenticOneGroupChat vs Agent Framework MagenticBuilder.
Demonstrates orchestrated multi-agent workflows with a central coordinator
managing specialized agents for complex tasks.
"""
# Load environment variables from .env file
load_dotenv()
@@ -1,14 +1,9 @@
# /// script
# requires-python = ">=3.10"
# dependencies = [
# "autogen-agentchat",
# "autogen-ext[openai]",
# ]
# ///
# Run with any PEP 723 compatible runner, e.g.:
# uv run samples/autogen-migration/single_agent/01_basic_assistant_agent.py
# Copyright (c) Microsoft. All rights reserved.
import asyncio
from dotenv import load_dotenv
"""Basic AutoGen AssistantAgent vs Agent Framework Agent.
Both samples expect OpenAI-compatible environment variables (OPENAI_API_KEY or
@@ -16,10 +11,6 @@ Azure OpenAI configuration). Update the prompts or client wiring to match your
model of choice before running.
"""
import asyncio
from dotenv import load_dotenv
# Load environment variables from .env file
load_dotenv()
@@ -1,24 +1,14 @@
# /// script
# requires-python = ">=3.10"
# dependencies = [
# "autogen-agentchat",
# "autogen-core",
# "autogen-ext[openai]",
# ]
# ///
# Run with any PEP 723 compatible runner, e.g.:
# uv run samples/autogen-migration/single_agent/02_assistant_agent_with_tool.py
# Copyright (c) Microsoft. All rights reserved.
"""AutoGen AssistantAgent vs Agent Framework Agent with function tools.
Demonstrates how to create and attach tools to agents in both frameworks.
"""
import asyncio
from dotenv import load_dotenv
"""AutoGen AssistantAgent vs Agent Framework Agent with function tools.
Demonstrates how to create and attach tools to agents in both frameworks.
"""
# Load environment variables from .env file
load_dotenv()
@@ -1,23 +1,14 @@
# /// script
# requires-python = ">=3.10"
# dependencies = [
# "autogen-agentchat",
# "autogen-ext[openai]",
# ]
# ///
# Run with any PEP 723 compatible runner, e.g.:
# uv run samples/autogen-migration/single_agent/03_assistant_agent_thread_and_stream.py
# Copyright (c) Microsoft. All rights reserved.
"""AutoGen vs Agent Framework: Thread management and streaming responses.
Demonstrates conversation state management and streaming in both frameworks.
"""
import asyncio
from dotenv import load_dotenv
"""AutoGen vs Agent Framework: Thread management and streaming responses.
Demonstrates conversation state management and streaming in both frameworks.
"""
# Load environment variables from .env file
load_dotenv()
@@ -1,24 +1,15 @@
# /// script
# requires-python = ">=3.10"
# dependencies = [
# "autogen-agentchat",
# "autogen-ext[openai]",
# ]
# ///
# Run with any PEP 723 compatible runner, e.g.:
# uv run samples/autogen-migration/single_agent/04_agent_as_tool.py
# Copyright (c) Microsoft. All rights reserved.
import asyncio
from dotenv import load_dotenv
"""AutoGen vs Agent Framework: Agent-as-a-Tool pattern.
Demonstrates hierarchical agent architectures where one agent delegates
work to specialized sub-agents wrapped as tools.
"""
import asyncio
from dotenv import load_dotenv
# Load environment variables from .env file
load_dotenv()
@@ -107,6 +98,7 @@ async def run_agent_framework() -> None:
if content.type == "function_call":
# Accumulate function call content as it streams in
call_id = content.call_id
assert call_id is not None, "Function call content must have a call_id"
if call_id in accumulated_calls:
# Add to existing call (arguments stream in gradually)
accumulated_calls[call_id] = accumulated_calls[call_id] + content
+6 -7
View File
@@ -165,18 +165,17 @@ Produces:
## Report Status Codes
| Status | Label | Description |
| ------- | --------- | ----------------------------------------- |
| SUCCESS | [PASS] | Sample ran to completion with exit code 0 |
| FAILURE | [FAIL] | Sample exited with non-zero code |
| TIMEOUT | [TIMEOUT] | Sample exceeded timeout limit |
| ERROR | [ERROR] | Exception during execution |
| Status | Label | Description |
| ------------- | --------------- | ----------------------------------------- |
| SUCCESS | [PASS] | Sample ran to completion with exit code 0 |
| FAILURE | [FAIL] | Sample did not complete successfully (non-zero exit code) |
| MISSING_SETUP | [MISSING_SETUP] | Sample skipped due to missing setup |
## Troubleshooting
### Agent output parsing errors
If an agent returns non-JSON content, that sample is marked as `ERROR` with parser details in the report.
If an agent returns non-JSON content, that sample is marked as `FAILURE` with parser details in the report.
### GitHub Copilot authentication or CLI issues
+9 -1
View File
@@ -75,6 +75,13 @@ Examples:
help="Custom name for the report files (without extension). If not provided, uses timestamp.",
)
parser.add_argument(
"--exclude",
nargs="+",
type=str,
help="Subdirectory paths to exclude (relative to the search directory set by --subdir)",
)
return parser.parse_args()
@@ -104,6 +111,7 @@ async def main() -> int:
samples_dir=samples_dir,
python_root=python_root,
subdir=args.subdir,
exclude=args.exclude,
max_parallel_workers=max(1, args.max_parallel_workers),
)
@@ -138,7 +146,7 @@ async def main() -> int:
print(f" JSON: {json_path}")
# Return appropriate exit code
failed = report.failure_count + report.timeout_count + report.error_count
failed = report.failure_count + report.missing_setup_count
return 1 if failed > 0 else 0
@@ -0,0 +1,224 @@
# Copyright (c) Microsoft. All rights reserved.
"""Aggregate validation reports across runs and produce a trend report.
Reads JSON reports from individual validation jobs, combines them with
cached history from previous runs, and produces a markdown trend report
showing per-sample status over the last 5 runs.
Usage:
python aggregate.py <reports-dir> <history-file> <output-file>
"""
import json
import sys
from datetime import datetime
from pathlib import Path
from typing import Any
MAX_HISTORY = 5
STATUS_EMOJI = {
"success": "âś…",
"failure": "❌",
"missing_setup": "⚠️",
}
def _format_run_label(timestamp: str) -> str:
"""Format a run timestamp as a compact column label (e.g. '03-24 18:05')."""
try:
dt = datetime.fromisoformat(timestamp)
return dt.strftime("%m-%d %H:%M")
except (ValueError, TypeError):
return timestamp[:16]
def load_current_run(reports_dir: Path) -> dict[str, Any]:
"""Load all JSON report files from the current run and merge them."""
combined_results: dict[str, str] = {}
total = success = failure = missing = 0
json_files = sorted(reports_dir.glob("*.json"))
if not json_files:
print(f"Warning: No JSON report files found in {reports_dir}")
return {
"timestamp": datetime.now().isoformat(),
"summary": {
"total_samples": 0,
"success_count": 0,
"failure_count": 0,
"missing_setup_count": 0,
},
"results": {},
}
for json_file in json_files:
print(f" Loading report: {json_file.name}")
with open(json_file, encoding="utf-8") as f:
report = json.load(f)
for result in report["results"]:
combined_results[result["path"]] = result["status"]
summary = report["summary"]
total += summary["total_samples"]
success += summary["success_count"]
failure += summary["failure_count"]
missing += summary["missing_setup_count"]
return {
"timestamp": datetime.now().isoformat(),
"summary": {
"total_samples": total,
"success_count": success,
"failure_count": failure,
"missing_setup_count": missing,
},
"results": combined_results,
}
def load_history(history_path: Path) -> list[dict[str, Any]]:
"""Load previous run history from cache."""
if history_path.exists():
with open(history_path, encoding="utf-8") as f:
data = json.load(f)
runs = data.get("runs", [])
print(f" Loaded {len(runs)} previous run(s) from history")
return runs
print(" No previous history found")
return []
def save_history(history_path: Path, runs: list[dict[str, Any]]) -> None:
"""Save run history, keeping only the last MAX_HISTORY entries."""
history_path.parent.mkdir(parents=True, exist_ok=True)
trimmed = runs[-MAX_HISTORY:]
with open(history_path, "w", encoding="utf-8") as f:
json.dump({"runs": trimmed}, f, indent=2)
print(f" Saved {len(trimmed)} run(s) to history")
def generate_trend_report(runs: list[dict[str, Any]]) -> str:
"""Generate a markdown trend report from run history."""
lines = [
"# Sample Validation Trend Report",
"",
f"*Generated: {datetime.now().strftime('%Y-%m-%d %H:%M UTC')}*",
"",
]
# --- Overall status table (most recent first) ---
lines.append("## Overall Status (Last 5 Runs)")
lines.append("")
lines.append("| Run | Success | Failure | Missing Setup | Total |")
lines.append("|-----|---------|---------|---------------|-------|")
for run in reversed(runs):
s = run["summary"]
label = _format_run_label(run["timestamp"])
lines.append(
f"| {label} | {s['success_count']}/{s['total_samples']} "
f"| {s['failure_count']}/{s['total_samples']} "
f"| {s['missing_setup_count']}/{s['total_samples']} "
f"| {s['total_samples']} |"
)
# Pad with N/A rows if fewer than 5 runs
for _ in range(MAX_HISTORY - len(runs)):
lines.append("| N/A | N/A | N/A | N/A | N/A |")
lines.append("")
# --- Per-sample results table ---
lines.append("## Per-Sample Results")
lines.append("")
# Collect all sample paths across all runs
all_paths: set[str] = set()
for run in runs:
all_paths.update(run["results"].keys())
if not all_paths:
lines.append("*No sample results available.*")
return "\n".join(lines)
# Build header (most recent run first)
header = "| Sample |"
separator = "|--------|"
for run in reversed(runs):
label = _format_run_label(run["timestamp"])
header += f" {label} |"
separator += "------------|"
for _ in range(MAX_HISTORY - len(runs)):
header += " N/A |"
separator += "-----|"
lines.append(header)
lines.append(separator)
for path in sorted(all_paths):
row = f"| `{path}` |"
for run in reversed(runs):
status = run["results"].get(path, "N/A")
emoji = STATUS_EMOJI.get(status, "N/A")
row += f" {emoji} |"
for _ in range(MAX_HISTORY - len(runs)):
row += " N/A |"
lines.append(row)
lines.append("")
lines.append("**Legend:** ✅ Success · ❌ Failure · ⚠️ Missing Setup · N/A Not available")
lines.append("")
return "\n".join(lines)
def main() -> int:
if len(sys.argv) != 4:
print("Usage: python aggregate.py <reports-dir> <history-file> <output-file>")
return 1
reports_dir = Path(sys.argv[1])
history_path = Path(sys.argv[2])
output_path = Path(sys.argv[3])
print("Aggregating validation results...")
# Load current run's reports
print(f"\nLoading reports from {reports_dir}:")
current_run = load_current_run(reports_dir)
s = current_run["summary"]
print(
f" Current run: {s['success_count']} success, "
f"{s['failure_count']} failure, "
f"{s['missing_setup_count']} missing setup "
f"(total: {s['total_samples']})"
)
# Load history and append current run
print(f"\nLoading history from {history_path}:")
runs = load_history(history_path)
runs.append(current_run)
runs = runs[-MAX_HISTORY:]
# Save updated history
print(f"\nSaving history to {history_path}:")
save_history(history_path, runs)
# Generate trend report
print("\nGenerating trend report...")
report = generate_trend_report(runs)
output_path.parent.mkdir(parents=True, exist_ok=True)
output_path.write_text(report, encoding="utf-8")
print(f"Trend report written to {output_path}")
# Also print the report to stdout
print("\n" + "=" * 80)
print(report)
return 0
if __name__ == "__main__":
sys.exit(main())
@@ -14,7 +14,8 @@ from agent_framework import (
handler,
)
from agent_framework.github import GitHubCopilotAgent
from copilot.types import PermissionRequest, PermissionRequestResult
from copilot.generated.session_events import PermissionRequest
from copilot.types import PermissionRequestResult
from pydantic import BaseModel
from typing_extensions import Never
@@ -36,6 +37,7 @@ class AgentResponseFormat(BaseModel):
status: str
output: str
error: str
fix: str
@dataclass
@@ -54,15 +56,20 @@ class BatchCompletion:
AgentInstruction = (
"You are validating exactly one Python sample.\n"
"Analyze the sample code and execute it. Based on the execution result, determine if it "
"runs successfully, fails, or times out. Feel free to install any required dependencies.\n"
"Analyze the sample code and execute it as it is. Based on the execution result, determine "
"if it runs successfully, fails, or is missing_setup. Use `missing_setup` if the sample reports "
"missing required environment variables. The environment you're given should contain the necessary "
"variables. Don't create new environment variables nor modify the sample code.\n"
"Feel free to install any required dependencies if needed.\n"
"The sample can be interactive. If it is interactive, respond to the sample when prompted "
"based on your analysis of the code. You do not need to consult human on what to respond.\n"
"If the sample fails, investigate the error and suggest a fix.\n"
"Return ONLY valid JSON with this schema:\n"
"{\n"
' "status": "success|failure|timeout|error",\n'
' "status": "success|failure|missing_setup",\n'
' "output": "short summary of the result and what you did if the sample was interactive",\n'
' "error": "error details or empty string"\n'
' "error": "error details or empty string",\n'
' "fix": "suggested code fix if the sample failed, otherwise empty string"\n'
"}\n\n"
)
@@ -87,16 +94,15 @@ def status_from_text(value: str) -> RunStatus:
for status in RunStatus:
if status.value == normalized:
return status
return RunStatus.ERROR
return RunStatus.FAILURE
def prompt_permission(
request: PermissionRequest, context: dict[str, str]
) -> PermissionRequestResult:
"""Permission handler that always approves."""
kind = request.get("kind", "unknown")
logger.debug(
f"[Permission Request: {kind}] ({context})Automatically approved for sample validation."
f"[Permission Request: {request.kind}] ({context})Automatically approved for sample validation."
)
return PermissionRequestResult(kind="approved")
@@ -108,39 +114,73 @@ class CustomAgentExecutor(Executor):
returned as error responses, otherwise an exception in one agent could crash the entire workflow.
"""
# Retry in case GitHub Copilot agent encounters transient errors unrelated to the sample execution.
RETRY_COUNT = 1
def __init__(self, agent: GitHubCopilotAgent):
super().__init__(id=agent.id)
self.agent = agent
self._session = agent.create_session()
@handler
async def handle_task(
self, sample: SampleInfo, ctx: WorkflowContext[WorkerFreed | RunResult]
) -> None:
"""Execute one sample task and notify collector + coordinator."""
try:
response = await self.agent.run(
[
Message(
role="user",
text=f"Validate the following sample:\n\n{sample.relative_path}",
current_retry = 0
while True:
try:
response = await self.agent.run(
[
Message(
role="user",
text=f"Validate the following sample:\n\n{sample.relative_path}",
)
],
session=self._session,
)
result_payload = parse_agent_json(response.text)
result = RunResult(
sample=sample,
status=status_from_text(result_payload.status),
output=result_payload.output,
error=result_payload.error,
fix=result_payload.fix,
)
break
except Exception as ex:
if current_retry < self.RETRY_COUNT:
logger.warning(
f"Error executing agent {self.agent.id} (attempt {current_retry + 1}/{self.RETRY_COUNT}): {ex}. Retrying..."
)
]
)
result_payload = parse_agent_json(response.text)
result = RunResult(
sample=sample,
status=status_from_text(result_payload.status),
output=result_payload.output,
error=result_payload.error,
)
except Exception as ex:
logger.error(f"Error executing agent {self.agent.id}: {ex}")
result = RunResult(
sample=sample,
status=RunStatus.ERROR,
output="",
error=str(ex),
)
try:
current_retry += 1
await self.agent.stop()
await self.agent.start()
self._session = self.agent.create_session() # Reset session for retry
continue
except Exception as restart_ex:
logger.error(
f"Error restarting agent {self.agent.id}: {restart_ex}. No more retries."
)
result = RunResult(
sample=sample,
status=RunStatus.FAILURE,
output="",
error=f"Original error: {ex}. Restart error: {restart_ex}",
fix="",
)
break
logger.error(f"Error executing agent {self.agent.id}: {ex}")
result = RunResult(
sample=sample,
status=RunStatus.FAILURE,
output="",
error=str(ex),
fix="",
)
break
await ctx.send_message(result, target_id="collector")
await ctx.send_message(WorkerFreed(worker_id=self.id), target_id="coordinator")
@@ -252,7 +292,7 @@ class CreateConcurrentValidationWorkflowExecutor(Executor):
instructions=AgentInstruction,
default_options={
"on_permission_request": prompt_permission,
"timeout": 180,
"timeout": 60,
}, # type: ignore
)
agents.append(agent)
+20 -4
View File
@@ -52,13 +52,18 @@ def _has_main_entrypoint_guard(path: Path) -> bool:
)
def discover_samples(samples_dir: Path, subdir: str | None = None) -> list[SampleInfo]:
def discover_samples(
samples_dir: Path,
subdir: str | None = None,
exclude: list[str] | None = None,
) -> list[SampleInfo]:
"""
Find all Python sample files in the samples directory.
Args:
samples_dir: Root samples directory
subdir: Optional subdirectory to filter to
exclude: Optional list of subdirectory paths (relative to the search directory) to exclude
Returns:
List of SampleInfo objects for each discovered sample
@@ -72,12 +77,21 @@ def discover_samples(samples_dir: Path, subdir: str | None = None) -> list[Sampl
else:
search_dir = samples_dir
# Resolve excluded paths to absolute for reliable comparison
exclude_paths = {(search_dir / exc).resolve() for exc in (exclude or [])}
python_files: list[Path] = []
# Walk through all subdirectories and find .py files
for root, dirs, files in os.walk(search_dir):
# Skip directories that start with _ (like _sample_validation)
dirs[:] = [d for d in dirs if not d.startswith("_") and d != "__pycache__"]
# Skip directories that start with _, __pycache__, or excluded paths
dirs[:] = [
d
for d in dirs
if not d.startswith("_")
and d != "__pycache__"
and (Path(root) / d).resolve() not in exclude_paths
]
for file in files:
# Skip files that start with _ and include only scripts with a main entrypoint guard
@@ -113,8 +127,10 @@ class DiscoverSamplesExecutor(Executor):
print(f"🔍 Discovering samples in {self.config.samples_dir}")
if self.config.subdir:
print(f" Filtering to subdirectory: {self.config.subdir}")
if self.config.exclude:
print(f" Excluding: {', '.join(self.config.exclude)}")
samples = discover_samples(self.config.samples_dir, self.config.subdir)
samples = discover_samples(self.config.samples_dir, self.config.subdir, self.config.exclude)
print(f" Found {len(samples)} samples")
await ctx.send_message(DiscoveryResult(samples=samples))
+9 -11
View File
@@ -18,6 +18,7 @@ class ValidationConfig:
samples_dir: Path
python_root: Path
subdir: str | None = None
exclude: list[str] | None = None
max_parallel_workers: int = 10
@@ -60,8 +61,7 @@ class RunStatus(Enum):
SUCCESS = "success"
FAILURE = "failure"
TIMEOUT = "timeout"
ERROR = "error"
MISSING_SETUP = "missing_setup"
@dataclass
@@ -72,6 +72,7 @@ class RunResult:
status: RunStatus
output: str
error: str
fix: str
@dataclass
@@ -89,8 +90,7 @@ class Report:
total_samples: int
success_count: int
failure_count: int
timeout_count: int
error_count: int
missing_setup_count: int
results: list[RunResult] = field(default_factory=list) # type: ignore
def to_markdown(self) -> str:
@@ -107,15 +107,14 @@ class Report:
f"| Total Samples | {self.total_samples} |",
f"| [PASS] Success | {self.success_count} |",
f"| [FAIL] Failure | {self.failure_count} |",
f"| [TIMEOUT] Timeout | {self.timeout_count} |",
f"| [ERROR] Error | {self.error_count} |",
f"| [MISSING_SETUP] Missing Setup | {self.missing_setup_count} |",
"",
"## Detailed Results",
"",
]
# Group by status
for status in [RunStatus.FAILURE, RunStatus.TIMEOUT, RunStatus.ERROR, RunStatus.SUCCESS]:
for status in [RunStatus.FAILURE, RunStatus.MISSING_SETUP, RunStatus.SUCCESS]:
status_results = [r for r in self.results if r.status == status]
if not status_results:
continue
@@ -123,8 +122,7 @@ class Report:
status_label = {
RunStatus.SUCCESS: "[PASS]",
RunStatus.FAILURE: "[FAIL]",
RunStatus.TIMEOUT: "[TIMEOUT]",
RunStatus.ERROR: "[ERROR]",
RunStatus.MISSING_SETUP: "[MISSING_SETUP]",
}
lines.append(f"### {status_label[status]} {status.value.title()} ({len(status_results)})")
@@ -148,8 +146,7 @@ class Report:
"total_samples": self.total_samples,
"success_count": self.success_count,
"failure_count": self.failure_count,
"timeout_count": self.timeout_count,
"error_count": self.error_count,
"missing_setup_count": self.missing_setup_count,
},
"results": [
{
@@ -157,6 +154,7 @@ class Report:
"status": r.status.value,
"output": r.output,
"error": r.error,
"fix": r.fix,
}
for r in self.results
],
+6 -10
View File
@@ -22,12 +22,11 @@ def generate_report(results: list[RunResult]) -> Report:
Returns:
Report object with aggregated statistics
"""
# Sort results: failures, timeouts, errors first, then successes
# Sort results: failures, missing setup first, then successes
status_priority = {
RunStatus.FAILURE: 0,
RunStatus.TIMEOUT: 1,
RunStatus.ERROR: 2,
RunStatus.SUCCESS: 3,
RunStatus.MISSING_SETUP: 1,
RunStatus.SUCCESS: 2,
}
sorted_results = sorted(results, key=lambda r: status_priority[r.status])
@@ -36,8 +35,7 @@ def generate_report(results: list[RunResult]) -> Report:
total_samples=len(results),
success_count=sum(1 for r in results if r.status == RunStatus.SUCCESS),
failure_count=sum(1 for r in results if r.status == RunStatus.FAILURE),
timeout_count=sum(1 for r in results if r.status == RunStatus.TIMEOUT),
error_count=sum(1 for r in results if r.status == RunStatus.ERROR),
missing_setup_count=sum(1 for r in results if r.status == RunStatus.MISSING_SETUP),
results=sorted_results,
)
@@ -86,8 +84,7 @@ def print_summary(report: Report) -> None:
if (
report.failure_count == 0
and report.timeout_count == 0
and report.error_count == 0
and report.missing_setup_count == 0
):
print("[PASS] ALL SAMPLES PASSED!")
else:
@@ -98,8 +95,7 @@ def print_summary(report: Report) -> None:
print("Results:")
print(f" [PASS] Success: {report.success_count}")
print(f" [FAIL] Failure: {report.failure_count}")
print(f" [TIMEOUT] Timeout: {report.timeout_count}")
print(f" [ERR] Errors: {report.error_count}")
print(f" [MISSING_SETUP] Missing Setup: {report.missing_setup_count}")
print("=" * 80)
# Print JSON output for GitHub Actions visibility
@@ -66,9 +66,10 @@ class RunDynamicValidationWorkflowExecutor(Executor):
fallback_results = [
RunResult(
sample=sample,
status=RunStatus.ERROR,
status=RunStatus.FAILURE,
output="",
error="Nested workflow did not return an ExecutionResult.",
fix="",
)
for sample in creation.samples
]