Compare commits

..
79 changed files with 1598 additions and 1425 deletions
@@ -24,9 +24,7 @@ runs:
using: "composite"
steps:
- name: Set up Node.js environment
uses: actions/setup-node@v6
with:
node-version: 22
uses: actions/setup-node@v4
- name: Install Copilot CLI
shell: bash
+8 -520
View File
@@ -41,13 +41,6 @@ jobs:
azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
os: ${{ runner.os }}
- name: Create .env for samples
run: |
echo "AZURE_AI_PROJECT_ENDPOINT=$AZURE_AI_PROJECT_ENDPOINT" >> .env
echo "AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME=$AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME" >> .env
echo "AZURE_OPENAI_ENDPOINT=$AZURE_OPENAI_ENDPOINT" >> .env
echo "AZURE_OPENAI_CHAT_DEPLOYMENT_NAME=$AZURE_OPENAI_CHAT_DEPLOYMENT_NAME" >> .env
- name: Run sample validation
run: |
cd scripts && uv run python -m sample_validation --subdir 01-get-started --save-report --report-name 01-get-started
@@ -57,7 +50,7 @@ jobs:
if: always()
with:
name: validation-report-01-get-started
path: python/samples/sample_validation/reports/
path: python/scripts/sample_validation/reports/
validate-02-agents:
name: Validate 02-agents
@@ -71,13 +64,10 @@ jobs:
AZURE_OPENAI_ENDPOINT: ${{ vars.AZUREOPENAI__ENDPOINT }}
AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__CHATDEPLOYMENTNAME }}
AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__EMBEDDINGDEPLOYMENTNAME }}
# OpenAI configuration
OPENAI_API_KEY: ${{ secrets.OPENAI__APIKEY }}
OPENAI_CHAT_MODEL_ID: ${{ vars.OPENAI__CHATMODELID }}
OPENAI_RESPONSES_MODEL_ID: ${{ vars.OPENAI__RESPONSESMODELID }}
# GitHub MCP
GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
# Observability
ENABLE_INSTRUMENTATION: "true"
defaults:
@@ -94,420 +84,16 @@ jobs:
azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
os: ${{ runner.os }}
- name: Create .env for samples
run: |
echo "AZURE_AI_PROJECT_ENDPOINT=$AZURE_AI_PROJECT_ENDPOINT" >> .env
echo "AZURE_AI_MODEL_DEPLOYMENT_NAME=$AZURE_AI_MODEL_DEPLOYMENT_NAME" >> .env
echo "AZURE_OPENAI_ENDPOINT=$AZURE_OPENAI_ENDPOINT" >> .env
echo "AZURE_OPENAI_CHAT_DEPLOYMENT_NAME=$AZURE_OPENAI_CHAT_DEPLOYMENT_NAME" >> .env
echo "AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME=$AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME" >> .env
echo "AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME=$AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME" >> .env
echo "OPENAI_API_KEY=$OPENAI_API_KEY" >> .env
echo "OPENAI_CHAT_MODEL_ID=$OPENAI_CHAT_MODEL_ID" >> .env
echo "OPENAI_RESPONSES_MODEL_ID=$OPENAI_RESPONSES_MODEL_ID" >> .env
echo "GITHUB_PAT=$GITHUB_PAT" >> .env
- name: Run sample validation
run: |
cd scripts && uv run python -m sample_validation --subdir 02-agents --exclude providers --save-report --report-name 02-agents
cd scripts && uv run python -m sample_validation --subdir 02-agents --save-report --report-name 02-agents
- name: Upload validation report
uses: actions/upload-artifact@v7
if: always()
with:
name: validation-report-02-agents
path: python/samples/sample_validation/reports/
validate-02-agents-openai:
name: Validate 02-agents/providers/openai
runs-on: ubuntu-latest
environment: integration
env:
OPENAI_API_KEY: ${{ secrets.OPENAI__APIKEY }}
OPENAI_CHAT_MODEL_ID: ${{ vars.OPENAI__CHATMODELID }}
OPENAI_RESPONSES_MODEL_ID: ${{ vars.OPENAI__RESPONSESMODELID }}
defaults:
run:
working-directory: python
steps:
- uses: actions/checkout@v6
- name: Setup environment
uses: ./.github/actions/sample-validation-setup
with:
azure-client-id: ${{ secrets.AZURE_CLIENT_ID }}
azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }}
azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
os: ${{ runner.os }}
- name: Create .env for samples
run: |
echo "OPENAI_API_KEY=$OPENAI_API_KEY" >> .env
echo "OPENAI_CHAT_MODEL_ID=$OPENAI_CHAT_MODEL_ID" >> .env
echo "OPENAI_RESPONSES_MODEL_ID=$OPENAI_RESPONSES_MODEL_ID" >> .env
- name: Run sample validation
run: |
cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/openai --save-report --report-name 02-agents-openai
- name: Upload validation report
uses: actions/upload-artifact@v7
if: always()
with:
name: validation-report-02-agents-openai
path: python/samples/sample_validation/reports/
validate-02-agents-azure-openai:
name: Validate 02-agents/providers/azure_openai
runs-on: ubuntu-latest
environment: integration
env:
AZURE_AI_PROJECT_ENDPOINT: ${{ vars.AZURE_AI_PROJECT_ENDPOINT }}
AZURE_OPENAI_ENDPOINT: ${{ vars.AZUREOPENAI__ENDPOINT }}
AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__CHATDEPLOYMENTNAME }}
AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
defaults:
run:
working-directory: python
steps:
- uses: actions/checkout@v6
- name: Setup environment
uses: ./.github/actions/sample-validation-setup
with:
azure-client-id: ${{ secrets.AZURE_CLIENT_ID }}
azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }}
azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
os: ${{ runner.os }}
- name: Create .env for samples
run: |
echo "AZURE_AI_PROJECT_ENDPOINT=$AZURE_AI_PROJECT_ENDPOINT" >> .env
echo "AZURE_OPENAI_ENDPOINT=$AZURE_OPENAI_ENDPOINT" >> .env
echo "AZURE_OPENAI_CHAT_DEPLOYMENT_NAME=$AZURE_OPENAI_CHAT_DEPLOYMENT_NAME" >> .env
echo "AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME=$AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME" >> .env
- name: Run sample validation
run: |
cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/azure_openai --save-report --report-name 02-agents-azure-openai
- name: Upload validation report
uses: actions/upload-artifact@v7
if: always()
with:
name: validation-report-02-agents-azure-openai
path: python/samples/sample_validation/reports/
validate-02-agents-azure-ai:
name: Validate 02-agents/providers/azure_ai
runs-on: ubuntu-latest
environment: integration
env:
AZURE_AI_PROJECT_ENDPOINT: ${{ vars.AZURE_AI_PROJECT_ENDPOINT }}
AZURE_AI_MODEL_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
AZURE_AI_CHAT_MODEL_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__CHATDEPLOYMENTNAME }}
AZURE_AI_EMBEDDING_MODEL_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__EMBEDDINGDEPLOYMENTNAME }}
BING_CONNECTION_ID: ${{ secrets.BING_CONNECTION_ID }}
defaults:
run:
working-directory: python
steps:
- uses: actions/checkout@v6
- name: Setup environment
uses: ./.github/actions/sample-validation-setup
with:
azure-client-id: ${{ secrets.AZURE_CLIENT_ID }}
azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }}
azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
os: ${{ runner.os }}
- name: Create .env for samples
run: |
echo "AZURE_AI_PROJECT_ENDPOINT=$AZURE_AI_PROJECT_ENDPOINT" >> .env
echo "AZURE_AI_MODEL_DEPLOYMENT_NAME=$AZURE_AI_MODEL_DEPLOYMENT_NAME" >> .env
echo "AZURE_AI_CHAT_MODEL_DEPLOYMENT_NAME=$AZURE_AI_CHAT_MODEL_DEPLOYMENT_NAME" >> .env
echo "AZURE_AI_EMBEDDING_MODEL_DEPLOYMENT_NAME=$AZURE_AI_EMBEDDING_MODEL_DEPLOYMENT_NAME" >> .env
echo "BING_CONNECTION_ID=$BING_CONNECTION_ID" >> .env
- name: Run sample validation
run: |
cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/azure_ai --save-report --report-name 02-agents-azure-ai
- name: Upload validation report
uses: actions/upload-artifact@v7
if: always()
with:
name: validation-report-02-agents-azure-ai
path: python/samples/sample_validation/reports/
validate-02-agents-azure-ai-agent:
name: Validate 02-agents/providers/azure_ai_agent
runs-on: ubuntu-latest
environment: integration
env:
AZURE_AI_PROJECT_ENDPOINT: ${{ vars.AZURE_AI_PROJECT_ENDPOINT }}
AZURE_AI_MODEL_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
defaults:
run:
working-directory: python
steps:
- uses: actions/checkout@v6
- name: Setup environment
uses: ./.github/actions/sample-validation-setup
with:
azure-client-id: ${{ secrets.AZURE_CLIENT_ID }}
azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }}
azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
os: ${{ runner.os }}
- name: Create .env for samples
run: |
echo "AZURE_AI_PROJECT_ENDPOINT=$AZURE_AI_PROJECT_ENDPOINT" >> .env
echo "AZURE_AI_MODEL_DEPLOYMENT_NAME=$AZURE_AI_MODEL_DEPLOYMENT_NAME" >> .env
- name: Run sample validation
run: |
cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/azure_ai_agent --save-report --report-name 02-agents-azure-ai-agent
- name: Upload validation report
uses: actions/upload-artifact@v7
if: always()
with:
name: validation-report-02-agents-azure-ai-agent
path: python/samples/sample_validation/reports/
validate-02-agents-anthropic:
name: Validate 02-agents/providers/anthropic
runs-on: ubuntu-latest
environment: integration
env:
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
ANTHROPIC_CHAT_MODEL_ID: ${{ vars.ANTHROPIC_CHAT_MODEL_ID }}
defaults:
run:
working-directory: python
steps:
- uses: actions/checkout@v6
- name: Setup environment
uses: ./.github/actions/sample-validation-setup
with:
azure-client-id: ${{ secrets.AZURE_CLIENT_ID }}
azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }}
azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
os: ${{ runner.os }}
- name: Create .env for samples
run: |
echo "ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY" >> .env
echo "ANTHROPIC_CHAT_MODEL_ID=$ANTHROPIC_CHAT_MODEL_ID" >> .env
- name: Run sample validation
run: |
cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/anthropic --save-report --report-name 02-agents-anthropic
- name: Upload validation report
uses: actions/upload-artifact@v7
if: always()
with:
name: validation-report-02-agents-anthropic
path: python/samples/sample_validation/reports/
validate-02-agents-github-copilot:
name: Validate 02-agents/providers/github_copilot
runs-on: ubuntu-latest
environment: integration
defaults:
run:
working-directory: python
steps:
- uses: actions/checkout@v6
- name: Setup environment
uses: ./.github/actions/sample-validation-setup
with:
azure-client-id: ${{ secrets.AZURE_CLIENT_ID }}
azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }}
azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
os: ${{ runner.os }}
- name: Run sample validation
run: |
cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/github_copilot --save-report --report-name 02-agents-github-copilot
- name: Upload validation report
uses: actions/upload-artifact@v7
if: always()
with:
name: validation-report-02-agents-github-copilot
path: python/samples/sample_validation/reports/
validate-02-agents-amazon:
name: Validate 02-agents/providers/amazon
if: false # Temporarily disabled - requires AWS credentials
runs-on: ubuntu-latest
environment: integration
env:
BEDROCK_CHAT_MODEL_ID: ${{ vars.BEDROCK__CHATMODELID }}
defaults:
run:
working-directory: python
steps:
- uses: actions/checkout@v6
- name: Setup environment
uses: ./.github/actions/sample-validation-setup
with:
azure-client-id: ${{ secrets.AZURE_CLIENT_ID }}
azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }}
azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
os: ${{ runner.os }}
- name: Run sample validation
run: |
cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/amazon --save-report --report-name 02-agents-amazon
- name: Upload validation report
uses: actions/upload-artifact@v7
if: always()
with:
name: validation-report-02-agents-amazon
path: python/samples/sample_validation/reports/
validate-02-agents-ollama:
name: Validate 02-agents/providers/ollama
if: false # Temporarily disabled - requires local Ollama server
runs-on: ubuntu-latest
environment: integration
env:
OLLAMA_MODEL: ${{ vars.OLLAMA__MODEL }}
defaults:
run:
working-directory: python
steps:
- uses: actions/checkout@v6
- name: Setup environment
uses: ./.github/actions/sample-validation-setup
with:
azure-client-id: ${{ secrets.AZURE_CLIENT_ID }}
azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }}
azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
os: ${{ runner.os }}
- name: Run sample validation
run: |
cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/ollama --save-report --report-name 02-agents-ollama
- name: Upload validation report
uses: actions/upload-artifact@v7
if: always()
with:
name: validation-report-02-agents-ollama
path: python/samples/sample_validation/reports/
validate-02-agents-foundry-local:
name: Validate 02-agents/providers/foundry_local
if: false # Temporarily disabled - requires local Foundry setup
runs-on: ubuntu-latest
environment: integration
defaults:
run:
working-directory: python
steps:
- uses: actions/checkout@v6
- name: Setup environment
uses: ./.github/actions/sample-validation-setup
with:
azure-client-id: ${{ secrets.AZURE_CLIENT_ID }}
azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }}
azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
os: ${{ runner.os }}
- name: Run sample validation
run: |
cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/foundry_local --save-report --report-name 02-agents-foundry-local
- name: Upload validation report
uses: actions/upload-artifact@v7
if: always()
with:
name: validation-report-02-agents-foundry-local
path: python/samples/sample_validation/reports/
validate-02-agents-copilotstudio:
name: Validate 02-agents/providers/copilotstudio
if: false # Temporarily disabled - requires Copilot Studio setup
runs-on: ubuntu-latest
environment: integration
env:
COPILOTSTUDIOAGENT__ENVIRONMENTID: ${{ secrets.COPILOTSTUDIOAGENT__ENVIRONMENTID }}
COPILOTSTUDIOAGENT__SCHEMANAME: ${{ secrets.COPILOTSTUDIOAGENT__SCHEMANAME }}
COPILOTSTUDIOAGENT__TENANTID: ${{ secrets.COPILOTSTUDIOAGENT__TENANTID }}
COPILOTSTUDIOAGENT__AGENTAPPID: ${{ secrets.COPILOTSTUDIOAGENT__AGENTAPPID }}
defaults:
run:
working-directory: python
steps:
- uses: actions/checkout@v6
- name: Setup environment
uses: ./.github/actions/sample-validation-setup
with:
azure-client-id: ${{ secrets.AZURE_CLIENT_ID }}
azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }}
azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
os: ${{ runner.os }}
- name: Create .env for samples
run: |
echo "COPILOTSTUDIOAGENT__ENVIRONMENTID=$COPILOTSTUDIOAGENT__ENVIRONMENTID" >> .env
echo "COPILOTSTUDIOAGENT__SCHEMANAME=$COPILOTSTUDIOAGENT__SCHEMANAME" >> .env
echo "COPILOTSTUDIOAGENT__TENANTID=$COPILOTSTUDIOAGENT__TENANTID" >> .env
echo "COPILOTSTUDIOAGENT__AGENTAPPID=$COPILOTSTUDIOAGENT__AGENTAPPID" >> .env
- name: Run sample validation
run: |
cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/copilotstudio --save-report --report-name 02-agents-copilotstudio
- name: Upload validation report
uses: actions/upload-artifact@v7
if: always()
with:
name: validation-report-02-agents-copilotstudio
path: python/samples/sample_validation/reports/
validate-02-agents-custom:
name: Validate 02-agents/providers/custom
runs-on: ubuntu-latest
environment: integration
defaults:
run:
working-directory: python
steps:
- uses: actions/checkout@v6
- name: Setup environment
uses: ./.github/actions/sample-validation-setup
with:
azure-client-id: ${{ secrets.AZURE_CLIENT_ID }}
azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }}
azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
os: ${{ runner.os }}
- name: Run sample validation
run: |
cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/custom --save-report --report-name 02-agents-custom
- name: Upload validation report
uses: actions/upload-artifact@v7
if: always()
with:
name: validation-report-02-agents-custom
path: python/samples/sample_validation/reports/
path: python/scripts/sample_validation/reports/
validate-03-workflows:
name: Validate 03-workflows
@@ -535,14 +121,6 @@ jobs:
azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
os: ${{ runner.os }}
- name: Create .env for samples
run: |
echo "AZURE_AI_PROJECT_ENDPOINT=$AZURE_AI_PROJECT_ENDPOINT" >> .env
echo "AZURE_AI_MODEL_DEPLOYMENT_NAME=$AZURE_AI_MODEL_DEPLOYMENT_NAME" >> .env
echo "AZURE_OPENAI_ENDPOINT=$AZURE_OPENAI_ENDPOINT" >> .env
echo "AZURE_OPENAI_CHAT_DEPLOYMENT_NAME=$AZURE_OPENAI_CHAT_DEPLOYMENT_NAME" >> .env
echo "AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME=$AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME" >> .env
- name: Run sample validation
run: |
cd scripts && uv run python -m sample_validation --subdir 03-workflows --save-report --report-name 03-workflows
@@ -552,7 +130,7 @@ jobs:
if: always()
with:
name: validation-report-03-workflows
path: python/samples/sample_validation/reports/
path: python/scripts/sample_validation/reports/
validate-04-hosting:
name: Validate 04-hosting
@@ -591,7 +169,7 @@ jobs:
if: always()
with:
name: validation-report-04-hosting
path: python/samples/sample_validation/reports/
path: python/scripts/sample_validation/reports/
validate-05-end-to-end:
name: Validate 05-end-to-end
@@ -635,7 +213,7 @@ jobs:
if: always()
with:
name: validation-report-05-end-to-end
path: python/samples/sample_validation/reports/
path: python/scripts/sample_validation/reports/
validate-autogen-migration:
name: Validate autogen-migration
@@ -666,16 +244,6 @@ jobs:
azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
os: ${{ runner.os }}
- name: Create .env for samples
run: |
echo "AZURE_AI_PROJECT_ENDPOINT=$AZURE_AI_PROJECT_ENDPOINT" >> .env
echo "AZURE_AI_MODEL_DEPLOYMENT_NAME=$AZURE_AI_MODEL_DEPLOYMENT_NAME" >> .env
echo "AZURE_OPENAI_ENDPOINT=$AZURE_OPENAI_ENDPOINT" >> .env
echo "AZURE_OPENAI_CHAT_DEPLOYMENT_NAME=$AZURE_OPENAI_CHAT_DEPLOYMENT_NAME" >> .env
echo "OPENAI_API_KEY=$OPENAI_API_KEY" >> .env
echo "OPENAI_CHAT_MODEL_ID=$OPENAI_CHAT_MODEL_ID" >> .env
echo "OPENAI_RESPONSES_MODEL_ID=$OPENAI_RESPONSES_MODEL_ID" >> .env
- name: Run sample validation
run: |
cd scripts && uv run python -m sample_validation --subdir autogen-migration --save-report --report-name autogen-migration
@@ -685,7 +253,7 @@ jobs:
if: always()
with:
name: validation-report-autogen-migration
path: python/samples/sample_validation/reports/
path: python/scripts/sample_validation/reports/
validate-semantic-kernel-migration:
name: Validate semantic-kernel-migration
@@ -722,21 +290,6 @@ jobs:
azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
os: ${{ runner.os }}
- name: Create .env for samples
run: |
echo "AZURE_AI_PROJECT_ENDPOINT=$AZURE_AI_PROJECT_ENDPOINT" >> .env
echo "AZURE_AI_MODEL_DEPLOYMENT_NAME=$AZURE_AI_MODEL_DEPLOYMENT_NAME" >> .env
echo "AZURE_OPENAI_ENDPOINT=$AZURE_OPENAI_ENDPOINT" >> .env
echo "AZURE_OPENAI_CHAT_DEPLOYMENT_NAME=$AZURE_OPENAI_CHAT_DEPLOYMENT_NAME" >> .env
echo "AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME=$AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME" >> .env
echo "OPENAI_API_KEY=$OPENAI_API_KEY" >> .env
echo "OPENAI_CHAT_MODEL_ID=$OPENAI_CHAT_MODEL_ID" >> .env
echo "OPENAI_RESPONSES_MODEL_ID=$OPENAI_RESPONSES_MODEL_ID" >> .env
echo "COPILOTSTUDIOAGENT__ENVIRONMENTID=$COPILOTSTUDIOAGENT__ENVIRONMENTID" >> .env
echo "COPILOTSTUDIOAGENT__SCHEMANAME=$COPILOTSTUDIOAGENT__SCHEMANAME" >> .env
echo "COPILOTSTUDIOAGENT__TENANTID=$COPILOTSTUDIOAGENT__TENANTID" >> .env
echo "COPILOTSTUDIOAGENT__AGENTAPPID=$COPILOTSTUDIOAGENT__AGENTAPPID" >> .env
- name: Run sample validation
run: |
cd scripts && uv run python -m sample_validation --subdir semantic-kernel-migration --save-report --report-name semantic-kernel-migration
@@ -746,69 +299,4 @@ jobs:
if: always()
with:
name: validation-report-semantic-kernel-migration
path: python/samples/sample_validation/reports/
aggregate-results:
name: Aggregate Results
runs-on: ubuntu-latest
if: always()
needs:
- validate-01-get-started
- validate-02-agents
- validate-02-agents-openai
- validate-02-agents-azure-openai
- validate-02-agents-azure-ai
- validate-02-agents-azure-ai-agent
- validate-02-agents-anthropic
- validate-02-agents-github-copilot
- validate-02-agents-amazon
- validate-02-agents-ollama
- validate-02-agents-foundry-local
- validate-02-agents-copilotstudio
- validate-02-agents-custom
- validate-03-workflows
- validate-04-hosting
- validate-05-end-to-end
- validate-autogen-migration
- validate-semantic-kernel-migration
steps:
- uses: actions/checkout@v6
- name: Download all validation reports
uses: actions/download-artifact@v7
with:
pattern: validation-report-*
path: reports/
merge-multiple: true
- name: Restore validation history
id: cache-restore
uses: actions/cache/restore@v4
with:
path: validation-history/
key: validation-history-${{ github.run_id }}
restore-keys: |
validation-history-
- name: Aggregate results and generate trend report
run: |
python3 python/scripts/sample_validation/aggregate.py \
reports/ \
validation-history/history.json \
trend-report.md
- name: Write trend report to job summary
run: cat trend-report.md >> "$GITHUB_STEP_SUMMARY"
- name: Save validation history
uses: actions/cache/save@v4
with:
path: validation-history/
key: validation-history-${{ github.run_id }}
- name: Upload trend report
uses: actions/upload-artifact@v7
if: always()
with:
name: validation-trend-report
path: trend-report.md
path: python/scripts/sample_validation/reports/
@@ -21,7 +21,7 @@ jobs:
steps:
- uses: actions/checkout@v6
- name: Download coverage report
uses: actions/download-artifact@v8
uses: actions/download-artifact@v7
with:
github-token: ${{ secrets.GH_ACTIONS_PR_WRITE }}
run-id: ${{ github.event.workflow_run.id }}
-1
View File
@@ -70,7 +70,6 @@
<PackageVersion Include="Microsoft.Extensions.AI.Evaluation.Safety" Version="10.3.0-preview.1.26109.11" />
<PackageVersion Include="Microsoft.Extensions.AI.OpenAI" Version="10.4.0" />
<PackageVersion Include="Microsoft.Extensions.Caching.Memory" Version="10.0.0" />
<PackageVersion Include="Microsoft.Extensions.Compliance.Abstractions" Version="10.4.0" />
<PackageVersion Include="Microsoft.Extensions.Configuration" Version="10.0.0" />
<PackageVersion Include="Microsoft.Extensions.Configuration.Binder" Version="10.0.0" />
<PackageVersion Include="Microsoft.Extensions.Configuration.EnvironmentVariables" Version="10.0.0" />
+1 -4
View File
@@ -309,6 +309,7 @@
<Project Path="samples/05-end-to-end/HostedAgents/AgentWithHostedMCP/AgentWithHostedMCP.csproj" />
<Project Path="samples/05-end-to-end/HostedAgents/AgentWithLocalTools/AgentWithLocalTools.csproj" />
<Project Path="samples/05-end-to-end/HostedAgents/AgentWithTextSearchRag/AgentWithTextSearchRag.csproj" />
<Project Path="samples/05-end-to-end/HostedAgents/AgentWithTools/AgentWithTools.csproj" />
<Project Path="samples/05-end-to-end/HostedAgents/FoundryMultiAgent/FoundryMultiAgent.csproj" />
<Project Path="samples/05-end-to-end/HostedAgents/FoundrySingleAgent/FoundrySingleAgent.csproj" />
</Folder>
@@ -452,10 +453,6 @@
<File Path="src/Shared/Samples/TextOutputHelperExtensions.cs" />
<File Path="src/Shared/Samples/XunitLogger.cs" />
</Folder>
<Folder Name="/Solution Items/src/Shared/Redaction/">
<File Path="src/Shared/Redaction/README.md" />
<File Path="src/Shared/Redaction/ReplacingRedactor.cs" />
</Folder>
<Folder Name="/Solution Items/src/Shared/Throw/">
<File Path="src/Shared/Throw/README.md" />
<File Path="src/Shared/Throw/Throw.cs" />
-3
View File
@@ -29,7 +29,4 @@
<ItemGroup Condition="'$(InjectSharedDiagnosticIds)' == 'true'">
<Compile Include="$(MSBuildThisFileDirectory)\..\..\src\Shared\DiagnosticIds\*.cs" LinkBase="Shared\DiagnosticIds" />
</ItemGroup>
<ItemGroup Condition="'$(InjectSharedRedaction)' == 'true'">
<Compile Include="$(MSBuildThisFileDirectory)\..\..\src\Shared\Redaction\*.cs" LinkBase="Shared\Redaction" />
</ItemGroup>
</Project>
@@ -36,10 +36,10 @@
</ItemGroup>
<ItemGroup>
<PackageReference Include="Azure.AI.AgentServer.AgentFramework" Version="1.0.0-beta.11" />
<PackageReference Include="Azure.AI.AgentServer.AgentFramework" Version="1.0.0-beta.9" />
<PackageReference Include="Azure.AI.OpenAI" Version="2.9.0-beta.1" />
<PackageReference Include="Azure.Identity" Version="1.17.1" />
<PackageReference Include="Microsoft.Agents.AI.OpenAI" Version="1.0.0-rc4" />
<PackageReference Include="Microsoft.Agents.AI.OpenAI" Version="1.0.0-rc1" />
</ItemGroup>
<!-- Add analyzers with compatible versions -->
@@ -1,4 +1,4 @@
<Project Sdk="Microsoft.NET.Sdk">
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
@@ -35,10 +35,11 @@
</ItemGroup>
<ItemGroup>
<PackageReference Include="Azure.AI.AgentServer.AgentFramework" Version="1.0.0-beta.11" />
<PackageReference Include="Azure.AI.OpenAI" Version="2.8.0-beta.1" />
<PackageReference Include="Azure.AI.AgentServer.AgentFramework" Version="1.0.0-beta.9" />
<PackageReference Include="Azure.AI.OpenAI" Version="2.9.0-beta.1" />
<PackageReference Include="Azure.Identity" Version="1.17.1" />
<PackageReference Include="Microsoft.Agents.AI.OpenAI" Version="1.0.0-rc4" />
<PackageReference Include="Microsoft.Agents.AI.OpenAI" Version="1.0.0-rc1" />
<PackageReference Include="Microsoft.Extensions.AI.OpenAI" Version="10.4.0" />
</ItemGroup>
<!-- Add analyzers with compatible versions -->
@@ -31,7 +31,8 @@ AITool mcpTool = new HostedMcpServerTool(serverName: "microsoft_learn", serverAd
AIAgent agent = new AzureOpenAIClient(
new Uri(endpoint),
new DefaultAzureCredential())
.GetResponsesClient(deploymentName)
.GetResponsesClient()
.AsIChatClient(deploymentName)
.AsAIAgent(
instructions: "You answer questions by searching the Microsoft Learn content only.",
name: "MicrosoftLearnAgent",
@@ -36,11 +36,11 @@
</ItemGroup>
<ItemGroup>
<PackageReference Include="Azure.AI.AgentServer.AgentFramework" Version="1.0.0-beta.11" />
<PackageReference Include="Azure.AI.Projects" Version="2.0.0-beta.1" />
<PackageReference Include="Azure.AI.AgentServer.AgentFramework" Version="1.0.0-beta.9" />
<PackageReference Include="Azure.AI.Projects" Version="1.2.0-beta.5" />
<PackageReference Include="Azure.AI.OpenAI" Version="2.9.0-beta.1" />
<PackageReference Include="Azure.Identity" Version="1.17.1" />
<PackageReference Include="Microsoft.Agents.AI.OpenAI" Version="1.0.0-rc4" />
<PackageReference Include="Microsoft.Agents.AI.OpenAI" Version="1.0.0-rc1" />
</ItemGroup>
<!-- Add analyzers with compatible versions -->
@@ -1,4 +1,4 @@
<Project Sdk="Microsoft.NET.Sdk">
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
@@ -35,10 +35,10 @@
</ItemGroup>
<ItemGroup>
<PackageReference Include="Azure.AI.AgentServer.AgentFramework" Version="1.0.0-beta.11" />
<PackageReference Include="Azure.AI.AgentServer.AgentFramework" Version="1.0.0-beta.9" />
<PackageReference Include="Azure.AI.OpenAI" Version="2.9.0-beta.1" />
<PackageReference Include="Azure.Identity" Version="1.17.1" />
<PackageReference Include="Microsoft.Agents.AI.OpenAI" Version="1.0.0-rc4" />
<PackageReference Include="Microsoft.Agents.AI.OpenAI" Version="1.0.0-rc1" />
</ItemGroup>
<!-- Add analyzers with compatible versions -->
@@ -0,0 +1,68 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFrameworks>net10.0</TargetFrameworks>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
<!--
Disable central package management for this project.
This project requires explicit package references with versions specified inline rather than
inheriting them from Directory.Packages.props. This is necessary because a Docker image will
be created from this project, and the Docker build process only has access to this folder
and cannot access parent folders where Directory.Packages.props resides.
-->
<ManagePackageVersionsCentrally>false</ManagePackageVersionsCentrally>
</PropertyGroup>
<!--
Remove analyzer PackageReference items inherited from Directory.Packages.props.
Note: ManagePackageVersionsCentrally only controls PackageVersion items, not PackageReference items.
Directory.Packages.props contains both PackageVersion and PackageReference entries for analyzers,
and the PackageReference items are always inherited through MSBuild imports regardless of the
ManagePackageVersionsCentrally setting. We must explicitly remove them before adding our own versions.
-->
<ItemGroup>
<PackageReference Remove="Microsoft.CodeAnalysis.NetAnalyzers" />
<PackageReference Remove="Microsoft.VisualStudio.Threading.Analyzers" />
<PackageReference Remove="xunit.analyzers" />
<PackageReference Remove="Moq.Analyzers" />
<PackageReference Remove="Roslynator.Analyzers" />
<PackageReference Remove="Roslynator.CodeAnalysis.Analyzers" />
<PackageReference Remove="Roslynator.Formatting.Analyzers" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Azure.AI.AgentServer.AgentFramework" Version="1.0.0-beta.9" />
<PackageReference Include="Azure.AI.OpenAI" Version="2.9.0-beta.1" />
<PackageReference Include="Azure.Identity" Version="1.17.1" />
<PackageReference Include="Microsoft.Agents.AI.OpenAI" Version="1.0.0-rc1" />
</ItemGroup>
<!-- Add analyzers with compatible versions -->
<ItemGroup>
<PackageReference Include="Microsoft.CodeAnalysis.NetAnalyzers" Version="10.0.100">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="Microsoft.VisualStudio.Threading.Analyzers" Version="17.14.15">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="Roslynator.Analyzers" Version="4.14.1">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="Roslynator.CodeAnalysis.Analyzers" Version="4.14.1">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="Roslynator.Formatting.Analyzers" Version="4.14.1">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
</ItemGroup>
</Project>
@@ -0,0 +1,20 @@
# Build the application
FROM mcr.microsoft.com/dotnet/sdk:10.0-alpine AS build
WORKDIR /src
# Copy files from the current directory on the host to the working directory in the container
COPY . .
RUN dotnet restore
RUN dotnet build -c Release --no-restore
RUN dotnet publish -c Release --no-build -o /app -f net10.0
# Run the application
FROM mcr.microsoft.com/dotnet/aspnet:10.0-alpine AS final
WORKDIR /app
# Copy everything needed to run the app from the "build" stage.
COPY --from=build /app .
EXPOSE 8088
ENTRYPOINT ["dotnet", "AgentWithTools.dll"]
@@ -0,0 +1,46 @@
// Copyright (c) Microsoft. All rights reserved.
// This sample demonstrates how to use Foundry tools (MCP and code interpreter)
// with an AI agent hosted using the Azure AI AgentServer SDK.
using Azure.AI.AgentServer.AgentFramework.Extensions;
using Azure.AI.OpenAI;
using Azure.Identity;
using Microsoft.Agents.AI;
using Microsoft.Extensions.AI;
string openAiEndpoint = Environment.GetEnvironmentVariable("AZURE_OPENAI_ENDPOINT") ?? throw new InvalidOperationException("AZURE_OPENAI_ENDPOINT is not set.");
string deploymentName = Environment.GetEnvironmentVariable("AZURE_OPENAI_DEPLOYMENT_NAME") ?? "gpt-4o-mini";
string toolConnectionId = Environment.GetEnvironmentVariable("MCP_TOOL_CONNECTION_ID") ?? throw new InvalidOperationException("MCP_TOOL_CONNECTION_ID is not set.");
// WARNING: DefaultAzureCredential is convenient for development but requires careful consideration in production.
// In production, consider using a specific credential (e.g., ManagedIdentityCredential) to avoid
// latency issues, unintended credential probing, and potential security risks from fallback mechanisms.
DefaultAzureCredential credential = new();
IChatClient chatClient = new AzureOpenAIClient(new Uri(openAiEndpoint), credential)
.GetChatClient(deploymentName)
.AsIChatClient()
.AsBuilder()
.UseFoundryTools(new { type = "mcp", project_connection_id = toolConnectionId }, new { type = "code_interpreter" })
.UseOpenTelemetry(sourceName: "Agents", configure: (cfg) => cfg.EnableSensitiveData = true)
.Build();
AIAgent agent = chatClient.AsAIAgent(
name: "AgentWithTools",
instructions: @"You are a helpful assistant with access to tools for fetching Microsoft documentation.
IMPORTANT: When the user asks about Microsoft Learn articles or documentation:
1. You MUST use the microsoft_docs_fetch tool to retrieve the actual content
2. Do NOT rely on your training data
3. Always fetch the latest information from the provided URL
Available tools:
- microsoft_docs_fetch: Fetches and converts Microsoft Learn documentation
- microsoft_docs_search: Searches Microsoft/Azure documentation
- microsoft_code_sample_search: Searches for code examples")
.AsBuilder()
.UseOpenTelemetry(sourceName: "Agents", configure: (cfg) => cfg.EnableSensitiveData = true)
.Build();
await agent.RunAIAgentAsync(telemetrySourceName: "Agents");
@@ -0,0 +1,45 @@
# What this sample demonstrates
This sample demonstrates how to use Foundry tools with an AI agent via the `UseFoundryTools` extension. The agent is configured with two tool types: an MCP (Model Context Protocol) connection for fetching Microsoft Learn documentation and a code interpreter for running code when needed.
Key features:
- Configuring Foundry tools using `UseFoundryTools` with MCP and code interpreter
- Connecting to an external MCP tool via a Foundry project connection
- Using `DefaultAzureCredential` for Azure authentication
- OpenTelemetry instrumentation for both the chat client and agent
> For common prerequisites and setup instructions, see the [Hosted Agent Samples README](../README.md).
## Prerequisites
In addition to the common prerequisites:
1. An **Azure AI Foundry project** with a chat model deployed (e.g., `gpt-5.2`, `gpt-4o-mini`)
2. The **Azure AI Developer** role assigned on the Foundry resource (includes the `agents/write` data action required by `UseFoundryTools`)
3. An **MCP tool connection** configured in your Foundry project pointing to `https://learn.microsoft.com/api/mcp`
## Environment Variables
In addition to the common environment variables in the root README:
```powershell
# Your Azure AI Foundry project endpoint (required by UseFoundryTools)
$env:AZURE_AI_PROJECT_ENDPOINT="https://your-resource.services.ai.azure.com/api/projects/your-project"
# Chat model deployment name (defaults to gpt-4o-mini if not set)
$env:AZURE_OPENAI_DEPLOYMENT_NAME="gpt-4o-mini"
# The MCP tool connection name (just the name, not the full ARM resource ID)
$env:MCP_TOOL_CONNECTION_ID="SampleMCPTool"
```
## How It Works
1. An `AzureOpenAIClient` is created with `DefaultAzureCredential` and used to get a chat client
2. The chat client is wrapped with `UseFoundryTools` which registers two Foundry tool types:
- **MCP connection**: Connects to an external MCP server (Microsoft Learn) via the project connection name, providing documentation fetch and search capabilities
- **Code interpreter**: Allows the agent to execute code snippets when needed
3. `UseFoundryTools` resolves the connection using `AZURE_AI_PROJECT_ENDPOINT` internally
4. A `ChatClientAgent` is created with instructions guiding it to use the MCP tools for documentation queries
5. The agent is hosted using `RunAIAgentAsync` which exposes the OpenAI Responses-compatible API endpoint
@@ -0,0 +1,31 @@
name: AgentWithTools
displayName: "Agent with Tools"
description: >
An AI agent that uses Foundry tools (MCP and code interpreter) with Azure OpenAI.
The agent can fetch Microsoft Learn documentation and run code when needed.
metadata:
authors:
- Microsoft Agent Framework Team
tags:
- Azure AI AgentServer
- Microsoft Agent Framework
- Tools
- MCP
- Code Interpreter
template:
kind: hosted
name: AgentWithTools
protocols:
- protocol: responses
version: v1
environment_variables:
- name: AZURE_OPENAI_ENDPOINT
value: ${AZURE_OPENAI_ENDPOINT}
- name: AZURE_OPENAI_DEPLOYMENT_NAME
value: gpt-4o-mini
- name: MCP_TOOL_CONNECTION_ID
value: ${MCP_TOOL_CONNECTION_ID}
resources:
- name: "gpt-4o-mini"
kind: model
id: gpt-4o-mini
@@ -0,0 +1,30 @@
@host = http://localhost:8088
@endpoint = {{host}}/responses
### Health Check
GET {{host}}/readiness
### Simple string input
POST {{endpoint}}
Content-Type: application/json
{
"input": "Please use the microsoft_docs_fetch tool to fetch and summarize the Microsoft Learn article at https://learn.microsoft.com/azure/ai-services/openai/overview"
}
### Explicit input
POST {{endpoint}}
Content-Type: application/json
{
"input": [
{
"type": "message",
"role": "user",
"content": [
{
"type": "input_text",
"text": "Please use the microsoft_docs_fetch tool to fetch and summarize the Microsoft Learn article at https://learn.microsoft.com/azure/ai-services/openai/overview"
}
]
}
]
}
@@ -35,10 +35,10 @@
</ItemGroup>
<ItemGroup>
<PackageReference Include="Azure.AI.AgentServer.AgentFramework" Version="1.0.0-beta.11" />
<PackageReference Include="Azure.AI.AgentServer.AgentFramework" Version="1.0.0-beta.9" />
<PackageReference Include="Azure.AI.OpenAI" Version="2.9.0-beta.1" />
<PackageReference Include="Azure.Identity" Version="1.17.1" />
<PackageReference Include="Microsoft.Agents.AI.OpenAI" Version="1.0.0-rc4" />
<PackageReference Include="Microsoft.Agents.AI.OpenAI" Version="1.0.0-rc1" />
</ItemGroup>
<!-- Add analyzers with compatible versions -->
@@ -1,4 +1,4 @@
<Project Sdk="Microsoft.NET.Sdk">
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFrameworks>net10.0</TargetFrameworks>
@@ -33,12 +33,12 @@
</ItemGroup>
<ItemGroup>
<PackageReference Include="Azure.AI.AgentServer.AgentFramework" Version="1.0.0-beta.11" />
<PackageReference Include="Azure.AI.Projects" Version="2.0.0-beta.1" />
<PackageReference Include="Azure.AI.AgentServer.AgentFramework" Version="1.0.0-beta.8" />
<PackageReference Include="Azure.AI.Projects" Version="1.2.0-beta.5" />
<PackageReference Include="Azure.Identity" Version="1.17.1" />
<PackageReference Include="Microsoft.Agents.AI" Version="1.0.0-rc4" />
<PackageReference Include="Microsoft.Agents.AI.AzureAI" Version="1.0.0-rc4" />
<PackageReference Include="Microsoft.Agents.AI.Workflows" Version="1.0.0-rc4" />
<PackageReference Include="Microsoft.Agents.AI" Version="1.0.0-preview.251219.1" />
<PackageReference Include="Microsoft.Agents.AI.AzureAI" Version="1.0.0-preview.251219.1" />
<PackageReference Include="Microsoft.Agents.AI.Workflows" Version="1.0.0-preview.251219.1" />
<PackageReference Include="OpenTelemetry" Version="1.12.0" />
<PackageReference Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" Version="1.12.0" />
</ItemGroup>
@@ -41,7 +41,7 @@ try
.Build();
Console.WriteLine("Starting Writer-Reviewer Workflow Agent Server on http://localhost:8088");
await workflow.AsAIAgent().RunAIAgentAsync();
await workflow.AsAgent().RunAIAgentAsync();
}
finally
{
@@ -33,11 +33,11 @@
</ItemGroup>
<ItemGroup>
<PackageReference Include="Azure.AI.AgentServer.AgentFramework" Version="1.0.0-beta.11" />
<PackageReference Include="Azure.AI.Projects" Version="2.0.0-beta.1" />
<PackageReference Include="Azure.AI.AgentServer.AgentFramework" Version="1.0.0-beta.8" />
<PackageReference Include="Azure.AI.Projects" Version="1.2.0-beta.5" />
<PackageReference Include="Azure.Identity" Version="1.17.1" />
<PackageReference Include="Microsoft.Agents.AI" Version="1.0.0-rc4" />
<PackageReference Include="Microsoft.Agents.AI.AzureAI" Version="1.0.0-rc4" />
<PackageReference Include="Microsoft.Agents.AI" Version="1.0.0-preview.251219.1" />
<PackageReference Include="Microsoft.Agents.AI.AzureAI" Version="1.0.0-preview.251219.1" />
</ItemGroup>
<!-- Add analyzers with compatible versions -->
@@ -6,6 +6,7 @@ These samples demonstrate how to build and host AI agents using the [Azure AI Ag
| Sample | Description |
|--------|-------------|
| [`AgentWithTools`](./AgentWithTools/) | Foundry tools (MCP + code interpreter) via `UseFoundryTools` |
| [`AgentWithLocalTools`](./AgentWithLocalTools/) | Local C# function tool execution (Seattle hotel search) |
| [`AgentThreadAndHITL`](./AgentThreadAndHITL/) | Human-in-the-loop with `ApprovalRequiredAIFunction` and thread persistence |
| [`AgentWithHostedMCP`](./AgentWithHostedMCP/) | Hosted MCP server tool (Microsoft Learn search) |
@@ -39,18 +40,19 @@ Most samples require one or more of these environment variables:
|----------|---------|-------------|
| `AZURE_OPENAI_ENDPOINT` | Most samples | Your Azure OpenAI resource endpoint URL |
| `AZURE_OPENAI_DEPLOYMENT_NAME` | Most samples | Chat model deployment name (defaults to `gpt-4o-mini`) |
| `AZURE_AI_PROJECT_ENDPOINT` | AgentWithLocalTools, FoundryMultiAgent, FoundrySingleAgent | Azure AI Foundry project endpoint |
| `AZURE_AI_PROJECT_ENDPOINT` | AgentWithTools, AgentWithLocalTools, FoundryMultiAgent, FoundrySingleAgent | Azure AI Foundry project endpoint |
| `MCP_TOOL_CONNECTION_ID` | AgentWithTools | Foundry MCP tool connection name |
| `MODEL_DEPLOYMENT_NAME` | AgentWithLocalTools, FoundryMultiAgent, FoundrySingleAgent | Chat model deployment name (defaults to `gpt-4o-mini`) |
See each sample's README for the specific variables required.
## Azure AI Foundry Setup (for samples that use Foundry)
Some samples (`AgentWithLocalTools`, `FoundrySingleAgent`, `FoundryMultiAgent`) connect to an Azure AI Foundry project. If you're using these samples, you'll need additional setup.
Some samples (`AgentWithTools`, `AgentWithLocalTools`) connect to an Azure AI Foundry project. If you're using these samples, you'll need additional setup.
### Azure AI Developer Role
Some Foundry operations require the **Azure AI Developer** role on the Cognitive Services resource. Even if you created the project, you may not have this role by default.
The `UseFoundryTools` extension requires the **Azure AI Developer** role on the Cognitive Services resource. Even if you created the project, you may not have this role by default.
```powershell
az role assignment create `
@@ -63,6 +65,23 @@ az role assignment create `
For more details on permissions, see [Azure AI Foundry Permissions](https://aka.ms/FoundryPermissions).
### Creating an MCP Tool Connection
The `AgentWithTools` sample requires an MCP tool connection configured in your Foundry project:
1. Go to the [Azure AI Foundry portal](https://ai.azure.com)
2. Navigate to your project
3. Go to **Connected resources** → **+ New connection** → **Model Context Protocol tool**
4. Fill in:
- **Name**: `SampleMCPTool` (or any name you prefer)
- **Remote MCP Server endpoint**: `https://learn.microsoft.com/api/mcp`
- **Authentication**: `Unauthenticated`
5. Click **Connect**
The connection **name** (e.g., `SampleMCPTool`) is used as the `MCP_TOOL_CONNECTION_ID` environment variable.
> **Important**: Use only the connection **name**, not the full ARM resource ID.
## Running a Sample
Each sample runs as a standalone hosted agent on `http://localhost:8088/`:
@@ -91,6 +110,14 @@ Each sample includes a `Dockerfile` and `agent.yaml` for deployment. To deploy y
Assign the **Azure AI Developer** role to your user. See [Azure AI Developer Role](#azure-ai-developer-role) above.
### `Project connection ... was not found`
Make sure `MCP_TOOL_CONNECTION_ID` contains only the connection **name** (e.g., `SampleMCPTool`), not the full ARM resource ID path.
### `AZURE_AI_PROJECT_ENDPOINT must be set`
The `UseFoundryTools` extension requires `AZURE_AI_PROJECT_ENDPOINT`. Set it to your Foundry project endpoint (e.g., `https://your-resource.services.ai.azure.com/api/projects/your-project`).
### Multi-framework error when running `dotnet run`
If you see "Your project targets multiple frameworks", specify the framework:
@@ -2,12 +2,10 @@
using System;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.AI;
using Microsoft.Shared.DiagnosticIds;
using Microsoft.Shared.Diagnostics;
namespace Microsoft.Agents.AI;
@@ -149,7 +147,6 @@ public abstract class AIContextProvider
// Create a filtered context for ProvideAIContextAsync, filtering input messages
// to exclude non-external messages (e.g. chat history, other AI context provider messages).
#pragma warning disable MAAI001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed.
var filteredContext = new InvokingContext(
context.Agent,
context.Session,
@@ -159,7 +156,6 @@ public abstract class AIContextProvider
Messages = inputContext.Messages is not null ? this.ProvideInputMessageFilter(inputContext.Messages) : null,
Tools = inputContext.Tools
});
#pragma warning restore MAAI001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed.
var provided = await this.ProvideAIContextAsync(filteredContext, cancellationToken).ConfigureAwait(false);
@@ -298,9 +294,7 @@ public abstract class AIContextProvider
return default;
}
#pragma warning disable MAAI001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed.
var subContext = new InvokedContext(context.Agent, context.Session, this.StoreInputRequestMessageFilter(context.RequestMessages), this.StoreInputResponseMessageFilter(context.ResponseMessages!));
#pragma warning restore MAAI001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed.
return this.StoreAIContextAsync(subContext, cancellationToken);
}
@@ -378,7 +372,6 @@ public abstract class AIContextProvider
/// <param name="session">The session associated with the agent invocation.</param>
/// <param name="aiContext">The AI context to be used by the agent for this invocation.</param>
/// <exception cref="ArgumentNullException"><paramref name="agent"/> or <paramref name="aiContext"/> is <see langword="null"/>.</exception>
[Experimental(DiagnosticIds.Experiments.AgentsAIExperiments)]
public InvokingContext(
AIAgent agent,
AgentSession? session,
@@ -438,7 +431,6 @@ public abstract class AIContextProvider
/// that were used by the agent for this invocation.</param>
/// <param name="responseMessages">The response messages generated during this invocation.</param>
/// <exception cref="ArgumentNullException"><paramref name="agent"/>, <paramref name="requestMessages"/>, or <paramref name="responseMessages"/> is <see langword="null"/>.</exception>
[Experimental(DiagnosticIds.Experiments.AgentsAIExperiments)]
public InvokedContext(
AIAgent agent,
AgentSession? session,
@@ -2,12 +2,10 @@
using System;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.AI;
using Microsoft.Shared.DiagnosticIds;
using Microsoft.Shared.Diagnostics;
namespace Microsoft.Agents.AI;
@@ -252,9 +250,7 @@ public abstract class ChatHistoryProvider
return default;
}
#pragma warning disable MAAI001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed.
var subContext = new InvokedContext(context.Agent, context.Session, this._storeInputRequestMessageFilter(context.RequestMessages), this._storeInputResponseMessageFilter(context.ResponseMessages!));
#pragma warning restore MAAI001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed.
return this.StoreChatHistoryAsync(subContext, cancellationToken);
}
@@ -344,7 +340,6 @@ public abstract class ChatHistoryProvider
/// <param name="session">The session associated with the agent invocation.</param>
/// <param name="requestMessages">The messages to be used by the agent for this invocation.</param>
/// <exception cref="ArgumentNullException"><paramref name="requestMessages"/> is <see langword="null"/>.</exception>
[Experimental(DiagnosticIds.Experiments.AgentsAIExperiments)]
public InvokingContext(
AIAgent agent,
AgentSession? session,
@@ -404,7 +399,6 @@ public abstract class ChatHistoryProvider
/// that were used by the agent for this invocation.</param>
/// <param name="responseMessages">The response messages generated during this invocation.</param>
/// <exception cref="ArgumentNullException"><paramref name="agent"/>, <paramref name="requestMessages"/>, or <paramref name="responseMessages"/> is <see langword="null"/>.</exception>
[Experimental(DiagnosticIds.Experiments.AgentsAIExperiments)]
public InvokedContext(
AIAgent agent,
AgentSession? session,
@@ -2,12 +2,10 @@
using System;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.AI;
using Microsoft.Shared.DiagnosticIds;
using Microsoft.Shared.Diagnostics;
namespace Microsoft.Agents.AI;
@@ -51,14 +49,12 @@ public abstract class MessageAIContextProvider : AIContextProvider
{
// Call ProvideMessagesAsync directly to return only additional messages.
// The base AIContextProvider.InvokingCoreAsync handles merging with the original input and stamping.
#pragma warning disable MAAI001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed.
return new AIContext
{
Messages = await this.ProvideMessagesAsync(
new InvokingContext(context.Agent, context.Session, context.AIContext.Messages ?? []),
cancellationToken).ConfigureAwait(false)
};
#pragma warning restore MAAI001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed.
}
/// <summary>
@@ -113,12 +109,10 @@ public abstract class MessageAIContextProvider : AIContextProvider
// Create a filtered context for ProvideMessagesAsync, filtering input messages
// to exclude non-external messages (e.g. chat history, other AI context provider messages).
#pragma warning disable MAAI001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed.
var filteredContext = new InvokingContext(
context.Agent,
context.Session,
this.ProvideInputMessageFilter(inputMessages));
#pragma warning restore MAAI001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed.
var providedMessages = await this.ProvideMessagesAsync(filteredContext, cancellationToken).ConfigureAwait(false);
@@ -169,7 +163,6 @@ public abstract class MessageAIContextProvider : AIContextProvider
/// <param name="session">The session associated with the agent invocation.</param>
/// <param name="requestMessages">The messages to be used by the agent for this invocation.</param>
/// <exception cref="ArgumentNullException"><paramref name="agent"/> or <paramref name="requestMessages"/> is <see langword="null"/>.</exception>
[Experimental(DiagnosticIds.Experiments.AgentsAIExperiments)]
public InvokingContext(
AIAgent agent,
AgentSession? session,
@@ -20,7 +20,8 @@
<!-- NuGet Package Settings -->
<Title>Microsoft Agent Framework AzureAI Persistent Agents</Title>
<Description>Provides Microsoft Agent Framework support for Azure AI Persistent Agents.</Description>
<!-- Disabled until Azure.AI.Agents.Persistent targets ME.AI 10.4.0+ (https://github.com/microsoft/agent-framework/issues/4769) -->
<IsPackable>false</IsPackable>
</PropertyGroup>
</Project>
@@ -1,3 +1,19 @@
# Microsoft.Agents.AI.AzureAI.Persistent
Provides integration between the Microsoft Agent Framework and Azure AI Agents Persistent (`Azure.AI.Agents.Persistent`).
## ⚠️ Known Compatibility Limitation
The underlying `Azure.AI.Agents.Persistent` package (currently 1.2.0-beta.9) targets `Microsoft.Extensions.AI.Abstractions` 10.1.x and references types that were renamed in 10.4.0 (e.g., `McpServerToolApprovalResponseContent` → `ToolApprovalResponseContent`). This causes `TypeLoadException` at runtime when used with ME.AI 10.4.0+.
**Compatible versions:**
| Package | Compatible Version |
|---|---|
| `Azure.AI.Agents.Persistent` | 1.2.0-beta.9 (targets ME.AI 10.1.x) |
| `Microsoft.Extensions.AI.Abstractions` | ≤ 10.3.0 |
| `OpenAI` | ≤ 2.8.0 |
**Resolution:** An updated version of `Azure.AI.Agents.Persistent` targeting ME.AI 10.4.0+ is expected in 1.2.0-beta.10. The upstream fix is tracked in [Azure/azure-sdk-for-net#56929](https://github.com/Azure/azure-sdk-for-net/pull/56929).
**Tracking issue:** [microsoft/agent-framework#4769](https://github.com/microsoft/agent-framework/issues/4769)
@@ -10,7 +10,6 @@ using System.Threading;
using System.Threading.Tasks;
using Azure.AI.Projects;
using Microsoft.Extensions.AI;
using Microsoft.Extensions.Compliance.Redaction;
using Microsoft.Extensions.Logging;
using Microsoft.Shared.DiagnosticIds;
using Microsoft.Shared.Diagnostics;
@@ -38,7 +37,7 @@ public sealed class FoundryMemoryProvider : AIContextProvider
private readonly string _memoryStoreName;
private readonly int _maxMemories;
private readonly int _updateDelay;
private readonly Redactor _redactor;
private readonly bool _enableSensitiveTelemetryData;
private readonly AIProjectClient _client;
private readonly ILogger<FoundryMemoryProvider>? _logger;
@@ -80,7 +79,7 @@ public sealed class FoundryMemoryProvider : AIContextProvider
this._memoryStoreName = memoryStoreName;
this._maxMemories = effectiveOptions.MaxMemories;
this._updateDelay = effectiveOptions.UpdateDelay;
this._redactor = effectiveOptions.EnableSensitiveTelemetryData ? NullRedactor.Instance : (effectiveOptions.Redactor ?? new ReplacingRedactor("<redacted>"));
this._enableSensitiveTelemetryData = effectiveOptions.EnableSensitiveTelemetryData;
}
/// <inheritdoc />
@@ -417,7 +416,7 @@ public sealed class FoundryMemoryProvider : AIContextProvider
private static bool IsAllowedRole(ChatRole role) =>
role == ChatRole.User || role == ChatRole.Assistant || role == ChatRole.System;
private string SanitizeLogData(string? data) => this._redactor.Redact(data);
private string? SanitizeLogData(string? data) => this._enableSensitiveTelemetryData ? data : "<redacted>";
/// <summary>
/// Represents the state of a <see cref="FoundryMemoryProvider"/> stored in the <see cref="AgentSession.StateBag"/>.
@@ -3,7 +3,6 @@
using System;
using System.Collections.Generic;
using Microsoft.Extensions.AI;
using Microsoft.Extensions.Compliance.Redaction;
namespace Microsoft.Agents.AI.FoundryMemory;
@@ -38,22 +37,8 @@ public sealed class FoundryMemoryProviderOptions
/// Gets or sets a value indicating whether sensitive data such as user ids and user messages may appear in logs.
/// </summary>
/// <value>Defaults to <see langword="false"/>.</value>
/// <remarks>
/// When set to <see langword="true"/>, sensitive data is passed through to logs unchanged and any
/// configured <see cref="Redactor"/> is ignored. This property takes precedence over <see cref="Redactor"/>.
/// </remarks>
public bool EnableSensitiveTelemetryData { get; set; }
/// <summary>
/// Gets or sets a custom <see cref="Redactor"/> used to redact sensitive data in log output.
/// </summary>
/// <value>
/// When <see langword="null"/> (the default), sensitive data is replaced with a placeholder.
/// When set, this redactor is used to transform sensitive values before they are logged.
/// Ignored when <see cref="EnableSensitiveTelemetryData"/> is <see langword="true"/>.
/// </value>
public Redactor? Redactor { get; set; }
/// <summary>
/// Gets or sets the key used to store the provider state in the session's <see cref="AgentSessionStateBag"/>.
/// </summary>
@@ -8,7 +8,6 @@
<PropertyGroup>
<InjectSharedThrow>true</InjectSharedThrow>
<InjectSharedDiagnosticIds>true</InjectSharedDiagnosticIds>
<InjectSharedRedaction>true</InjectSharedRedaction>
<InjectExperimentalAttributeOnLegacy>true</InjectExperimentalAttributeOnLegacy>
<InjectTrimAttributesOnLegacy>true</InjectTrimAttributesOnLegacy>
</PropertyGroup>
@@ -21,7 +20,6 @@
<ItemGroup>
<PackageReference Include="Azure.AI.Projects" />
<PackageReference Include="Microsoft.Extensions.Compliance.Abstractions" />
<PackageReference Include="OpenAI" />
</ItemGroup>
@@ -8,7 +8,6 @@ using System.Text.Json.Serialization;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.AI;
using Microsoft.Extensions.Compliance.Redaction;
using Microsoft.Extensions.Logging;
using Microsoft.Shared.Diagnostics;
@@ -52,7 +51,7 @@ public sealed class Mem0Provider : MessageAIContextProvider
private readonly ProviderSessionState<State> _sessionState;
private IReadOnlyList<string>? _stateKeys;
private readonly string _contextPrompt;
private readonly Redactor _redactor;
private readonly bool _enableSensitiveTelemetryData;
private readonly Mem0Client _client;
private readonly ILogger<Mem0Provider>? _logger;
@@ -92,7 +91,7 @@ public sealed class Mem0Provider : MessageAIContextProvider
this._client = new Mem0Client(httpClient);
this._contextPrompt = options?.ContextPrompt ?? DefaultContextPrompt;
this._redactor = options?.EnableSensitiveTelemetryData == true ? NullRedactor.Instance : (options?.Redactor ?? new ReplacingRedactor("<redacted>"));
this._enableSensitiveTelemetryData = options?.EnableSensitiveTelemetryData ?? false;
}
/// <inheritdoc />
@@ -298,5 +297,5 @@ public sealed class Mem0Provider : MessageAIContextProvider
public Mem0ProviderScope SearchScope { get; }
}
private string SanitizeLogData(string? data) => this._redactor.Redact(data);
private string? SanitizeLogData(string? data) => this._enableSensitiveTelemetryData ? data : "<redacted>";
}
@@ -3,7 +3,6 @@
using System;
using System.Collections.Generic;
using Microsoft.Extensions.AI;
using Microsoft.Extensions.Compliance.Redaction;
namespace Microsoft.Agents.AI.Mem0;
@@ -22,22 +21,8 @@ public sealed class Mem0ProviderOptions
/// Gets or sets a value indicating whether sensitive data such as user ids and user messages may appear in logs.
/// </summary>
/// <value>Defaults to <see langword="false"/>.</value>
/// <remarks>
/// When set to <see langword="true"/>, sensitive data is passed through to logs unchanged and any
/// configured <see cref="Redactor"/> is ignored. This property takes precedence over <see cref="Redactor"/>.
/// </remarks>
public bool EnableSensitiveTelemetryData { get; set; }
/// <summary>
/// Gets or sets a custom <see cref="Redactor"/> used to redact sensitive data in log output.
/// </summary>
/// <value>
/// When <see langword="null"/> (the default), sensitive data is replaced with a placeholder.
/// When set, this redactor is used to transform sensitive values before they are logged.
/// Ignored when <see cref="EnableSensitiveTelemetryData"/> is <see langword="true"/>.
/// </value>
public Redactor? Redactor { get; set; }
/// <summary>
/// Gets or sets the key used to store the provider state in the session's <see cref="AgentSessionStateBag"/>.
/// </summary>
@@ -6,7 +6,6 @@
<PropertyGroup>
<InjectSharedThrow>true</InjectSharedThrow>
<InjectSharedRedaction>true</InjectSharedRedaction>
<InjectTrimAttributesOnLegacy>true</InjectTrimAttributesOnLegacy>
</PropertyGroup>
@@ -24,10 +23,6 @@
<ProjectReference Include="..\Microsoft.Agents.AI.Abstractions\Microsoft.Agents.AI.Abstractions.csproj" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Compliance.Abstractions" />
</ItemGroup>
<PropertyGroup>
<!-- NuGet Package Settings -->
<Title>Microsoft Agent Framework - Mem0 integration</Title>
@@ -53,6 +53,9 @@ internal sealed class AsyncRunHandle : ICheckpointingHandle, IAsyncDisposable
public ValueTask<RunStatus> GetStatusAsync(CancellationToken cancellationToken = default)
=> this._eventStream.GetStatusAsync(cancellationToken);
internal bool TryGetResponsePortExecutorId(string portId, out string? executorId)
=> this._stepRunner.TryGetResponsePortExecutorId(portId, out executorId);
public async IAsyncEnumerable<WorkflowEvent> TakeEventStreamAsync(bool blockOnPendingRequest, [EnumeratorCancellation] CancellationToken cancellationToken = default)
{
//Debug.Assert(breakOnHalt);
@@ -3,6 +3,7 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
@@ -95,6 +96,18 @@ internal sealed class EdgeMap
return portRunner.ChaseEdgeAsync(new MessageEnvelope(response, ExecutorIdentity.None), this._stepTracer, cancellationToken);
}
internal bool TryGetResponsePortExecutorId(string portId, [NotNullWhen(true)] out string? executorId)
{
if (this._portEdgeRunners.TryGetValue(portId, out ResponseEdgeRunner? portRunner))
{
executorId = portRunner.ExecutorId;
return true;
}
executorId = null;
return false;
}
internal async ValueTask<Dictionary<EdgeId, PortableValue>> ExportStateAsync()
{
Dictionary<EdgeId, PortableValue> exportedStates = [];
@@ -19,6 +19,7 @@ internal interface ISuperStepRunner
bool HasUnprocessedMessages { get; }
ValueTask EnqueueResponseAsync(ExternalResponse response, CancellationToken cancellationToken = default);
bool TryGetResponsePortExecutorId(string portId, out string? executorId);
ValueTask<bool> IsValidInputTypeAsync<T>(CancellationToken cancellationToken = default);
ValueTask<bool> EnqueueMessageAsync<T>(T message, CancellationToken cancellationToken = default);
@@ -160,6 +160,8 @@ internal sealed class InProcessRunner : ISuperStepRunner, ICheckpointingHandle
bool ISuperStepRunner.HasUnservicedRequests => this.RunContext.HasUnservicedRequests;
bool ISuperStepRunner.HasUnprocessedMessages => this.RunContext.NextStepHasActions;
bool ISuperStepRunner.TryGetResponsePortExecutorId(string portId, out string? executorId)
=> this.RunContext.TryGetResponsePortExecutorId(portId, out executorId);
public bool IsCheckpointingEnabled => this.RunContext.IsCheckpointingEnabled;
@@ -296,6 +296,9 @@ internal sealed class InProcessRunnerContext : IRunnerContext
return this._externalRequests.TryRemove(requestId, out _);
}
internal bool TryGetResponsePortExecutorId(string portId, [NotNullWhen(true)] out string? executorId)
=> this._edgeMap.TryGetResponsePortExecutorId(portId, out executorId);
private IEventSink OutgoingEvents { get; }
internal StateManager StateManager { get; } = new();
@@ -68,10 +68,17 @@ internal sealed class AIAgentHostExecutor : ChatProtocolExecutor
throw new InvalidOperationException($"No pending ToolApprovalRequest found with id '{response.RequestId}'.");
}
List<ChatMessage> implicitTurnMessages = [new ChatMessage(ChatRole.User, [response])];
// Merge the external response with any already-buffered regular messages so mixed-content
// resumes can be processed in one invocation.
return this.ProcessTurnMessagesAsync(async (pendingMessages, ctx, ct) =>
{
pendingMessages.Add(new ChatMessage(ChatRole.User, [response]));
// ContinueTurnAsync owns failing to emit a TurnToken if this response does not clear up all remaining outstanding requests.
return this.ContinueTurnAsync(implicitTurnMessages, context, this._currentTurnEmitEvents ?? false, cancellationToken);
await this.ContinueTurnAsync(pendingMessages, ctx, this._currentTurnEmitEvents ?? false, ct).ConfigureAwait(false);
// Clear the buffered turn messages because they were consumed by ContinueTurnAsync.
return null;
}, context, cancellationToken);
}
private ValueTask HandleFunctionResultAsync(
@@ -84,8 +91,17 @@ internal sealed class AIAgentHostExecutor : ChatProtocolExecutor
throw new InvalidOperationException($"No pending FunctionCall found with id '{result.CallId}'.");
}
List<ChatMessage> implicitTurnMessages = [new ChatMessage(ChatRole.Tool, [result])];
return this.ContinueTurnAsync(implicitTurnMessages, context, this._currentTurnEmitEvents ?? false, cancellationToken);
// Merge the external response with any already-buffered regular messages so mixed-content
// resumes can be processed in one invocation.
return this.ProcessTurnMessagesAsync(async (pendingMessages, ctx, ct) =>
{
pendingMessages.Add(new ChatMessage(ChatRole.Tool, [result]));
await this.ContinueTurnAsync(pendingMessages, ctx, this._currentTurnEmitEvents ?? false, ct).ConfigureAwait(false);
// Clear the buffered turn messages because they were consumed by ContinueTurnAsync.
return null;
}, context, cancellationToken);
}
public bool ShouldEmitStreamingEvents(bool? emitEvents)
@@ -198,7 +214,7 @@ internal sealed class AIAgentHostExecutor : ChatProtocolExecutor
ExtractUnservicedRequests(response.Messages.SelectMany(message => message.Contents));
}
if (this._options.EmitAgentResponseEvents == true)
if (this._options.EmitAgentResponseEvents)
{
await context.YieldOutputAsync(response, cancellationToken).ConfigureAwait(false);
}
@@ -16,10 +16,12 @@ internal sealed class AIContentExternalHandler<TRequestContent, TResponseContent
where TResponseContent : AIContent
{
private readonly PortBinding? _portBinding;
private readonly string _portId;
private ConcurrentDictionary<string, TRequestContent> _pendingRequests = new();
public AIContentExternalHandler(ref ProtocolBuilder protocolBuilder, string portId, bool intercepted, Func<TResponseContent, IWorkflowContext, CancellationToken, ValueTask> handler)
{
this._portId = portId;
PortBinding? portBinding = null;
protocolBuilder = protocolBuilder.ConfigureRoutes(routeBuilder => ConfigureRoutes(routeBuilder, out portBinding));
this._portBinding = portBinding;
@@ -58,12 +60,14 @@ internal sealed class AIContentExternalHandler<TRequestContent, TResponseContent
{
if (!this._pendingRequests.TryAdd(id, requestContent))
{
throw new InvalidOperationException($"A pending request with ID '{id}' already exists.");
// Request is already pending; treat as an idempotent re-emission.
// Do not repost to the sink because request IDs must remain unique while pending.
return default;
}
return this.IsIntercepted
? context.SendMessageAsync(requestContent, cancellationToken: cancellationToken)
: this._portBinding.PostRequestAsync(requestContent, id, cancellationToken);
: this._portBinding.PostRequestAsync(requestContent, this.CreateExternalRequestId(id), cancellationToken);
}
public bool MarkRequestAsHandled(string id)
@@ -74,6 +78,8 @@ internal sealed class AIContentExternalHandler<TRequestContent, TResponseContent
[MemberNotNullWhen(false, nameof(_portBinding))]
private bool IsIntercepted => this._portBinding == null;
private string CreateExternalRequestId(string requestId) => $"{this._portId.Length}:{this._portId}:{requestId}";
private static string MakeKey(string id) => $"{id}_PendingRequests";
public async ValueTask OnCheckpointingAsync(string id, IWorkflowContext context, CancellationToken cancellationToken = default)
@@ -60,6 +60,9 @@ public sealed class StreamingRun : CheckpointableRunBase, IAsyncDisposable
internal ValueTask<bool> TrySendMessageUntypedAsync(object message, Type? declaredType = null)
=> this._runHandle.EnqueueMessageUntypedAsync(message, declaredType);
internal bool TryGetResponsePortExecutorId(string portId, out string? executorId)
=> this._runHandle.TryGetResponsePortExecutorId(portId, out executorId);
/// <summary>
/// Asynchronously streams workflow events as they occur during workflow execution.
/// </summary>
@@ -25,6 +25,25 @@ internal sealed class WorkflowSession : AgentSession
private InMemoryCheckpointManager? _inMemoryCheckpointManager;
/// <summary>
/// Tracks pending external requests by their workflow-facing request ID.
/// This mapping enables converting incoming response content back to <see cref="ExternalResponse"/>
/// when resuming a workflow from a checkpoint.
/// </summary>
/// <remarks>
/// <para>
/// Entries are added when a <see cref="RequestInfoEvent"/> is received during workflow execution,
/// and removed when a matching response is delivered via <see cref="SendMessagesWithResponseConversionAsync"/>.
/// </para>
/// <para>
/// The number of entries is bounded by the number of outstanding external requests in a single workflow run.
/// When a session is abandoned, all pending requests are released with the session object.
/// Request-level timeouts, if needed, should be implemented in the workflow definition itself
/// (e.g., using a timer racing against an external event).
/// </para>
/// </remarks>
private readonly Dictionary<string, ExternalRequest> _pendingRequests = [];
internal static bool VerifyCheckpointingConfiguration(IWorkflowExecutionEnvironment executionEnvironment, [NotNullWhen(true)] out InProcessExecutionEnvironment? inProcEnv)
{
inProcEnv = null;
@@ -90,6 +109,7 @@ internal sealed class WorkflowSession : AgentSession
this.LastCheckpoint = sessionState.LastCheckpoint;
this.StateBag = sessionState.StateBag;
this._pendingRequests = sessionState.PendingRequests ?? [];
}
public CheckpointInfo? LastCheckpoint { get; set; }
@@ -101,7 +121,8 @@ internal sealed class WorkflowSession : AgentSession
this.SessionId,
this.LastCheckpoint,
this._inMemoryCheckpointManager,
this.StateBag);
this.StateBag,
this._pendingRequests);
return marshaller.Marshal(info);
}
@@ -141,7 +162,7 @@ internal sealed class WorkflowSession : AgentSession
return update;
}
private async ValueTask<StreamingRun> CreateOrResumeRunAsync(List<ChatMessage> messages, CancellationToken cancellationToken = default)
private async ValueTask<ResumeRunResult> CreateOrResumeRunAsync(List<ChatMessage> messages, CancellationToken cancellationToken = default)
{
// The workflow is validated to be a ChatProtocol workflow by the WorkflowHostAgent before creating the session,
// and does not need to be checked again here.
@@ -154,18 +175,155 @@ internal sealed class WorkflowSession : AgentSession
cancellationToken)
.ConfigureAwait(false);
await run.TrySendMessageAsync(messages).ConfigureAwait(false);
return run;
// Process messages: convert response content to ExternalResponse, send regular messages as-is
ResumeDispatchInfo dispatchInfo = await this.SendMessagesWithResponseConversionAsync(run, messages).ConfigureAwait(false);
return new ResumeRunResult(run, dispatchInfo);
}
return await this._executionEnvironment
StreamingRun newRun = await this._executionEnvironment
.RunStreamingAsync(this._workflow,
messages,
this.SessionId,
cancellationToken)
.ConfigureAwait(false);
return new ResumeRunResult(newRun);
}
/// <summary>
/// Sends messages to the run, converting FunctionResultContent and UserInputResponseContent
/// to ExternalResponse when there's a matching pending request.
/// </summary>
/// <returns>
/// Structured information about how resume content was dispatched.
/// </returns>
private async ValueTask<ResumeDispatchInfo> SendMessagesWithResponseConversionAsync(StreamingRun run, List<ChatMessage> messages)
{
List<ChatMessage> regularMessages = [];
// Responses are deferred until after regular messages are queued so response handlers
// can merge buffered regular content in the same continuation turn.
List<(ExternalResponse Response, string RequestId)> externalResponses = [];
bool hasMatchedResponseForStartExecutor = false;
// Tracks content IDs already matched to pending requests within this invocation,
// preventing duplicate responses for the same ID from being sent to the workflow engine.
HashSet<string>? matchedContentIds = null;
foreach (ChatMessage message in messages)
{
List<AIContent> regularContents = [];
foreach (AIContent content in message.Contents)
{
string? contentId = GetResponseContentId(content);
// Skip duplicate response content for an already-matched content ID
if (contentId != null && matchedContentIds?.Contains(contentId) == true)
{
continue;
}
if (contentId != null
&& this.TryGetPendingRequest(contentId) is ExternalRequest pendingRequest)
{
// For intercepted/complex topologies the port may not be registered in the EdgeMap.
// Treat unknown port as non-start-executor (conservative): TurnToken will still be sent.
if (run.TryGetResponsePortExecutorId(pendingRequest.PortInfo.PortId, out string? responseExecutorId))
{
hasMatchedResponseForStartExecutor |= string.Equals(responseExecutorId, this._workflow.StartExecutorId, StringComparison.Ordinal);
}
AIContent normalizedResponseContent = NormalizeResponseContentForDelivery(content, pendingRequest);
externalResponses.Add((pendingRequest.CreateResponse(normalizedResponseContent), pendingRequest.RequestId));
(matchedContentIds ??= new(StringComparer.Ordinal)).Add(contentId);
}
else
{
regularContents.Add(content);
}
}
if (regularContents.Count > 0)
{
ChatMessage cloned = message.Clone();
cloned.Contents = regularContents;
regularMessages.Add(cloned);
}
}
// Send regular messages first so response handlers can merge them with responses.
bool hasRegularMessages = regularMessages.Count > 0;
if (hasRegularMessages)
{
await run.TrySendMessageAsync(regularMessages).ConfigureAwait(false);
}
// Send external responses after regular messages.
bool hasMatchedExternalResponses = false;
foreach ((ExternalResponse response, string requestId) in externalResponses)
{
await run.SendResponseAsync(response).ConfigureAwait(false);
hasMatchedExternalResponses = true;
this.RemovePendingRequest(requestId);
}
return new ResumeDispatchInfo(
hasRegularMessages,
hasMatchedExternalResponses,
hasMatchedResponseForStartExecutor);
}
/// <summary>
/// Creates the workflow-facing request content surfaced in response updates.
/// </summary>
private static AIContent CreateRequestContentForDelivery(ExternalRequest request) => request switch
{
ExternalRequest externalRequest when externalRequest.TryGetDataAs(out FunctionCallContent? functionCallContent)
=> CloneFunctionCallContent(functionCallContent, externalRequest.RequestId),
ExternalRequest externalRequest when externalRequest.TryGetDataAs(out ToolApprovalRequestContent? toolApprovalRequestContent)
=> CloneToolApprovalRequestContent(toolApprovalRequestContent, externalRequest.RequestId),
ExternalRequest externalRequest
=> externalRequest.ToFunctionCall(),
};
/// <summary>
/// Rewrites workflow-facing response content back to the original agent-owned content ID.
/// </summary>
private static AIContent NormalizeResponseContentForDelivery(AIContent content, ExternalRequest request) => content switch
{
FunctionResultContent functionResultContent when request.TryGetDataAs(out FunctionCallContent? functionCallContent)
=> CloneFunctionResultContent(functionResultContent, functionCallContent.CallId),
ToolApprovalResponseContent toolApprovalResponseContent when request.TryGetDataAs(out ToolApprovalRequestContent? toolApprovalRequestContent)
=> CloneToolApprovalResponseContent(toolApprovalResponseContent, toolApprovalRequestContent.RequestId),
_ => content,
};
/// <summary>
/// Gets the workflow-facing request ID from response content types.
/// </summary>
private static string? GetResponseContentId(AIContent content) => content switch
{
FunctionResultContent functionResultContent => functionResultContent.CallId,
ToolApprovalResponseContent toolApprovalResponseContent => toolApprovalResponseContent.RequestId,
_ => null
};
/// <summary>
/// Tries to get a pending request by workflow-facing request ID.
/// </summary>
private ExternalRequest? TryGetPendingRequest(string requestId) =>
this._pendingRequests.TryGetValue(requestId, out ExternalRequest? request) ? request : null;
/// <summary>
/// Adds a pending request indexed by workflow-facing request ID.
/// </summary>
private void AddPendingRequest(string requestId, ExternalRequest request) => this._pendingRequests[requestId] = request;
/// <summary>
/// Removes a pending request by workflow-facing request ID.
/// </summary>
private void RemovePendingRequest(string requestId) =>
this._pendingRequests.Remove(requestId);
internal async
IAsyncEnumerable<AgentResponseUpdate> InvokeStageAsync(
[EnumeratorCancellation] CancellationToken cancellationToken = default)
@@ -175,12 +333,25 @@ internal sealed class WorkflowSession : AgentSession
this.LastResponseId = Guid.NewGuid().ToString("N");
List<ChatMessage> messages = this.ChatHistoryProvider.GetFromBookmark(this).ToList();
#pragma warning disable CA2007 // Analyzer misfiring and not seeing .ConfigureAwait(false) below.
await using StreamingRun run =
ResumeRunResult resumeResult =
await this.CreateOrResumeRunAsync(messages, cancellationToken).ConfigureAwait(false);
#pragma warning disable CA2007 // Analyzer misfiring.
await using StreamingRun run = resumeResult.Run;
#pragma warning restore CA2007
await run.TrySendMessageAsync(new TurnToken(emitEvents: true)).ConfigureAwait(false);
ResumeDispatchInfo dispatchInfo = resumeResult.DispatchInfo;
// Send a TurnToken to the start executor unless the only activity is an external
// response directed at the start executor itself (which self-emits a TurnToken via
// ContinueTurnAsync). Non-start executors (e.g., RequestInfoExecutor) do not emit
// TurnTokens after processing responses, so the session must always provide one.
bool shouldSendTurnToken =
!dispatchInfo.HasMatchedExternalResponses
|| !dispatchInfo.HasMatchedResponseForStartExecutor;
if (shouldSendTurnToken)
{
await run.TrySendMessageAsync(new TurnToken(emitEvents: true)).ConfigureAwait(false);
}
await foreach (WorkflowEvent evt in run.WatchStreamAsync(blockOnPendingRequest: false, cancellationToken)
.ConfigureAwait(false)
.WithCancellation(cancellationToken))
@@ -192,8 +363,13 @@ internal sealed class WorkflowSession : AgentSession
break;
case RequestInfoEvent requestInfo:
FunctionCallContent fcContent = requestInfo.Request.ToFunctionCall();
AgentResponseUpdate update = this.CreateUpdate(this.LastResponseId, evt, fcContent);
AIContent requestContent = CreateRequestContentForDelivery(requestInfo.Request);
// Track the pending request so we can convert incoming responses back to ExternalResponse.
// External callers respond using the workflow-facing request ID, which is always RequestId.
this.AddPendingRequest(requestInfo.Request.RequestId, requestInfo.Request);
AgentResponseUpdate update = this.CreateUpdate(this.LastResponseId, evt, requestContent);
yield return update;
break;
@@ -267,15 +443,116 @@ internal sealed class WorkflowSession : AgentSession
/// <inheritdoc/>
public WorkflowChatHistoryProvider ChatHistoryProvider { get; }
/// <summary>
/// Captures the outcome of creating or resuming a workflow run,
/// indicating what types of messages were sent during resume.
/// </summary>
private readonly struct ResumeRunResult
{
/// <summary>The streaming run that was created or resumed.</summary>
public StreamingRun Run { get; }
/// <summary>How resume-time content was dispatched into the workflow runtime.</summary>
public ResumeDispatchInfo DispatchInfo { get; }
public ResumeRunResult(StreamingRun run, ResumeDispatchInfo dispatchInfo = default)
{
this.Run = Throw.IfNull(run);
this.DispatchInfo = dispatchInfo;
}
}
/// <summary>
/// Captures how resumed input was split across regular-message and external-response delivery paths.
/// </summary>
private readonly struct ResumeDispatchInfo
{
public ResumeDispatchInfo(bool hasRegularMessages, bool hasMatchedExternalResponses, bool hasMatchedResponseForStartExecutor)
{
this.HasRegularMessages = hasRegularMessages;
this.HasMatchedExternalResponses = hasMatchedExternalResponses;
this.HasMatchedResponseForStartExecutor = hasMatchedResponseForStartExecutor;
}
public bool HasRegularMessages { get; }
public bool HasMatchedExternalResponses { get; }
public bool HasMatchedResponseForStartExecutor { get; }
}
/// <summary>
/// Clones a <see cref="FunctionCallContent"/> with a workflow-facing call ID.
/// </summary>
private static FunctionCallContent CloneFunctionCallContent(FunctionCallContent content, string callId)
{
FunctionCallContent clone = new(callId, content.Name, content.Arguments)
{
Exception = content.Exception,
InformationalOnly = content.InformationalOnly,
};
return CopyContentMetadata(content, clone);
}
/// <summary>
/// Clones a <see cref="FunctionResultContent"/> with an agent-owned call ID.
/// </summary>
private static FunctionResultContent CloneFunctionResultContent(FunctionResultContent content, string callId)
{
FunctionResultContent clone = new(callId, content.Result)
{
Exception = content.Exception,
};
return CopyContentMetadata(content, clone);
}
/// <summary>
/// Clones a <see cref="ToolApprovalRequestContent"/> with a workflow-facing request ID.
/// </summary>
private static ToolApprovalRequestContent CloneToolApprovalRequestContent(ToolApprovalRequestContent content, string id)
{
ToolApprovalRequestContent clone = new(id, content.ToolCall);
return CopyContentMetadata(content, clone);
}
/// <summary>
/// Clones a <see cref="ToolApprovalResponseContent"/> with an agent-owned request ID.
/// </summary>
private static ToolApprovalResponseContent CloneToolApprovalResponseContent(ToolApprovalResponseContent content, string id)
{
ToolApprovalResponseContent clone = new(id, content.Approved, content.ToolCall)
{
Reason = content.Reason,
};
return CopyContentMetadata(content, clone);
}
/// <summary>
/// Copies shared <see cref="AIContent"/> metadata to a cloned content instance.
/// </summary>
private static TContent CopyContentMetadata<TContent>(AIContent source, TContent target)
where TContent : AIContent
{
target.AdditionalProperties = source.AdditionalProperties;
target.Annotations = source.Annotations;
target.RawRepresentation = source.RawRepresentation;
return target;
}
internal sealed class SessionState(
string sessionId,
CheckpointInfo? lastCheckpoint,
InMemoryCheckpointManager? checkpointManager = null,
AgentSessionStateBag? stateBag = null)
AgentSessionStateBag? stateBag = null,
Dictionary<string, ExternalRequest>? pendingRequests = null)
{
public string SessionId { get; } = sessionId;
public CheckpointInfo? LastCheckpoint { get; } = lastCheckpoint;
public InMemoryCheckpointManager? CheckpointManager { get; } = checkpointManager;
public AgentSessionStateBag StateBag { get; } = stateBag ?? new();
public Dictionary<string, ExternalRequest>? PendingRequests { get; } = pendingRequests;
}
}
@@ -7,7 +7,6 @@ using System.Linq.Expressions;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.AI;
using Microsoft.Extensions.Compliance.Redaction;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.VectorData;
using Microsoft.Shared.Diagnostics;
@@ -81,7 +80,7 @@ public sealed class ChatHistoryMemoryProvider : MessageAIContextProvider, IDispo
private readonly VectorStoreCollection<object, Dictionary<string, object?>> _collection;
private readonly int _maxResults;
private readonly string _contextPrompt;
private readonly Redactor _redactor;
private readonly bool _enableSensitiveTelemetryData;
private readonly ChatHistoryMemoryProviderOptions.SearchBehavior _searchTime;
private readonly string _toolName;
private readonly string _toolDescription;
@@ -119,7 +118,7 @@ public sealed class ChatHistoryMemoryProvider : MessageAIContextProvider, IDispo
options ??= new ChatHistoryMemoryProviderOptions();
this._maxResults = options.MaxResults.HasValue ? Throw.IfLessThanOrEqual(options.MaxResults.Value, 0) : DefaultMaxResults;
this._contextPrompt = options.ContextPrompt ?? DefaultContextPrompt;
this._redactor = options.EnableSensitiveTelemetryData ? NullRedactor.Instance : (options.Redactor ?? new ReplacingRedactor("<redacted>"));
this._enableSensitiveTelemetryData = options.EnableSensitiveTelemetryData;
this._searchTime = options.SearchTime;
this._logger = loggerFactory?.CreateLogger<ChatHistoryMemoryProvider>();
this._toolName = options.FunctionToolName ?? DefaultFunctionToolName;
@@ -486,7 +485,7 @@ public sealed class ChatHistoryMemoryProvider : MessageAIContextProvider, IDispo
GC.SuppressFinalize(this);
}
private string SanitizeLogData(string? data) => this._redactor.Redact(data);
private string? SanitizeLogData(string? data) => this._enableSensitiveTelemetryData ? data : "<redacted>";
/// <summary>
/// Rebinds a filter expression's body to use the specified shared parameter,
@@ -3,7 +3,6 @@
using System;
using System.Collections.Generic;
using Microsoft.Extensions.AI;
using Microsoft.Extensions.Compliance.Redaction;
namespace Microsoft.Agents.AI;
@@ -47,22 +46,8 @@ public sealed class ChatHistoryMemoryProviderOptions
/// Gets or sets a value indicating whether sensitive data such as user ids and user messages may appear in logs.
/// </summary>
/// <value>Defaults to <see langword="false"/>.</value>
/// <remarks>
/// When set to <see langword="true"/>, sensitive data is passed through to logs unchanged and any
/// configured <see cref="Redactor"/> is ignored. This property takes precedence over <see cref="Redactor"/>.
/// </remarks>
public bool EnableSensitiveTelemetryData { get; set; }
/// <summary>
/// Gets or sets a custom <see cref="Redactor"/> used to redact sensitive data in log output.
/// </summary>
/// <value>
/// When <see langword="null"/> (the default), sensitive data is replaced with a placeholder.
/// When set, this redactor is used to transform sensitive values before they are logged.
/// Ignored when <see cref="EnableSensitiveTelemetryData"/> is <see langword="true"/>.
/// </value>
public Redactor? Redactor { get; set; }
/// <summary>
/// Gets or sets the key used to store provider state in the <see cref="AgentSession.StateBag"/>.
/// </summary>
@@ -8,7 +8,6 @@
<PropertyGroup>
<InjectSharedThrow>true</InjectSharedThrow>
<InjectSharedDiagnosticIds>true</InjectSharedDiagnosticIds>
<InjectSharedRedaction>true</InjectSharedRedaction>
<InjectDiagnosticClassesOnLegacy>true</InjectDiagnosticClassesOnLegacy>
<InjectExperimentalAttributeOnLegacy>true</InjectExperimentalAttributeOnLegacy>
<InjectTrimAttributesOnLegacy>true</InjectTrimAttributesOnLegacy>
@@ -23,7 +22,6 @@
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.AI" />
<PackageReference Include="Microsoft.Extensions.Compliance.Abstractions" />
<PackageReference Include="Microsoft.Extensions.VectorData.Abstractions" />
<PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" />
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" />
@@ -7,7 +7,6 @@ using System.Text;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.AI;
using Microsoft.Extensions.Compliance.Redaction;
using Microsoft.Extensions.Logging;
using Microsoft.Shared.Diagnostics;
@@ -63,7 +62,6 @@ public sealed class TextSearchProvider : MessageAIContextProvider
private readonly string _contextPrompt;
private readonly string _citationsPrompt;
private readonly Func<IList<TextSearchResult>, string>? _contextFormatter;
private readonly Redactor _redactor;
/// <summary>
/// Initializes a new instance of the <see cref="TextSearchProvider"/> class.
@@ -91,7 +89,6 @@ public sealed class TextSearchProvider : MessageAIContextProvider
this._contextPrompt = options?.ContextPrompt ?? DefaultContextPrompt;
this._citationsPrompt = options?.CitationsPrompt ?? DefaultCitationsPrompt;
this._contextFormatter = options?.ContextFormatter;
this._redactor = options?.EnableSensitiveTelemetryData == true ? NullRedactor.Instance : (options?.Redactor ?? new ReplacingRedactor("<redacted>"));
// Create the on-demand search tool (only used if behavior is OnDemandFunctionCalling)
this._tools =
@@ -183,7 +180,7 @@ public sealed class TextSearchProvider : MessageAIContextProvider
if (this._logger?.IsEnabled(LogLevel.Trace) is true)
{
this._logger.LogTrace("TextSearchProvider: Search Results\nInput:{Input}\nOutput:{MessageText}", this.SanitizeLogData(input), this.SanitizeLogData(formatted));
this._logger.LogTrace("TextSearchProvider: Search Results\nInput:{Input}\nOutput:{MessageText}", input, formatted);
}
return [new ChatMessage(ChatRole.User, formatted)];
@@ -252,7 +249,7 @@ public sealed class TextSearchProvider : MessageAIContextProvider
if (this._logger.IsEnabled(LogLevel.Trace))
{
this._logger.LogTrace("TextSearchProvider Input:{UserQuestion}\nOutput:{MessageText}", this.SanitizeLogData(userQuestion), this.SanitizeLogData(outputText));
this._logger.LogTrace("TextSearchProvider Input:{UserQuestion}\nOutput:{MessageText}", userQuestion, outputText);
}
}
@@ -328,8 +325,6 @@ public sealed class TextSearchProvider : MessageAIContextProvider
public object? RawRepresentation { get; set; }
}
private string SanitizeLogData(string? data) => this._redactor.Redact(data);
/// <summary>
/// Represents the per-session state of a <see cref="TextSearchProvider"/> stored in the <see cref="AgentSession.StateBag"/>.
/// </summary>
@@ -3,7 +3,6 @@
using System;
using System.Collections.Generic;
using Microsoft.Extensions.AI;
using Microsoft.Extensions.Compliance.Redaction;
namespace Microsoft.Agents.AI;
@@ -118,26 +117,6 @@ public sealed class TextSearchProviderOptions
/// </value>
public List<ChatRole>? RecentMessageRolesIncluded { get; set; }
/// <summary>
/// Gets or sets a value indicating whether sensitive data such as user queries and search results may appear in logs.
/// </summary>
/// <value>Defaults to <see langword="false"/>.</value>
/// <remarks>
/// When set to <see langword="true"/>, sensitive data is passed through to logs unchanged and any
/// configured <see cref="Redactor"/> is ignored. This property takes precedence over <see cref="Redactor"/>.
/// </remarks>
public bool EnableSensitiveTelemetryData { get; set; }
/// <summary>
/// Gets or sets a custom <see cref="Redactor"/> used to redact sensitive data in log output.
/// </summary>
/// <value>
/// When <see langword="null"/> (the default), sensitive data is replaced with a placeholder.
/// When set, this redactor is used to transform sensitive values before they are logged.
/// Ignored when <see cref="EnableSensitiveTelemetryData"/> is <see langword="true"/>.
/// </value>
public Redactor? Redactor { get; set; }
/// <summary>
/// Behavior choices for the provider.
/// </summary>
-30
View File
@@ -1,30 +0,0 @@
# Redaction
Log data redaction utilities built on `Microsoft.Extensions.Compliance.Redaction.Redactor`.
Provides `ReplacingRedactor`, an internal `Redactor` implementation that replaces
any input with a fixed replacement string (e.g. `"<redacted>"`).
To use this in your project, add the following to your `.csproj` file:
```xml
<PropertyGroup>
<InjectSharedRedaction>true</InjectSharedRedaction>
</PropertyGroup>
```
You will also need to add a package reference to `Microsoft.Extensions.Compliance.Abstractions`:
```xml
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Compliance.Abstractions" />
</ItemGroup>
```
And finally, this also depends on the shared Throw class, so when using redaction, InjectSharedThrow should also be enabled:
```xml
<PropertyGroup>
<InjectSharedThrow>true</InjectSharedThrow>
</PropertyGroup>
```
@@ -1,35 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
using Microsoft.Extensions.Compliance.Redaction;
using Microsoft.Shared.Diagnostics;
namespace Microsoft.Agents.AI;
/// <summary>
/// A <see cref="Redactor"/> that replaces the entire input with a fixed replacement string.
/// </summary>
internal sealed class ReplacingRedactor : Redactor
{
private readonly string _replacementText;
/// <summary>
/// Initializes a new instance of the <see cref="ReplacingRedactor"/> class.
/// </summary>
/// <param name="replacementText">The text to substitute for any input value.</param>
/// <exception cref="ArgumentNullException">Thrown when <paramref name="replacementText"/> is <see langword="null"/>.</exception>
public ReplacingRedactor(string replacementText)
{
this._replacementText = Throw.IfNull(replacementText);
}
/// <inheritdoc />
public override int GetRedactedLength(ReadOnlySpan<char> input) => this._replacementText.Length;
/// <inheritdoc />
public override int Redact(ReadOnlySpan<char> source, Span<char> destination)
{
this._replacementText.AsSpan().CopyTo(destination);
return this._replacementText.Length;
}
}
@@ -4,7 +4,10 @@ using AgentConformance.IntegrationTests;
namespace AzureAIAgentsPersistent.IntegrationTests;
[Trait("Category", "Integration")]
// Disabled: Azure.AI.Agents.Persistent 1.2.0-beta.9 references McpServerToolApprovalResponseContent
// which was removed in ME.AI 10.4.0. Re-enable once Persistent targets ME.AI 10.4.0+ (expected in 1.2.0-beta.10).
// Tracking: https://github.com/microsoft/agent-framework/issues/4769
[Trait("Category", "IntegrationDisabled")]
public class AzureAIAgentsChatClientAgentRunStreamingTests() : ChatClientAgentRunStreamingTests<AzureAIAgentsPersistentFixture>(() => new())
{
}
@@ -4,7 +4,10 @@ using AgentConformance.IntegrationTests;
namespace AzureAIAgentsPersistent.IntegrationTests;
[Trait("Category", "Integration")]
// Disabled: Azure.AI.Agents.Persistent 1.2.0-beta.9 references McpServerToolApprovalResponseContent
// which was removed in ME.AI 10.4.0. Re-enable once Persistent targets ME.AI 10.4.0+ (expected in 1.2.0-beta.10).
// Tracking: https://github.com/microsoft/agent-framework/issues/4769
[Trait("Category", "IntegrationDisabled")]
public class AzureAIAgentsChatClientAgentRunTests() : ChatClientAgentRunTests<AzureAIAgentsPersistentFixture>(() => new())
{
}
@@ -14,7 +14,10 @@ using Shared.IntegrationTests;
namespace AzureAIAgentsPersistent.IntegrationTests;
[Trait("Category", "Integration")]
// Disabled: Azure.AI.Agents.Persistent 1.2.0-beta.9 references McpServerToolApprovalResponseContent
// which was removed in ME.AI 10.4.0. Re-enable once Persistent targets ME.AI 10.4.0+ (expected in 1.2.0-beta.10).
// Tracking: https://github.com/microsoft/agent-framework/issues/4769
[Trait("Category", "IntegrationDisabled")]
public class AzureAIAgentsPersistentCreateTests
{
private const string SkipCodeInterpreterReason = "Azure AI Code Interpreter intermittently fails to execute uploaded files in CI";
@@ -4,7 +4,10 @@ using AgentConformance.IntegrationTests;
namespace AzureAIAgentsPersistent.IntegrationTests;
[Trait("Category", "Integration")]
// Disabled: Azure.AI.Agents.Persistent 1.2.0-beta.9 references McpServerToolApprovalResponseContent
// which was removed in ME.AI 10.4.0. Re-enable once Persistent targets ME.AI 10.4.0+ (expected in 1.2.0-beta.10).
// Tracking: https://github.com/microsoft/agent-framework/issues/4769
[Trait("Category", "IntegrationDisabled")]
public class AzureAIAgentsPersistentRunStreamingTests() : RunStreamingTests<AzureAIAgentsPersistentFixture>(() => new())
{
}
@@ -4,7 +4,10 @@ using AgentConformance.IntegrationTests;
namespace AzureAIAgentsPersistent.IntegrationTests;
[Trait("Category", "Integration")]
// Disabled: Azure.AI.Agents.Persistent 1.2.0-beta.9 references McpServerToolApprovalResponseContent
// which was removed in ME.AI 10.4.0. Re-enable once Persistent targets ME.AI 10.4.0+ (expected in 1.2.0-beta.10).
// Tracking: https://github.com/microsoft/agent-framework/issues/4769
[Trait("Category", "IntegrationDisabled")]
public class AzureAIAgentsPersistentRunTests() : RunTests<AzureAIAgentsPersistentFixture>(() => new())
{
}
@@ -5,7 +5,10 @@ using AgentConformance.IntegrationTests;
namespace AzureAIAgentsPersistent.IntegrationTests;
[Trait("Category", "Integration")]
// Disabled: Azure.AI.Agents.Persistent 1.2.0-beta.9 references McpServerToolApprovalResponseContent
// which was removed in ME.AI 10.4.0. Re-enable once Persistent targets ME.AI 10.4.0+ (expected in 1.2.0-beta.10).
// Tracking: https://github.com/microsoft/agent-framework/issues/4769
[Trait("Category", "IntegrationDisabled")]
public class AzureAIAgentsPersistentStructuredOutputRunTests() : StructuredOutputRunTests<AzureAIAgentsPersistentFixture>(() => new())
{
private const string SkipReason = "Fails intermittently on the build agent/CI";
@@ -148,15 +148,11 @@ public sealed class Mem0ProviderTests : IDisposable
}
[Theory]
[InlineData(false, false, false, 4)]
[InlineData(false, false, true, 4)]
[InlineData(true, false, false, 4)]
[InlineData(true, false, true, 4)]
[InlineData(false, true, false, 2)]
[InlineData(false, true, true, 2)]
[InlineData(true, true, false, 2)]
[InlineData(true, true, true, 2)]
public async Task InvokingAsync_RedactsLogDataBasedOnOptionsAsync(bool enableSensitiveTelemetryData, bool requestThrows, bool useCustomRedactor, int expectedLogInvocations)
[InlineData(false, false, 4)]
[InlineData(true, false, 4)]
[InlineData(false, true, 2)]
[InlineData(true, true, 2)]
public async Task InvokingAsync_LogsUserIdBasedOnEnableSensitiveTelemetryDataAsync(bool enableSensitiveTelemetryData, bool requestThrows, int expectedLogInvocations)
{
// Arrange
if (requestThrows)
@@ -175,11 +171,7 @@ public sealed class Mem0ProviderTests : IDisposable
ThreadId = "session",
UserId = "user"
};
var options = new Mem0ProviderOptions
{
EnableSensitiveTelemetryData = enableSensitiveTelemetryData,
Redactor = useCustomRedactor ? new ReplacingRedactor("***") : null
};
var options = new Mem0ProviderOptions { EnableSensitiveTelemetryData = enableSensitiveTelemetryData };
var mockSession = new TestAgentSession();
var sut = new Mem0Provider(this._httpClient, _ => new Mem0Provider.State(storageScope), options: options, loggerFactory: this._loggerFactoryMock.Object);
@@ -188,8 +180,7 @@ public sealed class Mem0ProviderTests : IDisposable
// Act
await sut.InvokingAsync(invokingContext, CancellationToken.None);
// Assert — EnableSensitiveTelemetryData takes precedence over Redactor
string expectedRedaction = enableSensitiveTelemetryData ? "user" : (useCustomRedactor ? "***" : "<redacted>");
// Assert
Assert.Equal(expectedLogInvocations, this._loggerMock.Invocations.Count);
foreach (var logInvocation in this._loggerMock.Invocations)
{
@@ -200,18 +191,18 @@ public sealed class Mem0ProviderTests : IDisposable
var state = Assert.IsType<IReadOnlyList<KeyValuePair<string, object?>>>(logInvocation.Arguments[2], exactMatch: false);
var userIdValue = state.First(kvp => kvp.Key == "UserId").Value;
Assert.Equal(expectedRedaction, userIdValue);
Assert.Equal(enableSensitiveTelemetryData ? "user" : "<redacted>", userIdValue);
var inputValue = state.FirstOrDefault(kvp => kvp.Key == "Input").Value;
if (inputValue != null)
{
Assert.Equal(enableSensitiveTelemetryData ? "Who am I?" : expectedRedaction, inputValue);
Assert.Equal(enableSensitiveTelemetryData ? "Who am I?" : "<redacted>", inputValue);
}
var messageTextValue = state.FirstOrDefault(kvp => kvp.Key == "MessageText").Value;
if (messageTextValue != null)
{
Assert.Equal(enableSensitiveTelemetryData ? "## Memories\nConsider the following memories when answering user questions:\nName is Caoimhe" : expectedRedaction, messageTextValue);
Assert.Equal(enableSensitiveTelemetryData ? "## Memories\nConsider the following memories when answering user questions:\nName is Caoimhe" : "<redacted>", messageTextValue);
}
}
}
@@ -85,8 +85,7 @@ public sealed class TextSearchProviderTests
{
SearchTime = TextSearchProviderOptions.TextSearchBehavior.BeforeAIInvoke,
ContextPrompt = overrideContextPrompt,
CitationsPrompt = overrideCitationsPrompt,
EnableSensitiveTelemetryData = true
CitationsPrompt = overrideCitationsPrompt
};
var provider = new TextSearchProvider(SearchDelegateAsync, options, withLogging ? this._loggerFactoryMock.Object : null);
@@ -165,65 +164,6 @@ public sealed class TextSearchProviderTests
}
}
[Theory]
[InlineData(false, false)]
[InlineData(false, true)]
[InlineData(true, false)]
[InlineData(true, true)]
public async Task InvokingAsync_RedactsLogDataBasedOnOptionsAsync(bool enableSensitiveTelemetryData, bool useCustomRedactor)
{
// Arrange
List<TextSearchProvider.TextSearchResult> results =
[
new() { SourceName = "Doc1", SourceLink = "http://example.com/doc1", Text = "Content of Doc1" }
];
Task<IEnumerable<TextSearchProvider.TextSearchResult>> SearchDelegateAsync(string input, CancellationToken ct)
{
return Task.FromResult<IEnumerable<TextSearchProvider.TextSearchResult>>(results);
}
var options = new TextSearchProviderOptions
{
SearchTime = TextSearchProviderOptions.TextSearchBehavior.BeforeAIInvoke,
EnableSensitiveTelemetryData = enableSensitiveTelemetryData,
Redactor = useCustomRedactor ? new ReplacingRedactor("***") : null
};
var provider = new TextSearchProvider(SearchDelegateAsync, options, this._loggerFactoryMock.Object);
var invokingContext = new AIContextProvider.InvokingContext(
s_mockAgent,
new TestAgentSession(),
new AIContext { Messages = new List<ChatMessage> { new(ChatRole.User, "Sample user question?") } });
// Act
await provider.InvokingAsync(invokingContext, CancellationToken.None);
// Assert — EnableSensitiveTelemetryData takes precedence over Redactor
var traceInvocation = this._loggerMock.Invocations
.Where(i => i.Method.Name == nameof(ILogger.Log))
.FirstOrDefault(i => (LogLevel)i.Arguments[0]! == LogLevel.Trace);
Assert.NotNull(traceInvocation);
var state = Assert.IsType<IReadOnlyList<KeyValuePair<string, object?>>>(traceInvocation.Arguments[2], exactMatch: false);
var inputValue = state.First(kvp => kvp.Key == "Input").Value;
var messageTextValue = state.First(kvp => kvp.Key == "MessageText").Value;
if (enableSensitiveTelemetryData)
{
// EnableSensitiveTelemetryData=true: raw data passes through regardless of Redactor
Assert.Equal("Sample user question?", inputValue);
Assert.Contains("Content of Doc1", messageTextValue?.ToString()!);
}
else
{
// EnableSensitiveTelemetryData=false: custom redactor or default placeholder
string expectedRedaction = useCustomRedactor ? "***" : "<redacted>";
Assert.Equal(expectedRedaction, inputValue);
Assert.Equal(expectedRedaction, messageTextValue);
}
}
[Theory]
[InlineData(null, null, "Search", "Allows searching for additional information to help answer the user question.")]
[InlineData("CustomSearch", "CustomDescription", "CustomSearch", "CustomDescription")]
@@ -270,21 +270,16 @@ public class ChatHistoryMemoryProviderTests
}
[Theory]
[InlineData(false, false, false, 0)]
[InlineData(false, false, true, 0)]
[InlineData(true, false, false, 0)]
[InlineData(true, false, true, 0)]
[InlineData(false, true, false, 2)]
[InlineData(false, true, true, 2)]
[InlineData(true, true, false, 2)]
[InlineData(true, true, true, 2)]
public async Task InvokedAsync_RedactsLogDataBasedOnOptionsAsync(bool enableSensitiveTelemetryData, bool requestThrows, bool useCustomRedactor, int expectedLogInvocations)
[InlineData(false, false, 0)]
[InlineData(true, false, 0)]
[InlineData(false, true, 2)]
[InlineData(true, true, 2)]
public async Task InvokedAsync_LogsUserIdBasedOnEnableSensitiveTelemetryDataAsync(bool enableSensitiveTelemetryData, bool requestThrows, int expectedLogInvocations)
{
// Arrange
var options = new ChatHistoryMemoryProviderOptions
{
EnableSensitiveTelemetryData = enableSensitiveTelemetryData,
Redactor = useCustomRedactor ? new ReplacingRedactor("***") : null
EnableSensitiveTelemetryData = enableSensitiveTelemetryData
};
if (requestThrows)
@@ -314,7 +309,7 @@ public class ChatHistoryMemoryProviderTests
// Act
await provider.InvokedAsync(invokedContext, CancellationToken.None);
// Assert — EnableSensitiveTelemetryData takes precedence over Redactor
// Assert
Assert.Equal(expectedLogInvocations, this._loggerMock.Invocations.Count);
foreach (var logInvocation in this._loggerMock.Invocations)
{
@@ -325,8 +320,7 @@ public class ChatHistoryMemoryProviderTests
var state = Assert.IsType<IReadOnlyList<KeyValuePair<string, object?>>>(logInvocation.Arguments[2], exactMatch: false);
var userIdValue = state.First(kvp => kvp.Key == "UserId").Value;
string expectedRedaction = enableSensitiveTelemetryData ? "user1" : (useCustomRedactor ? "***" : "<redacted>");
Assert.Equal(expectedRedaction, userIdValue);
Assert.Equal(enableSensitiveTelemetryData ? "user1" : "<redacted>", userIdValue);
}
}
@@ -532,22 +526,17 @@ public class ChatHistoryMemoryProviderTests
}
[Theory]
[InlineData(false, false, false, 2)]
[InlineData(false, false, true, 2)]
[InlineData(true, false, false, 2)]
[InlineData(true, false, true, 2)]
[InlineData(false, true, false, 2)]
[InlineData(false, true, true, 2)]
[InlineData(true, true, false, 2)]
[InlineData(true, true, true, 2)]
public async Task InvokingAsync_RedactsLogDataBasedOnOptionsAsync(bool enableSensitiveTelemetryData, bool requestThrows, bool useCustomRedactor, int expectedLogInvocations)
[InlineData(false, false, 2)]
[InlineData(true, false, 2)]
[InlineData(false, true, 2)]
[InlineData(true, true, 2)]
public async Task InvokingAsync_LogsUserIdBasedOnEnableSensitiveTelemetryDataAsync(bool enableSensitiveTelemetryData, bool requestThrows, int expectedLogInvocations)
{
// Arrange
var options = new ChatHistoryMemoryProviderOptions
{
SearchTime = ChatHistoryMemoryProviderOptions.SearchBehavior.BeforeAIInvoke,
EnableSensitiveTelemetryData = enableSensitiveTelemetryData,
Redactor = useCustomRedactor ? new ReplacingRedactor("***") : null
EnableSensitiveTelemetryData = enableSensitiveTelemetryData
};
var scope = new ChatHistoryMemoryProviderScope
@@ -589,8 +578,7 @@ public class ChatHistoryMemoryProviderTests
// Act
await provider.InvokingAsync(invokingContext, CancellationToken.None);
// Assert — EnableSensitiveTelemetryData takes precedence over Redactor
string expectedRedaction = enableSensitiveTelemetryData ? "user1" : (useCustomRedactor ? "***" : "<redacted>");
// Assert
Assert.Equal(expectedLogInvocations, this._loggerMock.Invocations.Count);
foreach (var logInvocation in this._loggerMock.Invocations)
{
@@ -601,18 +589,18 @@ public class ChatHistoryMemoryProviderTests
var state = Assert.IsType<IReadOnlyList<KeyValuePair<string, object?>>>(logInvocation.Arguments[2], exactMatch: false);
var userIdValue = state.First(kvp => kvp.Key == "UserId").Value;
Assert.Equal(expectedRedaction, userIdValue);
Assert.Equal(enableSensitiveTelemetryData ? "user1" : "<redacted>", userIdValue);
var inputValue = state.FirstOrDefault(kvp => kvp.Key == "Input").Value;
if (inputValue != null)
{
Assert.Equal(enableSensitiveTelemetryData ? "Who am I?" : expectedRedaction, inputValue);
Assert.Equal(enableSensitiveTelemetryData ? "Who am I?" : "<redacted>", inputValue);
}
var messageTextValue = state.FirstOrDefault(kvp => kvp.Key == "MessageText").Value;
if (messageTextValue != null)
{
Assert.Equal(enableSensitiveTelemetryData ? "## Memories\nConsider the following memories when answering user questions:\nName is Caoimhe" : expectedRedaction, messageTextValue);
Assert.Equal(enableSensitiveTelemetryData ? "## Memories\nConsider the following memories when answering user questions:\nName is Caoimhe" : "<redacted>", messageTextValue);
}
}
}
@@ -673,6 +673,55 @@ public class JsonSerializationTests
ValidateCheckpoint(retrievedCheckpoint, prototype);
}
[Fact]
public void Test_SessionState_JsonRoundtrip_WithPendingRequests()
{
// Arrange
Dictionary<string, ExternalRequest> pendingRequests = new()
{
["call-1"] = TestExternalRequest,
["call-2"] = ExternalRequest.Create(TestPort, "Request2", "OtherData"),
};
WorkflowSession.SessionState prototype = new(
sessionId: "test-session-123",
lastCheckpoint: TestParentCheckpointInfo,
pendingRequests: pendingRequests);
// Act
WorkflowSession.SessionState result = RunJsonRoundtrip(prototype);
// Assert
result.SessionId.Should().Be(prototype.SessionId);
result.LastCheckpoint.Should().Be(prototype.LastCheckpoint);
result.StateBag.Should().NotBeNull();
result.PendingRequests.Should().NotBeNull()
.And.HaveCount(pendingRequests.Count);
foreach (string key in pendingRequests.Keys)
{
result.PendingRequests.Should().ContainKey(key);
ValidateExternalRequest(result.PendingRequests![key], pendingRequests[key]);
}
}
[Fact]
public void Test_SessionState_JsonRoundtrip_WithoutPendingRequests()
{
// Arrange
WorkflowSession.SessionState prototype = new(
sessionId: "test-session-456",
lastCheckpoint: null);
// Act
WorkflowSession.SessionState result = RunJsonRoundtrip(prototype);
// Assert
result.SessionId.Should().Be(prototype.SessionId);
result.LastCheckpoint.Should().BeNull();
result.PendingRequests.Should().BeNull();
}
/// <summary>
/// Verifies that the default behavior (without AllowOutOfOrderMetadataProperties) fails
/// when $type metadata is not the first property, demonstrating the PostgreSQL jsonb issue.
@@ -28,6 +28,184 @@ public sealed class ExpectedException : Exception
}
}
/// <summary>
/// A simple agent that emits a FunctionCallContent or ToolApprovalRequestContent request.
/// Used to test that RequestInfoEvent handling preserves the original content type.
/// </summary>
internal sealed class RequestEmittingAgent : AIAgent
{
private readonly AIContent _requestContent;
private readonly bool _completeOnResponse;
/// <summary>
/// Creates a new <see cref="RequestEmittingAgent"/> that emits the given request content.
/// </summary>
/// <param name="requestContent">The content to emit on each turn.</param>
/// <param name="completeOnResponse">
/// When <see langword="true"/>, the agent emits a text completion instead of re-emitting
/// the request when the incoming messages contain a <see cref="FunctionResultContent"/>
/// or <see cref="ToolApprovalResponseContent"/>. This models realistic agent behaviour
/// where the agent processes the tool result and produces a final answer.
/// </param>
public RequestEmittingAgent(AIContent requestContent, bool completeOnResponse = false)
{
this._requestContent = requestContent;
this._completeOnResponse = completeOnResponse;
}
private sealed class Session : AgentSession
{
public Session() { }
}
protected override ValueTask<AgentSession> DeserializeSessionCoreAsync(JsonElement serializedState, JsonSerializerOptions? jsonSerializerOptions = null, CancellationToken cancellationToken = default)
=> new(new Session());
protected override ValueTask<AgentSession> CreateSessionCoreAsync(CancellationToken cancellationToken = default)
=> new(new Session());
protected override ValueTask<JsonElement> SerializeSessionCoreAsync(AgentSession session, JsonSerializerOptions? jsonSerializerOptions = null, CancellationToken cancellationToken = default)
=> default;
protected override Task<AgentResponse> RunCoreAsync(IEnumerable<ChatMessage> messages, AgentSession? session = null, AgentRunOptions? options = null, CancellationToken cancellationToken = default)
=> this.RunStreamingAsync(messages, session, options, cancellationToken).ToAgentResponseAsync(cancellationToken);
protected override async IAsyncEnumerable<AgentResponseUpdate> RunCoreStreamingAsync(IEnumerable<ChatMessage> messages, AgentSession? session = null, AgentRunOptions? options = null, [EnumeratorCancellation] CancellationToken cancellationToken = default)
{
if (this._completeOnResponse && messages.Any(m => m.Contents.Any(c =>
c is FunctionResultContent || c is ToolApprovalResponseContent)))
{
yield return new AgentResponseUpdate(ChatRole.Assistant, [new TextContent("Request processed")]);
}
else
{
// Emit the request content
yield return new AgentResponseUpdate(ChatRole.Assistant, [this._requestContent]);
}
}
}
internal sealed class KickoffOnStartExecutor : ChatProtocolExecutor
{
private static readonly ChatProtocolExecutorOptions s_options = new()
{
AutoSendTurnToken = false,
};
private readonly string _downstreamExecutorId;
private readonly string _kickoffInputText;
private readonly string _kickoffMessageText;
private readonly string _regularResumeText;
private readonly string _regularProcessedText;
public KickoffOnStartExecutor(
string id,
string downstreamExecutorId,
string kickoffInputText,
string kickoffMessageText,
string regularResumeText,
string regularProcessedText)
: base(id, s_options)
{
this._downstreamExecutorId = downstreamExecutorId;
this._kickoffInputText = kickoffInputText;
this._kickoffMessageText = kickoffMessageText;
this._regularResumeText = regularResumeText;
this._regularProcessedText = regularProcessedText;
}
protected override async ValueTask TakeTurnAsync(List<ChatMessage> messages, IWorkflowContext context, bool? emitEvents, CancellationToken cancellationToken = default)
{
List<string> textContents =
[
.. messages
.SelectMany(message => message.Contents.OfType<TextContent>())
.Select(content => content.Text)
];
if (textContents.Contains(this._kickoffInputText, StringComparer.Ordinal))
{
await context.SendMessageAsync(
new List<ChatMessage> { new(ChatRole.User, this._kickoffMessageText) },
this._downstreamExecutorId,
cancellationToken).ConfigureAwait(false);
await context.SendMessageAsync(
new TurnToken(emitEvents),
this._downstreamExecutorId,
cancellationToken).ConfigureAwait(false);
}
if (textContents.Contains(this._regularResumeText, StringComparer.Ordinal))
{
AgentResponseUpdate update = new(ChatRole.Assistant, [new TextContent(this._regularProcessedText)])
{
CreatedAt = DateTimeOffset.UtcNow,
MessageId = Guid.NewGuid().ToString("N"),
ResponseId = Guid.NewGuid().ToString("N"),
Role = ChatRole.Assistant,
};
await context.AddEventAsync(new AgentResponseUpdateEvent(this.Id, update), cancellationToken).ConfigureAwait(false);
}
}
}
/// <summary>
/// A start executor that always emits a response update on every turn,
/// useful for verifying that a TurnToken was delivered by the session.
/// On the first turn (user messages present), it kicks off a downstream executor.
/// </summary>
internal sealed class TurnTrackingStartExecutor : ChatProtocolExecutor
{
private static readonly ChatProtocolExecutorOptions s_options = new()
{
AutoSendTurnToken = false,
};
private readonly string _downstreamExecutorId;
private readonly string _activatedMarker;
private int _activationCount;
/// <summary>Gets the number of times this executor has been activated (i.e., <see cref="TakeTurnAsync"/> called).</summary>
public int ActivationCount => this._activationCount;
public TurnTrackingStartExecutor(string id, string downstreamExecutorId, string activatedMarker)
: base(id, s_options)
{
this._downstreamExecutorId = downstreamExecutorId;
this._activatedMarker = activatedMarker;
}
protected override async ValueTask TakeTurnAsync(List<ChatMessage> messages, IWorkflowContext context, bool? emitEvents, CancellationToken cancellationToken = default)
{
Interlocked.Increment(ref this._activationCount);
// On the first turn, forward user messages and a TurnToken to the downstream executor.
if (messages.Any(m => m.Role == ChatRole.User))
{
await context.SendMessageAsync(
messages,
this._downstreamExecutorId,
cancellationToken).ConfigureAwait(false);
await context.SendMessageAsync(
new TurnToken(emitEvents),
this._downstreamExecutorId,
cancellationToken).ConfigureAwait(false);
}
// Always emit a marker to prove this executor was activated.
AgentResponseUpdate update = new(ChatRole.Assistant, [new TextContent(this._activatedMarker)])
{
CreatedAt = DateTimeOffset.UtcNow,
MessageId = Guid.NewGuid().ToString("N"),
ResponseId = Guid.NewGuid().ToString("N"),
Role = ChatRole.Assistant,
};
await context.AddEventAsync(new AgentResponseUpdateEvent(this.Id, update), cancellationToken).ConfigureAwait(false);
}
}
public class WorkflowHostSmokeTests
{
private sealed class AlwaysFailsAIAgent(bool failByThrowing) : AIAgent
@@ -112,4 +290,445 @@ public class WorkflowHostSmokeTests
hadErrorContent.Should().BeTrue();
}
/// <summary>
/// Tests that when a workflow emits a RequestInfoEvent with FunctionCallContent data,
/// the AgentResponseUpdate preserves the original FunctionCallContent type.
/// </summary>
[Fact]
public async Task Test_AsAgent_FunctionCallContentPreservedInRequestInfoAsync()
{
// Arrange
const string CallId = "test-call-id";
const string FunctionName = "testFunction";
FunctionCallContent originalContent = new(CallId, FunctionName);
RequestEmittingAgent requestAgent = new(originalContent);
ExecutorBinding agentBinding = requestAgent.BindAsExecutor(
new AIAgentHostOptions { InterceptUnterminatedFunctionCalls = false, EmitAgentUpdateEvents = true });
Workflow workflow = new WorkflowBuilder(agentBinding).Build();
// Act
List<AgentResponseUpdate> updates = await workflow.AsAIAgent("WorkflowAgent")
.RunStreamingAsync(new ChatMessage(ChatRole.User, "Hello"))
.ToListAsync();
// Assert
AgentResponseUpdate? updateWithFunctionCall = updates.FirstOrDefault(u =>
u.RawRepresentation is RequestInfoEvent && u.Contents.Any(c => c is FunctionCallContent));
updateWithFunctionCall.Should().NotBeNull("a FunctionCallContent should be present in the response updates");
FunctionCallContent retrievedContent = updateWithFunctionCall!.Contents
.OfType<FunctionCallContent>()
.Should().ContainSingle()
.Which;
retrievedContent.CallId.Should().NotBe(CallId);
retrievedContent.CallId.Should().EndWith($":{CallId}");
retrievedContent.Name.Should().Be(FunctionName);
}
/// <summary>
/// Tests that when a workflow emits a RequestInfoEvent with ToolApprovalRequestContent data,
/// the AgentResponseUpdate preserves the original ToolApprovalRequestContent type.
/// </summary>
[Fact]
public async Task Test_AsAgent_ToolApprovalRequestContentPreservedInRequestInfoAsync()
{
// Arrange
const string RequestId = "test-request-id";
McpServerToolCallContent mcpCall = new("call-id", "testToolName", "http://localhost");
ToolApprovalRequestContent originalContent = new(RequestId, mcpCall);
RequestEmittingAgent requestAgent = new(originalContent);
ExecutorBinding agentBinding = requestAgent.BindAsExecutor(
new AIAgentHostOptions { InterceptUserInputRequests = false, EmitAgentUpdateEvents = true });
Workflow workflow = new WorkflowBuilder(agentBinding).Build();
// Act
List<AgentResponseUpdate> updates = await workflow.AsAIAgent("WorkflowAgent")
.RunStreamingAsync(new ChatMessage(ChatRole.User, "Hello"))
.ToListAsync();
// Assert
AgentResponseUpdate? updateWithUserInput = updates.FirstOrDefault(u =>
u.RawRepresentation is RequestInfoEvent && u.Contents.Any(c => c is ToolApprovalRequestContent));
updateWithUserInput.Should().NotBeNull("a ToolApprovalRequestContent should be present in the response updates");
ToolApprovalRequestContent retrievedContent = updateWithUserInput!.Contents
.OfType<ToolApprovalRequestContent>()
.Should().ContainSingle()
.Which;
retrievedContent.Should().NotBeNull();
retrievedContent.RequestId.Should().NotBe(RequestId);
retrievedContent.RequestId.Should().EndWith($":{RequestId}");
}
/// <summary>
/// Tests the full roundtrip: workflow emits a request, external caller responds, workflow processes response.
/// </summary>
[Fact]
public async Task Test_AsAgent_FunctionCallRoundtrip_ResponseIsProcessedAsync()
{
// Arrange: Create an agent that emits a FunctionCallContent request
const string CallId = "roundtrip-call-id";
const string FunctionName = "testFunction";
FunctionCallContent requestContent = new(CallId, FunctionName);
RequestEmittingAgent requestAgent = new(requestContent, completeOnResponse: true);
ExecutorBinding agentBinding = requestAgent.BindAsExecutor(
new AIAgentHostOptions { InterceptUnterminatedFunctionCalls = false, EmitAgentUpdateEvents = true });
Workflow workflow = new WorkflowBuilder(agentBinding).Build();
AIAgent agent = workflow.AsAIAgent("WorkflowAgent");
// Act 1: First call - should receive the FunctionCallContent request
AgentSession session = await agent.CreateSessionAsync();
List<AgentResponseUpdate> firstCallUpdates = await agent.RunStreamingAsync(
new ChatMessage(ChatRole.User, "Start"),
session).ToListAsync();
// Assert 1: We should have received a FunctionCallContent
AgentResponseUpdate? updateWithRequest = firstCallUpdates.FirstOrDefault(u =>
u.RawRepresentation is RequestInfoEvent && u.Contents.Any(c => c is FunctionCallContent));
updateWithRequest.Should().NotBeNull("a FunctionCallContent should be present in the response updates");
FunctionCallContent receivedRequest = updateWithRequest!.Contents
.OfType<FunctionCallContent>()
.First();
receivedRequest.CallId.Should().EndWith($":{CallId}");
// Act 2: Send the response back
FunctionResultContent responseContent = new(receivedRequest.CallId, "test result");
ChatMessage responseMessage = new(ChatRole.Tool, [responseContent]);
// Act 2: Run the workflow with the response and capture the resulting updates
List<AgentResponseUpdate> secondCallUpdates = await agent.RunStreamingAsync(responseMessage, session).ToListAsync();
// Assert 2: The response should be processed and the original request should no longer be pending.
// Concretely, the workflow should not re-emit a FunctionCallContent with the same CallId.
secondCallUpdates.Should().NotBeNull("processing the response should produce updates");
secondCallUpdates.Should().NotBeEmpty("processing the response should progress the workflow");
secondCallUpdates
.Where(u => u.RawRepresentation is RequestInfoEvent)
.SelectMany(u => u.Contents.OfType<FunctionCallContent>())
.Should()
.NotContain(c => c.CallId == receivedRequest.CallId, "the external FunctionCallContent request should be cleared after processing the response");
}
/// <summary>
/// Tests the full roundtrip for ToolApprovalRequestContent: workflow emits request, external caller responds.
/// Verifying inbound ToolApprovalResponseContent conversion.
/// </summary>
[Fact]
public async Task Test_AsAgent_ToolApprovalRoundtrip_ResponseIsProcessedAsync()
{
// Arrange: Create an agent that emits a ToolApprovalRequestContent request
const string RequestId = "roundtrip-request-id";
McpServerToolCallContent mcpCall = new("mcp-call-id", "testMcpTool", "http://localhost");
ToolApprovalRequestContent requestContent = new(RequestId, mcpCall);
RequestEmittingAgent requestAgent = new(requestContent, completeOnResponse: true);
ExecutorBinding agentBinding = requestAgent.BindAsExecutor(
new AIAgentHostOptions { InterceptUserInputRequests = false, EmitAgentUpdateEvents = true });
Workflow workflow = new WorkflowBuilder(agentBinding).Build();
AIAgent agent = workflow.AsAIAgent("WorkflowAgent");
// Act 1: First call - should receive the ToolApprovalRequestContent request
AgentSession session = await agent.CreateSessionAsync();
List<AgentResponseUpdate> firstCallUpdates = await agent.RunStreamingAsync(
new ChatMessage(ChatRole.User, "Start"),
session).ToListAsync();
// Assert 1: We should have received a ToolApprovalRequestContent
AgentResponseUpdate? updateWithRequest = firstCallUpdates.FirstOrDefault(u =>
u.RawRepresentation is RequestInfoEvent && u.Contents.Any(c => c is ToolApprovalRequestContent));
updateWithRequest.Should().NotBeNull("a ToolApprovalRequestContent should be present in the response updates");
ToolApprovalRequestContent receivedRequest = updateWithRequest!.Contents
.OfType<ToolApprovalRequestContent>()
.First();
receivedRequest.RequestId.Should().EndWith($":{RequestId}");
// Act 2: Send the response back - use CreateResponse to get the right response type
ToolApprovalResponseContent responseContent = receivedRequest.CreateResponse(approved: true);
ChatMessage responseMessage = new(ChatRole.User, [responseContent]);
// Act 2: Run the workflow again with the response and capture the updates
List<AgentResponseUpdate> secondCallUpdates = await agent.RunStreamingAsync(responseMessage, session).ToListAsync();
// Assert 2: The response should be applied so that the original request is no longer pending
secondCallUpdates.Should().NotBeEmpty("handling the user input response should produce follow-up updates");
bool requestStillPresent = secondCallUpdates.Any(u =>
u.RawRepresentation is RequestInfoEvent
&& u.Contents.OfType<ToolApprovalRequestContent>().Any(r => r.RequestId == receivedRequest.RequestId));
requestStillPresent.Should().BeFalse("the original ToolApprovalRequestContent should not be re-emitted after its response is processed");
}
/// <summary>
/// Tests the mixed-message scenario: resume contains both an external response
/// (FunctionResultContent matching a pending request) and regular non-response content
/// in the same message.
/// Verifies that regular content is still processed and that no duplicate
/// pending-request errors, redundant FunctionCallContent re-emissions,
/// or workflow errors occur.
/// </summary>
[Fact]
public async Task Test_AsAgent_MixedResponseAndRegularMessage_BothProcessedAsync()
{
// Arrange: Create an agent that emits a FunctionCallContent request
const string CallId = "mixed-call-id";
const string FunctionName = "mixedTestFunction";
FunctionCallContent requestContent = new(CallId, FunctionName);
RequestEmittingAgent requestAgent = new(requestContent, completeOnResponse: true);
ExecutorBinding agentBinding = requestAgent.BindAsExecutor(
new AIAgentHostOptions { InterceptUnterminatedFunctionCalls = false, EmitAgentUpdateEvents = true });
Workflow workflow = new WorkflowBuilder(agentBinding).Build();
AIAgent agent = workflow.AsAIAgent("WorkflowAgent");
// Act 1: First call - should receive the FunctionCallContent request
AgentSession session = await agent.CreateSessionAsync();
List<AgentResponseUpdate> firstCallUpdates = await agent.RunStreamingAsync(
new ChatMessage(ChatRole.User, "Start"),
session).ToListAsync();
// Assert 1: We should have received a FunctionCallContent
AgentResponseUpdate requestUpdate = firstCallUpdates.First(u =>
u.RawRepresentation is RequestInfoEvent && u.Contents.Any(c => c is FunctionCallContent));
FunctionCallContent emittedRequest = requestUpdate.Contents.OfType<FunctionCallContent>().Single();
firstCallUpdates.Should().Contain(u => u.Contents.Any(c => c is FunctionCallContent),
"the first call should emit a FunctionCallContent request");
// Act 2: Send a mixed message containing both the function result AND regular non-response content
FunctionResultContent responseContent = new(emittedRequest.CallId, "tool output");
ChatMessage mixedMessage = new(ChatRole.Tool, [responseContent, new TextContent("additional context")]);
List<AgentResponseUpdate> secondCallUpdates = await agent.RunStreamingAsync(mixedMessage, session).ToListAsync();
// Assert 2: The workflow should have processed both parts without errors
secondCallUpdates.Should().NotBeEmpty("the mixed message should produce follow-up updates");
secondCallUpdates
.Where(u => u.RawRepresentation is RequestInfoEvent)
.SelectMany(u => u.Contents.OfType<FunctionCallContent>())
.Should()
.NotContain(c => c.CallId == emittedRequest.CallId, "the external FunctionCallContent should be cleared after the response is processed");
secondCallUpdates
.SelectMany(u => u.Contents.OfType<ErrorContent>())
.Should()
.BeEmpty("no workflow errors should occur when processing a mixed response-and-regular message");
}
[Fact]
public async Task Test_AsAgent_ResponseThenRegularAcrossMessages_NoDuplicateFunctionCallAsync()
{
const string CallId = "mixed-separate-call-id";
const string FunctionName = "mixedSeparateTestFunction";
RequestEmittingAgent requestAgent = new(new FunctionCallContent(CallId, FunctionName), completeOnResponse: true);
ExecutorBinding agentBinding = requestAgent.BindAsExecutor(
new AIAgentHostOptions { InterceptUnterminatedFunctionCalls = false, EmitAgentUpdateEvents = true });
Workflow workflow = new WorkflowBuilder(agentBinding).Build();
AIAgent agent = workflow.AsAIAgent("WorkflowAgent");
AgentSession session = await agent.CreateSessionAsync();
List<AgentResponseUpdate> firstCallUpdates = await agent.RunStreamingAsync(new ChatMessage(ChatRole.User, "Start"), session).ToListAsync();
FunctionCallContent emittedRequest = firstCallUpdates
.Where(u => u.RawRepresentation is RequestInfoEvent)
.SelectMany(u => u.Contents.OfType<FunctionCallContent>())
.Single();
ChatMessage[] resumeMessages =
[
new(ChatRole.Tool, [new FunctionResultContent(emittedRequest.CallId, "tool output")]),
new(ChatRole.Tool, [new TextContent("extra context in separate message")])
];
List<AgentResponseUpdate> secondCallUpdates = await agent.RunStreamingAsync(resumeMessages, session).ToListAsync();
secondCallUpdates.Should().NotBeEmpty();
secondCallUpdates
.Where(u => u.RawRepresentation is RequestInfoEvent)
.SelectMany(u => u.Contents.OfType<FunctionCallContent>())
.Should()
.NotContain(c => c.CallId == emittedRequest.CallId, "response+regular content split across messages should not re-emit the handled external request");
secondCallUpdates
.SelectMany(u => u.Contents.OfType<ErrorContent>())
.Should()
.BeEmpty();
}
[Fact]
public async Task Test_AsAgent_MatchingResponse_DoesNotCauseExtraTurnAsync()
{
const string CallId = "matching-response-call-id";
const string FunctionName = "matchingResponseFunction";
RequestEmittingAgent requestAgent = new(new FunctionCallContent(CallId, FunctionName), completeOnResponse: false);
ExecutorBinding agentBinding = requestAgent.BindAsExecutor(
new AIAgentHostOptions { InterceptUnterminatedFunctionCalls = false, EmitAgentUpdateEvents = true });
Workflow workflow = new WorkflowBuilder(agentBinding).Build();
AIAgent agent = workflow.AsAIAgent("WorkflowAgent");
AgentSession session = await agent.CreateSessionAsync();
List<AgentResponseUpdate> firstCallUpdates = await agent.RunStreamingAsync(new ChatMessage(ChatRole.User, "Start"), session).ToListAsync();
FunctionCallContent emittedRequest = firstCallUpdates
.Where(u => u.RawRepresentation is RequestInfoEvent)
.SelectMany(u => u.Contents.OfType<FunctionCallContent>())
.Single();
List<AgentResponseUpdate> secondCallUpdates = await agent.RunStreamingAsync(
new ChatMessage(ChatRole.Tool, [new FunctionResultContent(emittedRequest.CallId, "tool output")]),
session).ToListAsync();
int functionCallCount = secondCallUpdates
.Where(u => u.RawRepresentation is RequestInfoEvent)
.SelectMany(u => u.Contents.OfType<FunctionCallContent>())
.Count(c => c.CallId == emittedRequest.CallId);
functionCallCount.Should().Be(1, "a matching external response should not trigger an extra TurnToken-driven turn");
}
[Fact]
public async Task Test_AsAgent_MixedResponseAndRegularMessage_CrossExecutorStartExecutorIsReawakenedAsync()
{
const string StartExecutorId = "start-executor";
const string KickoffInputText = "Start";
const string KickoffMessageText = "kickoff downstream";
const string ResumeRegularText = "resume regular";
const string ResumeProcessedText = "regular message processed";
const string CallId = "cross-executor-call-id";
const string FunctionName = "crossExecutorFunction";
RequestEmittingAgent requestAgent = new(new FunctionCallContent(CallId, FunctionName), completeOnResponse: true);
ExecutorBinding requestBinding = requestAgent.BindAsExecutor(
new AIAgentHostOptions { InterceptUnterminatedFunctionCalls = false, EmitAgentUpdateEvents = true });
KickoffOnStartExecutor startExecutor = new(
StartExecutorId,
requestBinding.Id,
KickoffInputText,
KickoffMessageText,
ResumeRegularText,
ResumeProcessedText);
ExecutorBinding startBinding = startExecutor.BindExecutor();
Workflow workflow = new WorkflowBuilder(startBinding)
.AddEdge<List<ChatMessage>>(startBinding, requestBinding, messages =>
messages?.Any(message => message.Contents.OfType<TextContent>().Any(content => content.Text == KickoffMessageText)) == true)
.AddEdge<TurnToken>(startBinding, requestBinding, _ => true)
.Build();
AIAgent agent = workflow.AsAIAgent("WorkflowAgent");
AgentSession session = await agent.CreateSessionAsync();
List<AgentResponseUpdate> firstCallUpdates = await agent.RunStreamingAsync(
new ChatMessage(ChatRole.User, KickoffInputText),
session).ToListAsync();
FunctionCallContent emittedRequest = firstCallUpdates
.Where(u => u.RawRepresentation is RequestInfoEvent)
.SelectMany(u => u.Contents.OfType<FunctionCallContent>())
.Single();
ChatMessage[] resumeMessages =
[
new(ChatRole.Tool, [new FunctionResultContent(emittedRequest.CallId, "tool output")]),
new(ChatRole.User, ResumeRegularText)
];
List<AgentResponseUpdate> secondCallUpdates = await agent.RunStreamingAsync(resumeMessages, session).ToListAsync();
List<string> textContents = [.. secondCallUpdates.SelectMany(update => update.Contents.OfType<TextContent>()).Select(content => content.Text)];
textContents.Should().Contain(ResumeProcessedText, "the start executor should receive an explicit TurnToken when the matched response wakes a different executor");
textContents.Should().Contain("Request processed", "the matched external response should still be delivered to the downstream request owner");
secondCallUpdates
.Where(u => u.RawRepresentation is RequestInfoEvent)
.SelectMany(u => u.Contents.OfType<FunctionCallContent>())
.Should()
.NotContain(c => c.CallId == emittedRequest.CallId, "the handled external request should not be re-emitted while waking the start executor");
secondCallUpdates.SelectMany(u => u.Contents.OfType<ErrorContent>()).Should().BeEmpty();
}
[Fact]
public async Task Test_AsAgent_UnmatchedResponse_TriggersTurnAndKeepsProgressingAsync()
{
const string CallId = "unmatched-response-call-id";
const string FunctionName = "unmatchedResponseFunction";
RequestEmittingAgent requestAgent = new(new FunctionCallContent(CallId, FunctionName), completeOnResponse: false);
ExecutorBinding agentBinding = requestAgent.BindAsExecutor(
new AIAgentHostOptions { InterceptUnterminatedFunctionCalls = false, EmitAgentUpdateEvents = true });
Workflow workflow = new WorkflowBuilder(agentBinding).Build();
AIAgent agent = workflow.AsAIAgent("WorkflowAgent");
AgentSession session = await agent.CreateSessionAsync();
List<AgentResponseUpdate> firstCallUpdates = await agent.RunStreamingAsync(new ChatMessage(ChatRole.User, "Start"), session).ToListAsync();
firstCallUpdates.Should().Contain(u => u.Contents.Any(c => c is FunctionCallContent));
List<AgentResponseUpdate> secondCallUpdates = await agent.RunStreamingAsync(
new ChatMessage(ChatRole.Tool, [new FunctionResultContent("different-call-id", "tool output")]),
session).ToListAsync();
int functionCallCount = secondCallUpdates
.SelectMany(u => u.Contents.OfType<FunctionCallContent>())
.Count(c => c.CallId == CallId);
functionCallCount.Should().Be(1, "an unmatched response should be treated as regular input and still drive a TurnToken continuation without workflow errors");
secondCallUpdates.SelectMany(u => u.Contents.OfType<ErrorContent>()).Should().BeEmpty();
}
/// <summary>
/// Tests that when a resume contains only an external response directed at a non-start executor
/// (no regular messages), the start executor still receives a TurnToken and is activated.
/// This is a regression test for the case where the TurnToken was previously skipped because
/// <c>HasRegularMessages</c> was <see langword="false"/>, leaving the start executor dormant.
/// </summary>
[Fact]
public async Task Test_AsAgent_ResponseOnlyToNonStartExecutor_StartExecutorIsStillActivatedAsync()
{
// Arrange
const string StartExecutorId = "start-executor";
const string ActivatedMarker = "start-executor-activated";
const string CallId = "response-only-call-id";
const string FunctionName = "responseOnlyFunction";
RequestEmittingAgent requestAgent = new(new FunctionCallContent(CallId, FunctionName), completeOnResponse: true);
ExecutorBinding requestBinding = requestAgent.BindAsExecutor(
new AIAgentHostOptions { InterceptUnterminatedFunctionCalls = false, EmitAgentUpdateEvents = true });
TurnTrackingStartExecutor startExecutor = new(StartExecutorId, requestBinding.Id, ActivatedMarker);
ExecutorBinding startBinding = startExecutor.BindExecutor();
Workflow workflow = new WorkflowBuilder(startBinding)
.AddEdge<List<ChatMessage>>(startBinding, requestBinding, messages =>
messages?.Any(m => m.Contents.OfType<TextContent>().Any()) == true)
.AddEdge<TurnToken>(startBinding, requestBinding, _ => true)
.Build();
AIAgent agent = workflow.AsAIAgent("WorkflowAgent");
// Act 1: First call triggers the downstream FunctionCallContent request
AgentSession session = await agent.CreateSessionAsync();
List<AgentResponseUpdate> firstCallUpdates = await agent.RunStreamingAsync(
new ChatMessage(ChatRole.User, "Start"),
session).ToListAsync();
FunctionCallContent emittedRequest = firstCallUpdates
.Where(u => u.RawRepresentation is RequestInfoEvent)
.SelectMany(u => u.Contents.OfType<FunctionCallContent>())
.Single();
// Act 2: Resume with ONLY the external response (no regular messages)
List<AgentResponseUpdate> secondCallUpdates = await agent.RunStreamingAsync(
new ChatMessage(ChatRole.Tool, [new FunctionResultContent(emittedRequest.CallId, "tool output")]),
session).ToListAsync();
// Assert: Both the downstream and start executor should have been activated
List<string> textContents = [.. secondCallUpdates
.SelectMany(u => u.Contents.OfType<TextContent>())
.Select(c => c.Text)];
textContents.Should().Contain("Request processed",
"the downstream executor should process the external response");
textContents.Should().Contain(ActivatedMarker,
"the start executor should receive a TurnToken and be activated even when resume contains only an external response");
secondCallUpdates
.SelectMany(u => u.Contents.OfType<ErrorContent>())
.Should()
.BeEmpty();
}
}
@@ -466,15 +466,9 @@ class FunctionTool(SerializationMixin):
if func is None:
return create_model(f"{self.name}_input")
sig = inspect.signature(func)
try:
type_hints = typing.get_type_hints(func, include_extras=True)
except Exception:
type_hints = {}
fields: dict[str, Any] = {
pname: (
_parse_annotation(type_hints.get(pname, param.annotation))
if type_hints.get(pname, param.annotation) is not inspect.Parameter.empty
else str,
_parse_annotation(param.annotation) if param.annotation is not inspect.Parameter.empty else str,
param.default if param.default is not inspect.Parameter.empty else ...,
)
for pname, param in sig.parameters.items()
@@ -1,134 +0,0 @@
# Copyright (c) Microsoft. All rights reserved.
"""Tests for @tool with PEP 563 (from __future__ import annotations).
When ``from __future__ import annotations`` is active, all annotations
become strings. _resolve_input_model must resolve them via
typing.get_type_hints() before passing them to Pydantic's create_model.
"""
from __future__ import annotations
from pydantic import BaseModel
from agent_framework import tool
from agent_framework._middleware import FunctionInvocationContext
class SearchConfig(BaseModel):
max_results: int = 10
def test_tool_with_context_parameter():
"""FunctionInvocationContext parameter is excluded from schema under PEP 563."""
@tool
def get_weather(location: str, ctx: FunctionInvocationContext) -> str:
"""Get the weather for a given location."""
return f"Weather in {location}"
params = get_weather.parameters()
assert "ctx" not in params.get("properties", {})
assert "location" in params["properties"]
def test_tool_with_context_parameter_first():
"""FunctionInvocationContext as the first parameter is excluded under PEP 563."""
@tool
def get_weather(ctx: FunctionInvocationContext, location: str) -> str:
"""Get the weather for a given location."""
return f"Weather in {location}"
params = get_weather.parameters()
assert "ctx" not in params.get("properties", {})
assert "location" in params["properties"]
def test_tool_with_optional_param():
"""Optional[int] is resolved to the actual type, not left as a string."""
@tool
def search(query: str, limit: int | None = None) -> str:
"""Search for something."""
return query
params = search.parameters()
assert params["properties"]["query"]["type"] == "string"
limit_schema = params["properties"]["limit"]
limit_types = {t["type"] for t in limit_schema["anyOf"]}
assert limit_types == {"integer", "null"}
def test_tool_with_optional_param_and_context():
"""Optional param + FunctionInvocationContext both work under PEP 563."""
@tool
def search(query: str, limit: int | None = None, ctx: FunctionInvocationContext | None = None) -> str:
"""Search for something."""
return query
params = search.parameters()
assert params["properties"]["query"]["type"] == "string"
limit_schema = params["properties"]["limit"]
limit_types = {t["type"] for t in limit_schema["anyOf"]}
assert limit_types == {"integer", "null"}
assert "ctx" not in params.get("properties", {})
def test_tool_with_optional_custom_type():
"""Optional[CustomType] is resolved under PEP 563 (original bug pattern)."""
@tool
def search(query: str, config: SearchConfig | None = None) -> str:
"""Search for something."""
return query
params = search.parameters()
assert params["properties"]["query"]["type"] == "string"
config_schema = params["properties"]["config"]
config_types = [t.get("type") for t in config_schema["anyOf"]]
assert "null" in config_types
def test_tool_with_unresolvable_forward_ref():
"""Fallback to raw annotations when get_type_hints() fails."""
import types
# Build a function in an isolated namespace so get_type_hints() cannot resolve
# the forward reference, exercising the except-branch fallback.
ns: dict = {}
exec(
"def greet(name: str = 'world') -> str:\n '''Greet someone.'''\n return f'Hello {name}'\n",
ns,
)
func = ns["greet"]
# Place the function in a throwaway module so get_type_hints() will fail on
# any non-builtin forward ref while still having a valid __module__.
mod = types.ModuleType("_phantom")
func.__module__ = mod.__name__
t = tool(func)
params = t.parameters()
assert params["properties"]["name"]["type"] == "string"
async def test_tool_invoke_with_context():
"""Full invocation with FunctionInvocationContext under PEP 563."""
@tool
def get_weather(location: str, ctx: FunctionInvocationContext) -> str:
"""Get the weather for a given location."""
user = ctx.kwargs.get("user", "anon")
return f"Weather in {location} for {user}"
params = get_weather.parameters()
assert "ctx" not in params.get("properties", {})
context = FunctionInvocationContext(
function=get_weather,
arguments=get_weather.input_model(location="Seattle"),
kwargs={"user": "test_user"},
)
result = await get_weather.invoke(context=context)
assert result[0].text == "Weather in Seattle for test_user"
@@ -5,7 +5,7 @@ import os
from random import randint
from typing import Annotated, Any, Literal
from agent_framework import Message, SupportsChatGetResponse, tool
from agent_framework import SupportsChatGetResponse, tool
from agent_framework.azure import (
AzureAIAgentClient,
AzureOpenAIAssistantsClient,
@@ -117,37 +117,35 @@ async def main(client_name: ClientName = "openai_chat") -> None:
client = get_client(client_name)
# 1. Configure prompt and streaming mode.
message = Message("user", text="What's the weather in Amsterdam and in Paris?")
message = "What's the weather in Amsterdam and in Paris?"
stream = os.getenv("STREAM", "false").lower() == "true"
print(f"Client: {client_name}")
print(f"User: {message.text}")
print(f"User: {message}")
# 2. Run with context-managed clients.
if isinstance(client, OpenAIAssistantsClient | AzureOpenAIAssistantsClient | AzureAIAgentClient):
async with client:
if stream:
response_stream = client.get_response([message], stream=True, options={"tools": get_weather})
response_stream = client.get_response(message, stream=True, options={"tools": get_weather})
print("Assistant: ", end="")
async for chunk in response_stream:
if chunk.text:
print(chunk.text, end="")
print("")
else:
print(
f"Assistant: {await client.get_response([message], stream=False, options={'tools': get_weather})}"
)
print(f"Assistant: {await client.get_response(message, stream=False, options={'tools': get_weather})}")
return
# 3. Run with non-context-managed clients.
if stream:
response_stream = client.get_response([message], stream=True, options={"tools": get_weather})
response_stream = client.get_response(message, stream=True, options={"tools": get_weather})
print("Assistant: ", end="")
async for chunk in response_stream:
if chunk.text:
print(chunk.text, end="")
print("")
else:
print(f"Assistant: {await client.get_response([message], stream=False, options={'tools': get_weather})}")
print(f"Assistant: {await client.get_response(message, stream=False, options={'tools': get_weather})}")
if __name__ == "__main__":
@@ -1,17 +1,25 @@
# /// script
# requires-python = ">=3.10"
# dependencies = [
# "autogen-agentchat",
# "autogen-ext[openai]",
# ]
# ///
# Run with any PEP 723 compatible runner, e.g.:
# uv run samples/autogen-migration/orchestrations/01_round_robin_group_chat.py
# Copyright (c) Microsoft. All rights reserved.
import asyncio
from agent_framework import Message
from dotenv import load_dotenv
"""AutoGen RoundRobinGroupChat vs Agent Framework GroupChatBuilder/SequentialBuilder.
Demonstrates sequential agent orchestration where agents take turns processing
the task in a round-robin fashion.
"""
import asyncio
from agent_framework import Message
from dotenv import load_dotenv
# Load environment variables from .env file
load_dotenv()
@@ -90,7 +98,7 @@ async def run_agent_framework() -> None:
print("[Agent Framework] Sequential conversation:")
async for event in workflow.run("Create a brief summary about electric vehicles", stream=True):
if event.type == "output" and isinstance(event.data, list):
for message in event.data: # type: ignore
for message in event.data:
if isinstance(message, Message) and message.role == "assistant" and message.text:
print(f"---------- {message.author_name} ----------")
print(message.text)
@@ -136,7 +144,9 @@ async def run_agent_framework_with_cycle() -> None:
if last_message and "APPROVED" in last_message.text:
await context.yield_output("Content approved.")
else:
await context.send_message(AgentExecutorRequest(messages=response.full_conversation, should_respond=True))
await context.send_message(
AgentExecutorRequest(messages=response.full_conversation, should_respond=True)
)
workflow = (
WorkflowBuilder(start_executor=researcher)
@@ -1,17 +1,25 @@
# /// script
# requires-python = ">=3.10"
# dependencies = [
# "autogen-agentchat",
# "autogen-ext[openai]",
# ]
# ///
# Run with any PEP 723 compatible runner, e.g.:
# uv run samples/autogen-migration/orchestrations/02_selector_group_chat.py
# Copyright (c) Microsoft. All rights reserved.
import asyncio
from agent_framework import Message
from dotenv import load_dotenv
"""AutoGen SelectorGroupChat vs Agent Framework GroupChatBuilder.
Demonstrates LLM-based speaker selection where an orchestrator decides
which agent should speak next based on the conversation context.
"""
import asyncio
from agent_framework import Message
from dotenv import load_dotenv
# Load environment variables from .env file
load_dotenv()
@@ -105,7 +113,7 @@ async def run_agent_framework() -> None:
print("[Agent Framework] Group chat conversation:")
async for event in workflow.run("How do I connect to a PostgreSQL database using Python?", stream=True):
if event.type == "output" and isinstance(event.data, list):
for message in event.data: # type: ignore
for message in event.data:
if isinstance(message, Message) and message.role == "assistant" and message.text:
print(f"---------- {message.author_name} ----------")
print(message.text)
@@ -1,4 +1,19 @@
# /// script
# requires-python = ">=3.10"
# dependencies = [
# "autogen-agentchat",
# "autogen-ext[openai]",
# ]
# ///
# Run with any PEP 723 compatible runner, e.g.:
# uv run samples/autogen-migration/orchestrations/03_swarm.py
# Copyright (c) Microsoft. All rights reserved.
"""AutoGen Swarm pattern vs Agent Framework HandoffBuilder.
Demonstrates agent handoff coordination where agents can transfer control
to other specialized agents based on the task requirements.
"""
import asyncio
from typing import Any
@@ -6,12 +21,6 @@ from typing import Any
from agent_framework import AgentResponseUpdate, WorkflowEvent
from dotenv import load_dotenv
"""AutoGen Swarm pattern vs Agent Framework HandoffBuilder.
Demonstrates agent handoff coordination where agents can transfer control
to other specialized agents based on the task requirements.
"""
# Load environment variables from .env file
load_dotenv()
@@ -1,4 +1,19 @@
# /// script
# requires-python = ">=3.10"
# dependencies = [
# "autogen-agentchat",
# "autogen-ext[openai]",
# ]
# ///
# Run with any PEP 723 compatible runner, e.g.:
# uv run samples/autogen-migration/orchestrations/04_magentic_one.py
# Copyright (c) Microsoft. All rights reserved.
"""AutoGen MagenticOneGroupChat vs Agent Framework MagenticBuilder.
Demonstrates orchestrated multi-agent workflows with a central coordinator
managing specialized agents for complex tasks.
"""
import asyncio
import json
@@ -12,12 +27,6 @@ from agent_framework import (
from agent_framework.orchestrations import MagenticProgressLedger
from dotenv import load_dotenv
"""AutoGen MagenticOneGroupChat vs Agent Framework MagenticBuilder.
Demonstrates orchestrated multi-agent workflows with a central coordinator
managing specialized agents for complex tasks.
"""
# Load environment variables from .env file
load_dotenv()
@@ -1,9 +1,14 @@
# /// script
# requires-python = ">=3.10"
# dependencies = [
# "autogen-agentchat",
# "autogen-ext[openai]",
# ]
# ///
# Run with any PEP 723 compatible runner, e.g.:
# uv run samples/autogen-migration/single_agent/01_basic_assistant_agent.py
# Copyright (c) Microsoft. All rights reserved.
import asyncio
from dotenv import load_dotenv
"""Basic AutoGen AssistantAgent vs Agent Framework Agent.
Both samples expect OpenAI-compatible environment variables (OPENAI_API_KEY or
@@ -11,6 +16,10 @@ Azure OpenAI configuration). Update the prompts or client wiring to match your
model of choice before running.
"""
import asyncio
from dotenv import load_dotenv
# Load environment variables from .env file
load_dotenv()
@@ -1,14 +1,24 @@
# /// script
# requires-python = ">=3.10"
# dependencies = [
# "autogen-agentchat",
# "autogen-core",
# "autogen-ext[openai]",
# ]
# ///
# Run with any PEP 723 compatible runner, e.g.:
# uv run samples/autogen-migration/single_agent/02_assistant_agent_with_tool.py
# Copyright (c) Microsoft. All rights reserved.
import asyncio
from dotenv import load_dotenv
"""AutoGen AssistantAgent vs Agent Framework Agent with function tools.
Demonstrates how to create and attach tools to agents in both frameworks.
"""
import asyncio
from dotenv import load_dotenv
# Load environment variables from .env file
load_dotenv()
@@ -1,14 +1,23 @@
# /// script
# requires-python = ">=3.10"
# dependencies = [
# "autogen-agentchat",
# "autogen-ext[openai]",
# ]
# ///
# Run with any PEP 723 compatible runner, e.g.:
# uv run samples/autogen-migration/single_agent/03_assistant_agent_thread_and_stream.py
# Copyright (c) Microsoft. All rights reserved.
import asyncio
from dotenv import load_dotenv
"""AutoGen vs Agent Framework: Thread management and streaming responses.
Demonstrates conversation state management and streaming in both frameworks.
"""
import asyncio
from dotenv import load_dotenv
# Load environment variables from .env file
load_dotenv()
@@ -1,15 +1,24 @@
# /// script
# requires-python = ">=3.10"
# dependencies = [
# "autogen-agentchat",
# "autogen-ext[openai]",
# ]
# ///
# Run with any PEP 723 compatible runner, e.g.:
# uv run samples/autogen-migration/single_agent/04_agent_as_tool.py
# Copyright (c) Microsoft. All rights reserved.
import asyncio
from dotenv import load_dotenv
"""AutoGen vs Agent Framework: Agent-as-a-Tool pattern.
Demonstrates hierarchical agent architectures where one agent delegates
work to specialized sub-agents wrapped as tools.
"""
import asyncio
from dotenv import load_dotenv
# Load environment variables from .env file
load_dotenv()
@@ -98,7 +107,6 @@ async def run_agent_framework() -> None:
if content.type == "function_call":
# Accumulate function call content as it streams in
call_id = content.call_id
assert call_id is not None, "Function call content must have a call_id"
if call_id in accumulated_calls:
# Add to existing call (arguments stream in gradually)
accumulated_calls[call_id] = accumulated_calls[call_id] + content
+7 -6
View File
@@ -165,17 +165,18 @@ Produces:
## Report Status Codes
| Status | Label | Description |
| ------------- | --------------- | ----------------------------------------- |
| SUCCESS | [PASS] | Sample ran to completion with exit code 0 |
| FAILURE | [FAIL] | Sample did not complete successfully (non-zero exit code) |
| MISSING_SETUP | [MISSING_SETUP] | Sample skipped due to missing setup |
| Status | Label | Description |
| ------- | --------- | ----------------------------------------- |
| SUCCESS | [PASS] | Sample ran to completion with exit code 0 |
| FAILURE | [FAIL] | Sample exited with non-zero code |
| TIMEOUT | [TIMEOUT] | Sample exceeded timeout limit |
| ERROR | [ERROR] | Exception during execution |
## Troubleshooting
### Agent output parsing errors
If an agent returns non-JSON content, that sample is marked as `FAILURE` with parser details in the report.
If an agent returns non-JSON content, that sample is marked as `ERROR` with parser details in the report.
### GitHub Copilot authentication or CLI issues
+1 -9
View File
@@ -75,13 +75,6 @@ Examples:
help="Custom name for the report files (without extension). If not provided, uses timestamp.",
)
parser.add_argument(
"--exclude",
nargs="+",
type=str,
help="Subdirectory paths to exclude (relative to the search directory set by --subdir)",
)
return parser.parse_args()
@@ -111,7 +104,6 @@ async def main() -> int:
samples_dir=samples_dir,
python_root=python_root,
subdir=args.subdir,
exclude=args.exclude,
max_parallel_workers=max(1, args.max_parallel_workers),
)
@@ -146,7 +138,7 @@ async def main() -> int:
print(f" JSON: {json_path}")
# Return appropriate exit code
failed = report.failure_count + report.missing_setup_count
failed = report.failure_count + report.timeout_count + report.error_count
return 1 if failed > 0 else 0
@@ -1,224 +0,0 @@
# Copyright (c) Microsoft. All rights reserved.
"""Aggregate validation reports across runs and produce a trend report.
Reads JSON reports from individual validation jobs, combines them with
cached history from previous runs, and produces a markdown trend report
showing per-sample status over the last 5 runs.
Usage:
python aggregate.py <reports-dir> <history-file> <output-file>
"""
import json
import sys
from datetime import datetime
from pathlib import Path
from typing import Any
MAX_HISTORY = 5
STATUS_EMOJI = {
"success": "âś…",
"failure": "❌",
"missing_setup": "⚠️",
}
def _format_run_label(timestamp: str) -> str:
"""Format a run timestamp as a compact column label (e.g. '03-24 18:05')."""
try:
dt = datetime.fromisoformat(timestamp)
return dt.strftime("%m-%d %H:%M")
except (ValueError, TypeError):
return timestamp[:16]
def load_current_run(reports_dir: Path) -> dict[str, Any]:
"""Load all JSON report files from the current run and merge them."""
combined_results: dict[str, str] = {}
total = success = failure = missing = 0
json_files = sorted(reports_dir.glob("*.json"))
if not json_files:
print(f"Warning: No JSON report files found in {reports_dir}")
return {
"timestamp": datetime.now().isoformat(),
"summary": {
"total_samples": 0,
"success_count": 0,
"failure_count": 0,
"missing_setup_count": 0,
},
"results": {},
}
for json_file in json_files:
print(f" Loading report: {json_file.name}")
with open(json_file, encoding="utf-8") as f:
report = json.load(f)
for result in report["results"]:
combined_results[result["path"]] = result["status"]
summary = report["summary"]
total += summary["total_samples"]
success += summary["success_count"]
failure += summary["failure_count"]
missing += summary["missing_setup_count"]
return {
"timestamp": datetime.now().isoformat(),
"summary": {
"total_samples": total,
"success_count": success,
"failure_count": failure,
"missing_setup_count": missing,
},
"results": combined_results,
}
def load_history(history_path: Path) -> list[dict[str, Any]]:
"""Load previous run history from cache."""
if history_path.exists():
with open(history_path, encoding="utf-8") as f:
data = json.load(f)
runs = data.get("runs", [])
print(f" Loaded {len(runs)} previous run(s) from history")
return runs
print(" No previous history found")
return []
def save_history(history_path: Path, runs: list[dict[str, Any]]) -> None:
"""Save run history, keeping only the last MAX_HISTORY entries."""
history_path.parent.mkdir(parents=True, exist_ok=True)
trimmed = runs[-MAX_HISTORY:]
with open(history_path, "w", encoding="utf-8") as f:
json.dump({"runs": trimmed}, f, indent=2)
print(f" Saved {len(trimmed)} run(s) to history")
def generate_trend_report(runs: list[dict[str, Any]]) -> str:
"""Generate a markdown trend report from run history."""
lines = [
"# Sample Validation Trend Report",
"",
f"*Generated: {datetime.now().strftime('%Y-%m-%d %H:%M UTC')}*",
"",
]
# --- Overall status table (most recent first) ---
lines.append("## Overall Status (Last 5 Runs)")
lines.append("")
lines.append("| Run | Success | Failure | Missing Setup | Total |")
lines.append("|-----|---------|---------|---------------|-------|")
for run in reversed(runs):
s = run["summary"]
label = _format_run_label(run["timestamp"])
lines.append(
f"| {label} | {s['success_count']}/{s['total_samples']} "
f"| {s['failure_count']}/{s['total_samples']} "
f"| {s['missing_setup_count']}/{s['total_samples']} "
f"| {s['total_samples']} |"
)
# Pad with N/A rows if fewer than 5 runs
for _ in range(MAX_HISTORY - len(runs)):
lines.append("| N/A | N/A | N/A | N/A | N/A |")
lines.append("")
# --- Per-sample results table ---
lines.append("## Per-Sample Results")
lines.append("")
# Collect all sample paths across all runs
all_paths: set[str] = set()
for run in runs:
all_paths.update(run["results"].keys())
if not all_paths:
lines.append("*No sample results available.*")
return "\n".join(lines)
# Build header (most recent run first)
header = "| Sample |"
separator = "|--------|"
for run in reversed(runs):
label = _format_run_label(run["timestamp"])
header += f" {label} |"
separator += "------------|"
for _ in range(MAX_HISTORY - len(runs)):
header += " N/A |"
separator += "-----|"
lines.append(header)
lines.append(separator)
for path in sorted(all_paths):
row = f"| `{path}` |"
for run in reversed(runs):
status = run["results"].get(path, "N/A")
emoji = STATUS_EMOJI.get(status, "N/A")
row += f" {emoji} |"
for _ in range(MAX_HISTORY - len(runs)):
row += " N/A |"
lines.append(row)
lines.append("")
lines.append("**Legend:** ✅ Success · ❌ Failure · ⚠️ Missing Setup · N/A Not available")
lines.append("")
return "\n".join(lines)
def main() -> int:
if len(sys.argv) != 4:
print("Usage: python aggregate.py <reports-dir> <history-file> <output-file>")
return 1
reports_dir = Path(sys.argv[1])
history_path = Path(sys.argv[2])
output_path = Path(sys.argv[3])
print("Aggregating validation results...")
# Load current run's reports
print(f"\nLoading reports from {reports_dir}:")
current_run = load_current_run(reports_dir)
s = current_run["summary"]
print(
f" Current run: {s['success_count']} success, "
f"{s['failure_count']} failure, "
f"{s['missing_setup_count']} missing setup "
f"(total: {s['total_samples']})"
)
# Load history and append current run
print(f"\nLoading history from {history_path}:")
runs = load_history(history_path)
runs.append(current_run)
runs = runs[-MAX_HISTORY:]
# Save updated history
print(f"\nSaving history to {history_path}:")
save_history(history_path, runs)
# Generate trend report
print("\nGenerating trend report...")
report = generate_trend_report(runs)
output_path.parent.mkdir(parents=True, exist_ok=True)
output_path.write_text(report, encoding="utf-8")
print(f"Trend report written to {output_path}")
# Also print the report to stdout
print("\n" + "=" * 80)
print(report)
return 0
if __name__ == "__main__":
sys.exit(main())
@@ -14,8 +14,7 @@ from agent_framework import (
handler,
)
from agent_framework.github import GitHubCopilotAgent
from copilot.generated.session_events import PermissionRequest
from copilot.types import PermissionRequestResult
from copilot.types import PermissionRequest, PermissionRequestResult
from pydantic import BaseModel
from typing_extensions import Never
@@ -37,7 +36,6 @@ class AgentResponseFormat(BaseModel):
status: str
output: str
error: str
fix: str
@dataclass
@@ -56,20 +54,15 @@ class BatchCompletion:
AgentInstruction = (
"You are validating exactly one Python sample.\n"
"Analyze the sample code and execute it as it is. Based on the execution result, determine "
"if it runs successfully, fails, or is missing_setup. Use `missing_setup` if the sample reports "
"missing required environment variables. The environment you're given should contain the necessary "
"variables. Don't create new environment variables nor modify the sample code.\n"
"Feel free to install any required dependencies if needed.\n"
"Analyze the sample code and execute it. Based on the execution result, determine if it "
"runs successfully, fails, or times out. Feel free to install any required dependencies.\n"
"The sample can be interactive. If it is interactive, respond to the sample when prompted "
"based on your analysis of the code. You do not need to consult human on what to respond.\n"
"If the sample fails, investigate the error and suggest a fix.\n"
"Return ONLY valid JSON with this schema:\n"
"{\n"
' "status": "success|failure|missing_setup",\n'
' "status": "success|failure|timeout|error",\n'
' "output": "short summary of the result and what you did if the sample was interactive",\n'
' "error": "error details or empty string",\n'
' "fix": "suggested code fix if the sample failed, otherwise empty string"\n'
' "error": "error details or empty string"\n'
"}\n\n"
)
@@ -94,15 +87,16 @@ def status_from_text(value: str) -> RunStatus:
for status in RunStatus:
if status.value == normalized:
return status
return RunStatus.FAILURE
return RunStatus.ERROR
def prompt_permission(
request: PermissionRequest, context: dict[str, str]
) -> PermissionRequestResult:
"""Permission handler that always approves."""
kind = request.get("kind", "unknown")
logger.debug(
f"[Permission Request: {request.kind}] ({context})Automatically approved for sample validation."
f"[Permission Request: {kind}] ({context})Automatically approved for sample validation."
)
return PermissionRequestResult(kind="approved")
@@ -114,73 +108,39 @@ class CustomAgentExecutor(Executor):
returned as error responses, otherwise an exception in one agent could crash the entire workflow.
"""
# Retry in case GitHub Copilot agent encounters transient errors unrelated to the sample execution.
RETRY_COUNT = 1
def __init__(self, agent: GitHubCopilotAgent):
super().__init__(id=agent.id)
self.agent = agent
self._session = agent.create_session()
@handler
async def handle_task(
self, sample: SampleInfo, ctx: WorkflowContext[WorkerFreed | RunResult]
) -> None:
"""Execute one sample task and notify collector + coordinator."""
current_retry = 0
while True:
try:
response = await self.agent.run(
[
Message(
role="user",
text=f"Validate the following sample:\n\n{sample.relative_path}",
)
],
session=self._session,
)
result_payload = parse_agent_json(response.text)
result = RunResult(
sample=sample,
status=status_from_text(result_payload.status),
output=result_payload.output,
error=result_payload.error,
fix=result_payload.fix,
)
break
except Exception as ex:
if current_retry < self.RETRY_COUNT:
logger.warning(
f"Error executing agent {self.agent.id} (attempt {current_retry + 1}/{self.RETRY_COUNT}): {ex}. Retrying..."
try:
response = await self.agent.run(
[
Message(
role="user",
text=f"Validate the following sample:\n\n{sample.relative_path}",
)
try:
current_retry += 1
await self.agent.stop()
await self.agent.start()
self._session = self.agent.create_session() # Reset session for retry
continue
except Exception as restart_ex:
logger.error(
f"Error restarting agent {self.agent.id}: {restart_ex}. No more retries."
)
result = RunResult(
sample=sample,
status=RunStatus.FAILURE,
output="",
error=f"Original error: {ex}. Restart error: {restart_ex}",
fix="",
)
break
logger.error(f"Error executing agent {self.agent.id}: {ex}")
result = RunResult(
sample=sample,
status=RunStatus.FAILURE,
output="",
error=str(ex),
fix="",
)
break
]
)
result_payload = parse_agent_json(response.text)
result = RunResult(
sample=sample,
status=status_from_text(result_payload.status),
output=result_payload.output,
error=result_payload.error,
)
except Exception as ex:
logger.error(f"Error executing agent {self.agent.id}: {ex}")
result = RunResult(
sample=sample,
status=RunStatus.ERROR,
output="",
error=str(ex),
)
await ctx.send_message(result, target_id="collector")
await ctx.send_message(WorkerFreed(worker_id=self.id), target_id="coordinator")
@@ -292,7 +252,7 @@ class CreateConcurrentValidationWorkflowExecutor(Executor):
instructions=AgentInstruction,
default_options={
"on_permission_request": prompt_permission,
"timeout": 60,
"timeout": 180,
}, # type: ignore
)
agents.append(agent)
+4 -20
View File
@@ -52,18 +52,13 @@ def _has_main_entrypoint_guard(path: Path) -> bool:
)
def discover_samples(
samples_dir: Path,
subdir: str | None = None,
exclude: list[str] | None = None,
) -> list[SampleInfo]:
def discover_samples(samples_dir: Path, subdir: str | None = None) -> list[SampleInfo]:
"""
Find all Python sample files in the samples directory.
Args:
samples_dir: Root samples directory
subdir: Optional subdirectory to filter to
exclude: Optional list of subdirectory paths (relative to the search directory) to exclude
Returns:
List of SampleInfo objects for each discovered sample
@@ -77,21 +72,12 @@ def discover_samples(
else:
search_dir = samples_dir
# Resolve excluded paths to absolute for reliable comparison
exclude_paths = {(search_dir / exc).resolve() for exc in (exclude or [])}
python_files: list[Path] = []
# Walk through all subdirectories and find .py files
for root, dirs, files in os.walk(search_dir):
# Skip directories that start with _, __pycache__, or excluded paths
dirs[:] = [
d
for d in dirs
if not d.startswith("_")
and d != "__pycache__"
and (Path(root) / d).resolve() not in exclude_paths
]
# Skip directories that start with _ (like _sample_validation)
dirs[:] = [d for d in dirs if not d.startswith("_") and d != "__pycache__"]
for file in files:
# Skip files that start with _ and include only scripts with a main entrypoint guard
@@ -127,10 +113,8 @@ class DiscoverSamplesExecutor(Executor):
print(f"🔍 Discovering samples in {self.config.samples_dir}")
if self.config.subdir:
print(f" Filtering to subdirectory: {self.config.subdir}")
if self.config.exclude:
print(f" Excluding: {', '.join(self.config.exclude)}")
samples = discover_samples(self.config.samples_dir, self.config.subdir, self.config.exclude)
samples = discover_samples(self.config.samples_dir, self.config.subdir)
print(f" Found {len(samples)} samples")
await ctx.send_message(DiscoveryResult(samples=samples))
+11 -9
View File
@@ -18,7 +18,6 @@ class ValidationConfig:
samples_dir: Path
python_root: Path
subdir: str | None = None
exclude: list[str] | None = None
max_parallel_workers: int = 10
@@ -61,7 +60,8 @@ class RunStatus(Enum):
SUCCESS = "success"
FAILURE = "failure"
MISSING_SETUP = "missing_setup"
TIMEOUT = "timeout"
ERROR = "error"
@dataclass
@@ -72,7 +72,6 @@ class RunResult:
status: RunStatus
output: str
error: str
fix: str
@dataclass
@@ -90,7 +89,8 @@ class Report:
total_samples: int
success_count: int
failure_count: int
missing_setup_count: int
timeout_count: int
error_count: int
results: list[RunResult] = field(default_factory=list) # type: ignore
def to_markdown(self) -> str:
@@ -107,14 +107,15 @@ class Report:
f"| Total Samples | {self.total_samples} |",
f"| [PASS] Success | {self.success_count} |",
f"| [FAIL] Failure | {self.failure_count} |",
f"| [MISSING_SETUP] Missing Setup | {self.missing_setup_count} |",
f"| [TIMEOUT] Timeout | {self.timeout_count} |",
f"| [ERROR] Error | {self.error_count} |",
"",
"## Detailed Results",
"",
]
# Group by status
for status in [RunStatus.FAILURE, RunStatus.MISSING_SETUP, RunStatus.SUCCESS]:
for status in [RunStatus.FAILURE, RunStatus.TIMEOUT, RunStatus.ERROR, RunStatus.SUCCESS]:
status_results = [r for r in self.results if r.status == status]
if not status_results:
continue
@@ -122,7 +123,8 @@ class Report:
status_label = {
RunStatus.SUCCESS: "[PASS]",
RunStatus.FAILURE: "[FAIL]",
RunStatus.MISSING_SETUP: "[MISSING_SETUP]",
RunStatus.TIMEOUT: "[TIMEOUT]",
RunStatus.ERROR: "[ERROR]",
}
lines.append(f"### {status_label[status]} {status.value.title()} ({len(status_results)})")
@@ -146,7 +148,8 @@ class Report:
"total_samples": self.total_samples,
"success_count": self.success_count,
"failure_count": self.failure_count,
"missing_setup_count": self.missing_setup_count,
"timeout_count": self.timeout_count,
"error_count": self.error_count,
},
"results": [
{
@@ -154,7 +157,6 @@ class Report:
"status": r.status.value,
"output": r.output,
"error": r.error,
"fix": r.fix,
}
for r in self.results
],
+10 -6
View File
@@ -22,11 +22,12 @@ def generate_report(results: list[RunResult]) -> Report:
Returns:
Report object with aggregated statistics
"""
# Sort results: failures, missing setup first, then successes
# Sort results: failures, timeouts, errors first, then successes
status_priority = {
RunStatus.FAILURE: 0,
RunStatus.MISSING_SETUP: 1,
RunStatus.SUCCESS: 2,
RunStatus.TIMEOUT: 1,
RunStatus.ERROR: 2,
RunStatus.SUCCESS: 3,
}
sorted_results = sorted(results, key=lambda r: status_priority[r.status])
@@ -35,7 +36,8 @@ def generate_report(results: list[RunResult]) -> Report:
total_samples=len(results),
success_count=sum(1 for r in results if r.status == RunStatus.SUCCESS),
failure_count=sum(1 for r in results if r.status == RunStatus.FAILURE),
missing_setup_count=sum(1 for r in results if r.status == RunStatus.MISSING_SETUP),
timeout_count=sum(1 for r in results if r.status == RunStatus.TIMEOUT),
error_count=sum(1 for r in results if r.status == RunStatus.ERROR),
results=sorted_results,
)
@@ -84,7 +86,8 @@ def print_summary(report: Report) -> None:
if (
report.failure_count == 0
and report.missing_setup_count == 0
and report.timeout_count == 0
and report.error_count == 0
):
print("[PASS] ALL SAMPLES PASSED!")
else:
@@ -95,7 +98,8 @@ def print_summary(report: Report) -> None:
print("Results:")
print(f" [PASS] Success: {report.success_count}")
print(f" [FAIL] Failure: {report.failure_count}")
print(f" [MISSING_SETUP] Missing Setup: {report.missing_setup_count}")
print(f" [TIMEOUT] Timeout: {report.timeout_count}")
print(f" [ERR] Errors: {report.error_count}")
print("=" * 80)
# Print JSON output for GitHub Actions visibility
@@ -66,10 +66,9 @@ class RunDynamicValidationWorkflowExecutor(Executor):
fallback_results = [
RunResult(
sample=sample,
status=RunStatus.FAILURE,
status=RunStatus.ERROR,
output="",
error="Nested workflow did not return an ExecutionResult.",
fix="",
)
for sample in creation.samples
]