mirror of
https://github.com/microsoft/agent-framework.git
synced 2026-06-16 21:04:09 +08:00
.NET: CI hardening — split Functions tests, re-enable skipped integration tests (#5717)
* Split DurableTask/AzureFunctions integration tests into dedicated CI job - Add -TestProjectNameExclude parameter to New-FilteredSolution.ps1 - Add 'functions' and 'core' path filters to paths-filter job - Exclude DurableTask/AzureFunctions from main dotnet-test job - Remove emulator setup from dotnet-test (no longer needed) - Add new dotnet-test-functions job (ubuntu/net10.0 only, path-conditional) - Update merge gate and report job to include dotnet-test-functions Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Address PR feedback: add Workflows.Generators to core filter, drop dotnetChanges gate from functions job Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Re-enable Anthropic integration tests Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Upgrade Anthropic SDK 12.13.0 -> 12.20.0 to fix M.E.AI incompatibility Fixes MissingMethodException on WebSearchToolResultContent.get_Results() caused by Anthropic 12.13.0 being compiled against an older Microsoft.Extensions.AI.Abstractions version. Suppress RT0003 in AI.Abstractions.csproj as the transitive reference from the upgraded Anthropic SDK conflicts with the explicit one. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Fix Anthropic unit test mocks for SDK 12.20.0 interface changes Add missing interface members: IAnthropicClient.WebhookKey, IBetaService.MemoryStores, IBetaService.Webhooks, IBetaService.UserProfiles Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Re-enable CheckSystem declarative integration tests The CheckSystem.yaml tests were temporarily skipped in PR #4270 during the Azure.AI.Projects 2.0.0-beta.1 SDK update. Since then, the system variable plumbing (SystemScope, SetLastMessageAsync, conversation initialization) has been significantly updated and stabilized. The other tests in these same files pass reliably using the same infrastructure. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Fix CheckSystem test case to expect 1 response The CheckSystem workflow sends a 'PASSED!' SendActivity when all system variables are populated, producing 1 AgentResponseEvent. The test case had min_response_count: 0 with no max, so the assertion defaulted max to 0 and failed with 'Response count greater than expected: 0 (Actual: 1)'. Updated to expect exactly 1 response, matching the SendActivity pattern. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Re-enable Foundry OpenAPI server-side tool integration test Remove Skip="For manual testing only" from AsAIAgent_WithOpenAPITool_NativeSDKCreation_InvokesServerSideToolAsync. The test already uses RetryFact(3 retries, 5s delay) to handle transient failures from the external restcountries.com API. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Include workflow file in functions/core path filters A PR editing only dotnet-build-and-test.yml would skip dotnet-test-functions because the workflow path was missing from both the functions and core path filter lists. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Rename filter parameters for consistency TestProjectNameFilter -> TestProjectNameIncludeFilter TestProjectNameExclude -> TestProjectNameExcludeFilter Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Remove unnecessary RT0003 warning suppression The RT0003 suppression was added during the Anthropic SDK 12.20.0 upgrade but the warning no longer fires. Removing it to keep the NoWarn list minimal. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Remove duplicate WebhookKey properties from merge Both our branch and main added WebhookKey to the Anthropic test mock classes, resulting in CS0102 duplicate definition errors. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
Giles Odigwe
committed by
GitHub
Unverified
GitHub
parent
3b6a4574eb
commit
cfd3dfe40b
@@ -38,6 +38,8 @@ jobs:
|
||||
dotnetChanges: ${{ steps.filter.outputs.dotnet }}
|
||||
cosmosDbChanges: ${{ steps.filter.outputs.cosmosdb }}
|
||||
foundryHostingChanges: ${{ steps.filter.outputs.foundryHosting }}
|
||||
functionsChanged: ${{ steps.filter.outputs.functions }}
|
||||
coreChanged: ${{ steps.filter.outputs.core }}
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: dorny/paths-filter@v3
|
||||
@@ -64,6 +66,24 @@ jobs:
|
||||
- 'dotnet/Directory.Packages.props'
|
||||
- 'dotnet/tests/Foundry.Hosting.IntegrationTests/scripts/it-build-image.ps1'
|
||||
- '.github/workflows/dotnet-build-and-test.yml'
|
||||
functions:
|
||||
- 'dotnet/src/Microsoft.Agents.AI.DurableTask/**'
|
||||
- 'dotnet/src/Microsoft.Agents.AI.Hosting.AzureFunctions/**'
|
||||
- 'dotnet/tests/Microsoft.Agents.AI.DurableTask.IntegrationTests/**'
|
||||
- 'dotnet/tests/Microsoft.Agents.AI.Hosting.AzureFunctions.IntegrationTests/**'
|
||||
- '.github/actions/azure-functions-integration-setup/**'
|
||||
- '.github/workflows/dotnet-build-and-test.yml'
|
||||
core:
|
||||
- 'dotnet/src/Microsoft.Agents.AI/**'
|
||||
- 'dotnet/src/Microsoft.Agents.AI.Abstractions/**'
|
||||
- 'dotnet/src/Microsoft.Agents.AI.OpenAI/**'
|
||||
- 'dotnet/src/Microsoft.Agents.AI.Workflows/**'
|
||||
- 'dotnet/src/Microsoft.Agents.AI.Workflows.Generators/**'
|
||||
- 'dotnet/eng/scripts/New-FilteredSolution.ps1'
|
||||
- 'dotnet/tests/Directory.Build.props'
|
||||
- 'dotnet/Directory.Packages.props'
|
||||
- 'dotnet/global.json'
|
||||
- '.github/workflows/dotnet-build-and-test.yml'
|
||||
# run only if 'dotnet' files were changed
|
||||
- name: dotnet tests
|
||||
if: steps.filter.outputs.dotnet == 'true'
|
||||
@@ -211,10 +231,11 @@ jobs:
|
||||
Verbose = $true
|
||||
}
|
||||
./dotnet/eng/scripts/New-FilteredSolution.ps1 @commonArgs `
|
||||
-TestProjectNameFilter "*UnitTests*" `
|
||||
-TestProjectNameIncludeFilter "*UnitTests*" `
|
||||
-OutputPath dotnet/filtered-unit.slnx
|
||||
./dotnet/eng/scripts/New-FilteredSolution.ps1 @commonArgs `
|
||||
-TestProjectNameFilter "*IntegrationTests*" `
|
||||
-TestProjectNameIncludeFilter "*IntegrationTests*" `
|
||||
-TestProjectNameExcludeFilter "*DurableTask.IntegrationTests*","*AzureFunctions.IntegrationTests*" `
|
||||
-OutputPath dotnet/filtered-integration.slnx
|
||||
|
||||
- name: Run Unit Tests
|
||||
@@ -256,14 +277,6 @@ jobs:
|
||||
tenant-id: ${{ secrets.AZURE_TENANT_ID }}
|
||||
subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
|
||||
|
||||
# This setup action is required for both Durable Task and Azure Functions integration tests.
|
||||
# We only run it on Ubuntu since the Durable Task and Azure Functions features are not available
|
||||
# on .NET Framework (net472) which is what we use the Windows runner for.
|
||||
- name: Set up Durable Task and Azure Functions Integration Test Emulators
|
||||
if: github.event_name != 'pull_request' && matrix.integration-tests && matrix.os == 'ubuntu-latest'
|
||||
uses: ./.github/actions/azure-functions-integration-setup
|
||||
id: azure-functions-setup
|
||||
|
||||
- name: Run Integration Tests
|
||||
shell: pwsh
|
||||
working-directory: dotnet
|
||||
@@ -416,11 +429,110 @@ jobs:
|
||||
AZURE_SEARCH_INDEX_NAME: ${{ secrets.AZURE_SEARCH_INDEX_NAME }}
|
||||
# IT_HOSTED_AGENT_IMAGE was exported into $GITHUB_ENV by the previous step.
|
||||
|
||||
# DurableTask and AzureFunctions integration tests (ubuntu/net10.0 only).
|
||||
# Split from main dotnet-test job for path-based filtering and parallelism.
|
||||
dotnet-test-functions:
|
||||
needs: [paths-filter]
|
||||
if: >
|
||||
github.event_name != 'pull_request' &&
|
||||
(needs.paths-filter.outputs.functionsChanged == 'true' ||
|
||||
needs.paths-filter.outputs.coreChanged == 'true' ||
|
||||
github.event_name == 'schedule' ||
|
||||
github.event_name == 'workflow_dispatch')
|
||||
runs-on: ubuntu-latest
|
||||
environment: integration
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
persist-credentials: false
|
||||
sparse-checkout: |
|
||||
.
|
||||
.github
|
||||
dotnet
|
||||
python
|
||||
declarative-agents
|
||||
|
||||
- name: Setup dotnet
|
||||
uses: actions/setup-dotnet@v5.2.0
|
||||
with:
|
||||
global-json-file: ${{ github.workspace }}/dotnet/global.json
|
||||
|
||||
- name: Build functions integration test projects
|
||||
shell: bash
|
||||
working-directory: dotnet
|
||||
run: |
|
||||
dotnet build ./tests/Microsoft.Agents.AI.DurableTask.IntegrationTests -c Release -f net10.0 --warnaserror
|
||||
dotnet build ./tests/Microsoft.Agents.AI.Hosting.AzureFunctions.IntegrationTests -c Release -f net10.0 --warnaserror
|
||||
|
||||
- name: Azure CLI Login
|
||||
uses: azure/login@v2
|
||||
with:
|
||||
client-id: ${{ secrets.AZURE_CLIENT_ID }}
|
||||
tenant-id: ${{ secrets.AZURE_TENANT_ID }}
|
||||
subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
|
||||
|
||||
- name: Set up Durable Task and Azure Functions Integration Test Emulators
|
||||
uses: ./.github/actions/azure-functions-integration-setup
|
||||
id: azure-functions-setup
|
||||
|
||||
- name: Run Functions Integration Tests
|
||||
shell: pwsh
|
||||
working-directory: dotnet
|
||||
run: |
|
||||
# Run DurableTask integration tests
|
||||
dotnet test `
|
||||
--project ./tests/Microsoft.Agents.AI.DurableTask.IntegrationTests `
|
||||
-f net10.0 `
|
||||
-c Release `
|
||||
--no-build -v Normal `
|
||||
--report-xunit-trx `
|
||||
--report-junit `
|
||||
--results-directory ../IntegrationTestResults/ `
|
||||
--ignore-exit-code 8 `
|
||||
--filter-not-trait "Category=IntegrationDisabled" `
|
||||
--parallel-algorithm aggressive `
|
||||
--max-threads 2.0x
|
||||
|
||||
# Run AzureFunctions integration tests
|
||||
dotnet test `
|
||||
--project ./tests/Microsoft.Agents.AI.Hosting.AzureFunctions.IntegrationTests `
|
||||
-f net10.0 `
|
||||
-c Release `
|
||||
--no-build -v Normal `
|
||||
--report-xunit-trx `
|
||||
--report-junit `
|
||||
--results-directory ../IntegrationTestResults/ `
|
||||
--ignore-exit-code 8 `
|
||||
--filter-not-trait "Category=IntegrationDisabled" `
|
||||
--parallel-algorithm aggressive `
|
||||
--max-threads 2.0x
|
||||
env:
|
||||
# OpenAI Models
|
||||
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
||||
OPENAI_CHAT_MODEL_NAME: ${{ vars.OPENAI_CHAT_MODEL_NAME }}
|
||||
OPENAI_REASONING_MODEL_NAME: ${{ vars.OPENAI_REASONING_MODEL_NAME }}
|
||||
# Azure OpenAI Models
|
||||
AZURE_OPENAI_DEPLOYMENT_NAME: ${{ vars.AZURE_OPENAI_DEPLOYMENT_NAME }}
|
||||
AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ vars.AZURE_OPENAI_DEPLOYMENT_NAME }}
|
||||
AZURE_OPENAI_ENDPOINT: ${{ vars.AZURE_OPENAI_ENDPOINT }}
|
||||
# Azure AI Foundry
|
||||
AZURE_AI_PROJECT_ENDPOINT: ${{ vars.AZURE_AI_PROJECT_ENDPOINT }}
|
||||
AZURE_AI_MODEL_DEPLOYMENT_NAME: ${{ vars.AZURE_AI_MODEL_DEPLOYMENT_NAME }}
|
||||
AZURE_AI_BING_CONNECTION_ID: ${{ vars.AZURE_AI_BING_CONNECTION_ID }}
|
||||
|
||||
- name: Upload functions test results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: dotnet-test-results-functions-net10.0-ubuntu-latest
|
||||
path: IntegrationTestResults/**/*.junit
|
||||
if-no-files-found: ignore
|
||||
|
||||
# This final job is required to satisfy the merge queue. It must only run (or succeed) if no tests failed
|
||||
dotnet-build-and-test-check:
|
||||
if: always()
|
||||
runs-on: ubuntu-latest
|
||||
needs: [dotnet-build, dotnet-test, dotnet-foundry-hosted-it]
|
||||
needs: [dotnet-build, dotnet-test, dotnet-foundry-hosted-it, dotnet-test-functions]
|
||||
steps:
|
||||
- name: Get Date
|
||||
shell: bash
|
||||
@@ -467,7 +579,7 @@ jobs:
|
||||
github.event_name != 'pull_request' &&
|
||||
(contains(join(needs.*.result, ','), 'success') ||
|
||||
contains(join(needs.*.result, ','), 'failure'))
|
||||
needs: [dotnet-test]
|
||||
needs: [dotnet-test, dotnet-test-functions]
|
||||
runs-on: ubuntu-latest
|
||||
defaults:
|
||||
run:
|
||||
|
||||
@@ -21,10 +21,15 @@
|
||||
.PARAMETER Configuration
|
||||
Optional MSBuild configuration used when querying TargetFrameworks. Defaults to Debug.
|
||||
|
||||
.PARAMETER TestProjectNameFilter
|
||||
.PARAMETER TestProjectNameIncludeFilter
|
||||
Optional wildcard pattern to filter test project names (e.g., *UnitTests*, *IntegrationTests*).
|
||||
When specified, only test projects whose filename matches this pattern are kept.
|
||||
|
||||
.PARAMETER TestProjectNameExcludeFilter
|
||||
Optional wildcard pattern(s) to exclude test projects by name (e.g., *DurableTask.IntegrationTests*).
|
||||
When specified, test projects whose filename matches any of these patterns are removed.
|
||||
Applied after TestProjectNameIncludeFilter. Can be a single string or an array of strings.
|
||||
|
||||
.PARAMETER ExcludeSamples
|
||||
When specified, removes all projects under the samples/ directory from the solution.
|
||||
|
||||
@@ -38,11 +43,15 @@
|
||||
|
||||
.EXAMPLE
|
||||
# Generate a solution with only unit test projects
|
||||
./dotnet/eng/scripts/New-FilteredSolution.ps1 -Solution dotnet/agent-framework-dotnet.slnx -TargetFramework net10.0 -TestProjectNameFilter "*UnitTests*" -OutputPath filtered-unit.slnx
|
||||
./dotnet/eng/scripts/New-FilteredSolution.ps1 -Solution dotnet/agent-framework-dotnet.slnx -TargetFramework net10.0 -TestProjectNameIncludeFilter "*UnitTests*" -OutputPath filtered-unit.slnx
|
||||
|
||||
.EXAMPLE
|
||||
# Inline usage with dotnet test (PowerShell)
|
||||
dotnet test --solution (./dotnet/eng/scripts/New-FilteredSolution.ps1 -Solution dotnet/agent-framework-dotnet.slnx -TargetFramework net472) --no-build -f net472
|
||||
|
||||
.EXAMPLE
|
||||
# Generate integration tests excluding DurableTask and AzureFunctions
|
||||
./dotnet/eng/scripts/New-FilteredSolution.ps1 -Solution dotnet/agent-framework-dotnet.slnx -TargetFramework net10.0 -TestProjectNameIncludeFilter "*IntegrationTests*" -TestProjectNameExcludeFilter "*DurableTask.IntegrationTests*","*AzureFunctions.IntegrationTests*" -OutputPath filtered-other-integration.slnx
|
||||
#>
|
||||
|
||||
[CmdletBinding()]
|
||||
@@ -55,7 +64,9 @@ param(
|
||||
|
||||
[string]$Configuration = "Debug",
|
||||
|
||||
[string]$TestProjectNameFilter,
|
||||
[string]$TestProjectNameIncludeFilter,
|
||||
|
||||
[string[]]$TestProjectNameExcludeFilter,
|
||||
|
||||
[switch]$ExcludeSamples,
|
||||
|
||||
@@ -100,13 +111,30 @@ foreach ($proj in $allProjects) {
|
||||
$isTestProject = $projRelPath -like "*tests/*"
|
||||
|
||||
# Filter test projects by name pattern if specified
|
||||
if ($isTestProject -and $TestProjectNameFilter -and ($projFileName -notlike $TestProjectNameFilter)) {
|
||||
if ($isTestProject -and $TestProjectNameIncludeFilter -and ($projFileName -notlike $TestProjectNameIncludeFilter)) {
|
||||
Write-Verbose "Removing (name filter): $projRelPath"
|
||||
$removed += $projRelPath
|
||||
$proj.ParentNode.RemoveChild($proj) | Out-Null
|
||||
continue
|
||||
}
|
||||
|
||||
# Exclude test projects matching any exclusion pattern
|
||||
if ($isTestProject -and $TestProjectNameExcludeFilter) {
|
||||
$excluded = $false
|
||||
foreach ($pattern in $TestProjectNameExcludeFilter) {
|
||||
if ($projFileName -like $pattern) {
|
||||
$excluded = $true
|
||||
break
|
||||
}
|
||||
}
|
||||
if ($excluded) {
|
||||
Write-Verbose "Removing (exclude filter): $projRelPath"
|
||||
$removed += $projRelPath
|
||||
$proj.ParentNode.RemoveChild($proj) | Out-Null
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
if (-not (Test-Path $projFullPath)) {
|
||||
Write-Verbose "Project not found, keeping in solution: $projRelPath"
|
||||
$kept += $projRelPath
|
||||
|
||||
@@ -183,7 +183,7 @@ public class FoundryVersionedAgentCreateTests
|
||||
/// invokes the server-side OpenAPI function through <c>RunAsync</c>.
|
||||
/// Regression test for https://github.com/microsoft/agent-framework/issues/4883.
|
||||
/// </summary>
|
||||
[RetryFact(Constants.RetryCount, Constants.RetryDelay, Skip = "For manual testing only")]
|
||||
[RetryFact(Constants.RetryCount, Constants.RetryDelay)]
|
||||
public async Task AsAIAgent_WithOpenAPITool_NativeSDKCreation_InvokesServerSideToolAsync()
|
||||
{
|
||||
// Arrange — create agent version with OpenAPI tool using native Azure.AI.Projects SDK types.
|
||||
|
||||
+1
-1
@@ -14,7 +14,7 @@ namespace Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests;
|
||||
public sealed class DeclarativeCodeGenTest(ITestOutputHelper output) : WorkflowTest(output)
|
||||
{
|
||||
[Theory]
|
||||
[InlineData("CheckSystem.yaml", "CheckSystem.json", Skip = "Temporarily skipped")]
|
||||
[InlineData("CheckSystem.yaml", "CheckSystem.json")]
|
||||
[InlineData("SendActivity.yaml", "SendActivity.json")]
|
||||
[InlineData("InvokeAgent.yaml", "InvokeAgent.json")]
|
||||
[InlineData("InvokeAgent.yaml", "InvokeAgent.json", true)]
|
||||
|
||||
+1
-1
@@ -15,7 +15,7 @@ namespace Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests;
|
||||
public sealed class DeclarativeWorkflowTest(ITestOutputHelper output) : WorkflowTest(output)
|
||||
{
|
||||
[Theory]
|
||||
[InlineData("CheckSystem.yaml", "CheckSystem.json", Skip = "Temporarily skipped")]
|
||||
[InlineData("CheckSystem.yaml", "CheckSystem.json")]
|
||||
[InlineData("ConversationMessages.yaml", "ConversationMessages.json")]
|
||||
[InlineData("ConversationMessages.yaml", "ConversationMessages.json", true)]
|
||||
[InlineData("InputArguments.yaml", "InputArguments.json")]
|
||||
|
||||
+4
-1
@@ -10,7 +10,10 @@
|
||||
"conversation_count": 1,
|
||||
"min_action_count": 2,
|
||||
"max_action_count": -1,
|
||||
"min_response_count": 0,
|
||||
"min_response_count": 1,
|
||||
"max_response_count": 1,
|
||||
"min_message_count": 0,
|
||||
"max_message_count": 0,
|
||||
"actions": {
|
||||
"start": [
|
||||
"check_system"
|
||||
|
||||
Reference in New Issue
Block a user