agent-framework/.github/workflows/dotnet-build-and-test.yml

#
# This workflow will build all .slnx files in the dotnet folder, and run all unit tests and integration tests using dotnet docker containers,
# each targeting a single version of the dotnet SDK.
#

name: dotnet-build-and-test

on:
  workflow_dispatch:
  pull_request:
    branches: ["main", "feature*"]
  merge_group:
    branches: ["main", "feature*"]
  push:
    branches: ["main", "feature*"]
  schedule:
    - cron: "0 0 * * *" # Run at midnight UTC daily

env:
  COVERAGE_THRESHOLD: 80
  COVERAGE_FRAMEWORK: net10.0 # framework target for which we run/report code coverage

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

permissions:
  contents: read
  id-token: "write"

jobs:
  paths-filter:
    runs-on: ubuntu-latest
    permissions:
      contents: read
      pull-requests: read
    outputs:
      dotnetChanges: ${{ steps.filter.outputs.dotnet }}
      cosmosDbChanges: ${{ steps.filter.outputs.cosmosdb }}
      foundryHostingChanges: ${{ steps.filter.outputs.foundryHosting }}
    steps:
      - uses: actions/checkout@v6
      - uses: dorny/paths-filter@v3
        id: filter
        with:
          filters: |
            dotnet:
              - 'dotnet/**'
            cosmosdb:
              - 'dotnet/src/Microsoft.Agents.AI.CosmosNoSql/**'
            # The Foundry hosted-agent IT is costly (builds a container, pushes to ACR,
            # provisions live agents). Only run it when the project under test, its
            # dependency chain, the test container, the test fixture, or their tooling
            # changed. Keep this list in sync with $hashedDirs in scripts/it-build-image.ps1.
            foundryHosting:
              - 'dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/**'
              - 'dotnet/src/Microsoft.Agents.AI.Foundry/**'
              - 'dotnet/src/Microsoft.Agents.AI/**'
              - 'dotnet/src/Microsoft.Agents.AI.Abstractions/**'
              - 'dotnet/src/Microsoft.Agents.AI.Workflows/**'
              - 'dotnet/tests/Foundry.Hosting.IntegrationTests/**'
              - 'dotnet/tests/Foundry.Hosting.IntegrationTests.TestContainer/**'
              - 'dotnet/Directory.Packages.props'
              - 'dotnet/tests/Foundry.Hosting.IntegrationTests/scripts/it-build-image.ps1'
              - '.github/workflows/dotnet-build-and-test.yml'
      # run only if 'dotnet' files were changed
      - name: dotnet tests
        if: steps.filter.outputs.dotnet == 'true'
        run: echo "Dotnet file"
      - name: dotnet CosmosDB tests
        if: steps.filter.outputs.cosmosdb == 'true'
        run: echo "Dotnet CosmosDB changes"
      # run only if not 'dotnet' files were changed
      - name: not dotnet tests
        if: steps.filter.outputs.dotnet != 'true'
        run: echo "NOT dotnet file"

  # Build the full solution (including samples) on all TFMs. No tests.
  dotnet-build:
    needs: paths-filter
    if: needs.paths-filter.outputs.dotnetChanges == 'true'
    strategy:
      fail-fast: false
      matrix:
        include:
          - { targetFramework: "net10.0", os: "ubuntu-latest", configuration: Release }
          - { targetFramework: "net9.0", os: "windows-latest", configuration: Debug }
          - { targetFramework: "net8.0", os: "ubuntu-latest", configuration: Release }
          - { targetFramework: "net472", os: "windows-latest", configuration: Release }

    runs-on: ${{ matrix.os }}
    steps:
      - uses: actions/checkout@v6
        with:
            persist-credentials: false
            sparse-checkout: |
              .
              .github
              dotnet
              python
              declarative-agents

      - name: Setup dotnet
        uses: actions/setup-dotnet@v5.2.0
        with:
          global-json-file: ${{ github.workspace }}/dotnet/global.json
      - name: Build dotnet solutions
        shell: bash
        run: |
          export SOLUTIONS=$(find ./dotnet/ -type f -name "*.slnx" | tr '\n' ' ')
          for solution in $SOLUTIONS; do
            dotnet build $solution -c ${{ matrix.configuration }} --warnaserror
          done
      - name: Package install check
        shell: bash
        # All frameworks are only built for the release configuration, so we only run this step for the release configuration
        # and dotnet new doesn't support net472
        if: matrix.configuration == 'Release' && matrix.targetFramework != 'net472'
        run: |
          TEMP_DIR=$(mktemp -d)

          export SOLUTIONS=$(find ./dotnet/ -type f -name "*.slnx" | tr '\n' ' ')
          for solution in $SOLUTIONS; do
            dotnet pack $solution /property:TargetFrameworks=${{ matrix.targetFramework }} -c ${{ matrix.configuration }} --no-build --no-restore --output "$TEMP_DIR/artifacts"
          done

          pushd "$TEMP_DIR"

          # Create a new console app to test the package installation
          dotnet new console -f ${{ matrix.targetFramework }} --name packcheck --output consoleapp

          # Create minimal nuget.config and use only dotnet nuget commands
          echo '<?xml version="1.0" encoding="utf-8"?><configuration><packageSources><clear /></packageSources></configuration>' > consoleapp/nuget.config

          # Add sources with local first using dotnet nuget commands
          dotnet nuget add source ../artifacts --name local --configfile consoleapp/nuget.config
          dotnet nuget add source https://api.nuget.org/v3/index.json --name nuget.org --configfile consoleapp/nuget.config

          # Change to project directory to ensure local nuget.config is used
          pushd consoleapp
          dotnet add packcheck.csproj package Microsoft.Agents.AI --prerelease
          dotnet build -f ${{ matrix.targetFramework }} -c ${{ matrix.configuration }} packcheck.csproj

          # Clean up
          popd
          popd
          rm -rf "$TEMP_DIR"

  # Build src+tests only (no samples) for a single TFM and run tests.
  dotnet-test:
    needs: paths-filter
    if: needs.paths-filter.outputs.dotnetChanges == 'true'
    strategy:
      fail-fast: false
      matrix:
        include:
          - { targetFramework: "net10.0", os: "ubuntu-latest", configuration: Release, integration-tests: true, environment: "integration" }
          - { targetFramework: "net472", os: "windows-latest", configuration: Release, integration-tests: true, environment: "integration" }

    runs-on: ${{ matrix.os }}
    environment: ${{ matrix.environment }}
    steps:
      - uses: actions/checkout@v6
        with:
            persist-credentials: false
            sparse-checkout: |
              .
              .github
              dotnet
              python
              declarative-agents

      # Start Cosmos DB Emulator for all integration tests and only for unit tests when CosmosDB changes happened)
      - name: Start Azure Cosmos DB Emulator
        if: ${{ runner.os == 'Windows' && (needs.paths-filter.outputs.cosmosDbChanges == 'true' || (github.event_name != 'pull_request' && matrix.integration-tests)) }}
        shell: pwsh
        run: |
          Write-Host "Launching Azure Cosmos DB Emulator"
          Import-Module "$env:ProgramFiles\Azure Cosmos DB Emulator\PSModules\Microsoft.Azure.CosmosDB.Emulator"
          Start-CosmosDbEmulator -NoUI -Key "C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw=="
          echo "COSMOSDB_EMULATOR_AVAILABLE=true" >> $env:GITHUB_ENV

      - name: Setup dotnet
        uses: actions/setup-dotnet@v5.2.0
        with:
          global-json-file: ${{ github.workspace }}/dotnet/global.json

      - name: Generate test solution (no samples)
        shell: pwsh
        run: |
          ./dotnet/eng/scripts/New-FilteredSolution.ps1 `
            -Solution dotnet/agent-framework-dotnet.slnx `
            -TargetFramework ${{ matrix.targetFramework }} `
            -Configuration ${{ matrix.configuration }} `
            -ExcludeSamples `
            -OutputPath dotnet/filtered.slnx `
            -Verbose

      - name: Build src and tests
        shell: bash
        run: dotnet build dotnet/filtered.slnx -c ${{ matrix.configuration }} -f ${{ matrix.targetFramework }} --warnaserror

      - name: Generate test-type filtered solutions
        shell: pwsh
        run: |
          $commonArgs = @{
            Solution       = "dotnet/filtered.slnx"
            TargetFramework = "${{ matrix.targetFramework }}"
            Configuration   = "${{ matrix.configuration }}"
            Verbose         = $true
          }
          ./dotnet/eng/scripts/New-FilteredSolution.ps1 @commonArgs `
            -TestProjectNameFilter "*UnitTests*" `
            -OutputPath dotnet/filtered-unit.slnx
          ./dotnet/eng/scripts/New-FilteredSolution.ps1 @commonArgs `
            -TestProjectNameFilter "*IntegrationTests*" `
            -OutputPath dotnet/filtered-integration.slnx

      - name: Run Unit Tests
        shell: pwsh
        working-directory: dotnet
        run: |
          $coverageSettings = Join-Path $PWD "tests/coverage.runsettings"
          $coverageArgs = @()
          if ("${{ matrix.targetFramework }}" -eq "${{ env.COVERAGE_FRAMEWORK }}") {
            $coverageArgs = @(
              "--coverage",
              "--coverage-output-format", "cobertura",
              "--coverage-settings", $coverageSettings,
              "--results-directory", "../TestResults/Coverage/"
            )
          }

          dotnet test --solution ./filtered-unit.slnx `
            -f ${{ matrix.targetFramework }} `
            -c ${{ matrix.configuration }} `
            --no-build -v Normal `
            --report-xunit-trx `
            --ignore-exit-code 8 `
            @coverageArgs
        env:
          # Cosmos DB Emulator connection settings
          COSMOSDB_ENDPOINT: https://localhost:8081
          COSMOSDB_KEY: C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==

      - name: Log event name and matrix integration-tests
        shell: bash
        run: echo "github.event_name:${{ github.event_name }} matrix.integration-tests:${{ matrix.integration-tests }} github.event.action:${{ github.event.action }} github.event.pull_request.merged:${{ github.event.pull_request.merged }}"

      - name: Azure CLI Login
        if: github.event_name != 'pull_request' && matrix.integration-tests
        uses: azure/login@v2
        with:
          client-id: ${{ secrets.AZURE_CLIENT_ID }}
          tenant-id: ${{ secrets.AZURE_TENANT_ID }}
          subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}

        # This setup action is required for both Durable Task and Azure Functions integration tests.
        # We only run it on Ubuntu since the Durable Task and Azure Functions features are not available
        # on .NET Framework (net472) which is what we use the Windows runner for.
      - name: Set up Durable Task and Azure Functions Integration Test Emulators
        if: github.event_name != 'pull_request' && matrix.integration-tests && matrix.os == 'ubuntu-latest'
        uses: ./.github/actions/azure-functions-integration-setup
        id: azure-functions-setup

      - name: Run Integration Tests
        shell: pwsh
        working-directory: dotnet
        if: github.event_name != 'pull_request' && matrix.integration-tests
        run: |
          dotnet test --solution ./filtered-integration.slnx `
            -f ${{ matrix.targetFramework }} `
            -c ${{ matrix.configuration }} `
            --no-build -v Normal `
            --report-xunit-trx `
            --report-junit `
            --results-directory ../IntegrationTestResults/ `
            --ignore-exit-code 8 `
            --filter-not-trait "Category=IntegrationDisabled" `
            --filter-not-trait "Category=FoundryHostedAgents" `
            --parallel-algorithm aggressive `
            --max-threads 2.0x
        env:
          # Cosmos DB Emulator connection settings
          COSMOSDB_ENDPOINT: https://localhost:8081
          COSMOSDB_KEY: C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==
          # OpenAI Models
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
          OPENAI_CHAT_MODEL_NAME: ${{ vars.OPENAI_CHAT_MODEL_NAME }}
          OPENAI_REASONING_MODEL_NAME: ${{ vars.OPENAI_REASONING_MODEL_NAME }}
          # Azure OpenAI Models
          AZURE_OPENAI_DEPLOYMENT_NAME: ${{ vars.AZURE_OPENAI_DEPLOYMENT_NAME }}
          AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ vars.AZURE_OPENAI_DEPLOYMENT_NAME }}
          AZURE_OPENAI_ENDPOINT: ${{ vars.AZURE_OPENAI_ENDPOINT }}
          # Azure AI Foundry
          AZURE_AI_PROJECT_ENDPOINT: ${{ vars.AZURE_AI_PROJECT_ENDPOINT }}
          AZURE_AI_MODEL_DEPLOYMENT_NAME: ${{ vars.AZURE_AI_MODEL_DEPLOYMENT_NAME }}
          AZURE_AI_BING_CONNECTION_ID: ${{ vars.AZURE_AI_BING_CONNECTION_ID }}
          # Anthropic Models
          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
          ANTHROPIC_CHAT_MODEL_NAME: ${{ vars.ANTHROPIC_CHAT_MODEL_NAME }}
          ANTHROPIC_REASONING_MODEL_NAME: ${{ vars.ANTHROPIC_REASONING_MODEL_NAME }}

      # Generate test reports and check coverage
      - name: Generate test reports
        if: matrix.targetFramework == env.COVERAGE_FRAMEWORK
        uses: danielpalme/ReportGenerator-GitHub-Action@5.5.3
        with:
          reports: "./TestResults/Coverage/**/*.cobertura.xml"
          targetdir: "./TestResults/Reports"
          reporttypes: "HtmlInline;JsonSummary"

      - name: Upload coverage report artifact
        if: matrix.targetFramework == env.COVERAGE_FRAMEWORK
        uses: actions/upload-artifact@v7
        with:
          name: CoverageReport-${{ matrix.os }}-${{ matrix.targetFramework }}-${{ matrix.configuration }} # Artifact name
          path: ./TestResults/Reports # Directory containing files to upload

      - name: Check coverage
        if: matrix.targetFramework == env.COVERAGE_FRAMEWORK
        shell: pwsh
        run: ./dotnet/eng/scripts/dotnet-check-coverage.ps1 -JsonReportPath "TestResults/Reports/Summary.json" -CoverageThreshold $env:COVERAGE_THRESHOLD

      - name: Upload integration test results
        if: always() && github.event_name != 'pull_request' && matrix.integration-tests
        uses: actions/upload-artifact@v7
        with:
          name: dotnet-test-results-${{ matrix.targetFramework }}-${{ matrix.os }}
          path: IntegrationTestResults/**/*.junit
          if-no-files-found: ignore

  # The Foundry hosted-agent IT is costly (it builds a container, pushes to ACR, and provisions
  # live agents on a separate Foundry project). Running it in its own job keeps the overall
  # workflow time roughly flat: it executes in parallel to dotnet-build and dotnet-test and is
  # gated on paths-filter.outputs.foundryHostingChanges so unrelated edits skip the work.
  dotnet-foundry-hosted-it:
    needs: paths-filter
    if: github.event_name != 'pull_request' && needs.paths-filter.outputs.foundryHostingChanges == 'true'
    runs-on: ubuntu-latest
    environment: integration
    env:
      targetFramework: net10.0
      configuration: Release
    steps:
      - uses: actions/checkout@v6
        with:
          persist-credentials: false
          sparse-checkout: |
            .
            .github
            dotnet
            python

      - name: Setup dotnet
        uses: actions/setup-dotnet@v5.2.0
        with:
          global-json-file: ${{ github.workspace }}/dotnet/global.json

      - name: Generate test solution (no samples)
        shell: pwsh
        run: |
          ./dotnet/eng/scripts/New-FilteredSolution.ps1 `
            -Solution dotnet/agent-framework-dotnet.slnx `
            -TargetFramework $env:targetFramework `
            -Configuration $env:configuration `
            -ExcludeSamples `
            -OutputPath dotnet/filtered.slnx `
            -Verbose

      - name: Generate Foundry hosted IT filtered solution
        shell: pwsh
        run: |
          ./dotnet/eng/scripts/New-FilteredSolution.ps1 `
            -Solution dotnet/filtered.slnx `
            -TargetFramework $env:targetFramework `
            -Configuration $env:configuration `
            -TestProjectNameFilter "Foundry.Hosting.IntegrationTests*" `
            -OutputPath dotnet/filtered-foundry-hosted.slnx `
            -Verbose

      - name: Build Foundry hosted IT (and its deps)
        shell: bash
        run: dotnet build dotnet/filtered-foundry-hosted.slnx -c "$configuration" -f "$targetFramework" --warnaserror

      - name: Azure CLI Login
        uses: azure/login@v2
        with:
          client-id: ${{ secrets.AZURE_CLIENT_ID }}
          tenant-id: ${{ secrets.AZURE_TENANT_ID }}
          subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}

      # We rebuild and push the test container image on every IT run so framework code changes
      # are picked up; the image tag is content-hashed across the test container source AND its
      # framework project references, so identical content is a no-op push.
      #
      # `-UsePrebuiltProjectReferences` opts into the no-rebuild fast path: publish skips
      # rebuilding ProjectReferences and consumes the DLLs the prior "Build Foundry hosted IT
      # (and its deps)" step already produced. This avoids MSB3026 ("file is being used by
      # another process") collisions caused by the previous build's shared-compilation server
      # still holding file handles to those DLLs. Safe in CI because the prebuild step ran in
      # the same job against the same source. Do not remove the prebuild step (the subsequent
      # `dotnet test --no-build` step depends on it too).
      - name: Build and push Foundry Hosted Agents test container
        id: build-foundry-hosted-image
        shell: pwsh
        working-directory: ${{ github.workspace }}
        run: |
          $registry = "${{ vars.IT_HOSTED_AGENT_REGISTRY }}"
          if ([string]::IsNullOrWhiteSpace($registry)) {
            throw "IT_HOSTED_AGENT_REGISTRY not set in the integration environment."
          }
          & "${{ github.workspace }}/dotnet/tests/Foundry.Hosting.IntegrationTests/scripts/it-build-image.ps1" -Registry $registry -UsePrebuiltProjectReferences | Tee-Object -FilePath $env:GITHUB_ENV -Append

      - name: Run Foundry Hosted Agents Integration Tests
        shell: pwsh
        working-directory: dotnet
        run: |
          dotnet test --solution ./filtered-foundry-hosted.slnx `
            -f $env:targetFramework `
            -c $env:configuration `
            --no-build -v Normal `
            --report-xunit-trx `
            --ignore-exit-code 8 `
            --filter-trait "Category=FoundryHostedAgents"
        env:
          AZURE_AI_PROJECT_ENDPOINT: ${{ vars.IT_HOSTED_AGENT_PROJECT_ENDPOINT }}
          AZURE_AI_MODEL_DEPLOYMENT_NAME: ${{ vars.IT_HOSTED_AGENT_MODEL_DEPLOYMENT_NAME }}
          # IT_HOSTED_AGENT_IMAGE was exported into $GITHUB_ENV by the previous step.

  # This final job is required to satisfy the merge queue. It must only run (or succeed) if no tests failed
  dotnet-build-and-test-check:
    if: always()
    runs-on: ubuntu-latest
    needs: [dotnet-build, dotnet-test, dotnet-foundry-hosted-it]
    steps:
      - name: Get Date
        shell: bash
        run: |
          echo "date=$(date +'%m/%d/%Y %H:%M:%S')" >> "$GITHUB_ENV"

      - name: Run Type is Daily
        if: ${{ github.event_name == 'schedule' }}
        shell: bash
        run: |
          echo "run_type=Daily" >> "$GITHUB_ENV"

      - name: Run Type is Manual
        if: ${{ github.event_name == 'workflow_dispatch' }}
        shell: bash
        run: |
          echo "run_type=Manual" >> "$GITHUB_ENV"

      - name: Run Type is ${{ github.event_name }}
        if: ${{ github.event_name != 'schedule' && github.event_name != 'workflow_dispatch'}}
        shell: bash
        run: |
          echo "run_type=${{ github.event_name }}" >> "$GITHUB_ENV"

      - name: Fail workflow if tests failed
        id: check_tests_failed
        if: contains(join(needs.*.result, ','), 'failure')
        uses: actions/github-script@v8
        with:
          script: core.setFailed('Integration Tests Failed!')

      - name: Fail workflow if tests cancelled
        id: check_tests_cancelled
        if: contains(join(needs.*.result, ','), 'cancelled')
        uses: actions/github-script@v8
        with:
          script: core.setFailed('Integration Tests Cancelled!')

  # Integration test trend report (aggregates JUnit XML results from dotnet test jobs)
  dotnet-integration-test-report:
    name: Integration Test Report
    if: >
      always() &&
      github.event_name != 'pull_request' &&
      (contains(join(needs.*.result, ','), 'success') ||
       contains(join(needs.*.result, ','), 'failure'))
    needs: [dotnet-test]
    runs-on: ubuntu-latest
    defaults:
      run:
        working-directory: python
    steps:
      - uses: actions/checkout@v6
        with:
          persist-credentials: false
          sparse-checkout: |
            .github/actions/python-setup
            python
      - name: Set up python and install the project
        uses: ./.github/actions/python-setup
        with:
          python-version: "3.13"
          os: ${{ runner.os }}
      - name: Download all test results from current run
        uses: actions/download-artifact@v4
        with:
          pattern: dotnet-test-results-*
          path: dotnet-test-results/
      - name: Restore report history cache
        uses: actions/cache/restore@v4
        with:
          path: python/dotnet-integration-report-history.json
          key: dotnet-integration-report-history-${{ github.run_id }}
          restore-keys: |
            dotnet-integration-report-history-
      - name: Generate trend report
        run: >
          uv run python scripts/integration_test_report/aggregate.py
          ../dotnet-test-results/
          dotnet-integration-report-history.json
          dotnet-integration-test-report.md
      - name: Post to Job Summary
        if: always()
        run: cat dotnet-integration-test-report.md >> $GITHUB_STEP_SUMMARY
      - name: Save report history cache
        if: always()
        uses: actions/cache/save@v4
        with:
          path: python/dotnet-integration-report-history.json
          key: dotnet-integration-report-history-${{ github.run_id }}
      - name: Upload trend report
        if: always()
        uses: actions/upload-artifact@v7
        with:
          name: dotnet-integration-test-report
          path: |
            python/dotnet-integration-test-report.md
            python/dotnet-integration-report-history.json