Python: add better test coverage to individual tests, and all-tests task, gh … (#400)

* add better test coverage to individual tests, and all-tests task, gh action to surface

* remove cache location

* test version-file

* updated uv setup for consistency

* mypy fix

* update naming

* temporarily removed mypy from workflow

.github/workflows/python-code-quality.yml

@@ -8,7 +8,8 @@ on:
       - "python/**"
 
 env:
-  UV_SEMVER: "0.8.x"
+  # Configure a constant location for the uv cache
+  UV_CACHE_DIR: /tmp/.uv-cache
 
 jobs:
   pre-commit:
@@ -24,15 +25,13 @@ jobs:
       run:
         working-directory: ./python
     env:
-      # Configure a constant location for the uv cache
-      UV_CACHE_DIR: /tmp/.uv-cache
       UV_PYTHON: ${{ matrix.python-version }}
     steps:
       - uses: actions/checkout@v4
       - name: Set up uv
         uses: astral-sh/setup-uv@v6
         with:
-          version: ${{ env.UV_SEMVER }}
+          version-file: "python/pyproject.toml"
           enable-cache: true
           cache-suffix: ${{ runner.os }}-${{ matrix.python-version }}
           cache-dependency-glob: "**/uv.lock"

.github/workflows/python-merge-tests.yml

@@ -17,7 +17,6 @@ env:
   # Configure a constant location for the uv cache
   UV_CACHE_DIR: /tmp/.uv-cache
   RUN_INTEGRATION_TESTS: "true"
-  UV_SEMVER: "0.8.x"
 
 jobs:
   paths-filter:
@@ -68,7 +67,7 @@ jobs:
       - name: Set up uv
         uses: astral-sh/setup-uv@v6
         with:
-          version: ${{ env.UV_SEMVER }}
+          version-file: "python/pyproject.toml"
           enable-cache: true
           cache-suffix: ${{ runner.os }}-${{ matrix.python-version }}
           cache-dependency-glob: "**/uv.lock"
@@ -123,7 +122,7 @@ jobs:
       - name: Set up uv
         uses: astral-sh/setup-uv@v6
         with:
-          version: ${{ env.UV_SEMVER }}
+          version-file: "python/pyproject.toml"
           enable-cache: true
           cache-suffix: ${{ runner.os }}-${{ matrix.python-version }}
           cache-dependency-glob: "**/uv.lock"
@@ -184,7 +183,7 @@ jobs:
       - name: Set up uv
         uses: astral-sh/setup-uv@v6
         with:
-          version: ${{ env.UV_SEMVER }}
+          version-file: "python/pyproject.toml"
           enable-cache: true
           cache-suffix: ${{ runner.os }}-${{ matrix.python-version }}
           cache-dependency-glob: "**/uv.lock"

.github/workflows/python-release.yml

@@ -7,9 +7,9 @@ on:
 permissions:
   contents: write
   id-token: write
-
 env:
-  UV_SEMVER: "0.8.x"
+  # Configure a constant location for the uv cache
+  UV_CACHE_DIR: /tmp/.uv-cache
 
 jobs:
   python-build-assets:
@@ -27,7 +27,7 @@ jobs:
       - name: Set up uv
         uses: astral-sh/setup-uv@v6
         with:
-          version: ${{ env.UV_SEMVER }}
+          version-file: "python/pyproject.toml"
           enable-cache: true
           cache-suffix: ${{ runner.os }}-${{ env.UV_PYTHON }}
           cache-dependency-glob: "**/uv.lock"

.github/workflows/python-test-coverage-report.yml

@@ -0,0 +1,52 @@
+name: Python - Test Coverage Report
+
+on:
+  workflow_run:
+    workflows: ["Python - Test Coverage"]
+    types:
+      - completed
+
+permissions:
+  contents: read
+  pull-requests: write
+
+jobs:
+  python-test-coverage-report:
+    runs-on: ubuntu-latest
+    if: github.event.workflow_run.conclusion == 'success'
+    continue-on-error: false
+    defaults:
+      run:
+        working-directory: python
+    steps:
+      - uses: actions/checkout@v4
+      - name: Download coverage report
+        uses: actions/download-artifact@v4
+        with:
+          github-token: ${{ secrets.GH_ACTIONS_PR_WRITE }}
+          run-id: ${{ github.event.workflow_run.id }}
+          path: ./python
+          merge-multiple: true
+      - name: Display structure of downloaded files
+        run: ls
+      - name: Read and set PR number
+        # Need to read the PR number from the file saved in the previous workflow
+        # because the workflow_run event does not have access to the PR number
+        # The PR number is needed to post the comment on the PR
+        run: |
+          PR_NUMBER=$(cat pr_number)
+          echo "PR number: $PR_NUMBER"
+          echo "PR_NUMBER=$PR_NUMBER" >> $GITHUB_ENV
+      - name: Pytest coverage comment
+        id: coverageComment
+        uses: MishaKav/pytest-coverage-comment@v1.1.53
+        with:
+          github-token: ${{ secrets.GH_ACTIONS_PR_WRITE }}
+          issue-number: ${{ env.PR_NUMBER }}
+          pytest-xml-coverage-path: python/python-coverage.xml
+          title: "Python Test Coverage Report"
+          badge-title: "Python Test Coverage"
+          junitxml-title: "Python Unit Test Overview"
+          junitxml-path: python/pytest.xml
+          default-branch: "main"
+          report-only-changed-files: true

.github/workflows/python-test-coverage.yml

@@ -0,0 +1,49 @@
+name: Python - Test Coverage
+
+on:
+  pull_request:
+    branches: ["main", "feature*"]
+    paths:
+      - "python/packages/**"
+      - "python/tests/unit/**"
+env:
+  # Configure a constant location for the uv cache
+  UV_CACHE_DIR: /tmp/.uv-cache
+
+jobs:
+  python-tests-coverage:
+    runs-on: ubuntu-latest
+    continue-on-error: false
+    defaults:
+      run:
+        working-directory: python
+    env:
+      UV_PYTHON: "3.10"
+    steps:
+      - uses: actions/checkout@v4
+      # Save the PR number to a file since the workflow_run event
+      # in the coverage report workflow does not have access to it
+      - name: Save PR number
+        run: |
+          echo ${{ github.event.number }} > ./pr_number
+      - name: Set up uv
+        uses: astral-sh/setup-uv@v6
+        with:
+          version-file: "python/pyproject.toml"
+          enable-cache: true
+          cache-suffix: ${{ runner.os }}-${{ env.UV_PYTHON }}
+          cache-dependency-glob: "**/uv.lock"
+      - name: Install the project
+        run: uv sync --all-extras --dev
+      - name: Run all tests with coverage report
+        run: uv run poe all-tests --cov-report=xml:python-coverage.xml -q --junitxml=pytest.xml
+      - name: Upload coverage report
+        uses: actions/upload-artifact@v4
+        with:
+          path: |
+            python/python-coverage.xml
+            python/pytest.xml
+            python/pr_number
+          overwrite: true
+          retention-days: 1
+          if-no-files-found: error

.github/workflows/python-tests.yml

@@ -8,7 +8,6 @@ on:
 env:
   # Configure a constant location for the uv cache
   UV_CACHE_DIR: /tmp/.uv-cache
-  UV_SEMVER: "0.8.x"
 
 jobs:
   python-tests:
@@ -31,7 +30,7 @@ jobs:
       - name: Set up uv
         uses: astral-sh/setup-uv@v6
         with:
-          version: ${{ env.UV_SEMVER }}
+          version-file: "python/pyproject.toml"
           enable-cache: true
           cache-suffix: ${{ runner.os }}-${{ matrix.python-version }}
           cache-dependency-glob: "**/uv.lock"

python/packages/azure/pyproject.toml

@@ -48,6 +48,11 @@ timeout = 120
 [tool.ruff]
 extend = "../../pyproject.toml"
 
+[tool.coverage.run]
+omit = [
+    "**/__init__.py"
+]
+
 [tool.pyright]
 extend = "../../pyproject.toml"
 exclude = ['tests']
@@ -76,6 +81,7 @@ executor.type = "uv"
 include = "../../shared_tasks.toml"
 [tool.poe.tasks]
 mypy = "mypy --config-file $POE_ROOT/pyproject.toml agent_framework_azure"
+test = "pytest --cov=agent_framework_azure --cov-report=term-missing:skip-covered tests"
 
 [tool.uv.build-backend]
 module-name = "agent_framework_azure"

python/packages/foundry/pyproject.toml

@@ -50,6 +50,11 @@ timeout = 120
 [tool.ruff]
 extend = "../../pyproject.toml"
 
+[tool.coverage.run]
+omit = [
+    "**/__init__.py"
+]
+
 [tool.pyright]
 extend = "../../pyproject.toml"
 exclude = ['tests']
@@ -78,6 +83,7 @@ executor.type = "uv"
 include = "../../shared_tasks.toml"
 [tool.poe.tasks]
 mypy = "mypy --config-file $POE_ROOT/pyproject.toml agent_framework_foundry"
+test = "pytest --cov=agent_framework_foundry --cov-report=term-missing:skip-covered tests"
 
 [tool.uv.build-backend]
 module-name = "agent_framework_foundry"

python/packages/main/pyproject.toml

@@ -61,6 +61,11 @@ asyncio_default_fixture_loop_scope = "function"
 filterwarnings = []
 timeout = 120
 
+[tool.coverage.run]
+omit = [
+    "**/__init__.py"
+]
+
 [tool.ruff]
 extend = "../../pyproject.toml"
 
@@ -92,6 +97,7 @@ executor.type = "uv"
 include = "../../shared_tasks.toml"
 [tool.poe.tasks]
 mypy = "mypy --config-file $POE_ROOT/pyproject.toml agent_framework"
+test = "pytest --cov=agent_framework --cov-report=term-missing:skip-covered tests"
 
 [tool.uv.build-backend]
 module-name = "agent_framework"

python/packages/workflow/pyproject.toml

@@ -47,6 +47,11 @@ timeout = 120
 [tool.ruff]
 extend = "../../pyproject.toml"
 
+[tool.coverage.run]
+omit = [
+    "**/__init__.py"
+]
+
 [tool.pyright]
 extend = "../../pyproject.toml"
 exclude = ['tests']
@@ -73,6 +78,8 @@ exclude_dirs = ["tests"]
 [tool.poe]
 executor.type = "uv"
 include = "../../shared_tasks.toml"
+[tool.poe.tasks]
+test = "pytest --cov=agent_framework_workflow --cov-report=term-missing:skip-covered tests"
 
 [tool.uv.build-backend]
 module-name = "agent_framework_workflow"

python/pyproject.toml

@@ -123,6 +123,19 @@ convention = "google"
 notice-rgx = "^# Copyright \\(c\\) Microsoft\\. All rights reserved\\."
 min-file-size = 1
 
+[tool.pytest.ini_options]
+testpaths = 'tests'
+addopts = "-ra -q -r fEX"
+asyncio_mode = "auto"
+asyncio_default_fixture_loop_scope = "function"
+filterwarnings = []
+timeout = 120
+
+[tool.coverage.run]
+omit = [
+    "**/__init__.py"
+]
+
 [tool.pyright]
 include = ["agent_framework", "samples"]
 exclude = ["**/tests/**", "docs", "**/.venv/**"]
@@ -172,6 +185,7 @@ build = "python run_tasks_in_packages_if_exists.py build"
 # combined checks
 check = ["fmt", "lint", "pyright", "mypy", "test", "markdown-code-lint", "samples-code-check"]
 pre-commit-check = ["fmt", "lint", "pyright", "markdown-code-lint", "samples-code-check"]
+all-tests = "pytest --cov=agent_framework --cov=agent_framework_azure --cov=agent_framework_foundry --cov=agent_framework_workflow --cov-report=term-missing:skip-covered packages/**/tests"
 
 [tool.poe.tasks.venv]
 cmd = "uv venv --clear --python $python"
@@ -184,24 +198,7 @@ sequence = [
 ]
 args = [{ name = "python", default = "3.13", options = ['-p', '--python'] }]
 
-
 [tool.setuptools.packages.find]
 where = ["packages"]
 include = ["agent_framework**"]
 namespaces = true
-
-[tool.tox]
-requires = ["tox>=4", "tox-uv>=1"]
-env_list = [ "3.13"] #, "3.12", "3.11", "3.10" ]
-
-[tool.tox.env_run_base]
-no_package = true
-description = "run tests"
-dependency_groups = [
-  "dev",
-]
-deps = [
-    "-e packages/azure",
-    "-e packages/main"
-]
-commands = [["pytest", { replace = "posargs", default = ["packages/main/tests"], extend = true }]]

python/shared_tasks.toml

@@ -4,4 +4,3 @@ format.ref = "fmt"
 lint = "ruff check"
 pyright = "pyright"
 build = "uv build"
-test = "pytest --cov=agent_framework --cov-report=term-missing:skip-covered tests"