From 53866218d2cdc1807e388525fd6e7322e4efb30a Mon Sep 17 00:00:00 2001 From: Eduard van Valkenburg Date: Tue, 12 Aug 2025 20:35:36 +0200 Subject: [PATCH] =?UTF-8?q?Python:=20add=20better=20test=20coverage=20to?= =?UTF-8?q?=20individual=20tests,=20and=20all-tests=20task,=20gh=20?= =?UTF-8?q?=E2=80=A6=20(#400)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * add better test coverage to individual tests, and all-tests task, gh action to surface * remove cache location * test version-file * updated uv setup for consistency * mypy fix * update naming * temporarily removed mypy from workflow --- .github/workflows/python-code-quality.yml | 7 ++- .github/workflows/python-merge-tests.yml | 7 ++- .github/workflows/python-release.yml | 6 +-- .../workflows/python-test-coverage-report.yml | 52 +++++++++++++++++++ .github/workflows/python-test-coverage.yml | 49 +++++++++++++++++ .github/workflows/python-tests.yml | 3 +- python/packages/azure/pyproject.toml | 6 +++ python/packages/foundry/pyproject.toml | 6 +++ python/packages/main/pyproject.toml | 6 +++ python/packages/workflow/pyproject.toml | 7 +++ python/pyproject.toml | 31 +++++------ python/shared_tasks.toml | 1 - 12 files changed, 150 insertions(+), 31 deletions(-) create mode 100644 .github/workflows/python-test-coverage-report.yml create mode 100644 .github/workflows/python-test-coverage.yml diff --git a/.github/workflows/python-code-quality.yml b/.github/workflows/python-code-quality.yml index a4b58629ea..d940513986 100644 --- a/.github/workflows/python-code-quality.yml +++ b/.github/workflows/python-code-quality.yml @@ -8,7 +8,8 @@ on: - "python/**" env: - UV_SEMVER: "0.8.x" + # Configure a constant location for the uv cache + UV_CACHE_DIR: /tmp/.uv-cache jobs: pre-commit: @@ -24,15 +25,13 @@ jobs: run: working-directory: ./python env: - # Configure a constant location for the uv cache - UV_CACHE_DIR: /tmp/.uv-cache UV_PYTHON: ${{ matrix.python-version }} steps: - uses: actions/checkout@v4 - name: Set up uv uses: astral-sh/setup-uv@v6 with: - version: ${{ env.UV_SEMVER }} + version-file: "python/pyproject.toml" enable-cache: true cache-suffix: ${{ runner.os }}-${{ matrix.python-version }} cache-dependency-glob: "**/uv.lock" diff --git a/.github/workflows/python-merge-tests.yml b/.github/workflows/python-merge-tests.yml index 8568b19134..e4cae65fa3 100644 --- a/.github/workflows/python-merge-tests.yml +++ b/.github/workflows/python-merge-tests.yml @@ -17,7 +17,6 @@ env: # Configure a constant location for the uv cache UV_CACHE_DIR: /tmp/.uv-cache RUN_INTEGRATION_TESTS: "true" - UV_SEMVER: "0.8.x" jobs: paths-filter: @@ -68,7 +67,7 @@ jobs: - name: Set up uv uses: astral-sh/setup-uv@v6 with: - version: ${{ env.UV_SEMVER }} + version-file: "python/pyproject.toml" enable-cache: true cache-suffix: ${{ runner.os }}-${{ matrix.python-version }} cache-dependency-glob: "**/uv.lock" @@ -123,7 +122,7 @@ jobs: - name: Set up uv uses: astral-sh/setup-uv@v6 with: - version: ${{ env.UV_SEMVER }} + version-file: "python/pyproject.toml" enable-cache: true cache-suffix: ${{ runner.os }}-${{ matrix.python-version }} cache-dependency-glob: "**/uv.lock" @@ -184,7 +183,7 @@ jobs: - name: Set up uv uses: astral-sh/setup-uv@v6 with: - version: ${{ env.UV_SEMVER }} + version-file: "python/pyproject.toml" enable-cache: true cache-suffix: ${{ runner.os }}-${{ matrix.python-version }} cache-dependency-glob: "**/uv.lock" diff --git a/.github/workflows/python-release.yml b/.github/workflows/python-release.yml index 3c57e01b55..571af6c148 100644 --- a/.github/workflows/python-release.yml +++ b/.github/workflows/python-release.yml @@ -7,9 +7,9 @@ on: permissions: contents: write id-token: write - env: - UV_SEMVER: "0.8.x" + # Configure a constant location for the uv cache + UV_CACHE_DIR: /tmp/.uv-cache jobs: python-build-assets: @@ -27,7 +27,7 @@ jobs: - name: Set up uv uses: astral-sh/setup-uv@v6 with: - version: ${{ env.UV_SEMVER }} + version-file: "python/pyproject.toml" enable-cache: true cache-suffix: ${{ runner.os }}-${{ env.UV_PYTHON }} cache-dependency-glob: "**/uv.lock" diff --git a/.github/workflows/python-test-coverage-report.yml b/.github/workflows/python-test-coverage-report.yml new file mode 100644 index 0000000000..3f3256db89 --- /dev/null +++ b/.github/workflows/python-test-coverage-report.yml @@ -0,0 +1,52 @@ +name: Python - Test Coverage Report + +on: + workflow_run: + workflows: ["Python - Test Coverage"] + types: + - completed + +permissions: + contents: read + pull-requests: write + +jobs: + python-test-coverage-report: + runs-on: ubuntu-latest + if: github.event.workflow_run.conclusion == 'success' + continue-on-error: false + defaults: + run: + working-directory: python + steps: + - uses: actions/checkout@v4 + - name: Download coverage report + uses: actions/download-artifact@v4 + with: + github-token: ${{ secrets.GH_ACTIONS_PR_WRITE }} + run-id: ${{ github.event.workflow_run.id }} + path: ./python + merge-multiple: true + - name: Display structure of downloaded files + run: ls + - name: Read and set PR number + # Need to read the PR number from the file saved in the previous workflow + # because the workflow_run event does not have access to the PR number + # The PR number is needed to post the comment on the PR + run: | + PR_NUMBER=$(cat pr_number) + echo "PR number: $PR_NUMBER" + echo "PR_NUMBER=$PR_NUMBER" >> $GITHUB_ENV + - name: Pytest coverage comment + id: coverageComment + uses: MishaKav/pytest-coverage-comment@v1.1.53 + with: + github-token: ${{ secrets.GH_ACTIONS_PR_WRITE }} + issue-number: ${{ env.PR_NUMBER }} + pytest-xml-coverage-path: python/python-coverage.xml + title: "Python Test Coverage Report" + badge-title: "Python Test Coverage" + junitxml-title: "Python Unit Test Overview" + junitxml-path: python/pytest.xml + default-branch: "main" + report-only-changed-files: true diff --git a/.github/workflows/python-test-coverage.yml b/.github/workflows/python-test-coverage.yml new file mode 100644 index 0000000000..59c8b4b4bf --- /dev/null +++ b/.github/workflows/python-test-coverage.yml @@ -0,0 +1,49 @@ +name: Python - Test Coverage + +on: + pull_request: + branches: ["main", "feature*"] + paths: + - "python/packages/**" + - "python/tests/unit/**" +env: + # Configure a constant location for the uv cache + UV_CACHE_DIR: /tmp/.uv-cache + +jobs: + python-tests-coverage: + runs-on: ubuntu-latest + continue-on-error: false + defaults: + run: + working-directory: python + env: + UV_PYTHON: "3.10" + steps: + - uses: actions/checkout@v4 + # Save the PR number to a file since the workflow_run event + # in the coverage report workflow does not have access to it + - name: Save PR number + run: | + echo ${{ github.event.number }} > ./pr_number + - name: Set up uv + uses: astral-sh/setup-uv@v6 + with: + version-file: "python/pyproject.toml" + enable-cache: true + cache-suffix: ${{ runner.os }}-${{ env.UV_PYTHON }} + cache-dependency-glob: "**/uv.lock" + - name: Install the project + run: uv sync --all-extras --dev + - name: Run all tests with coverage report + run: uv run poe all-tests --cov-report=xml:python-coverage.xml -q --junitxml=pytest.xml + - name: Upload coverage report + uses: actions/upload-artifact@v4 + with: + path: | + python/python-coverage.xml + python/pytest.xml + python/pr_number + overwrite: true + retention-days: 1 + if-no-files-found: error diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index 5989408bba..2001cb9997 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -8,7 +8,6 @@ on: env: # Configure a constant location for the uv cache UV_CACHE_DIR: /tmp/.uv-cache - UV_SEMVER: "0.8.x" jobs: python-tests: @@ -31,7 +30,7 @@ jobs: - name: Set up uv uses: astral-sh/setup-uv@v6 with: - version: ${{ env.UV_SEMVER }} + version-file: "python/pyproject.toml" enable-cache: true cache-suffix: ${{ runner.os }}-${{ matrix.python-version }} cache-dependency-glob: "**/uv.lock" diff --git a/python/packages/azure/pyproject.toml b/python/packages/azure/pyproject.toml index 152cb15ddb..c2f2a718f1 100644 --- a/python/packages/azure/pyproject.toml +++ b/python/packages/azure/pyproject.toml @@ -48,6 +48,11 @@ timeout = 120 [tool.ruff] extend = "../../pyproject.toml" +[tool.coverage.run] +omit = [ + "**/__init__.py" +] + [tool.pyright] extend = "../../pyproject.toml" exclude = ['tests'] @@ -76,6 +81,7 @@ executor.type = "uv" include = "../../shared_tasks.toml" [tool.poe.tasks] mypy = "mypy --config-file $POE_ROOT/pyproject.toml agent_framework_azure" +test = "pytest --cov=agent_framework_azure --cov-report=term-missing:skip-covered tests" [tool.uv.build-backend] module-name = "agent_framework_azure" diff --git a/python/packages/foundry/pyproject.toml b/python/packages/foundry/pyproject.toml index b8621931e8..2d12831c69 100644 --- a/python/packages/foundry/pyproject.toml +++ b/python/packages/foundry/pyproject.toml @@ -50,6 +50,11 @@ timeout = 120 [tool.ruff] extend = "../../pyproject.toml" +[tool.coverage.run] +omit = [ + "**/__init__.py" +] + [tool.pyright] extend = "../../pyproject.toml" exclude = ['tests'] @@ -78,6 +83,7 @@ executor.type = "uv" include = "../../shared_tasks.toml" [tool.poe.tasks] mypy = "mypy --config-file $POE_ROOT/pyproject.toml agent_framework_foundry" +test = "pytest --cov=agent_framework_foundry --cov-report=term-missing:skip-covered tests" [tool.uv.build-backend] module-name = "agent_framework_foundry" diff --git a/python/packages/main/pyproject.toml b/python/packages/main/pyproject.toml index ad7ec7601b..1cd1b5d97e 100644 --- a/python/packages/main/pyproject.toml +++ b/python/packages/main/pyproject.toml @@ -61,6 +61,11 @@ asyncio_default_fixture_loop_scope = "function" filterwarnings = [] timeout = 120 +[tool.coverage.run] +omit = [ + "**/__init__.py" +] + [tool.ruff] extend = "../../pyproject.toml" @@ -92,6 +97,7 @@ executor.type = "uv" include = "../../shared_tasks.toml" [tool.poe.tasks] mypy = "mypy --config-file $POE_ROOT/pyproject.toml agent_framework" +test = "pytest --cov=agent_framework --cov-report=term-missing:skip-covered tests" [tool.uv.build-backend] module-name = "agent_framework" diff --git a/python/packages/workflow/pyproject.toml b/python/packages/workflow/pyproject.toml index e97aad3534..73577d3a45 100644 --- a/python/packages/workflow/pyproject.toml +++ b/python/packages/workflow/pyproject.toml @@ -47,6 +47,11 @@ timeout = 120 [tool.ruff] extend = "../../pyproject.toml" +[tool.coverage.run] +omit = [ + "**/__init__.py" +] + [tool.pyright] extend = "../../pyproject.toml" exclude = ['tests'] @@ -73,6 +78,8 @@ exclude_dirs = ["tests"] [tool.poe] executor.type = "uv" include = "../../shared_tasks.toml" +[tool.poe.tasks] +test = "pytest --cov=agent_framework_workflow --cov-report=term-missing:skip-covered tests" [tool.uv.build-backend] module-name = "agent_framework_workflow" diff --git a/python/pyproject.toml b/python/pyproject.toml index 3cb5c8591a..b42e390638 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -123,6 +123,19 @@ convention = "google" notice-rgx = "^# Copyright \\(c\\) Microsoft\\. All rights reserved\\." min-file-size = 1 +[tool.pytest.ini_options] +testpaths = 'tests' +addopts = "-ra -q -r fEX" +asyncio_mode = "auto" +asyncio_default_fixture_loop_scope = "function" +filterwarnings = [] +timeout = 120 + +[tool.coverage.run] +omit = [ + "**/__init__.py" +] + [tool.pyright] include = ["agent_framework", "samples"] exclude = ["**/tests/**", "docs", "**/.venv/**"] @@ -172,6 +185,7 @@ build = "python run_tasks_in_packages_if_exists.py build" # combined checks check = ["fmt", "lint", "pyright", "mypy", "test", "markdown-code-lint", "samples-code-check"] pre-commit-check = ["fmt", "lint", "pyright", "markdown-code-lint", "samples-code-check"] +all-tests = "pytest --cov=agent_framework --cov=agent_framework_azure --cov=agent_framework_foundry --cov=agent_framework_workflow --cov-report=term-missing:skip-covered packages/**/tests" [tool.poe.tasks.venv] cmd = "uv venv --clear --python $python" @@ -184,24 +198,7 @@ sequence = [ ] args = [{ name = "python", default = "3.13", options = ['-p', '--python'] }] - [tool.setuptools.packages.find] where = ["packages"] include = ["agent_framework**"] namespaces = true - -[tool.tox] -requires = ["tox>=4", "tox-uv>=1"] -env_list = [ "3.13"] #, "3.12", "3.11", "3.10" ] - -[tool.tox.env_run_base] -no_package = true -description = "run tests" -dependency_groups = [ - "dev", -] -deps = [ - "-e packages/azure", - "-e packages/main" -] -commands = [["pytest", { replace = "posargs", default = ["packages/main/tests"], extend = true }]] diff --git a/python/shared_tasks.toml b/python/shared_tasks.toml index f0f52d99ad..e7ec8c8bf9 100644 --- a/python/shared_tasks.toml +++ b/python/shared_tasks.toml @@ -4,4 +4,3 @@ format.ref = "fmt" lint = "ruff check" pyright = "pyright" build = "uv build" -test = "pytest --cov=agent_framework --cov-report=term-missing:skip-covered tests"