Python: Simplify Python Poe tasks and unify package selectors (#4722)

* updated automation tasks and commands, with alias for the time being

* Restore aggregate test exclusions

Preserve the legacy all-tests scope for test --all by excluding lab and devui from the default aggregate sweep, while still allowing explicit package selection. Also ignore hidden/generated test directories such as .mypy_cache during aggregate discovery.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

* updated versions in pre-commit

---------

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
Eduard van Valkenburg
2026-03-18 19:39:11 +01:00
committed by GitHub
Unverified
parent d3d0100822
commit f48c4512d3
60 changed files with 1704 additions and 527 deletions
+8 -8
View File
@@ -46,14 +46,14 @@ These are the normal user-facing entrypoints:
uv run poe upgrade-dev-dependency-pins
uv run poe upgrade-dev-dependencies
uv run poe validate-dependency-bounds-test
uv run poe validate-dependency-bounds-test --project <workspace-package-name>
uv run poe validate-dependency-bounds-project --mode both --project <workspace-package-name> --dependency "<dependency-name>"
uv run poe validate-dependency-bounds-test --package core
uv run poe validate-dependency-bounds-project --mode both --package core --dependency "<dependency-name>"
```
- `upgrade-dev-dependency-pins` only refreshes exact dev pins in `pyproject.toml` files.
- `upgrade-dev-dependencies` refreshes dev pins (using task above), runs `uv lock --upgrade`, reinstalls from the frozen lockfile, then runs `check`, `typing`, and `test`.
- `validate-dependency-bounds-test` runs the repo-wide lower/upper smoke gate.
- `validate-dependency-bounds-project` is the single package-scoped task; use `--mode lower`, `--mode upper`, or `--mode both` for the target package/dependency pair. Its `--project` argument defaults to `*`, and `--dependency` is optional, so automation can also use it for repo-wide upper-bound runs.
- `validate-dependency-bounds-project` is the single package-scoped task; use `--mode lower`, `--mode upper`, or `--mode both` for the target package/dependency pair. Its `--package` argument defaults to `*`, and `--dependency` is optional, so automation can also use it for repo-wide upper-bound runs.
### GitHub Actions workflows
@@ -61,7 +61,7 @@ These workflows call the Poe tasks:
- `.github/workflows/python-dependency-range-validation.yml`
- Trigger: `workflow_dispatch`
- Runs `uv run poe validate-dependency-bounds-project --mode upper --project "*"`
- Runs `uv run poe validate-dependency-bounds-project --mode upper --package "*"`
- Uploads `python/scripts/dependencies/dependency-range-results.json`
- Creates issues for failing candidate versions and opens/updates a PR for passing range updates
@@ -76,10 +76,10 @@ These are useful for debugging or targeted manual runs:
```bash
python -m scripts.dependencies.upgrade_dev_dependencies --dry-run --version-source lock
python -m scripts.dependencies.validate_dependency_bounds --mode test --package packages/core --dry-run
python -m scripts.dependencies.validate_dependency_bounds --mode both --package packages/core --dependencies openai --dry-run
python -m scripts.dependencies._dependency_bounds_lower_impl --packages packages/core --dependencies openai --dry-run
python -m scripts.dependencies._dependency_bounds_upper_impl --packages packages/core --dependencies openai --dry-run
python -m scripts.dependencies.validate_dependency_bounds --mode test --package core --dry-run
python -m scripts.dependencies.validate_dependency_bounds --mode both --package core --dependencies openai --dry-run
python -m scripts.dependencies._dependency_bounds_lower_impl --packages core --dependencies openai --dry-run
python -m scripts.dependencies._dependency_bounds_upper_impl --packages core --dependencies openai --dry-run
```
Use the direct lower/upper implementation modules mainly for debugging or development of the optimizers themselves. For normal usage, prefer the Poe tasks or `validate_dependency_bounds.py`.
@@ -1,5 +1,5 @@
# Copyright (c) Microsoft. All rights reserved.
# ruff: noqa: INP001, S404, S603
# ruff: noqa: S404, S603
"""Lower dependency bounds, validate, and persist the oldest passing set."""
@@ -21,14 +21,15 @@ from urllib import error as urllib_error
from urllib import request as urllib_request
import tomli
from packaging.requirements import InvalidRequirement, Requirement
from packaging.version import InvalidVersion, Version
from rich import print
from scripts.dependencies._dependency_bounds_runtime import (
extend_command_with_runtime_tools,
extend_command_with_task,
)
from packaging.requirements import InvalidRequirement, Requirement
from packaging.version import InvalidVersion, Version
from rich import print
from scripts.task_runner import discover_projects, extract_poe_tasks
from scripts.task_runner import discover_projects, extract_poe_tasks, project_filter_matches
CHECK_TASK_PRIORITY = ("check", "typing", "pyright", "mypy", "lint")
REQ_PATTERN = r"^\s*([A-Za-z0-9_.-]+(?:\[[^\]]+\])?)\s*(.*?)\s*$"
@@ -937,7 +938,7 @@ def main() -> None:
"--packages",
nargs="*",
default=None,
help="Optional package filters by workspace path (e.g., packages/core) or package name.",
help="Optional package filters by short name (for example core), workspace path, or package name.",
)
parser.add_argument(
"--dependencies",
@@ -1001,7 +1002,11 @@ def main() -> None:
project_section = package_config.get("project", {})
optional_dependencies = project_section.get("optional-dependencies", {}) or {}
dependency_groups = package_config.get("dependency-groups", {}) or {}
if package_filters and str(project_path) not in package_filters and package_name not in package_filters:
# Reuse the shared selector matcher so direct optimizer runs accept the
# same short-name package filters as the contributor-facing Poe tasks.
if package_filters and not any(
project_filter_matches(project_path, package_filter, [package_name]) for package_filter in package_filters
):
continue
plans.append(
PackagePlan(
@@ -1,5 +1,5 @@
# Copyright (c) Microsoft. All rights reserved.
# ruff: noqa: INP001, S404, S603
# ruff: noqa: S404, S603
"""Raise dependency upper bounds, validate, and persist the latest passing set."""
@@ -22,15 +22,16 @@ from urllib import error as urllib_error
from urllib import request as urllib_request
import tomli
from packaging.requirements import InvalidRequirement, Requirement
from packaging.version import InvalidVersion, Version
from rich import print
from scripts.dependencies._dependency_bounds_runtime import (
extend_command_with_runtime_tools,
extend_command_with_task,
next_zero_major_minor_boundary,
)
from packaging.requirements import InvalidRequirement, Requirement
from packaging.version import InvalidVersion, Version
from rich import print
from scripts.task_runner import discover_projects, extract_poe_tasks
from scripts.task_runner import discover_projects, extract_poe_tasks, project_filter_matches
CHECK_TASK_PRIORITY = ("check", "typing", "pyright", "mypy", "lint")
REQ_PATTERN = r"^\s*([A-Za-z0-9_.-]+(?:\[[^\]]+\])?)\s*(.*?)\s*$"
@@ -1088,7 +1089,7 @@ def main() -> None:
"--packages",
nargs="*",
default=None,
help="Optional package filters by workspace path (e.g., packages/core) or package name.",
help="Optional package filters by short name (for example core), workspace path, or package name.",
)
parser.add_argument(
"--dependencies",
@@ -1153,7 +1154,11 @@ def main() -> None:
project_section = package_config.get("project", {})
optional_dependencies = project_section.get("optional-dependencies", {}) or {}
dependency_groups = package_config.get("dependency-groups", {}) or {}
if package_filters and str(project_path) not in package_filters and package_name not in package_filters:
# Reuse the shared selector matcher so direct optimizer runs accept the
# same short-name package filters as the contributor-facing Poe tasks.
if package_filters and not any(
project_filter_matches(project_path, package_filter, [package_name]) for package_filter in package_filters
):
continue
plans.append(
PackagePlan(
@@ -0,0 +1,118 @@
# Copyright (c) Microsoft. All rights reserved.
# ruff: noqa: S603
"""Add a dependency to one workspace package selected by short name or path.
``uv add --package`` expects the published workspace distribution name, while
the root Poe surface intentionally speaks in short repo package names such as
``core``. This wrapper keeps the user-facing selector stable and translates it
just before delegating to uv.
"""
from __future__ import annotations
import argparse
import subprocess
from dataclasses import dataclass
from pathlib import Path
import tomli
from rich import print
from scripts.task_runner import discover_projects, project_filter_matches
@dataclass(frozen=True)
class WorkspacePackage:
"""Workspace package metadata needed for `uv add --package`."""
short_name: str
project_path: Path
distribution_name: str
def _load_distribution_name(pyproject_file: Path) -> str:
with pyproject_file.open("rb") as f:
data = tomli.load(f)
return str(data.get("project", {}).get("name", "")).strip()
def _discover_workspace_packages(workspace_root: Path) -> list[WorkspacePackage]:
workspace_pyproject = workspace_root / "pyproject.toml"
packages: list[WorkspacePackage] = []
for project_path in sorted(discover_projects(workspace_pyproject), key=str):
pyproject_file = workspace_root / project_path / "pyproject.toml"
if not pyproject_file.exists():
continue
distribution_name = _load_distribution_name(pyproject_file)
if not distribution_name:
continue
packages.append(
WorkspacePackage(
short_name=project_path.name,
project_path=project_path,
distribution_name=distribution_name,
)
)
return packages
def _resolve_workspace_package(workspace_root: Path, project_filter: str) -> WorkspacePackage:
"""Resolve one workspace package from a user-facing selector.
The wrapper accepts the same short-name/path/distribution-name vocabulary as
the other root tasks, but errors on ambiguous matches so dependency edits
never hit the wrong package.
"""
matches = [
package
for package in _discover_workspace_packages(workspace_root)
if project_filter_matches(package.project_path, project_filter, [package.short_name, package.distribution_name])
]
if not matches:
raise SystemExit(f"No workspace package matched selector '{project_filter}'.")
if len(matches) > 1:
names = ", ".join(sorted(package.short_name for package in matches))
raise SystemExit(
f"Package selector '{project_filter}' matched multiple workspace packages: {names}. "
"Use a more specific short name or path."
)
return matches[0]
def main() -> None:
"""Resolve a workspace project selector, then delegate to `uv add`."""
parser = argparse.ArgumentParser(
description="Add a dependency to a single workspace package selected by short name, path, or package name."
)
parser.add_argument(
"-P",
"--package",
dest="project",
metavar="PACKAGE",
required=True,
help="Workspace package selector, such as `core`.",
)
# Keep the old long flag as a silent alias while downstream automation
# finishes moving to the user-facing ``--package`` spelling.
parser.add_argument("--project", dest="project", help=argparse.SUPPRESS)
parser.add_argument("-D", "--dependency", required=True, help="Dependency specifier to add.")
args = parser.parse_args()
workspace_root = Path(__file__).resolve().parents[2]
package = _resolve_workspace_package(workspace_root, args.project)
print(
f"[cyan]Adding {args.dependency} to {package.short_name} "
f"({package.distribution_name})[/cyan]"
)
result = subprocess.run(
["uv", "add", "--package", package.distribution_name, args.dependency],
cwd=workspace_root,
check=False,
)
if result.returncode:
raise SystemExit(result.returncode)
if __name__ == "__main__":
main()
@@ -1,5 +1,5 @@
# Copyright (c) Microsoft. All rights reserved.
# ruff: noqa: INP001, S404, S603
# ruff: noqa: S404, S603
"""Unified dependency-bound validation entrypoint.
@@ -8,6 +8,10 @@ Modes:
- lower: run lower-bound expansion for one package.
- upper: run upper-bound expansion for one package.
- both: run lower then upper expansion for one package.
Package filters intentionally reuse the root task selector semantics so the
same short package names (for example ``core``) work in both contributor
commands and direct debugging entrypoints.
"""
from __future__ import annotations
@@ -23,6 +27,7 @@ from pathlib import Path
import tomli
from rich import print
from scripts.dependencies._dependency_bounds_runtime import (
extend_command_with_runtime_tools,
extend_command_with_task,
@@ -33,7 +38,7 @@ from scripts.dependencies._dependency_bounds_upper_impl import (
_load_package_name,
_resolve_internal_editables,
)
from scripts.task_runner import discover_projects, extract_poe_tasks
from scripts.task_runner import discover_projects, extract_poe_tasks, project_filter_matches
_LOWER_IMPL_MODULE = "scripts.dependencies._dependency_bounds_lower_impl"
_UPPER_IMPL_MODULE = "scripts.dependencies._dependency_bounds_upper_impl"
@@ -76,10 +81,10 @@ def _coerce_subprocess_output(output: str | bytes | None) -> str:
def _build_test_plans(workspace_root: Path, package_filter: str | None) -> list[PackageTestPlan]:
"""Build per-package test plans for the requested workspace selector."""
workspace_pyproject = workspace_root / "pyproject.toml"
package_map = _build_workspace_package_map(workspace_root)
internal_graph = _build_internal_graph(workspace_root, package_map)
normalized_filter = None if package_filter in {None, "", "*"} else package_filter
plans: list[PackageTestPlan] = []
missing_tasks: list[str] = []
@@ -89,7 +94,14 @@ def _build_test_plans(workspace_root: Path, package_filter: str | None) -> list[
continue
package_name = _load_package_name(pyproject_file)
if normalized_filter and str(project_path) != normalized_filter and package_name != normalized_filter:
# Reuse the shared matcher so dependency-bound test mode accepts the
# same short names and legacy path-style selectors as the root Poe
# commands.
if (
package_filter
and package_filter != "*"
and not project_filter_matches(project_path, package_filter, [package_name])
):
continue
available_tasks = extract_poe_tasks(pyproject_file)
@@ -366,7 +378,10 @@ def main() -> None:
parser.add_argument(
"--package",
default=None,
help="Optional workspace package path/name filter for all modes. Use '*' or omit it for the whole workspace.",
help=(
"Optional workspace package selector for all modes, such as `core`. "
"Use '*' or omit it for the whole workspace."
),
)
parser.add_argument(
"--dependencies",
+80 -12
View File
@@ -1,6 +1,11 @@
# Copyright (c) Microsoft. All rights reserved.
"""Shared utilities for running poe tasks across workspace packages in parallel."""
"""Shared utilities for running Poe tasks across workspace packages.
These helpers centralize workspace discovery, selector matching, and execution
mode so the root task dispatcher and dependency tooling interpret package
filters the same way.
"""
import concurrent.futures
import glob
@@ -8,6 +13,8 @@ import os
import subprocess
import sys
import time
from collections.abc import Sequence
from fnmatch import fnmatch
from pathlib import Path
import tomli
@@ -70,12 +77,67 @@ def build_work_items(projects: list[Path], task_names: list[str]) -> list[tuple[
return work_items
def _run_task_subprocess(project: Path, task: str, workspace_root: Path) -> tuple[Path, str, int, str, str, float]:
def normalize_project_filter(value: str) -> str:
"""Normalize a user-supplied workspace selector.
Strip presentation differences so short names, relative paths, and globs can
be compared with one matcher.
"""
normalized = value.strip().strip("/").replace("\\", "/")
return normalized or "."
def build_project_filter_candidates(project: Path | str, aliases: Sequence[str] = ()) -> set[str]:
"""Return accepted selector values for one workspace project.
We accept the workspace path, short package name, and any supplied aliases
so user-facing ``--package core`` stays stable even when underlying tools
still need paths or distribution names.
"""
normalized_path = normalize_project_filter(str(project))
candidates = {normalized_path}
if normalized_path == ".":
candidates.update({"./", "root"})
else:
# Accept bare short names like ``core`` alongside ``packages/core`` and
# ``./packages/core`` so callers do not have to care which form a
# downstream script prefers.
path = Path(normalized_path)
candidates.add(path.name)
candidates.add(f"./{normalized_path}")
for alias in aliases:
normalized_alias = normalize_project_filter(alias)
if normalized_alias and normalized_alias != ".":
candidates.add(normalized_alias)
return {candidate.lower() for candidate in candidates}
def project_filter_matches(project: Path | str, pattern: str, aliases: Sequence[str] = ()) -> bool:
"""Return whether a project matches a user-supplied selector or glob.
Matching happens against the normalized candidate set so CLI callers can use
the same selector vocabulary everywhere.
"""
normalized_pattern = normalize_project_filter(pattern).lower()
return any(
fnmatch(candidate, normalized_pattern)
for candidate in build_project_filter_candidates(project, aliases)
)
def _run_task_subprocess(
project: Path,
task: str,
workspace_root: Path,
task_args: Sequence[str] = (),
) -> tuple[Path, str, int, str, str, float]:
"""Run a single poe task in a project directory via subprocess."""
start = time.monotonic()
cwd = workspace_root / project
result = subprocess.run(
["uv", "run", "poe", task],
["uv", "run", "poe", task, *task_args],
cwd=cwd,
capture_output=True,
text=True,
@@ -84,20 +146,20 @@ def _run_task_subprocess(project: Path, task: str, workspace_root: Path) -> tupl
return (project, task, result.returncode, result.stdout, result.stderr, elapsed)
def _run_sequential(work_items: list[tuple[Path, str]]) -> None:
def _run_sequential(work_items: list[tuple[Path, str]], task_args: Sequence[str] = ()) -> None:
"""Run tasks sequentially using in-process PoeThePoet (streaming output)."""
from poethepoet.app import PoeThePoet
for project, task in work_items:
print(f"Running task {task} in {project}")
app = PoeThePoet(cwd=project)
result = app(cli_args=[task])
result = app(cli_args=[task, *task_args])
if result:
sys.exit(result)
def _run_parallel(work_items: list[tuple[Path, str]], workspace_root: Path) -> None:
"""Run all (package × task) combinations in parallel via subprocesses."""
def _run_parallel(work_items: list[tuple[Path, str]], workspace_root: Path, task_args: Sequence[str] = ()) -> None:
"""Run all (package x task) combinations in parallel via subprocesses."""
max_workers = min(len(work_items), os.cpu_count() or 4)
failures: list[tuple[Path, str, str, str]] = []
completed = 0
@@ -107,7 +169,7 @@ def _run_parallel(work_items: list[tuple[Path, str]], workspace_root: Path) -> N
with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
futures = {
executor.submit(_run_task_subprocess, project, task, workspace_root): (project, task)
executor.submit(_run_task_subprocess, project, task, workspace_root, task_args): (project, task)
for project, task in work_items
}
for future in concurrent.futures.as_completed(futures):
@@ -123,7 +185,7 @@ def _run_parallel(work_items: list[tuple[Path, str]], workspace_root: Path) -> N
if failures:
print(f"\n[red]{len(failures)} task(s) failed:[/red]")
for project, task, stdout, stderr in failures:
print(f"\n[red]{'='*60}[/red]")
print(f"\n[red]{'=' * 60}[/red]")
print(f"[red]FAILED: {task} in {project}[/red]")
if stdout.strip():
print(stdout)
@@ -134,7 +196,13 @@ def _run_parallel(work_items: list[tuple[Path, str]], workspace_root: Path) -> N
print(f"\n[green]All {total} task(s) passed ✓[/green]")
def run_tasks(work_items: list[tuple[Path, str]], workspace_root: Path, *, sequential: bool = False) -> None:
def run_tasks(
work_items: list[tuple[Path, str]],
workspace_root: Path,
*,
sequential: bool = False,
task_args: Sequence[str] = (),
) -> None:
"""Run work items either in parallel or sequentially.
Single items use in-process PoeThePoet for streaming output.
@@ -145,6 +213,6 @@ def run_tasks(work_items: list[tuple[Path, str]], workspace_root: Path, *, seque
return
if sequential or len(work_items) == 1:
_run_sequential(work_items)
_run_sequential(work_items, task_args)
else:
_run_parallel(work_items, workspace_root)
_run_parallel(work_items, workspace_root, task_args)
+698
View File
@@ -0,0 +1,698 @@
# Copyright (c) Microsoft. All rights reserved.
"""Dispatch contributor-facing workspace tasks with consistent scope flags.
This script is the single root-task entrypoint used by ``python/pyproject.toml``.
It keeps selector semantics, aggregate-vs-fan-out behaviour, and compatibility
aliases in one place so docs and automation can share the same command surface.
"""
from __future__ import annotations
import argparse
import os
import subprocess
import sys
from dataclasses import dataclass
from pathlib import Path
import tomli
from packaging.specifiers import SpecifierSet
from packaging.version import Version
from rich import print
from task_runner import build_work_items, discover_projects, project_filter_matches, run_tasks
WORKSPACE_ROOT = Path(__file__).resolve().parent.parent
WORKSPACE_PYPROJECT = WORKSPACE_ROOT / "pyproject.toml"
CURRENT_PYTHON = Version(f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}")
SAMPLE_EXCLUDES = "samples/autogen-migration,samples/semantic-kernel-migration"
SAMPLE_RUFF_IGNORE = "E501,ASYNC,B901,TD002"
MARKDOWN_EXCLUDES = [
"cookiecutter-agent-framework-lab",
"tau2",
"packages/devui/frontend",
"context_providers/azure_ai_search",
]
DEFAULT_AGGREGATE_TEST_EXCLUDES = {"devui", "lab"}
@dataclass(frozen=True)
class WorkspaceProject:
"""Metadata about a workspace package."""
path: Path
name: str
distribution_name: str
requires_python: str | None
def parse_args(argv: list[str]) -> tuple[argparse.Namespace, list[str]]:
"""Parse the workspace command and return any pass-through arguments."""
parser = argparse.ArgumentParser(description="Dispatch workspace Poe tasks with consistent scope flags.")
subparsers = parser.add_subparsers(dest="command", required=True)
def add_project_option(command: argparse.ArgumentParser) -> None:
command.add_argument(
"-P",
"--package",
dest="project",
default="*",
metavar="PACKAGE",
help="Workspace package selector or glob pattern, such as `core`.",
)
# Keep a hidden compatibility alias while old automation and local
# muscle memory migrate from ``--project`` to ``--package``.
command.add_argument("--project", dest="project", help=argparse.SUPPRESS)
def add_syntax_mode_options(command: argparse.ArgumentParser) -> None:
command.add_argument("-F", "--format", action="store_true", help="Run formatting only.")
command.add_argument("-C", "--check", action="store_true", help="Run lint checks only.")
def add_all_option(command: argparse.ArgumentParser) -> None:
command.add_argument("-A", "--all", action="store_true", help="Run a single aggregate workspace sweep.")
def add_samples_option(command: argparse.ArgumentParser) -> None:
command.add_argument("-S", "--samples", action="store_true", help="Target samples/ instead of packages.")
def add_cov_option(command: argparse.ArgumentParser) -> None:
command.add_argument("-C", "--cov", action="store_true", help="Enable coverage output.")
syntax = subparsers.add_parser("syntax")
add_project_option(syntax)
add_samples_option(syntax)
add_syntax_mode_options(syntax)
for command_name in ("fmt", "build", "clean-dist", "check-packages"):
command = subparsers.add_parser(command_name)
add_project_option(command)
lint = subparsers.add_parser("lint")
add_project_option(lint)
add_samples_option(lint)
pyright = subparsers.add_parser("pyright")
add_project_option(pyright)
add_all_option(pyright)
add_samples_option(pyright)
mypy = subparsers.add_parser("mypy")
add_project_option(mypy)
add_all_option(mypy)
typing = subparsers.add_parser("typing")
add_project_option(typing)
add_all_option(typing)
test = subparsers.add_parser("test")
add_project_option(test)
add_all_option(test)
add_cov_option(test)
check = subparsers.add_parser("check")
add_project_option(check)
add_samples_option(check)
prek_check = subparsers.add_parser("prek-check")
prek_check.add_argument("files", nargs="*", default=["."], help="Files reported by pre-commit.")
subparsers.add_parser("ci-mypy")
return parser.parse_known_args(argv)
def load_toml(file_path: Path) -> dict:
"""Load a TOML file."""
with file_path.open("rb") as file:
return tomli.load(file)
def discover_workspace_projects() -> list[WorkspaceProject]:
"""Return workspace packages together with their Python-version metadata."""
projects: list[WorkspaceProject] = []
for project_path in discover_projects(WORKSPACE_PYPROJECT):
pyproject = load_toml(WORKSPACE_ROOT / project_path / "pyproject.toml")
requires_python = pyproject.get("project", {}).get("requires-python")
distribution_name = str(pyproject.get("project", {}).get("name", "")).strip()
projects.append(
WorkspaceProject(
path=project_path,
name=project_path.name,
distribution_name=distribution_name,
requires_python=requires_python,
)
)
return projects
def supports_current_python(project: WorkspaceProject) -> bool:
"""Return whether the current interpreter satisfies the project's Python requirement."""
if not project.requires_python:
return True
return SpecifierSet(project.requires_python).contains(CURRENT_PYTHON, prereleases=True)
def select_projects(pattern: str) -> list[WorkspaceProject]:
"""Select supported workspace projects that match the supplied pattern.
The shared matcher accepts short names such as ``core``, legacy path-style
values, and distribution names so every root task family speaks the same
selector dialect.
"""
matched_projects = [
project
for project in discover_workspace_projects()
if project_filter_matches(project.path, pattern, aliases=[project.name, project.distribution_name])
]
if not matched_projects:
print(f"[red]No workspace projects matched pattern '{pattern}'.[/red]")
raise SystemExit(2)
supported_projects = [project for project in matched_projects if supports_current_python(project)]
unsupported_projects = [project.name for project in matched_projects if not supports_current_python(project)]
if unsupported_projects:
version = f"{sys.version_info.major}.{sys.version_info.minor}"
print(
"[yellow]Skipping packages not supported by "
f"Python {version}: {', '.join(sorted(unsupported_projects))}[/yellow]"
)
return supported_projects
def relative_path(path: Path) -> str:
"""Convert a workspace path to a stable relative string."""
return path.relative_to(WORKSPACE_ROOT).as_posix()
def collect_source_dirs(projects: list[WorkspaceProject]) -> list[Path]:
"""Collect top-level import package directories for the selected projects."""
source_dirs: set[Path] = set()
for project in projects:
project_root = WORKSPACE_ROOT / project.path
for init_file in project_root.rglob("__init__.py"):
package_dir = init_file.parent
if package_dir.name.startswith("agent_framework"):
source_dirs.add(package_dir)
return sorted(source_dirs)
def collect_test_dirs(projects: list[WorkspaceProject]) -> list[Path]:
"""Collect test directories for the selected projects."""
test_dirs: set[Path] = set()
for project in projects:
project_root = WORKSPACE_ROOT / project.path
for directory_name in ("tests", "ag_ui_tests"):
for test_dir in project_root.rglob(directory_name):
relative_test_dir = test_dir.relative_to(project_root)
# Ignore hidden/generated trees such as ``.mypy_cache`` so the
# aggregate sweep only targets real repository test directories.
if test_dir.is_dir() and not any(part.startswith(".") for part in relative_test_dir.parts):
test_dirs.add(test_dir)
return sorted(test_dirs)
def run_command(command: list[str]) -> None:
"""Run a subprocess from the workspace root and stream its output."""
result = subprocess.run(command, cwd=WORKSPACE_ROOT, check=False)
if result.returncode:
raise SystemExit(result.returncode)
def run_fan_out(task_names: list[str], project_pattern: str, task_args: list[str]) -> None:
"""Run package-local Poe tasks across the selected projects."""
selected_projects = select_projects(project_pattern)
if not selected_projects:
print("[yellow]No selected projects support the current Python version, skipping.[/yellow]")
return
work_items = build_work_items([project.path for project in selected_projects], task_names)
run_tasks(work_items, WORKSPACE_ROOT, task_args=task_args)
def sample_pyright_config() -> str:
"""Return the sample Pyright configuration for the current interpreter."""
if sys.version_info < (3, 11):
return "pyrightconfig.samples.py310.json"
return "pyrightconfig.samples.json"
def run_sample_lint(extra_args: list[str]) -> None:
"""Run linting against samples/."""
command = [
"uv",
"run",
"ruff",
"check",
"samples",
"--fix",
"--exclude",
SAMPLE_EXCLUDES,
"--ignore",
SAMPLE_RUFF_IGNORE,
*extra_args,
]
run_command(command)
def run_sample_format(extra_args: list[str]) -> None:
"""Run formatting against samples/."""
command = [
"uv",
"run",
"ruff",
"format",
"samples",
"--exclude",
SAMPLE_EXCLUDES,
*extra_args,
]
run_command(command)
def run_sample_pyright(extra_args: list[str]) -> None:
"""Run sample syntax/import validation."""
command = ["uv", "run", "pyright", "-p", sample_pyright_config(), "--warnings", *extra_args]
run_command(command)
def run_markdown_code_lint(files: list[str] | None = None) -> None:
"""Run markdown code-block linting globally or for the changed markdown files only."""
command = [
"uv",
"run",
"python",
"scripts/check_md_code_blocks.py",
]
if files is None:
command.extend([
"README.md",
"./packages/**/README.md",
"./samples/**/*.md",
])
else:
if not files:
print("[yellow]No markdown files changed, skipping markdown code lint.[/yellow]")
return
command.extend(files)
command.append("--no-glob")
for excluded_path in MARKDOWN_EXCLUDES:
command.extend(["--exclude", excluded_path])
run_command(command)
def run_aggregate_pyright(project_pattern: str, extra_args: list[str]) -> None:
"""Run a single Pyright sweep across the selected project roots."""
projects = select_projects(project_pattern)
if not projects:
print("[yellow]No selected projects support the current Python version, skipping.[/yellow]")
return
project_paths = [relative_path(WORKSPACE_ROOT / project.path) for project in projects]
run_command(["uv", "run", "pyright", *extra_args, *project_paths])
def run_aggregate_mypy(project_pattern: str, extra_args: list[str]) -> None:
"""Run a single MyPy sweep across the selected project import roots."""
projects = select_projects(project_pattern)
if not projects:
print("[yellow]No selected projects support the current Python version, skipping.[/yellow]")
return
source_dirs = [relative_path(path) for path in collect_source_dirs(projects)]
if not source_dirs:
print("[yellow]No import roots found for the selected projects, skipping MyPy.[/yellow]")
return
run_command(["uv", "run", "mypy", "--config-file", "pyproject.toml", *extra_args, *source_dirs])
def run_aggregate_test(project_pattern: str, cov: bool, extra_args: list[str]) -> None:
"""Run a single pytest sweep across the selected project test directories."""
projects = select_projects(project_pattern)
if not projects:
print("[yellow]No selected projects support the current Python version, skipping.[/yellow]")
return
if project_pattern == "*":
# Preserve the legacy ``all-tests`` contract when ``test --all`` runs with
# the default selector: experimental packages stay opt-in instead of
# suddenly joining every PR unit-test sweep.
projects = [project for project in projects if project.name not in DEFAULT_AGGREGATE_TEST_EXCLUDES]
if not projects:
print("[yellow]No aggregate-test projects remain after applying default exclusions.[/yellow]")
return
test_dirs = [relative_path(path) for path in collect_test_dirs(projects)]
if not test_dirs:
print("[yellow]No test directories found for the selected projects, skipping pytest.[/yellow]")
return
command = [
"uv",
"run",
"pytest",
"--import-mode=importlib",
"-m",
"not integration",
"-rs",
"-n",
"logical",
"--dist",
"worksteal",
]
if cov:
for source_dir in collect_source_dirs(projects):
command.append(f"--cov={source_dir.name}")
command.extend(["--cov-config=pyproject.toml", "--cov-report=term-missing:skip-covered"])
command.extend(extra_args)
command.extend(test_dirs)
run_command(command)
def normalize_changed_file(file_path: str) -> str:
"""Normalize changed-file paths passed from git or pre-commit."""
normalized = file_path.replace("\\", "/")
if normalized.startswith("python/"):
return normalized[7:]
return normalized
def has_changed_sample_files(files: list[str]) -> bool:
"""Return whether any changed file lives under samples/."""
return any(normalize_changed_file(file_path).startswith("samples/") for file_path in files)
def changed_markdown_files(files: list[str]) -> list[str]:
"""Return markdown files from the provided change list."""
markdown_files = [normalize_changed_file(file_path) for file_path in files]
return sorted({file_path for file_path in markdown_files if file_path.endswith(".md")})
def run_changed_package_tasks(task_names: list[str], files: list[str]) -> None:
"""Run package-local tasks only in packages affected by the provided file list."""
command = [
"uv",
"run",
"python",
"scripts/run_tasks_in_changed_packages.py",
*task_names,
"--files",
*files,
]
run_command(command)
def run_prek_check(files: list[str]) -> None:
"""Run the lightweight pre-commit task surface."""
normalized_files = [normalize_changed_file(file_path) for file_path in files] or ["."]
run_changed_package_tasks(["fmt", "lint"], normalized_files)
run_markdown_code_lint(changed_markdown_files(normalized_files))
if has_changed_sample_files(normalized_files):
print("[cyan]Sample files changed, running sample checks.[/cyan]")
run_sample_lint([])
run_sample_pyright([])
else:
print("[yellow]No sample files changed, skipping sample checks.[/yellow]")
def git_diff_name_only(*revisions: str) -> list[str] | None:
"""Try a git diff strategy and return changed files if it succeeds."""
result = subprocess.run(
["git", "diff", "--name-only", *revisions, "--", "."],
cwd=WORKSPACE_ROOT,
capture_output=True,
text=True,
check=False,
)
if result.returncode != 0:
return None
return [line for line in result.stdout.splitlines() if line]
def detect_ci_changed_files() -> list[str]:
"""Detect changed files for change-based mypy runs."""
base_ref = os.environ.get("GITHUB_BASE_REF")
if base_ref:
subprocess.run(
["git", "fetch", "origin", base_ref, "--depth=1"],
cwd=WORKSPACE_ROOT,
capture_output=True,
text=True,
check=False,
)
strategies = [
(f"origin/{base_ref}...HEAD",),
("FETCH_HEAD...HEAD",),
("HEAD^...HEAD",),
]
else:
strategies = [
("origin/main...HEAD",),
("main...HEAD",),
("HEAD~1",),
]
for strategy in strategies:
changed_files = git_diff_name_only(*strategy)
if changed_files is not None:
return changed_files or ["."]
return ["."]
def run_ci_mypy() -> None:
"""Run MyPy only where changes require it, mirroring CI behaviour."""
changed_files = detect_ci_changed_files()
print("[cyan]Changed files for CI mypy:[/cyan]")
for file_path in changed_files:
print(f" {file_path}")
run_changed_package_tasks(["mypy"], changed_files)
def ensure_no_extra_args(command_name: str, extra_args: list[str]) -> None:
"""Reject unsupported pass-through arguments for commands that do not forward them."""
if extra_args:
joined_args = " ".join(extra_args)
print(f"[red]Command '{command_name}' does not accept extra arguments: {joined_args}[/red]")
raise SystemExit(2)
def resolve_syntax_modes(*, format_selected: bool, check_selected: bool) -> tuple[bool, bool]:
"""Resolve which syntax steps to run."""
if not format_selected and not check_selected:
return True, True
return format_selected, check_selected
def run_syntax(
*,
project_pattern: str,
samples: bool,
format_selected: bool,
check_selected: bool,
extra_args: list[str],
) -> None:
"""Run formatting and/or lint checking for packages or samples.
Combined package mode deliberately dispatches ``fmt`` and ``lint`` together
so the shared task runner can start both legs in parallel.
"""
run_format, run_check = resolve_syntax_modes(
format_selected=format_selected,
check_selected=check_selected,
)
if run_format and run_check and extra_args:
joined_args = " ".join(extra_args)
print(
"[red]Extra arguments are only supported when syntax runs a single mode; "
f"use either --format or --check with: {joined_args}[/red]"
)
raise SystemExit(2)
if samples and project_pattern != "*":
print("[red]--samples cannot be combined with --package.[/red]")
raise SystemExit(2)
format_args = extra_args if run_format and not run_check else []
check_args = extra_args if run_check and not run_format else []
if samples:
if run_format:
run_sample_format(format_args)
if run_check:
run_sample_lint(check_args)
return
if run_format and run_check:
# Fan out both legs in one call so task_runner can parallelize format
# and lint work across the same selected package set.
run_fan_out(["fmt", "lint"], project_pattern, [])
return
if run_format:
run_fan_out(["fmt"], project_pattern, format_args)
if run_check:
run_fan_out(["lint"], project_pattern, check_args)
def main() -> None:
"""Dispatch the requested workspace task."""
args, extra_args = parse_args(sys.argv[1:])
if args.command == "syntax":
run_syntax(
project_pattern=args.project,
samples=args.samples,
format_selected=args.format,
check_selected=args.check,
extra_args=extra_args,
)
return
if args.command == "fmt":
run_syntax(
project_pattern=args.project,
samples=False,
format_selected=True,
check_selected=False,
extra_args=extra_args,
)
return
if args.command == "lint":
if args.samples:
run_syntax(
project_pattern=args.project,
samples=True,
format_selected=False,
check_selected=True,
extra_args=extra_args,
)
return
run_syntax(
project_pattern=args.project,
samples=False,
format_selected=False,
check_selected=True,
extra_args=extra_args,
)
return
if args.command == "pyright":
if args.samples:
if args.all or args.project != "*":
print("[red]--samples cannot be combined with --all or --package.[/red]")
raise SystemExit(2)
run_sample_pyright(extra_args)
return
if args.all:
run_aggregate_pyright(args.project, extra_args)
return
run_fan_out(["pyright"], args.project, extra_args)
return
if args.command == "mypy":
if args.all:
run_aggregate_mypy(args.project, extra_args)
return
run_fan_out(["mypy"], args.project, extra_args)
return
if args.command == "typing":
ensure_no_extra_args(args.command, extra_args)
if args.all:
# Start MyPy first so combined typing runs follow the requested
# ordering even though completion still depends on runtime duration.
run_aggregate_mypy(args.project, [])
run_aggregate_pyright(args.project, [])
return
# Preserve the same "MyPy first" ordering for the per-package fan-out
# path as well.
run_fan_out(["mypy", "pyright"], args.project, [])
return
if args.command == "test":
if args.all:
run_aggregate_test(args.project, args.cov, extra_args)
return
run_fan_out(["test"], args.project, extra_args)
return
if args.command == "build":
ensure_no_extra_args(args.command, extra_args)
run_fan_out(["build"], args.project, [])
return
if args.command == "clean-dist":
ensure_no_extra_args(args.command, extra_args)
run_fan_out(["clean-dist"], args.project, [])
return
if args.command == "check-packages":
ensure_no_extra_args(args.command, extra_args)
run_syntax(
project_pattern=args.project,
samples=False,
format_selected=False,
check_selected=False,
extra_args=[],
)
run_fan_out(["pyright"], args.project, [])
return
if args.command == "check":
ensure_no_extra_args(args.command, extra_args)
if args.samples:
if args.project != "*":
print("[red]--samples cannot be combined with --package.[/red]")
raise SystemExit(2)
run_syntax(
project_pattern="*",
samples=True,
format_selected=False,
check_selected=False,
extra_args=[],
)
run_sample_pyright([])
return
run_syntax(
project_pattern=args.project,
samples=False,
format_selected=False,
check_selected=False,
extra_args=[],
)
run_fan_out(["pyright"], args.project, [])
run_fan_out(["test"], args.project, [])
# Sample validation and markdown lint are intentionally workspace-wide;
# a package-scoped check should stay focused on the selected package set.
if args.project == "*":
run_syntax(
project_pattern="*",
samples=True,
format_selected=False,
check_selected=False,
extra_args=[],
)
run_sample_pyright([])
run_markdown_code_lint()
return
if args.command == "prek-check":
ensure_no_extra_args(args.command, extra_args)
run_prek_check(args.files)
return
if args.command == "ci-mypy":
ensure_no_extra_args(args.command, extra_args)
run_ci_mypy()
return
print(f"[red]Unsupported command: {args.command}[/red]")
raise SystemExit(2)
if __name__ == "__main__":
main()