Compare commits

..
Author SHA1 Message Date
github-actions[bot] 587356c778 Python: chore: upgrade dev dependencies 2026-03-16 15:36:52 +00:00
1815 changed files with 55450 additions and 101903 deletions
-1
View File
@@ -3,7 +3,6 @@
"image": "mcr.microsoft.com/devcontainers/dotnet",
"features": {
"ghcr.io/devcontainers/features/azure-cli:1.2.9": {},
"ghcr.io/devcontainers/features/docker-in-docker:2": {},
"ghcr.io/devcontainers/features/github-cli:1": {
"version": "2"
},
+1 -1
View File
@@ -47,7 +47,7 @@ body:
attributes:
label: Package Versions
description: List the agent-framework-* packages and versions you are using
placeholder: "e.g., agent-framework-core: 1.0.0, agent-framework-foundry: 1.0.0"
placeholder: "e.g., agent-framework-core: 1.0.0, agent-framework-azure-ai: 1.0.0"
validations:
required: true
@@ -24,9 +24,7 @@ runs:
using: "composite"
steps:
- name: Set up Node.js environment
uses: actions/setup-node@v6
with:
node-version: 22
uses: actions/setup-node@v4
- name: Install Copilot CLI
shell: bash
@@ -34,7 +32,7 @@ runs:
- name: Test Copilot CLI
shell: bash
run: copilot --version && copilot -p "What can you do in one sentence?"
run: copilot -p "What can you do in one sentence?"
- name: Azure CLI Login
uses: azure/login@v2
@@ -1,166 +0,0 @@
name: Setup Local MCP Server
description: Start and validate a local streamable HTTP MCP server for integration tests
inputs:
fallback_url:
description: Existing LOCAL_MCP_URL value to keep as a fallback if local startup fails
required: false
default: ''
host:
description: Host interface to bind the local MCP server
required: false
default: '127.0.0.1'
port:
description: Port to bind the local MCP server
required: false
default: '8011'
mount_path:
description: Mount path for the local streamable HTTP MCP endpoint
required: false
default: '/mcp'
outputs:
effective_url:
description: Local MCP URL when startup succeeds, otherwise the provided fallback URL
value: ${{ steps.start.outputs.effective_url }}
local_url:
description: URL of the local MCP server
value: ${{ steps.start.outputs.local_url }}
started:
description: Whether the local MCP server started and passed validation
value: ${{ steps.start.outputs.started }}
pid:
description: PID of the local MCP server process when startup succeeded
value: ${{ steps.start.outputs.pid }}
runs:
using: composite
steps:
- name: Start and validate local MCP server
id: start
shell: bash
run: |
set -euo pipefail
host="${{ inputs.host }}"
port="${{ inputs.port }}"
mount_path="${{ inputs.mount_path }}"
fallback_url="${{ inputs.fallback_url }}"
if [[ ! "$mount_path" =~ ^/ ]]; then
mount_path="/$mount_path"
fi
local_url="http://${host}:${port}${mount_path}"
health_url="http://${host}:${port}/healthz"
log_file="$RUNNER_TEMP/local-mcp-server.log"
pid_file="$RUNNER_TEMP/local-mcp-server.pid"
rm -f "$log_file" "$pid_file"
server_pid="$(
python3 - "$GITHUB_WORKSPACE/python" "$log_file" "$host" "$port" "$mount_path" <<'PY'
from __future__ import annotations
import subprocess
import sys
workspace, log_file, host, port, mount_path = sys.argv[1:]
with open(log_file, "w", encoding="utf-8") as log:
process = subprocess.Popen(
[
"uv",
"run",
"python",
"scripts/local_mcp_streamable_http_server.py",
"--host",
host,
"--port",
port,
"--mount-path",
mount_path,
],
cwd=workspace,
stdout=log,
stderr=subprocess.STDOUT,
start_new_session=True,
)
print(process.pid)
PY
)"
echo "$server_pid" > "$pid_file"
started=false
for _ in $(seq 1 30); do
if curl --silent --fail "$health_url" >/dev/null; then
started=true
break
fi
if ! kill -0 "$server_pid" 2>/dev/null; then
break
fi
sleep 1
done
if [[ "$started" == "true" ]]; then
if ! (
cd "$GITHUB_WORKSPACE/python"
LOCAL_MCP_URL="$local_url" uv run python - <<'PY'
from __future__ import annotations
import asyncio
import os
from agent_framework import Content, MCPStreamableHTTPTool
def result_to_text(result: str | list[Content]) -> str:
if isinstance(result, str):
return result
return "\n".join(content.text for content in result if content.type == "text" and content.text)
async def main() -> None:
tool = MCPStreamableHTTPTool(
name="local_ci_mcp",
url=os.environ["LOCAL_MCP_URL"],
approval_mode="never_require",
)
async with tool:
assert tool.functions, "Local MCP server did not expose any tools."
result = result_to_text(await tool.functions[0].invoke(query="What is Agent Framework?"))
assert result, "Local MCP server returned an empty response."
asyncio.run(main())
PY
); then
started=false
fi
fi
effective_url="$local_url"
pid="$server_pid"
if [[ "$started" != "true" ]]; then
effective_url="$fallback_url"
pid=""
if kill -0 "$server_pid" 2>/dev/null; then
kill -TERM -- "-$server_pid" 2>/dev/null || kill -TERM "$server_pid" || true
sleep 1
kill -KILL -- "-$server_pid" 2>/dev/null || kill -KILL "$server_pid" || true
fi
echo "Local MCP server was unavailable; continuing with fallback LOCAL_MCP_URL."
if [[ -f "$log_file" ]]; then
tail -n 100 "$log_file" || true
fi
else
echo "Using local MCP server at $local_url"
fi
echo "started=$started" >> "$GITHUB_OUTPUT"
echo "local_url=$local_url" >> "$GITHUB_OUTPUT"
echo "effective_url=$effective_url" >> "$GITHUB_OUTPUT"
echo "pid=$pid" >> "$GITHUB_OUTPUT"
-216
View File
@@ -1,216 +0,0 @@
# Copyright (c) Microsoft. All rights reserved.
"""Scan open issues and PRs labeled 'waiting-for-author' for stale follow-ups.
Team members manually add the 'waiting-for-author' label when they need a
response from the external author. If the author hasn't replied within
DAYS_THRESHOLD days of the last team comment, post a reminder and add the
'requested-info' label to prevent duplicate pings.
"""
from __future__ import annotations
import os
import sys
import time
from datetime import datetime, timezone
from github import Auth, Github, GithubException
from github.Issue import Issue
from github.IssueComment import IssueComment
PING_COMMENT = (
"@{author}, friendly reminder — this issue is waiting on your response. "
"Please share any updates when you get a chance. (This is an automated message.)"
)
TRIGGER_LABEL = "waiting-for-author"
PINGED_LABEL = "requested-info"
def get_team_members(g: Github, org: str, team_slug: str) -> set[str]:
"""Fetch active team member usernames."""
try:
org_obj = g.get_organization(org)
team = org_obj.get_team_by_slug(team_slug)
return {m.login for m in team.get_members()}
except GithubException as exc:
if exc.status in (403, 404):
print(
f"ERROR: Failed to fetch team members for {org}/{team_slug} "
f"(HTTP {exc.status}). Check that the token has the 'read:org' "
f"scope and that the team slug '{team_slug}' is correct."
)
else:
print(f"ERROR: Failed to fetch team members for {org}/{team_slug}: {exc}")
sys.exit(1)
except Exception as exc:
print(f"ERROR: Failed to fetch team members for {org}/{team_slug}: {exc}")
sys.exit(1)
def find_last_team_comment(
comments: list[IssueComment], team_members: set[str]
) -> IssueComment | None:
"""Return the most recent comment from a team member, or None."""
for comment in reversed(comments):
if comment.user and comment.user.login in team_members:
return comment
return None
def author_replied_after(
comments: list[IssueComment], author: str, after: datetime
) -> bool:
"""Check if the issue author commented after the given timestamp."""
for comment in comments:
if (
comment.user
and comment.user.login == author
and comment.created_at > after
):
return True
return False
def should_ping(
issue: Issue,
team_members: set[str],
days_threshold: int,
now: datetime,
) -> bool:
"""Determine whether this issue/PR should be pinged.
Only issues/PRs carrying the 'waiting-for-author' label are candidates.
"""
author = issue.user.login
# Skip if the trigger label is not present
if not any(label.name == TRIGGER_LABEL for label in issue.labels):
return False
# Skip if author is a team member
if author in team_members:
return False
# Skip if already pinged
if any(label.name == PINGED_LABEL for label in issue.labels):
return False
# Skip if no comments at all
if issue.comments == 0:
return False
# Fetch comments once for both lookups
comments = list(issue.get_comments())
# Find last team member comment
last_team_comment = find_last_team_comment(comments, team_members)
if last_team_comment is None:
return False
# Skip if author replied after the last team comment
if author_replied_after(comments, author, last_team_comment.created_at):
return False
# Check if enough days have passed
days_since = (now - last_team_comment.created_at.astimezone(timezone.utc)).days
if days_since < days_threshold:
return False
return True
def ping(issue: Issue, dry_run: bool) -> bool:
"""Post a reminder comment and add the 'requested-info' label. Returns True on success."""
author = issue.user.login
kind = "PR" if issue.pull_request else "Issue"
if dry_run:
print(f" [DRY RUN] Would ping {kind} #{issue.number} (@{author})")
return True
max_retries = 3
commented = False
labeled = False
for attempt in range(1, max_retries + 1):
try:
if not commented:
issue.create_comment(PING_COMMENT.format(author=author))
commented = True
if not labeled:
issue.add_to_labels(PINGED_LABEL)
labeled = True
print(f" Pinged {kind} #{issue.number} (@{author})")
return True
except Exception as exc:
if attempt < max_retries:
wait = 2 ** attempt # 2s, 4s
print(f" WARN: Attempt {attempt}/{max_retries} failed for {kind} #{issue.number}: {exc}. Retrying in {wait}s...")
time.sleep(wait)
else:
print(f" ERROR: Failed to ping {kind} #{issue.number} after {max_retries} attempts: {exc}")
return False
def main() -> None:
token = os.environ.get("GITHUB_TOKEN")
if not token:
print("ERROR: GITHUB_TOKEN environment variable is required")
sys.exit(1)
repository = os.environ.get("GITHUB_REPOSITORY")
if not repository:
print("ERROR: GITHUB_REPOSITORY environment variable is required")
sys.exit(1)
team_slug = os.environ.get("TEAM_SLUG")
if not team_slug:
print("ERROR: TEAM_SLUG environment variable is required")
sys.exit(1)
days_threshold_raw = os.environ.get("DAYS_THRESHOLD", "4")
try:
days_threshold = int(days_threshold_raw)
except ValueError:
print(f"ERROR: DAYS_THRESHOLD must be a numeric value, got '{days_threshold_raw}'")
sys.exit(1)
dry_run = os.environ.get("DRY_RUN", "false").lower() == "true"
org = repository.split("/")[0]
if dry_run:
print("Running in DRY RUN mode — no comments or labels will be applied.\n")
g = Github(auth=Auth.Token(token))
repo = g.get_repo(repository)
print(f"Fetching team members for {org}/{team_slug}...")
team_members = get_team_members(g, org, team_slug)
print(f"Found {len(team_members)} team members.\n")
now = datetime.now(timezone.utc)
pinged = []
failed = []
scanned = 0
print(f"Scanning open issues and PRs labeled '{TRIGGER_LABEL}' (threshold: {days_threshold} days)...\n")
for issue in repo.get_issues(state="open", labels=[TRIGGER_LABEL]):
scanned += 1
if should_ping(issue, team_members, days_threshold, now):
if ping(issue, dry_run):
pinged.append(issue.number)
else:
failed.append(issue.number)
print(f"\nDone. Scanned {scanned} items, pinged {len(pinged)}, failed {len(failed)}.")
if pinged:
print(f"Pinged: {', '.join(f'#{n}' for n in pinged)}")
if failed:
print(f"Failed: {', '.join(f'#{n}' for n in failed)}")
sys.exit(1)
if __name__ == "__main__":
main()
-297
View File
@@ -1,297 +0,0 @@
# Copyright (c) Microsoft. All rights reserved.
"""Tests for stale_issue_pr_ping.py."""
from __future__ import annotations
import os
import sys
from datetime import datetime, timezone, timedelta
from unittest.mock import MagicMock, patch
import pytest
# Ensure the script directory is importable
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "scripts"))
from stale_issue_pr_ping import (
PINGED_LABEL,
PING_COMMENT,
TRIGGER_LABEL,
author_replied_after,
find_last_team_comment,
get_team_members,
main,
ping,
should_ping,
)
TEAM = {"alice", "bob"}
NOW = datetime(2026, 3, 15, 12, 0, 0, tzinfo=timezone.utc)
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _make_comment(login: str | None, created_at: datetime) -> MagicMock:
"""Create a mock IssueComment."""
c = MagicMock()
if login is None:
c.user = None
else:
c.user = MagicMock()
c.user.login = login
c.created_at = created_at
return c
def _make_label(name: str) -> MagicMock:
lbl = MagicMock()
lbl.name = name
return lbl
def _make_issue(
author: str = "external",
labels: list[str] | None = None,
comment_count: int = 1,
comments: list[MagicMock] | None = None,
pull_request: bool = False,
number: int = 42,
) -> MagicMock:
issue = MagicMock()
issue.user = MagicMock()
issue.user.login = author
issue.number = number
# Default to having the trigger label, since the API query pre-filters.
if labels is None:
labels = [TRIGGER_LABEL]
issue.labels = [_make_label(n) for n in labels]
issue.comments = comment_count
issue.pull_request = MagicMock() if pull_request else None
if comments is not None:
issue.get_comments.return_value = comments
return issue
# ---------------------------------------------------------------------------
# find_last_team_comment
# ---------------------------------------------------------------------------
class TestFindLastTeamComment:
def test_returns_last_team_comment(self):
c1 = _make_comment("alice", datetime(2026, 3, 1, tzinfo=timezone.utc))
c2 = _make_comment("external", datetime(2026, 3, 2, tzinfo=timezone.utc))
c3 = _make_comment("bob", datetime(2026, 3, 3, tzinfo=timezone.utc))
assert find_last_team_comment([c1, c2, c3], TEAM) is c3
def test_returns_none_when_no_team_comments(self):
c1 = _make_comment("external", datetime(2026, 3, 1, tzinfo=timezone.utc))
assert find_last_team_comment([c1], TEAM) is None
def test_returns_none_for_empty_list(self):
assert find_last_team_comment([], TEAM) is None
def test_skips_deleted_user(self):
c1 = _make_comment(None, datetime(2026, 3, 1, tzinfo=timezone.utc))
c2 = _make_comment("alice", datetime(2026, 3, 2, tzinfo=timezone.utc))
assert find_last_team_comment([c1, c2], TEAM) is c2
def test_only_deleted_users(self):
c1 = _make_comment(None, datetime(2026, 3, 1, tzinfo=timezone.utc))
assert find_last_team_comment([c1], TEAM) is None
# ---------------------------------------------------------------------------
# author_replied_after
# ---------------------------------------------------------------------------
class TestAuthorRepliedAfter:
def test_author_replied(self):
after = datetime(2026, 3, 1, tzinfo=timezone.utc)
c1 = _make_comment("external", datetime(2026, 3, 2, tzinfo=timezone.utc))
assert author_replied_after([c1], "external", after) is True
def test_author_not_replied(self):
after = datetime(2026, 3, 5, tzinfo=timezone.utc)
c1 = _make_comment("external", datetime(2026, 3, 2, tzinfo=timezone.utc))
assert author_replied_after([c1], "external", after) is False
def test_different_user_replied(self):
after = datetime(2026, 3, 1, tzinfo=timezone.utc)
c1 = _make_comment("someone_else", datetime(2026, 3, 2, tzinfo=timezone.utc))
assert author_replied_after([c1], "external", after) is False
def test_deleted_user_comment(self):
after = datetime(2026, 3, 1, tzinfo=timezone.utc)
c1 = _make_comment(None, datetime(2026, 3, 2, tzinfo=timezone.utc))
assert author_replied_after([c1], "external", after) is False
# ---------------------------------------------------------------------------
# should_ping
# ---------------------------------------------------------------------------
class TestShouldPing:
def test_should_ping_stale_issue(self):
team_comment = _make_comment("alice", NOW - timedelta(days=5))
issue = _make_issue(comments=[team_comment], comment_count=1)
assert should_ping(issue, TEAM, 4, NOW) is True
def test_skip_team_member_author(self):
issue = _make_issue(author="alice", labels=[TRIGGER_LABEL], comment_count=1)
assert should_ping(issue, TEAM, 4, NOW) is False
def test_skip_already_pinged(self):
issue = _make_issue(labels=[TRIGGER_LABEL, PINGED_LABEL], comment_count=1)
assert should_ping(issue, TEAM, 4, NOW) is False
def test_skip_no_comments(self):
issue = _make_issue(comment_count=0)
assert should_ping(issue, TEAM, 4, NOW) is False
def test_skip_no_team_comment(self):
c = _make_comment("external", NOW - timedelta(days=5))
issue = _make_issue(comments=[c], comment_count=1)
assert should_ping(issue, TEAM, 4, NOW) is False
def test_skip_author_replied(self):
team_c = _make_comment("alice", NOW - timedelta(days=5))
author_c = _make_comment("external", NOW - timedelta(days=3))
issue = _make_issue(comments=[team_c, author_c], comment_count=2)
assert should_ping(issue, TEAM, 4, NOW) is False
def test_skip_not_enough_days(self):
team_comment = _make_comment("alice", NOW - timedelta(days=2))
issue = _make_issue(comments=[team_comment], comment_count=1)
assert should_ping(issue, TEAM, 4, NOW) is False
def test_aware_datetime_handled(self):
"""Timezone-aware datetimes should not be mangled by astimezone."""
aware_dt = (NOW - timedelta(days=5)).replace(tzinfo=timezone.utc)
team_comment = _make_comment("alice", aware_dt)
issue = _make_issue(comments=[team_comment], comment_count=1)
assert should_ping(issue, TEAM, 4, NOW) is True
def test_naive_datetime_handled(self):
"""Naive datetimes (pre-PyGithub 2.x) should be handled by astimezone."""
naive_dt = (NOW - timedelta(days=5)).replace(tzinfo=None)
team_comment = _make_comment("alice", naive_dt)
issue = _make_issue(comments=[team_comment], comment_count=1)
# astimezone on naive datetime treats it as local time; just verify no crash
should_ping(issue, TEAM, 4, NOW)
# ---------------------------------------------------------------------------
# ping
# ---------------------------------------------------------------------------
class TestPing:
def test_dry_run(self, capsys):
issue = _make_issue()
assert ping(issue, dry_run=True) is True
issue.create_comment.assert_not_called()
assert "DRY RUN" in capsys.readouterr().out
def test_success(self, capsys):
issue = _make_issue()
assert ping(issue, dry_run=False) is True
issue.create_comment.assert_called_once()
issue.add_to_labels.assert_called_once_with(PINGED_LABEL)
@patch("stale_issue_pr_ping.time.sleep")
def test_retry_on_failure(self, mock_sleep):
issue = _make_issue()
issue.create_comment.side_effect = [Exception("net error"), None]
assert ping(issue, dry_run=False) is True
assert issue.create_comment.call_count == 2
mock_sleep.assert_called_once()
@patch("stale_issue_pr_ping.time.sleep")
def test_idempotent_retry_skips_comment_on_label_failure(self, mock_sleep):
"""If create_comment succeeds but add_to_labels fails, retry should not re-comment."""
issue = _make_issue()
issue.add_to_labels.side_effect = [Exception("label error"), None]
assert ping(issue, dry_run=False) is True
# Comment should only be created once even though there were 2 attempts
assert issue.create_comment.call_count == 1
assert issue.add_to_labels.call_count == 2
@patch("stale_issue_pr_ping.time.sleep")
def test_all_retries_fail(self, mock_sleep):
issue = _make_issue()
issue.create_comment.side_effect = Exception("permanent error")
assert ping(issue, dry_run=False) is False
assert issue.create_comment.call_count == 3
# ---------------------------------------------------------------------------
# get_team_members
# ---------------------------------------------------------------------------
class TestGetTeamMembers:
def test_success(self):
g = MagicMock()
member = MagicMock()
member.login = "alice"
g.get_organization.return_value.get_team_by_slug.return_value.get_members.return_value = [member]
assert get_team_members(g, "org", "my-team") == {"alice"}
def test_403_error_message(self, capsys):
from github import GithubException
g = MagicMock()
g.get_organization.return_value.get_team_by_slug.side_effect = GithubException(
403, {"message": "Forbidden"}, None
)
with pytest.raises(SystemExit):
get_team_members(g, "org", "my-team")
out = capsys.readouterr().out
assert "read:org" in out
assert "403" in out
def test_404_error_message(self, capsys):
from github import GithubException
g = MagicMock()
g.get_organization.return_value.get_team_by_slug.side_effect = GithubException(
404, {"message": "Not Found"}, None
)
with pytest.raises(SystemExit):
get_team_members(g, "org", "bad-slug")
out = capsys.readouterr().out
assert "read:org" in out
assert "bad-slug" in out
def test_generic_error(self, capsys):
g = MagicMock()
g.get_organization.side_effect = RuntimeError("boom")
with pytest.raises(SystemExit):
get_team_members(g, "org", "team")
# ---------------------------------------------------------------------------
# main env var validation
# ---------------------------------------------------------------------------
class TestMain:
@patch.dict(os.environ, {
"GITHUB_TOKEN": "tok",
"GITHUB_REPOSITORY": "org/repo",
"TEAM_SLUG": "my-team",
"DAYS_THRESHOLD": "abc",
}, clear=True)
def test_invalid_days_threshold(self, capsys):
with pytest.raises(SystemExit):
main()
assert "numeric" in capsys.readouterr().out
@patch.dict(os.environ, {
"GITHUB_TOKEN": "tok",
"GITHUB_REPOSITORY": "org/repo",
}, clear=True)
def test_missing_team_slug(self, capsys):
with pytest.raises(SystemExit):
main()
assert "TEAM_SLUG" in capsys.readouterr().out
+6 -6
View File
@@ -82,10 +82,10 @@ jobs:
.github
dotnet
python
declarative-agents
workflow-samples
- name: Setup dotnet
uses: actions/setup-dotnet@v5.2.0
uses: actions/setup-dotnet@v5.1.0
with:
global-json-file: ${{ github.workspace }}/dotnet/global.json
- name: Build dotnet solutions
@@ -152,7 +152,7 @@ jobs:
.github
dotnet
python
declarative-agents
workflow-samples
# Start Cosmos DB Emulator for all integration tests and only for unit tests when CosmosDB changes happened)
- name: Start Azure Cosmos DB Emulator
@@ -165,7 +165,7 @@ jobs:
echo "COSMOSDB_EMULATOR_AVAILABLE=true" >> $env:GITHUB_ENV
- name: Setup dotnet
uses: actions/setup-dotnet@v5.2.0
uses: actions/setup-dotnet@v5.1.0
with:
global-json-file: ${{ github.workspace }}/dotnet/global.json
@@ -281,7 +281,7 @@ jobs:
# Generate test reports and check coverage
- name: Generate test reports
if: matrix.targetFramework == env.COVERAGE_FRAMEWORK
uses: danielpalme/ReportGenerator-GitHub-Action@5.5.3
uses: danielpalme/ReportGenerator-GitHub-Action@5.5.1
with:
reports: "./TestResults/Coverage/**/*.cobertura.xml"
targetdir: "./TestResults/Reports"
@@ -289,7 +289,7 @@ jobs:
- name: Upload coverage report artifact
if: matrix.targetFramework == env.COVERAGE_FRAMEWORK
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: CoverageReport-${{ matrix.os }}-${{ matrix.targetFramework }}-${{ matrix.configuration }} # Artifact name
path: ./TestResults/Reports # Directory containing files to upload
@@ -38,7 +38,7 @@ jobs:
.github
dotnet
python
declarative-agents
workflow-samples
- name: Start Azure Cosmos DB Emulator
if: runner.os == 'Windows'
@@ -50,7 +50,7 @@ jobs:
echo "COSMOS_EMULATOR_AVAILABLE=true" >> $env:GITHUB_ENV
- name: Setup dotnet
uses: actions/setup-dotnet@v5.2.0
uses: actions/setup-dotnet@v5.1.0
with:
global-json-file: ${{ github.workspace }}/dotnet/global.json
-137
View File
@@ -1,137 +0,0 @@
#
# Runs the .NET sample verification tool, which builds and executes sample projects
# and verifies their output using deterministic checks and AI-powered verification.
#
# Results are displayed as a GitHub Job Summary and the CSV report is uploaded as an artifact.
#
name: dotnet-verify-samples
on:
workflow_dispatch:
inputs:
category:
description: "Sample category to run (blank for all)"
required: false
type: choice
options:
- ""
- "01-get-started"
- "02-agents"
- "03-workflows"
parallelism:
description: "Max parallel sample runs"
required: false
default: "8"
type: string
schedule:
- cron: "0 6 * * 1-5" # Weekdays at 6:00 UTC
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
permissions:
contents: read
id-token: write
jobs:
verify-samples:
runs-on: ubuntu-latest
environment: 'integration'
timeout-minutes: 90
steps:
- uses: actions/checkout@v6
with:
persist-credentials: false
sparse-checkout: |
.
.github
dotnet
python
declarative-agents
- name: Setup dotnet
uses: actions/setup-dotnet@v5.2.0
with:
global-json-file: ${{ github.workspace }}/dotnet/global.json
- name: Azure CLI Login
if: github.event_name != 'pull_request'
uses: azure/login@v2
with:
client-id: ${{ secrets.AZURE_CLIENT_ID }}
tenant-id: ${{ secrets.AZURE_TENANT_ID }}
subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
- name: Generate filtered solution
shell: pwsh
run: |
./dotnet/eng/scripts/New-FilteredSolution.ps1 `
-Solution dotnet/agent-framework-dotnet.slnx `
-TargetFramework net10.0 `
-Configuration Debug `
-OutputPath dotnet/filtered.slnx `
-Verbose
- name: Build solution
shell: bash
run: dotnet build dotnet/filtered.slnx -f net10.0 --warnaserror
- name: Run verify-samples
id: verify
working-directory: dotnet
shell: bash
run: |
CATEGORY_ARG=""
if [ -n "$CATEGORY_INPUT" ]; then
CATEGORY_ARG="--category $CATEGORY_INPUT"
fi
dotnet run --project eng/verify-samples -- \
$CATEGORY_ARG \
--parallel "$PARALLELISM" \
--md results.md \
--csv results.csv \
--log results.log
env:
CATEGORY_INPUT: ${{ github.event.inputs.category || '' }}
PARALLELISM: ${{ github.event.inputs.parallelism || '8' }}
# OpenAI Models
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
OPENAI_CHAT_MODEL_NAME: ${{ vars.OPENAI_CHAT_MODEL_NAME }}
OPENAI_REASONING_MODEL_NAME: ${{ vars.OPENAI_REASONING_MODEL_NAME }}
# Azure OpenAI Models
AZURE_OPENAI_DEPLOYMENT_NAME: ${{ vars.AZURE_OPENAI_DEPLOYMENT_NAME }}
AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ vars.AZURE_OPENAI_DEPLOYMENT_NAME }}
AZURE_OPENAI_ENDPOINT: ${{ vars.AZURE_OPENAI_ENDPOINT }}
# Azure AI Foundry
AZURE_AI_PROJECT_ENDPOINT: ${{ vars.AZURE_AI_PROJECT_ENDPOINT }}
AZURE_AI_MODEL_DEPLOYMENT_NAME: ${{ vars.AZURE_AI_MODEL_DEPLOYMENT_NAME }}
AZURE_AI_BING_CONNECTION_ID: ${{ vars.AZURE_AI_BING_CONNECTION_ID }}
- name: Write Job Summary
if: always()
working-directory: dotnet
shell: bash
run: |
if [ -f results.md ]; then
cat results.md >> "$GITHUB_STEP_SUMMARY"
else
echo "⚠️ No results.md generated — verify-samples may have failed to start." >> "$GITHUB_STEP_SUMMARY"
fi
- name: Upload results
if: always()
uses: actions/upload-artifact@v7
with:
name: verify-samples-results
path: |
dotnet/results.csv
dotnet/results.log
if-no-files-found: warn
- name: Fail if samples failed
if: always() && steps.verify.outcome == 'failure'
shell: bash
run: exit 1
+6 -5
View File
@@ -34,14 +34,15 @@ from dataclasses import dataclass
# (e.g., "packages/core/agent_framework/observability.py")
# =============================================================================
ENFORCED_TARGETS: set[str] = {
# Packages (sorted alphabetically)
"packages.anthropic.agent_framework_anthropic",
"packages.azure-ai-search.agent_framework_azure_ai_search",
# Packages
"packages.azure-ai.agent_framework_azure_ai",
"packages.core.agent_framework",
"packages.core.agent_framework._workflows",
"packages.foundry.agent_framework_foundry",
"packages.openai.agent_framework_openai",
"packages.purview.agent_framework_purview",
"packages.anthropic.agent_framework_anthropic",
"packages.azure-ai-search.agent_framework_azure_ai_search",
"packages.core.agent_framework.azure",
"packages.core.agent_framework.openai",
# Individual files (if you want to enforce specific files instead of whole packages)
"packages/core/agent_framework/observability.py",
# Add more targets here as coverage improves
+6 -4
View File
@@ -75,7 +75,7 @@ jobs:
os: ${{ runner.os }}
env:
UV_CACHE_DIR: /tmp/.uv-cache
- name: Run syntax and pyright across packages
- name: Run fmt, lint, pyright in parallel across packages
run: uv run poe check-packages
samples-markdown:
@@ -104,8 +104,10 @@ jobs:
os: ${{ runner.os }}
env:
UV_CACHE_DIR: /tmp/.uv-cache
- name: Run samples checks
run: uv run poe check -S
- name: Run samples lint
run: uv run poe samples-lint
- name: Run samples syntax check
run: uv run poe samples-syntax
- name: Run markdown code lint
run: uv run poe markdown-code-lint
@@ -138,4 +140,4 @@ jobs:
- name: Run Mypy
env:
GITHUB_BASE_REF: ${{ github.event.pull_request.base.ref || github.base_ref || 'main' }}
run: uv run python scripts/workspace_poe_tasks.py ci-mypy
run: uv run poe ci-mypy
@@ -38,13 +38,13 @@ jobs:
id: validate_ranges
# Keep workflow running so we can still publish diagnostics from this run.
continue-on-error: true
run: uv run poe validate-dependency-bounds-project --mode upper --package "*"
run: uv run poe validate-dependency-bounds-project --mode upper --project "*"
working-directory: ./python
- name: Upload dependency range report
# Always publish the report so failures are inspectable even when validation fails.
if: always()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v4
with:
name: dependency-range-results
path: python/scripts/dependencies/dependency-range-results.json
@@ -203,7 +203,7 @@ jobs:
cat > "${PR_BODY_FILE}" <<'EOF'
This PR was generated by the dependency range validation workflow.
- Ran `uv run poe validate-dependency-bounds-project --mode upper --package "*"`
- Ran `uv run poe validate-dependency-bounds-project --mode upper --project "*"`
- Updated package dependency bounds
- Refreshed `python/uv.lock` with `uv lock --upgrade`
EOF
+25 -73
View File
@@ -48,8 +48,9 @@ jobs:
os: ${{ runner.os }}
- name: Test with pytest (unit tests only)
run: >
uv run poe test -A
uv run poe all-tests
-m "not integration"
-n logical --dist worksteal
--timeout=120 --session-timeout=900 --timeout_method thread
--retries 2 --retry-delay 5
@@ -60,10 +61,9 @@ jobs:
environment: integration
timeout-minutes: 60
env:
OPENAI_CHAT_COMPLETION_MODEL: ${{ vars.OPENAI__CHATMODELID }}
OPENAI_CHAT_MODEL: ${{ vars.OPENAI__RESPONSESMODELID }}
OPENAI_MODEL: ${{ vars.OPENAI__RESPONSESMODELID }}
OPENAI_EMBEDDING_MODEL: ${{ vars.OPENAI_EMBEDDING_MODEL_ID }}
OPENAI_CHAT_MODEL_ID: ${{ vars.OPENAI__CHATMODELID }}
OPENAI_RESPONSES_MODEL_ID: ${{ vars.OPENAI__RESPONSESMODELID }}
OPENAI_EMBEDDINGS_MODEL_ID: ${{ vars.OPENAI_EMBEDDING_MODEL_ID }}
OPENAI_API_KEY: ${{ secrets.OPENAI__APIKEY }}
defaults:
run:
@@ -82,8 +82,8 @@ jobs:
- name: Test with pytest (OpenAI integration)
run: >
uv run pytest --import-mode=importlib
packages/openai/tests
-m "integration and not azure"
packages/core/tests/openai
-m integration
-n logical --dist worksteal
--timeout=120 --session-timeout=900 --timeout_method thread
--retries 2 --retry-delay 5
@@ -95,10 +95,9 @@ jobs:
environment: integration
timeout-minutes: 60
env:
AZURE_OPENAI_CHAT_COMPLETION_MODEL: ${{ vars.AZUREOPENAI__CHATDEPLOYMENTNAME }}
AZURE_OPENAI_CHAT_MODEL: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
AZURE_OPENAI_MODEL: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
AZURE_OPENAI_EMBEDDING_MODEL: ${{ vars.AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME }}
AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__CHATDEPLOYMENTNAME }}
AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__EMBEDDINGDEPLOYMENTNAME }}
AZURE_OPENAI_ENDPOINT: ${{ vars.AZUREOPENAI__ENDPOINT }}
defaults:
run:
@@ -123,9 +122,7 @@ jobs:
- name: Test with pytest (Azure OpenAI integration)
run: >
uv run pytest --import-mode=importlib
packages/openai/tests/openai/test_openai_chat_completion_client_azure.py
packages/openai/tests/openai/test_openai_chat_client_azure.py
packages/openai/tests/openai/test_openai_embedding_client_azure.py
packages/core/tests/azure
-m integration
-n logical --dist worksteal
--timeout=120 --session-timeout=900 --timeout_method thread
@@ -139,7 +136,7 @@ jobs:
timeout-minutes: 60
env:
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
ANTHROPIC_CHAT_MODEL: ${{ vars.ANTHROPIC_CHAT_MODEL_ID }}
ANTHROPIC_CHAT_MODEL_ID: ${{ vars.ANTHROPIC_CHAT_MODEL_ID }}
LOCAL_MCP_URL: ${{ vars.LOCAL_MCP__URL }}
defaults:
run:
@@ -155,13 +152,6 @@ jobs:
with:
python-version: ${{ env.UV_PYTHON }}
os: ${{ runner.os }}
- name: Start local MCP server
id: local-mcp
uses: ./.github/actions/setup-local-mcp-server
with:
fallback_url: ${{ env.LOCAL_MCP_URL }}
- name: Prefer local MCP URL when available
run: echo "LOCAL_MCP_URL=${{ steps.local-mcp.outputs.effective_url }}" >> "$GITHUB_ENV"
- name: Test with pytest (Anthropic, Ollama, MCP integration)
run: >
uv run pytest --import-mode=importlib
@@ -171,27 +161,7 @@ jobs:
-m integration
-n logical --dist worksteal
--timeout=120 --session-timeout=900 --timeout_method thread
--retries 2 --retry-delay 30
- name: Stop local MCP server
if: always()
shell: bash
run: |
set -euo pipefail
server_pid="${{ steps.local-mcp.outputs.pid }}"
if [[ -z "$server_pid" ]]; then
exit 0
fi
if ! kill -0 "$server_pid" 2>/dev/null; then
exit 0
fi
kill -TERM -- "-$server_pid" 2>/dev/null || kill -TERM "$server_pid" 2>/dev/null || true
for _ in $(seq 1 10); do
if ! kill -0 "$server_pid" 2>/dev/null; then
exit 0
fi
sleep 1
done
kill -KILL -- "-$server_pid" 2>/dev/null || kill -KILL "$server_pid" 2>/dev/null || true
--retries 2 --retry-delay 5
# Azure Functions + Durable Task integration tests
python-tests-functions:
@@ -201,17 +171,12 @@ jobs:
timeout-minutes: 60
env:
UV_PYTHON: "3.11"
OPENAI_CHAT_COMPLETION_MODEL: ${{ vars.OPENAI__CHATMODELID }}
OPENAI_CHAT_MODEL: ${{ vars.OPENAI__RESPONSESMODELID }}
OPENAI_MODEL: ${{ vars.OPENAI__RESPONSESMODELID }}
OPENAI_EMBEDDING_MODEL: ${{ vars.OPENAI_EMBEDDING_MODEL_ID }}
OPENAI_CHAT_MODEL_ID: ${{ vars.OPENAI__CHATMODELID }}
OPENAI_RESPONSES_MODEL_ID: ${{ vars.OPENAI__RESPONSESMODELID }}
OPENAI_API_KEY: ${{ secrets.OPENAI__APIKEY }}
AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__CHATDEPLOYMENTNAME }}
AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
AZURE_OPENAI_ENDPOINT: ${{ vars.AZUREOPENAI__ENDPOINT }}
AZURE_OPENAI_MODEL: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
AZURE_OPENAI_CHAT_MODEL: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
AZURE_OPENAI_CHAT_COMPLETION_MODEL: ${{ vars.AZUREOPENAI__CHATDEPLOYMENTNAME }}
FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT }}
FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL }}
FUNCTIONS_WORKER_RUNTIME: "python"
DURABLE_TASK_SCHEDULER_CONNECTION_STRING: "Endpoint=http://localhost:8080;TaskHub=default;Authentication=None"
AzureWebJobsStorage: "UseDevelopmentStorage=true"
@@ -245,25 +210,18 @@ jobs:
packages/durabletask/tests/integration_tests
-m integration
-n logical --dist worksteal
-x
--timeout=360 --session-timeout=900 --timeout_method thread
--timeout=120 --session-timeout=900 --timeout_method thread
--retries 2 --retry-delay 5
# Foundry integration tests
python-tests-foundry:
name: Python Integration Tests - Foundry
# Azure AI integration tests
python-tests-azure-ai:
name: Python Integration Tests - Azure AI
runs-on: ubuntu-latest
environment: integration
timeout-minutes: 60
env:
FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT }}
FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL }}
FOUNDRY_AGENT_NAME: ${{ vars.FOUNDRY_AGENT_NAME }}
FOUNDRY_AGENT_VERSION: ${{ vars.FOUNDRY_AGENT_VERSION }}
FOUNDRY_MODELS_ENDPOINT: ${{ vars.FOUNDRY_MODELS_ENDPOINT || '' }}
FOUNDRY_MODELS_API_KEY: ${{ secrets.FOUNDRY_MODELS_API_KEY || '' }}
FOUNDRY_EMBEDDING_MODEL: ${{ vars.FOUNDRY_EMBEDDING_MODEL || '' }}
FOUNDRY_IMAGE_EMBEDDING_MODEL: ${{ vars.FOUNDRY_IMAGE_EMBEDDING_MODEL || '' }}
AZURE_AI_PROJECT_ENDPOINT: ${{ secrets.AZUREAI__ENDPOINT }}
AZURE_AI_MODEL_DEPLOYMENT_NAME: ${{ vars.AZUREAI__DEPLOYMENTNAME }}
LOCAL_MCP_URL: ${{ vars.LOCAL_MCP__URL }}
defaults:
run:
@@ -287,13 +245,7 @@ jobs:
subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
- name: Test with pytest
timeout-minutes: 15
run: >
uv run pytest --import-mode=importlib
packages/foundry/tests
-m integration
-n logical --dist worksteal
--timeout=120 --session-timeout=900 --timeout_method thread
--retries 2 --retry-delay 5
run: uv run --directory packages/azure-ai poe integration-tests -n logical --dist worksteal --timeout=120 --session-timeout=900 --timeout_method thread --retries 2 --retry-delay 5
# Azure Cosmos integration tests
python-tests-cosmos:
@@ -350,7 +302,7 @@ jobs:
python-tests-azure-openai,
python-tests-misc-integration,
python-tests-functions,
python-tests-foundry,
python-tests-azure-ai,
python-tests-cosmos
]
steps:
+43 -99
View File
@@ -37,7 +37,7 @@ jobs:
azureChanged: ${{ steps.filter.outputs.azure }}
miscChanged: ${{ steps.filter.outputs.misc }}
functionsChanged: ${{ steps.filter.outputs.functions }}
foundryChanged: ${{ steps.filter.outputs.foundry }}
azureAiChanged: ${{ steps.filter.outputs.azure-ai }}
cosmosChanged: ${{ steps.filter.outputs.cosmos }}
steps:
- uses: actions/checkout@v6
@@ -47,9 +47,6 @@ jobs:
filters: |
python:
- 'python/**'
- '.github/actions/setup-local-mcp-server/**'
- '.github/workflows/python-merge-tests.yml'
- '.github/workflows/python-integration-tests.yml'
core:
- 'python/packages/core/agent_framework/_*.py'
- 'python/packages/core/agent_framework/_workflows/**'
@@ -57,28 +54,20 @@ jobs:
- 'python/packages/core/agent_framework/observability.py'
openai:
- 'python/packages/core/agent_framework/openai/**'
- 'python/packages/openai/**'
- 'python/samples/**/providers/openai/**'
- 'python/packages/core/tests/openai/**'
azure:
- 'python/packages/openai/**'
- 'python/packages/core/agent_framework/azure/**'
- 'python/samples/**/providers/azure/**'
- 'python/packages/core/tests/azure/**'
misc:
- 'python/packages/anthropic/**'
- 'python/packages/ollama/**'
- 'python/packages/core/agent_framework/_mcp.py'
- 'python/packages/core/tests/core/test_mcp.py'
- 'python/scripts/local_mcp_streamable_http_server.py'
- '.github/actions/setup-local-mcp-server/**'
- '.github/workflows/python-merge-tests.yml'
- '.github/workflows/python-integration-tests.yml'
functions:
- 'python/packages/azurefunctions/**'
- 'python/packages/durabletask/**'
foundry:
- 'python/packages/foundry/**'
- 'python/samples/**/providers/foundry/**'
- 'python/samples/02-agents/embeddings/foundry_embeddings.py'
azure-ai:
- 'python/packages/azure-ai/**'
cosmos:
- 'python/packages/azure-cosmos/**'
# run only if 'python' files were changed
@@ -111,17 +100,17 @@ jobs:
os: ${{ runner.os }}
- name: Test with pytest (unit tests only)
run: >
uv run poe test -A
uv run poe all-tests
-m "not integration"
-n logical --dist worksteal
--timeout=120 --session-timeout=900 --timeout_method thread
--retries 2 --retry-delay 5
--junitxml=pytest.xml
working-directory: ./python
- name: Surface failing tests
if: always()
uses: pmeier/pytest-results-action@v0.7.2
with:
path: ./python/pytest.xml
path: ./python/**.xml
summary: true
display-options: fEX
fail-on-empty: false
@@ -140,10 +129,9 @@ jobs:
runs-on: ubuntu-latest
environment: integration
env:
OPENAI_CHAT_COMPLETION_MODEL: ${{ vars.OPENAI__CHATMODELID }}
OPENAI_CHAT_MODEL: ${{ vars.OPENAI__RESPONSESMODELID }}
OPENAI_MODEL: ${{ vars.OPENAI__RESPONSESMODELID }}
OPENAI_EMBEDDING_MODEL: ${{ vars.OPENAI_EMBEDDING_MODEL_ID }}
OPENAI_CHAT_MODEL_ID: ${{ vars.OPENAI__CHATMODELID }}
OPENAI_RESPONSES_MODEL_ID: ${{ vars.OPENAI__RESPONSESMODELID }}
OPENAI_EMBEDDINGS_MODEL_ID: ${{ vars.OPENAI_EMBEDDING_MODEL_ID }}
OPENAI_API_KEY: ${{ secrets.OPENAI__APIKEY }}
defaults:
run:
@@ -159,12 +147,11 @@ jobs:
- name: Test with pytest (OpenAI integration)
run: >
uv run pytest --import-mode=importlib
packages/openai/tests
-m "integration and not azure"
packages/core/tests/openai
-m integration
-n logical --dist worksteal
--timeout=120 --session-timeout=900 --timeout_method thread
--retries 2 --retry-delay 5
--junitxml=pytest.xml
working-directory: ./python
- name: Test OpenAI samples
timeout-minutes: 10
@@ -175,7 +162,7 @@ jobs:
if: always()
uses: pmeier/pytest-results-action@v0.7.2
with:
path: ./python/pytest.xml
path: ./python/**.xml
summary: true
display-options: fEX
fail-on-empty: false
@@ -194,10 +181,9 @@ jobs:
runs-on: ubuntu-latest
environment: integration
env:
AZURE_OPENAI_CHAT_COMPLETION_MODEL: ${{ vars.AZUREOPENAI__CHATDEPLOYMENTNAME }}
AZURE_OPENAI_CHAT_MODEL: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
AZURE_OPENAI_MODEL: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
AZURE_OPENAI_EMBEDDING_MODEL: ${{ vars.AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME }}
AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__CHATDEPLOYMENTNAME }}
AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__EMBEDDINGDEPLOYMENTNAME }}
AZURE_OPENAI_ENDPOINT: ${{ vars.AZUREOPENAI__ENDPOINT }}
defaults:
run:
@@ -220,14 +206,11 @@ jobs:
- name: Test with pytest (Azure OpenAI integration)
run: >
uv run pytest --import-mode=importlib
packages/openai/tests/openai/test_openai_chat_completion_client_azure.py
packages/openai/tests/openai/test_openai_chat_client_azure.py
packages/openai/tests/openai/test_openai_embedding_client_azure.py
packages/core/tests/azure
-m integration
-n logical --dist worksteal
--timeout=120 --session-timeout=900 --timeout_method thread
--retries 2 --retry-delay 5
--junitxml=pytest.xml
working-directory: ./python
- name: Test Azure samples
timeout-minutes: 10
@@ -238,7 +221,7 @@ jobs:
if: always()
uses: pmeier/pytest-results-action@v0.7.2
with:
path: ./python/pytest.xml
path: ./python/**.xml
summary: true
display-options: fEX
fail-on-empty: false
@@ -258,7 +241,7 @@ jobs:
environment: integration
env:
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
ANTHROPIC_CHAT_MODEL: ${{ vars.ANTHROPIC_CHAT_MODEL_ID }}
ANTHROPIC_CHAT_MODEL_ID: ${{ vars.ANTHROPIC_CHAT_MODEL_ID }}
LOCAL_MCP_URL: ${{ vars.LOCAL_MCP__URL }}
defaults:
run:
@@ -271,13 +254,6 @@ jobs:
with:
python-version: ${{ env.UV_PYTHON }}
os: ${{ runner.os }}
- name: Start local MCP server
id: local-mcp
uses: ./.github/actions/setup-local-mcp-server
with:
fallback_url: ${{ env.LOCAL_MCP_URL }}
- name: Prefer local MCP URL when available
run: echo "LOCAL_MCP_URL=${{ steps.local-mcp.outputs.effective_url }}" >> "$GITHUB_ENV"
- name: Test with pytest (Anthropic, Ollama, MCP integration)
run: >
uv run pytest --import-mode=importlib
@@ -287,34 +263,13 @@ jobs:
-m integration
-n logical --dist worksteal
--timeout=120 --session-timeout=900 --timeout_method thread
--retries 2 --retry-delay 30
--junitxml=pytest.xml
--retries 2 --retry-delay 5
working-directory: ./python
- name: Stop local MCP server
if: always()
shell: bash
run: |
set -euo pipefail
server_pid="${{ steps.local-mcp.outputs.pid }}"
if [[ -z "$server_pid" ]]; then
exit 0
fi
if ! kill -0 "$server_pid" 2>/dev/null; then
exit 0
fi
kill -TERM -- "-$server_pid" 2>/dev/null || kill -TERM "$server_pid" 2>/dev/null || true
for _ in $(seq 1 10); do
if ! kill -0 "$server_pid" 2>/dev/null; then
exit 0
fi
sleep 1
done
kill -KILL -- "-$server_pid" 2>/dev/null || kill -KILL "$server_pid" 2>/dev/null || true
- name: Surface failing tests
if: always()
uses: pmeier/pytest-results-action@v0.7.2
with:
path: ./python/pytest.xml
path: ./python/**.xml
summary: true
display-options: fEX
fail-on-empty: false
@@ -334,17 +289,12 @@ jobs:
environment: integration
env:
UV_PYTHON: "3.11"
OPENAI_CHAT_COMPLETION_MODEL: ${{ vars.OPENAI__CHATMODELID }}
OPENAI_CHAT_MODEL: ${{ vars.OPENAI__RESPONSESMODELID }}
OPENAI_MODEL: ${{ vars.OPENAI__RESPONSESMODELID }}
OPENAI_EMBEDDING_MODEL: ${{ vars.OPENAI_EMBEDDING_MODEL_ID }}
OPENAI_CHAT_MODEL_ID: ${{ vars.OPENAI__CHATMODELID }}
OPENAI_RESPONSES_MODEL_ID: ${{ vars.OPENAI__RESPONSESMODELID }}
OPENAI_API_KEY: ${{ secrets.OPENAI__APIKEY }}
AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__CHATDEPLOYMENTNAME }}
AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
AZURE_OPENAI_ENDPOINT: ${{ vars.AZUREOPENAI__ENDPOINT }}
AZURE_OPENAI_MODEL: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
AZURE_OPENAI_CHAT_MODEL: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
AZURE_OPENAI_CHAT_COMPLETION_MODEL: ${{ vars.AZUREOPENAI__CHATDEPLOYMENTNAME }}
FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT }}
FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL }}
FUNCTIONS_WORKER_RUNTIME: "python"
DURABLE_TASK_SCHEDULER_CONNECTION_STRING: "Endpoint=http://localhost:8080;TaskHub=default;Authentication=None"
AzureWebJobsStorage: "UseDevelopmentStorage=true"
@@ -376,37 +326,33 @@ jobs:
packages/durabletask/tests/integration_tests
-m integration
-n logical --dist worksteal
-x
--timeout=360 --session-timeout=900 --timeout_method thread
--timeout=120 --session-timeout=900 --timeout_method thread
--retries 2 --retry-delay 5
--junitxml=pytest.xml
working-directory: ./python
- name: Surface failing tests
if: always()
uses: pmeier/pytest-results-action@v0.7.2
with:
path: ./python/pytest.xml
path: ./python/**.xml
summary: true
display-options: fEX
fail-on-empty: false
title: Functions integration test results
python-tests-foundry:
name: Python Integration Tests - Foundry
python-tests-azure-ai:
name: Python Tests - Azure AI
needs: paths-filter
if: >
github.event_name != 'pull_request' &&
needs.paths-filter.outputs.pythonChanges == 'true' &&
(github.event_name != 'merge_group' ||
needs.paths-filter.outputs.foundryChanged == 'true' ||
needs.paths-filter.outputs.azureAiChanged == 'true' ||
needs.paths-filter.outputs.coreChanged == 'true')
runs-on: ubuntu-latest
environment: integration
env:
FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT }}
FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL }}
FOUNDRY_AGENT_NAME: ${{ vars.FOUNDRY_AGENT_NAME }}
FOUNDRY_AGENT_VERSION: ${{ vars.FOUNDRY_AGENT_VERSION }}
AZURE_AI_PROJECT_ENDPOINT: ${{ secrets.AZUREAI__ENDPOINT }}
AZURE_AI_MODEL_DEPLOYMENT_NAME: ${{ vars.AZUREAI__DEPLOYMENTNAME }}
LOCAL_MCP_URL: ${{ vars.LOCAL_MCP__URL }}
defaults:
run:
@@ -428,20 +374,18 @@ jobs:
subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
- name: Test with pytest
timeout-minutes: 15
run: >
uv run pytest --import-mode=importlib
packages/foundry/tests
-m integration
-n logical --dist worksteal
--timeout=120 --session-timeout=900 --timeout_method thread
--retries 2 --retry-delay 5
--junitxml=pytest.xml
run: uv run --directory packages/azure-ai poe integration-tests -n logical --dist worksteal --timeout=120 --session-timeout=900 --timeout_method thread --retries 2 --retry-delay 5
working-directory: ./python
- name: Test Azure AI samples
timeout-minutes: 10
if: env.RUN_SAMPLES_TESTS == 'true'
run: uv run pytest tests/samples/ -m "azure-ai"
working-directory: ./python
- name: Surface failing tests
if: always()
uses: pmeier/pytest-results-action@v0.7.2
with:
path: ./python/pytest.xml
path: ./python/**.xml
summary: true
display-options: fEX
fail-on-empty: false
@@ -495,13 +439,13 @@ jobs:
echo "Cosmos DB emulator did not become ready in time." >&2
exit 1
- name: Test with pytest (Cosmos integration)
run: uv run --directory packages/azure-cosmos poe integration-tests -n logical --dist worksteal --timeout=120 --session-timeout=900 --timeout_method thread --retries 2 --retry-delay 5 --junitxml=pytest.xml
run: uv run --directory packages/azure-cosmos poe integration-tests -n logical --dist worksteal --timeout=120 --session-timeout=900 --timeout_method thread --retries 2 --retry-delay 5
working-directory: ./python
- name: Surface failing tests
if: always()
uses: pmeier/pytest-results-action@v0.7.2
with:
path: ./python/pytest.xml
path: ./python/**.xml
summary: true
display-options: fEX
fail-on-empty: false
@@ -517,7 +461,7 @@ jobs:
python-tests-azure-openai,
python-tests-misc-integration,
python-tests-functions,
python-tests-foundry,
python-tests-azure-ai,
python-tests-cosmos,
]
steps:
+61 -491
View File
@@ -23,8 +23,10 @@ jobs:
environment: integration
env:
# Required configuration for get-started samples
FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT || vars.AZURE_AI_PROJECT_ENDPOINT }}
FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL || vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
AZURE_AI_PROJECT_ENDPOINT: ${{ vars.AZURE_AI_PROJECT_ENDPOINT }}
AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
AZURE_OPENAI_ENDPOINT: ${{ vars.AZUREOPENAI__ENDPOINT }}
AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__CHATDEPLOYMENTNAME }}
defaults:
run:
working-directory: python
@@ -39,43 +41,33 @@ jobs:
azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
os: ${{ runner.os }}
- name: Create .env for samples
run: |
echo "FOUNDRY_PROJECT_ENDPOINT=$FOUNDRY_PROJECT_ENDPOINT" >> .env
echo "FOUNDRY_MODEL=$FOUNDRY_MODEL" >> .env
- name: Run sample validation
run: |
cd scripts && uv run python -m sample_validation --subdir 01-get-started --save-report --report-name 01-get-started
- name: Upload validation report
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v4
if: always()
with:
name: validation-report-01-get-started
path: python/samples/sample_validation/reports/
path: python/scripts/sample_validation/reports/
validate-02-agents:
name: Validate 02-agents
runs-on: ubuntu-latest
environment: integration
env:
# Foundry configuration
FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT || vars.AZURE_AI_PROJECT_ENDPOINT }}
FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL || vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
# Azure AI configuration
AZURE_AI_PROJECT_ENDPOINT: ${{ vars.AZURE_AI_PROJECT_ENDPOINT }}
AZURE_AI_MODEL_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
# Azure OpenAI configuration
AZURE_OPENAI_ENDPOINT: ${{ vars.AZUREOPENAI__ENDPOINT }}
AZURE_OPENAI_MODEL: ${{ vars.AZURE_OPENAI_DEPLOYMENT_NAME || vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
AZURE_OPENAI_CHAT_COMPLETION_MODEL: ${{ vars.AZUREOPENAI__CHATDEPLOYMENTNAME }}
AZURE_OPENAI_CHAT_MODEL: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
AZURE_OPENAI_EMBEDDING_MODEL: ${{ vars.AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME || vars.AZUREOPENAI__EMBEDDINGDEPLOYMENTNAME }}
AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__CHATDEPLOYMENTNAME }}
AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
# OpenAI configuration
OPENAI_API_KEY: ${{ secrets.OPENAI__APIKEY }}
OPENAI_CHAT_COMPLETION_MODEL: ${{ vars.OPENAI__CHATMODELID }}
OPENAI_CHAT_MODEL: ${{ vars.OPENAI__RESPONSESMODELID }}
# GitHub MCP
GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
OPENAI_MODEL: ${{ vars.OPENAI__RESPONSESMODELID }}
OPENAI_CHAT_MODEL_ID: ${{ vars.OPENAI__CHATMODELID }}
OPENAI_RESPONSES_MODEL_ID: ${{ vars.OPENAI__RESPONSESMODELID }}
# Observability
ENABLE_INSTRUMENTATION: "true"
defaults:
@@ -92,361 +84,29 @@ jobs:
azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
os: ${{ runner.os }}
- name: Create .env for samples
run: |
echo "FOUNDRY_PROJECT_ENDPOINT=$FOUNDRY_PROJECT_ENDPOINT" >> .env
echo "FOUNDRY_MODEL=$FOUNDRY_MODEL" >> .env
echo "AZURE_OPENAI_ENDPOINT=$AZURE_OPENAI_ENDPOINT" >> .env
echo "AZURE_OPENAI_MODEL=$AZURE_OPENAI_MODEL" >> .env
echo "AZURE_OPENAI_CHAT_COMPLETION_MODEL=$AZURE_OPENAI_CHAT_COMPLETION_MODEL" >> .env
echo "AZURE_OPENAI_CHAT_MODEL=$AZURE_OPENAI_CHAT_MODEL" >> .env
echo "AZURE_OPENAI_EMBEDDING_MODEL=$AZURE_OPENAI_EMBEDDING_MODEL" >> .env
echo "OPENAI_API_KEY=$OPENAI_API_KEY" >> .env
echo "OPENAI_CHAT_COMPLETION_MODEL=$OPENAI_CHAT_COMPLETION_MODEL" >> .env
echo "OPENAI_CHAT_MODEL=$OPENAI_CHAT_MODEL" >> .env
echo "GITHUB_PAT=$GITHUB_PAT" >> .env
- name: Run sample validation
run: |
cd scripts && uv run python -m sample_validation --subdir 02-agents --exclude providers --save-report --report-name 02-agents
cd scripts && uv run python -m sample_validation --subdir 02-agents --save-report --report-name 02-agents
- name: Upload validation report
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v4
if: always()
with:
name: validation-report-02-agents
path: python/samples/sample_validation/reports/
validate-02-agents-openai:
name: Validate 02-agents/providers/openai
runs-on: ubuntu-latest
environment: integration
env:
OPENAI_API_KEY: ${{ secrets.OPENAI__APIKEY }}
OPENAI_MODEL: ${{ vars.OPENAI__CHATMODELID }}
OPENAI_CHAT_COMPLETION_MODEL: ${{ vars.OPENAI__CHATMODELID }}
OPENAI_CHAT_MODEL: ${{ vars.OPENAI__RESPONSESMODELID }}
defaults:
run:
working-directory: python
steps:
- uses: actions/checkout@v6
- name: Setup environment
uses: ./.github/actions/sample-validation-setup
with:
azure-client-id: ${{ secrets.AZURE_CLIENT_ID }}
azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }}
azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
os: ${{ runner.os }}
- name: Create .env for samples
run: |
echo "OPENAI_API_KEY=$OPENAI_API_KEY" >> .env
echo "OPENAI_MODEL=$OPENAI_MODEL" >> .env
echo "OPENAI_CHAT_COMPLETION_MODEL=$OPENAI_CHAT_COMPLETION_MODEL" >> .env
echo "OPENAI_CHAT_MODEL=$OPENAI_CHAT_MODEL" >> .env
- name: Run sample validation
run: |
cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/openai --save-report --report-name 02-agents-openai
- name: Upload validation report
uses: actions/upload-artifact@v7
if: always()
with:
name: validation-report-02-agents-openai
path: python/samples/sample_validation/reports/
validate-02-agents-azure:
name: Validate 02-agents/providers/azure
runs-on: ubuntu-latest
environment: integration
env:
AZURE_OPENAI_ENDPOINT: ${{ vars.AZUREOPENAI__ENDPOINT }}
AZURE_OPENAI_MODEL: ${{ vars.AZURE_OPENAI_DEPLOYMENT_NAME || vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
AZURE_OPENAI_API_VERSION: ${{ vars.AZURE_OPENAI_API_VERSION || '' }}
defaults:
run:
working-directory: python
steps:
- uses: actions/checkout@v6
- name: Setup environment
uses: ./.github/actions/sample-validation-setup
with:
azure-client-id: ${{ secrets.AZURE_CLIENT_ID }}
azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }}
azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
os: ${{ runner.os }}
- name: Create .env for samples
run: |
echo "AZURE_OPENAI_ENDPOINT=$AZURE_OPENAI_ENDPOINT" >> .env
echo "AZURE_OPENAI_MODEL=$AZURE_OPENAI_MODEL" >> .env
echo "AZURE_OPENAI_API_VERSION=$AZURE_OPENAI_API_VERSION" >> .env
- name: Run sample validation
run: |
cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/azure --save-report --report-name 02-agents-azure
- name: Upload validation report
uses: actions/upload-artifact@v7
if: always()
with:
name: validation-report-02-agents-azure
path: python/samples/sample_validation/reports/
validate-02-agents-anthropic:
name: Validate 02-agents/providers/anthropic
runs-on: ubuntu-latest
environment: integration
env:
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
ANTHROPIC_CHAT_MODEL: ${{ vars.ANTHROPIC_CHAT_MODEL_ID }}
defaults:
run:
working-directory: python
steps:
- uses: actions/checkout@v6
- name: Setup environment
uses: ./.github/actions/sample-validation-setup
with:
azure-client-id: ${{ secrets.AZURE_CLIENT_ID }}
azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }}
azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
os: ${{ runner.os }}
- name: Create .env for samples
run: |
echo "ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY" >> .env
echo "ANTHROPIC_CHAT_MODEL=$ANTHROPIC_CHAT_MODEL" >> .env
- name: Run sample validation
run: |
cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/anthropic --save-report --report-name 02-agents-anthropic
- name: Upload validation report
uses: actions/upload-artifact@v7
if: always()
with:
name: validation-report-02-agents-anthropic
path: python/samples/sample_validation/reports/
validate-02-agents-github-copilot:
name: Validate 02-agents/providers/github_copilot
runs-on: ubuntu-latest
environment: integration
defaults:
run:
working-directory: python
steps:
- uses: actions/checkout@v6
- name: Setup environment
uses: ./.github/actions/sample-validation-setup
with:
azure-client-id: ${{ secrets.AZURE_CLIENT_ID }}
azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }}
azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
os: ${{ runner.os }}
- name: Run sample validation
run: |
cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/github_copilot --save-report --report-name 02-agents-github-copilot
- name: Upload validation report
uses: actions/upload-artifact@v7
if: always()
with:
name: validation-report-02-agents-github-copilot
path: python/samples/sample_validation/reports/
validate-02-agents-amazon:
name: Validate 02-agents/providers/amazon
if: false # Temporarily disabled - requires AWS credentials
runs-on: ubuntu-latest
environment: integration
env:
BEDROCK_CHAT_MODEL: ${{ vars.BEDROCK__CHATMODELID }}
defaults:
run:
working-directory: python
steps:
- uses: actions/checkout@v6
- name: Setup environment
uses: ./.github/actions/sample-validation-setup
with:
azure-client-id: ${{ secrets.AZURE_CLIENT_ID }}
azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }}
azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
os: ${{ runner.os }}
- name: Run sample validation
run: |
cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/amazon --save-report --report-name 02-agents-amazon
- name: Upload validation report
uses: actions/upload-artifact@v7
if: always()
with:
name: validation-report-02-agents-amazon
path: python/samples/sample_validation/reports/
validate-02-agents-ollama:
name: Validate 02-agents/providers/ollama
if: false # Temporarily disabled - requires local Ollama server
runs-on: ubuntu-latest
environment: integration
env:
OLLAMA_MODEL: ${{ vars.OLLAMA__MODEL }}
defaults:
run:
working-directory: python
steps:
- uses: actions/checkout@v6
- name: Setup environment
uses: ./.github/actions/sample-validation-setup
with:
azure-client-id: ${{ secrets.AZURE_CLIENT_ID }}
azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }}
azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
os: ${{ runner.os }}
- name: Run sample validation
run: |
cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/ollama --save-report --report-name 02-agents-ollama
- name: Upload validation report
uses: actions/upload-artifact@v7
if: always()
with:
name: validation-report-02-agents-ollama
path: python/samples/sample_validation/reports/
validate-02-agents-foundry:
name: Validate 02-agents/providers/foundry
if: false # Temporarily disabled - provider folder also contains the local Foundry sample
runs-on: ubuntu-latest
environment: integration
env:
FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT || vars.AZURE_AI_PROJECT_ENDPOINT }}
FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL || vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
FOUNDRY_AGENT_NAME: ${{ vars.FOUNDRY_AGENT_NAME || '' }}
FOUNDRY_AGENT_VERSION: ${{ vars.FOUNDRY_AGENT_VERSION || '' }}
defaults:
run:
working-directory: python
steps:
- uses: actions/checkout@v6
- name: Setup environment
uses: ./.github/actions/sample-validation-setup
with:
azure-client-id: ${{ secrets.AZURE_CLIENT_ID }}
azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }}
azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
os: ${{ runner.os }}
- name: Create .env for samples
run: |
echo "FOUNDRY_PROJECT_ENDPOINT=$FOUNDRY_PROJECT_ENDPOINT" >> .env
echo "FOUNDRY_MODEL=$FOUNDRY_MODEL" >> .env
echo "FOUNDRY_AGENT_NAME=$FOUNDRY_AGENT_NAME" >> .env
echo "FOUNDRY_AGENT_VERSION=$FOUNDRY_AGENT_VERSION" >> .env
- name: Run sample validation
run: |
cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/foundry --save-report --report-name 02-agents-foundry
- name: Upload validation report
uses: actions/upload-artifact@v7
if: always()
with:
name: validation-report-02-agents-foundry
path: python/samples/sample_validation/reports/
validate-02-agents-copilotstudio:
name: Validate 02-agents/providers/copilotstudio
if: false # Temporarily disabled - requires Copilot Studio setup
runs-on: ubuntu-latest
environment: integration
env:
COPILOTSTUDIOAGENT__ENVIRONMENTID: ${{ secrets.COPILOTSTUDIOAGENT__ENVIRONMENTID }}
COPILOTSTUDIOAGENT__SCHEMANAME: ${{ secrets.COPILOTSTUDIOAGENT__SCHEMANAME }}
COPILOTSTUDIOAGENT__TENANTID: ${{ secrets.COPILOTSTUDIOAGENT__TENANTID }}
COPILOTSTUDIOAGENT__AGENTAPPID: ${{ secrets.COPILOTSTUDIOAGENT__AGENTAPPID }}
defaults:
run:
working-directory: python
steps:
- uses: actions/checkout@v6
- name: Setup environment
uses: ./.github/actions/sample-validation-setup
with:
azure-client-id: ${{ secrets.AZURE_CLIENT_ID }}
azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }}
azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
os: ${{ runner.os }}
- name: Create .env for samples
run: |
echo "COPILOTSTUDIOAGENT__ENVIRONMENTID=$COPILOTSTUDIOAGENT__ENVIRONMENTID" >> .env
echo "COPILOTSTUDIOAGENT__SCHEMANAME=$COPILOTSTUDIOAGENT__SCHEMANAME" >> .env
echo "COPILOTSTUDIOAGENT__TENANTID=$COPILOTSTUDIOAGENT__TENANTID" >> .env
echo "COPILOTSTUDIOAGENT__AGENTAPPID=$COPILOTSTUDIOAGENT__AGENTAPPID" >> .env
- name: Run sample validation
run: |
cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/copilotstudio --save-report --report-name 02-agents-copilotstudio
- name: Upload validation report
uses: actions/upload-artifact@v7
if: always()
with:
name: validation-report-02-agents-copilotstudio
path: python/samples/sample_validation/reports/
validate-02-agents-custom:
name: Validate 02-agents/providers/custom
runs-on: ubuntu-latest
environment: integration
defaults:
run:
working-directory: python
steps:
- uses: actions/checkout@v6
- name: Setup environment
uses: ./.github/actions/sample-validation-setup
with:
azure-client-id: ${{ secrets.AZURE_CLIENT_ID }}
azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }}
azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
os: ${{ runner.os }}
- name: Run sample validation
run: |
cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/custom --save-report --report-name 02-agents-custom
- name: Upload validation report
uses: actions/upload-artifact@v7
if: always()
with:
name: validation-report-02-agents-custom
path: python/samples/sample_validation/reports/
path: python/scripts/sample_validation/reports/
validate-03-workflows:
name: Validate 03-workflows
runs-on: ubuntu-latest
environment: integration
env:
FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT || vars.AZURE_AI_PROJECT_ENDPOINT }}
FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL || vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
# Azure AI configuration
AZURE_AI_PROJECT_ENDPOINT: ${{ vars.AZURE_AI_PROJECT_ENDPOINT }}
AZURE_AI_MODEL_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
# Azure OpenAI configuration
AZURE_OPENAI_ENDPOINT: ${{ vars.AZUREOPENAI__ENDPOINT }}
AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__CHATDEPLOYMENTNAME }}
AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
defaults:
run:
working-directory: python
@@ -461,30 +121,29 @@ jobs:
azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
os: ${{ runner.os }}
- name: Create .env for samples
run: |
echo "FOUNDRY_PROJECT_ENDPOINT=$FOUNDRY_PROJECT_ENDPOINT" >> .env
echo "FOUNDRY_MODEL=$FOUNDRY_MODEL" >> .env
- name: Run sample validation
run: |
cd scripts && uv run python -m sample_validation --subdir 03-workflows --save-report --report-name 03-workflows
- name: Upload validation report
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v4
if: always()
with:
name: validation-report-03-workflows
path: python/samples/sample_validation/reports/
path: python/scripts/sample_validation/reports/
validate-04-hosting:
name: Validate 04-hosting
if: false # Temporarily disabled because of sample complexity
if: false # Temporarily disabled because of sample complexity
runs-on: ubuntu-latest
environment: integration
env:
FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT || vars.AZURE_AI_PROJECT_ENDPOINT }}
FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL || vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
# Azure AI configuration
AZURE_AI_PROJECT_ENDPOINT: ${{ vars.AZURE_AI_PROJECT_ENDPOINT }}
AZURE_AI_MODEL_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
# Azure OpenAI configuration
AZURE_OPENAI_ENDPOINT: ${{ vars.AZUREOPENAI__ENDPOINT }}
AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
# A2A configuration
A2A_AGENT_HOST: http://localhost:5001/
defaults:
@@ -506,30 +165,31 @@ jobs:
cd scripts && uv run python -m sample_validation --subdir 04-hosting --save-report --report-name 04-hosting
- name: Upload validation report
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v4
if: always()
with:
name: validation-report-04-hosting
path: python/samples/sample_validation/reports/
path: python/scripts/sample_validation/reports/
validate-05-end-to-end:
name: Validate 05-end-to-end
if: false # Temporarily disabled because of sample complexity
if: false # Temporarily disabled because of sample complexity
runs-on: ubuntu-latest
environment: integration
env:
FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT || vars.AZURE_AI_PROJECT_ENDPOINT }}
FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL || vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
# Azure AI configuration
AZURE_AI_PROJECT_ENDPOINT: ${{ vars.AZURE_AI_PROJECT_ENDPOINT }}
AZURE_AI_MODEL_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
# Azure OpenAI configuration
AZURE_OPENAI_ENDPOINT: ${{ vars.AZUREOPENAI__ENDPOINT }}
AZURE_OPENAI_MODEL: ${{ vars.AZURE_OPENAI_DEPLOYMENT_NAME || vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__CHATDEPLOYMENTNAME }}
AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
# Azure AI Search (for evaluation samples)
AZURE_SEARCH_ENDPOINT: ${{ secrets.AZURE_SEARCH_ENDPOINT }}
AZURE_SEARCH_API_KEY: ${{ secrets.AZURE_SEARCH_API_KEY }}
AZURE_SEARCH_INDEX_NAME: ${{ secrets.AZURE_SEARCH_INDEX_NAME }}
# Evaluation sample
FOUNDRY_MODEL_WORKFLOW: ${{ vars.FOUNDRY_MODEL_WORKFLOW || vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
FOUNDRY_MODEL_EVAL: ${{ vars.FOUNDRY_MODEL_EVAL || vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
AZURE_AI_MODEL_DEPLOYMENT_NAME_WORKFLOW: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
defaults:
run:
working-directory: python
@@ -549,27 +209,27 @@ jobs:
cd scripts && uv run python -m sample_validation --subdir 05-end-to-end --save-report --report-name 05-end-to-end
- name: Upload validation report
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v4
if: always()
with:
name: validation-report-05-end-to-end
path: python/samples/sample_validation/reports/
path: python/scripts/sample_validation/reports/
validate-autogen-migration:
name: Validate autogen-migration
runs-on: ubuntu-latest
environment: integration
env:
FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT || vars.AZURE_AI_PROJECT_ENDPOINT }}
FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL || vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
# Azure AI configuration
AZURE_AI_PROJECT_ENDPOINT: ${{ vars.AZURE_AI_PROJECT_ENDPOINT }}
AZURE_AI_MODEL_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
# Azure OpenAI configuration
AZURE_OPENAI_ENDPOINT: ${{ vars.AZUREOPENAI__ENDPOINT }}
AZURE_OPENAI_MODEL: ${{ vars.AZURE_OPENAI_DEPLOYMENT_NAME || vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__CHATDEPLOYMENTNAME }}
# OpenAI configuration
OPENAI_API_KEY: ${{ secrets.OPENAI__APIKEY }}
OPENAI_CHAT_COMPLETION_MODEL: ${{ vars.OPENAI__CHATMODELID }}
OPENAI_CHAT_MODEL: ${{ vars.OPENAI__RESPONSESMODELID }}
OPENAI_MODEL: ${{ vars.OPENAI__RESPONSESMODELID }}
OPENAI_CHAT_MODEL_ID: ${{ vars.OPENAI__CHATMODELID }}
OPENAI_RESPONSES_MODEL_ID: ${{ vars.OPENAI__RESPONSESMODELID }}
defaults:
run:
working-directory: python
@@ -584,46 +244,33 @@ jobs:
azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
os: ${{ runner.os }}
- name: Create .env for samples
run: |
echo "FOUNDRY_PROJECT_ENDPOINT=$FOUNDRY_PROJECT_ENDPOINT" >> .env
echo "FOUNDRY_MODEL=$FOUNDRY_MODEL" >> .env
echo "AZURE_OPENAI_ENDPOINT=$AZURE_OPENAI_ENDPOINT" >> .env
echo "AZURE_OPENAI_MODEL=$AZURE_OPENAI_MODEL" >> .env
echo "OPENAI_API_KEY=$OPENAI_API_KEY" >> .env
echo "OPENAI_CHAT_COMPLETION_MODEL=$OPENAI_CHAT_COMPLETION_MODEL" >> .env
echo "OPENAI_CHAT_MODEL=$OPENAI_CHAT_MODEL" >> .env
- name: Run sample validation
run: |
cd scripts && uv run python -m sample_validation --subdir autogen-migration --save-report --report-name autogen-migration
- name: Upload validation report
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v4
if: always()
with:
name: validation-report-autogen-migration
path: python/samples/sample_validation/reports/
path: python/scripts/sample_validation/reports/
validate-semantic-kernel-migration:
name: Validate semantic-kernel-migration
runs-on: ubuntu-latest
environment: integration
env:
FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT || vars.AZURE_AI_PROJECT_ENDPOINT }}
FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL || vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
# Azure OpenAI configuration for AF
# Azure AI configuration
AZURE_AI_PROJECT_ENDPOINT: ${{ vars.AZURE_AI_PROJECT_ENDPOINT }}
AZURE_AI_MODEL_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
# Azure OpenAI configuration
AZURE_OPENAI_ENDPOINT: ${{ vars.AZUREOPENAI__ENDPOINT }}
AZURE_OPENAI_MODEL: ${{ vars.AZURE_OPENAI_DEPLOYMENT_NAME || vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
# Azure OpenAI configuration for SK
AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ vars.AZURE_OPENAI_DEPLOYMENT_NAME }}
# OpenAI key
AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__CHATDEPLOYMENTNAME }}
AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }}
# OpenAI configuration
OPENAI_API_KEY: ${{ secrets.OPENAI__APIKEY }}
OPENAI_CHAT_COMPLETION_MODEL: ${{ vars.OPENAI__CHATMODELID }}
OPENAI_CHAT_MODEL: ${{ vars.OPENAI__RESPONSESMODELID }}
OPENAI_MODEL: ${{ vars.OPENAI__RESPONSESMODELID }}
# OpenAI configuration for SK
OPENAI_CHAT_MODEL_ID: ${{ vars.OPENAI__CHATMODELID }}
OPENAI_RESPONSES_MODEL_ID: ${{ vars.OPENAI__RESPONSESMODELID }}
# Copilot Studio
COPILOTSTUDIOAGENT__ENVIRONMENTID: ${{ secrets.COPILOTSTUDIOAGENT__ENVIRONMENTID }}
COPILOTSTUDIOAGENT__SCHEMANAME: ${{ secrets.COPILOTSTUDIOAGENT__SCHEMANAME }}
@@ -643,90 +290,13 @@ jobs:
azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
os: ${{ runner.os }}
- name: Create .env for samples
run: |
echo "FOUNDRY_PROJECT_ENDPOINT=$FOUNDRY_PROJECT_ENDPOINT" >> .env
echo "FOUNDRY_MODEL=$FOUNDRY_MODEL" >> .env
echo "AZURE_OPENAI_ENDPOINT=$AZURE_OPENAI_ENDPOINT" >> .env
echo "AZURE_OPENAI_MODEL=$AZURE_OPENAI_MODEL" >> .env
echo "OPENAI_API_KEY=$OPENAI_API_KEY" >> .env
echo "OPENAI_CHAT_COMPLETION_MODEL=$OPENAI_CHAT_COMPLETION_MODEL" >> .env
echo "OPENAI_CHAT_MODEL=$OPENAI_CHAT_MODEL" >> .env
echo "COPILOTSTUDIOAGENT__ENVIRONMENTID=$COPILOTSTUDIOAGENT__ENVIRONMENTID" >> .env
echo "COPILOTSTUDIOAGENT__SCHEMANAME=$COPILOTSTUDIOAGENT__SCHEMANAME" >> .env
echo "COPILOTSTUDIOAGENT__TENANTID=$COPILOTSTUDIOAGENT__TENANTID" >> .env
echo "COPILOTSTUDIOAGENT__AGENTAPPID=$COPILOTSTUDIOAGENT__AGENTAPPID" >> .env
- name: Run sample validation
run: |
cd scripts && uv run python -m sample_validation --subdir semantic-kernel-migration --save-report --report-name semantic-kernel-migration
- name: Upload validation report
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v4
if: always()
with:
name: validation-report-semantic-kernel-migration
path: python/samples/sample_validation/reports/
aggregate-results:
name: Aggregate Results
runs-on: ubuntu-latest
if: always()
needs:
- validate-01-get-started
- validate-02-agents
- validate-02-agents-openai
- validate-02-agents-azure
- validate-02-agents-anthropic
- validate-02-agents-github-copilot
- validate-02-agents-amazon
- validate-02-agents-ollama
- validate-02-agents-foundry
- validate-02-agents-copilotstudio
- validate-02-agents-custom
- validate-03-workflows
- validate-04-hosting
- validate-05-end-to-end
- validate-autogen-migration
- validate-semantic-kernel-migration
steps:
- uses: actions/checkout@v6
- name: Download all validation reports
uses: actions/download-artifact@v7
with:
pattern: validation-report-*
path: reports/
merge-multiple: true
- name: Restore validation history
id: cache-restore
uses: actions/cache/restore@v4
with:
path: validation-history/
key: validation-history-${{ github.run_id }}
restore-keys: |
validation-history-
- name: Aggregate results and generate trend report
run: |
python3 python/scripts/sample_validation/aggregate.py \
reports/ \
validation-history/history.json \
trend-report.md
- name: Write trend report to job summary
run: cat trend-report.md >> "$GITHUB_STEP_SUMMARY"
- name: Save validation history
uses: actions/cache/save@v4
with:
path: validation-history/
key: validation-history-${{ github.run_id }}
- name: Upload trend report
uses: actions/upload-artifact@v7
if: always()
with:
name: validation-trend-report
path: trend-report.md
path: python/scripts/sample_validation/reports/
@@ -21,7 +21,7 @@ jobs:
steps:
- uses: actions/checkout@v6
- name: Download coverage report
uses: actions/download-artifact@v8
uses: actions/download-artifact@v7
with:
github-token: ${{ secrets.GH_ACTIONS_PR_WRITE }}
run-id: ${{ github.event.workflow_run.id }}
@@ -46,7 +46,7 @@ jobs:
echo "PR_NUMBER=$PR_NUMBER" >> "$GITHUB_ENV"
- name: Pytest coverage comment
id: coverageComment
uses: MishaKav/pytest-coverage-comment@v1.6.0
uses: MishaKav/pytest-coverage-comment@v1.2.0
with:
github-token: ${{ secrets.GH_ACTIONS_PR_WRITE }}
issue-number: ${{ env.PR_NUMBER }}
+3 -3
View File
@@ -32,17 +32,17 @@ jobs:
id: python-setup
uses: ./.github/actions/python-setup
with:
python-version: ${{ env.UV_PYTHON }}
python-version: ${{ matrix.python-version }}
os: ${{ runner.os }}
env:
# Configure a constant location for the uv cache
UV_CACHE_DIR: /tmp/.uv-cache
- name: Run all tests with coverage report
run: uv run poe test -A -C --cov-report=xml:python-coverage.xml -q --junitxml=pytest.xml
run: uv run poe all-tests-cov --cov-report=xml:python-coverage.xml -q --junitxml=pytest.xml
- name: Check coverage threshold
run: python ${{ github.workspace }}/.github/workflows/python-check-coverage.py python-coverage.xml ${{ env.COVERAGE_THRESHOLD }}
- name: Upload coverage report
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
path: |
python/python-coverage.xml
+2 -2
View File
@@ -40,7 +40,7 @@ jobs:
UV_CACHE_DIR: /tmp/.uv-cache
# Unit tests
- name: Run all tests
run: uv run poe test -A --junitxml=pytest.xml
run: uv run poe all-tests ${{ matrix.python-version == '3.10' && '--ignore-glob=packages/github_copilot/**' || '' }}
working-directory: ./python
# Surface failing tests
@@ -48,7 +48,7 @@ jobs:
if: always()
uses: pmeier/pytest-results-action@v0.7.2
with:
path: ./python/pytest.xml
path: ./python/**.xml
summary: true
display-options: fEX
fail-on-empty: false
-49
View File
@@ -1,49 +0,0 @@
name: Stale issue and PR ping
on:
schedule:
- cron: '0 0 * * *' # Midnight UTC daily
workflow_dispatch:
inputs:
days_threshold:
description: 'Days of silence before pinging the author'
required: false
default: '4'
dry_run:
description: 'Log what would be pinged without taking action'
required: false
default: 'false'
type: choice
options:
- 'false'
- 'true'
concurrency:
group: stale-issue-pr-ping
cancel-in-progress: true
jobs:
ping_stale:
name: "Ping stale issues and PRs"
runs-on: ubuntu-latest
permissions:
contents: read
issues: write
pull-requests: write
steps:
- uses: actions/checkout@v6
- uses: actions/setup-python@v5
with:
python-version: '3.13'
- name: Install dependencies
run: pip install PyGithub==2.6.0
- name: Run stale issue/PR ping
run: python .github/scripts/stale_issue_pr_ping.py
env:
GITHUB_TOKEN: ${{ secrets.GH_ACTIONS_PR_WRITE }}
TEAM_SLUG: ${{ secrets.DEVELOPER_TEAM }}
DAYS_THRESHOLD: ${{ github.event.inputs.days_threshold || '4' }}
DRY_RUN: ${{ github.event.inputs.dry_run || 'false' }}
-5
View File
@@ -47,8 +47,6 @@ htmlcov/
.cache
nosetests.xml
coverage.xml
pytest.xml
python-coverage.xml
*.cover
*.py,cover
.hypothesis/
@@ -232,6 +230,3 @@ local.settings.json
# Database files
*.db
python/dotnet-ref
# Generated filtered solution files (created by eng/scripts/New-FilteredSolution.ps1)
dotnet/filtered-*.slnx
+8 -47
View File
@@ -74,37 +74,6 @@ Contributions must maintain API signature and behavioral compatibility. Contribu
that include breaking changes will be rejected. Please file an issue to discuss
your idea or change if you believe that a breaking change is warranted.
#### Automated API Compatibility Validation
The .NET projects use [Package Validation](https://learn.microsoft.com/dotnet/fundamentals/package-validation/overview)
to automatically detect API breaking changes. This validation runs during `dotnet build`
(Release configuration) and `dotnet pack`, comparing the current API surface against the
latest published NuGet baseline version.
**What gets validated:** By default, packable RC packages (`IsReleaseCandidate=true`) and
GA packages (`IsGenerallyAvailable=true`) that have a published NuGet baseline and do not
override validation settings are automatically validated. The shared baseline version and
default validation settings are defined in `dotnet/nuget/nuget-package.props`, but
individual projects may opt out (for example by setting `EnablePackageValidation=false`).
**If the build fails with CP errors (e.g., CP0001, CP0002):**
1. **Unintentional breaking change** — Refactor your code to maintain backward compatibility.
2. **Intentional breaking change** (approved by maintainers) — Generate a suppression file:
```bash
dotnet build <project>.csproj -c Release /p:ApiCompatGenerateSuppressionFile=true
```
This creates or updates a `CompatibilitySuppressions.xml` in the project directory.
Include this file in your PR with justification for the breaking change.
**After each release:**
1. Delete all `CompatibilitySuppressions.xml` files from validated projects.
2. Update `PackageValidationBaselineVersion` in `dotnet/nuget/nuget-package.props` to the
newly published version.
For more details, see the [Package Validation diagnostic IDs](https://learn.microsoft.com/dotnet/fundamentals/package-validation/diagnostic-ids).
### Suggested Workflow
We use and recommend the following workflow:
@@ -123,30 +92,22 @@ We use and recommend the following workflow:
"issue-123" or "githubhandle-issue".
4. Make and commit your changes to your branch.
5. Add new tests corresponding to your change, if applicable.
6. Run the relevant scripts in [the section below](#development-setup) to ensure that your build is clean and all tests are passing.
6. Run the relevant scripts in [the section below](#development-scripts) to ensure that your build is clean and all tests are passing.
7. Create a PR against the repository's **main** branch.
- State in the description what issue or improvement your change is addressing.
- Verify that all the Continuous Integration checks are passing.
8. Wait for feedback or approval of your changes from the code maintainers.
9. When area owners have signed off, and all checks are green, your PR will be merged.
### Development Setup
### Development scripts
Each language has its own dev setup guide, coding standards, and build scripts:
The scripts below are used to build, test, and lint within the project.
- **Python**: [Dev Setup](./python/DEV_SETUP.md) · [Coding Standard](./python/CODING_STANDARD.md) · [README](./python/README.md)
- From the `./python` directory:
- Build: `uv run poe build`
- Unit tests: `uv run poe test -A -m "not integration"`
- Integration tests: `uv run poe test -A -m integration` (requires API keys/endpoints)
- Format + lint: `uv run poe syntax`
- All checks: `uv run poe check`
- **.NET**: [README](./dotnet/README.md) · [Agent Instructions](./dotnet/AGENTS.md)
- From the `./dotnet` directory:
- Build: `dotnet build`
- Unit tests: `dotnet test --filter-query "/*UnitTests*/*/*/*"`
- Integration tests: `dotnet test --filter-query "/*IntegrationTests*/*/*/*"` (requires API keys/endpoints)
- Linting (auto-fix): `dotnet format`
- Python: see [python/DEV_SETUP.md](./python/DEV_SETUP.md).
- .NET:
- Build: `dotnet build`
- Test: `dotnet test`
- Linting (auto-fix): `dotnet format`
### PR - CI Process
+47 -77
View File
@@ -2,7 +2,7 @@
# Welcome to Microsoft Agent Framework!
[![Microsoft Foundry Discord](https://dcbadge.limes.pink/api/server/b5zjErwbQM?style=flat)](https://discord.gg/b5zjErwbQM)
[![Microsoft Azure AI Foundry Discord](https://dcbadge.limes.pink/api/server/b5zjErwbQM?style=flat)](https://discord.gg/b5zjErwbQM)
[![MS Learn Documentation](https://img.shields.io/badge/MS%20Learn-Documentation-blue)](https://learn.microsoft.com/en-us/agent-framework/)
[![PyPI](https://img.shields.io/pypi/v/agent-framework)](https://pypi.org/project/agent-framework/)
[![NuGet](https://img.shields.io/nuget/v/Microsoft.Agents.AI)](https://www.nuget.org/profiles/MicrosoftAgentFramework/)
@@ -28,7 +28,7 @@ Welcome to Microsoft's comprehensive multi-language framework for building, orch
Python
```bash
pip install agent-framework
pip install agent-framework --pre
# This will install all sub-packages, see `python/packages` for individual packages.
# It may take a minute on first install on Windows.
```
@@ -90,27 +90,27 @@ Still have questions? Join our [weekly office hours](./COMMUNITY.md#public-commu
Create a simple Azure Responses Agent that writes a haiku about the Microsoft Agent Framework
```python
# pip install agent-framework
# pip install agent-framework --pre
# Use `az login` to authenticate with Azure CLI
import os
import asyncio
from agent_framework import Agent
from agent_framework.foundry import FoundryChatClient
from agent_framework.azure import AzureOpenAIResponsesClient
from azure.identity import AzureCliCredential
async def main():
# Initialize a chat agent with Microsoft Foundry
# Initialize a chat agent with Azure OpenAI Responses
# the endpoint, deployment name, and api version can be set via environment variables
# or they can be passed in directly to the FoundryChatClient constructor
agent = Agent(
client=FoundryChatClient(
credential=AzureCliCredential(),
# project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"],
# model=os.environ["FOUNDRY_MODEL_DEPLOYMENT_NAME"],
),
name="HaikuBot",
instructions="You are an upbeat assistant that writes beautifully.",
# or they can be passed in directly to the AzureOpenAIResponsesClient constructor
agent = AzureOpenAIResponsesClient(
# endpoint=os.environ["AZURE_OPENAI_ENDPOINT"],
# deployment_name=os.environ["AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME"],
# api_version=os.environ["AZURE_OPENAI_API_VERSION"],
# api_key=os.environ["AZURE_OPENAI_API_KEY"], # Optional if using AzureCliCredential
credential=AzureCliCredential(), # Optional, if using api_key
).as_agent(
name="HaikuBot",
instructions="You are an upbeat assistant that writes beautifully.",
)
print(await agent.run("Write a haiku about Microsoft Agent Framework."))
@@ -120,38 +120,41 @@ if __name__ == "__main__":
```
### Basic Agent - .NET
Create a simple Agent, using Microsoft Foundry with token-based auth, that writes a haiku about the Microsoft Agent Framework
```c#
// dotnet add package Microsoft.Agents.AI.Foundry
// Use `az login` to authenticate with Azure CLI
using Azure.AI.Projects;
using Azure.Identity;
using System;
using Azure.AI.Projects;
using Azure.Identity;
var endpoint = Environment.GetEnvironmentVariable("AZURE_AI_PROJECT_ENDPOINT") ?? throw new InvalidOperationException("AZURE_AI_PROJECT_ENDPOINT is not set.");
var deploymentName = Environment.GetEnvironmentVariable("AZURE_AI_MODEL_DEPLOYMENT_NAME") ?? "gpt-5.4-mini";
var agent = new AIProjectClient(new Uri(endpoint), new DefaultAzureCredential())
.AsAIAgent(model: deploymentName, name: "HaikuBot", instructions: "You are an upbeat assistant that writes beautifully.");
Console.WriteLine(await agent.RunAsync("Write a haiku about Microsoft Agent Framework."));
```
Create a simple Agent, using OpenAI Responses, that writes a haiku about the Microsoft Agent Framework
```c#
// dotnet add package Microsoft.Agents.AI.OpenAI
using System;
// dotnet add package Microsoft.Agents.AI.OpenAI --prerelease
using Microsoft.Agents.AI;
using OpenAI;
using OpenAI.Responses;
// Replace the <apikey> with your OpenAI API key.
var agent = new OpenAIClient("<apikey>")
.GetResponsesClient()
.AsAIAgent(model: "gpt-5.4-mini", name: "HaikuBot", instructions: "You are an upbeat assistant that writes beautifully.");
.GetResponsesClient("gpt-4o-mini")
.AsAIAgent(name: "HaikuBot", instructions: "You are an upbeat assistant that writes beautifully.");
Console.WriteLine(await agent.RunAsync("Write a haiku about Microsoft Agent Framework."));
```
Create a simple Agent, using Azure OpenAI Responses with token based auth, that writes a haiku about the Microsoft Agent Framework
```c#
// dotnet add package Microsoft.Agents.AI.OpenAI --prerelease
// dotnet add package Azure.Identity
// Use `az login` to authenticate with Azure CLI
using System.ClientModel.Primitives;
using Azure.Identity;
using Microsoft.Agents.AI;
using OpenAI;
using OpenAI.Responses;
// Replace <resource> and gpt-4o-mini with your Azure OpenAI resource name and deployment name.
var agent = new OpenAIClient(
new BearerTokenPolicy(new AzureCliCredential(), "https://ai.azure.com/.default"),
new OpenAIClientOptions() { Endpoint = new Uri("https://<resource>.openai.azure.com/openai/v1") })
.GetResponsesClient("gpt-4o-mini")
.AsAIAgent(name: "HaikuBot", instructions: "You are an upbeat assistant that writes beautifully.");
Console.WriteLine(await agent.RunAsync("Write a haiku about Microsoft Agent Framework."));
```
@@ -160,43 +163,15 @@ Console.WriteLine(await agent.RunAsync("Write a haiku about Microsoft Agent Fram
### Python
- [Getting Started](./python/samples/01-get-started): progressive tutorial from hello-world to hosting
- [Getting Started with Agents](./python/samples/01-get-started): progressive tutorial from hello-world to hosting
- [Agent Concepts](./python/samples/02-agents): deep-dive samples by topic (tools, middleware, providers, etc.)
- [Workflows](./python/samples/03-workflows): workflow creation and integration with agents
- [Hosting](./python/samples/04-hosting): A2A, Azure Functions, Durable Task hosting
- [End-to-End](./python/samples/05-end-to-end): full applications, evaluation, and demos
- [Getting Started with Workflows](./python/samples/03-workflows): workflow creation and integration with agents
### .NET
- [Getting Started](./dotnet/samples/01-get-started): progressive tutorial from hello agent to hosting
- [Agent Concepts](./dotnet/samples/02-agents/Agents): basic agent creation and tool usage
- [Agent Providers](./dotnet/samples/02-agents/AgentProviders): samples showing different agent providers
- [Workflows](./dotnet/samples/03-workflows): advanced multi-agent patterns and workflow orchestration
- [Hosting](./dotnet/samples/04-hosting): A2A, Durable Agents, Durable Workflows
- [End-to-End](./dotnet/samples/05-end-to-end): full applications and demos
## Troubleshooting
### Authentication
| Problem | Cause | Fix |
|---------|-------|-----|
| Authentication errors when using Azure credentials | Not signed in to Azure CLI | Run `az login` before starting your app |
| API key errors | Wrong or missing API key | Verify the key and ensure it's for the correct resource/provider |
> **Tip:** `DefaultAzureCredential` is convenient for development but in production, consider using a specific credential (e.g., `ManagedIdentityCredential`) to avoid latency issues, unintended credential probing, and potential security risks from fallback mechanisms.
### Environment Variables
The samples typically read configuration from environment variables. Common required variables:
| Variable | Used by | Purpose |
|----------|---------|---------|
| `AZURE_OPENAI_ENDPOINT` | Azure OpenAI samples | Your Azure OpenAI resource URL |
| `AZURE_OPENAI_DEPLOYMENT_NAME` | Azure OpenAI samples | Model deployment name (e.g. `gpt-4o-mini`) |
| `AZURE_AI_PROJECT_ENDPOINT` | Microsoft Foundry samples | Your Microsoft Foundry project endpoint |
| `AZURE_AI_MODEL_DEPLOYMENT_NAME` | Microsoft Foundry samples | Model deployment name |
| `OPENAI_API_KEY` | OpenAI (non-Azure) samples | Your OpenAI platform API key |
- [Getting Started with Agents](./dotnet/samples/02-agents/Agents): basic agent creation and tool usage
- [Agent Provider Samples](./dotnet/samples/02-agents/AgentProviders): samples showing different agent providers
- [Workflow Samples](./dotnet/samples/03-workflows): advanced multi-agent patterns and workflow orchestration
## Contributor Resources
@@ -207,9 +182,4 @@ The samples typically read configuration from environment variables. Common requ
## Important Notes
> [!IMPORTANT]
> If you use Microsoft Agent Framework to build applications that operate with any third-party servers, agents, code, or non-Azure Direct models (“Third-Party Systems”), you do so at your own risk. Third-Party Systems are Non-Microsoft Products under the Microsoft Product Terms and are governed by their own third-party license terms. You are responsible for any usage and associated costs.
>
>We recommend reviewing all data being shared with and received from Third-Party Systems and being cognizant of third-party practices for handling, sharing, retention and location of data. It is your responsibility to manage whether your data will flow outside of your organizations Azure compliance and geographic boundaries and any related implications, and that appropriate permissions, boundaries and approvals are provisioned.
>
>You are responsible for carefully reviewing and testing applications you build using Microsoft Agent Framework in the context of your specific use cases, and making all appropriate decisions and customizations. This includes implementing your own responsible AI mitigations such as metaprompt, content filters, or other safety systems, and ensuring your applications meet appropriate quality, reliability, security, and trustworthiness standards. See also: [Transparency FAQ](./TRANSPARENCY_FAQ.md)
If you use the Microsoft Agent Framework to build applications that operate with third-party servers or agents, you do so at your own risk. We recommend reviewing all data being shared with third-party servers or agents and being cognizant of third-party practices for retention and location of data. It is your responsibility to manage whether your data will flow outside of your organization's Azure compliance and geographic boundaries and any related implications.
@@ -1,3 +1,3 @@
# Declarative Agents
This folder contains sample agent definitions that can be run using the declarative agent support, for python see the [declarative agent python sample folder](../../python/samples/02-agents/declarative/).
This folder contains sample agent definitions that can be run using the declarative agent support, for python see the [declarative agent python sample folder](../python/samples/02-agents/declarative/).
@@ -3,13 +3,13 @@ name: MicrosoftLearnAgent
description: Microsoft Learn Agent
instructions: You answer questions by searching the Microsoft Learn content only.
model:
id: =Env.FOUNDRY_MODEL
id: =Env.AZURE_FOUNDRY_PROJECT_MODEL_ID
options:
temperature: 0.9
topP: 0.95
connection:
kind: remote
endpoint: =Env.FOUNDRY_PROJECT_ENDPOINT
endpoint: =Env.AZURE_FOUNDRY_PROJECT_ENDPOINT
tools:
- kind: mcp
name: microsoft_learn
@@ -1,125 +0,0 @@
---
status: accepted
contact: rogerbarreto
date: 2026-03-06
deciders: rogerbarreto, alliscode
consulted: ""
informed: ""
---
# Foundry agent surface stays centered on `ChatClientAgent`
## Context
The Microsoft Foundry integration exposes two distinct usage patterns:
1. Direct Responses usage, where callers provide model, instructions, and tools at runtime.
2. Server-side versioned agents, where callers create and manage `AgentVersion` resources through `AIProjectClient.Agents`.
We briefly explored adding public wrapper types such as `FoundryAgent`, `FoundryVersionedAgent`, and `FoundryResponsesChatClient` to make those paths feel more specialized. That direction created extra public types, duplicated existing `ChatClientAgent` behavior, and pushed samples toward compatibility helpers instead of the native Azure SDK flow.
## Decision
Keep the public surface centered on `ChatClientAgent`.
- Direct Responses scenarios use `AIProjectClient.AsAIAgent(...)`.
- Server-side versioned scenarios use native `AIProjectClient.Agents` APIs to create or retrieve agent resources, then wrap `AgentRecord` or `AgentVersion` with `AIProjectClient.AsAIAgent(...)`.
- Compatibility helpers such as `AIProjectClient.CreateAIAgentAsync(...)` and `AIProjectClient.GetAIAgentAsync(...)` remain only as obsolete migration shims.
- Public wrapper types `FoundryAgent`, `FoundryVersionedAgent`, `FoundryResponsesChatClient`, and `FoundryResponsesChatClientAgent` are not part of the chosen direction.
## Why
- `ChatClientAgent` is already the framework abstraction used everywhere else.
- `AIProjectClient` is the native Azure SDK entry point for versioned agent lifecycle operations.
- A single agent abstraction avoids parallel type hierarchies for the same backend.
- Samples become clearer when they show either:
- direct Responses construction via `AIProjectClient.AsAIAgent(...)`, or
- native Foundry resource management via `AIProjectClient.Agents`.
## Consequences
### Direct Responses path
Use the convenience overloads on `AIProjectClient`:
```csharp
AIProjectClient aiProjectClient = new(new Uri(endpoint), credential);
ChatClientAgent agent = aiProjectClient.AsAIAgent(
model: deploymentName,
instructions: "You are good at telling jokes.",
name: "JokerAgent");
```
Or use composed `ChatClientAgent`
```csharp
ProjectResponsesClient projectResponsesClient = new(new Uri(endpoint), new DefaultAzureCredential(), new AgentReference($"model:{deploymentName}"));
ChatClientAgent agent = new(
chatClient: projectResponsesClient.AsIChatClient(),
instructions: "You are good at telling jokes.",
name: "JokerAgent");
```
This path is code-first and does not create a persistent server-side agent.
### Versioned agent path
Use the convenience overloads on `AIProjectClient`:
```csharp
AIProjectClient aiProjectClient = new(new Uri(endpoint), credential);
AgentVersion version = await aiProjectClient.Agents.CreateAgentVersionAsync(
"JokerAgent",
new AgentVersionCreationOptions(
new PromptAgentDefinition(deploymentName)
{
Instructions = "You are good at telling jokes."
}));
ChatClientAgent agent = aiProjectClient.AsAIAgent(version);
```
Or use composed `ChatClientAgent`
```csharp
AIProjectClient aiProjectClient = new(new Uri(endpoint), credential);
AgentVersion version = await aiProjectClient.Agents.CreateAgentVersionAsync(
"JokerAgent",
new AgentVersionCreationOptions(
new PromptAgentDefinition(deploymentName)
{
Instructions = "You are good at telling jokes."
}));
ProjectResponsesClient projectResponsesClient = aiProjectClient
.GetProjectOpenAIClient()
.GetProjectResponsesClientForAgent(new AgentReference(version.Name, version.Version));
ChatClientAgent agent = new(
chatClient: projectResponsesClient.AsIChatClient(),
name: "JokerAgent");
```
### Samples
- `FoundryAgents/` samples show the direct Responses path with `AIProjectClient.AsAIAgent(...)`.
- `FoundryVersionedAgents/` samples should show native `AIProjectClient.Agents` create/get/delete flows plus `AsAIAgent(...)`.
### Compatibility APIs
Obsolete helper extensions remain only to ease migration of existing code. New samples and new guidance should not be written against them.
## Rejected direction
Do not introduce or preserve separate public wrapper types whose main purpose is to forward to `ChatClientAgent` while carrying Foundry-specific naming.
That approach:
- duplicates lifecycle concepts already present on `AIProjectClient`,
- fragments the public API,
- complicates samples and docs,
- and makes migration harder by encouraging wrapper-specific affordances.
-960
View File
@@ -1,960 +0,0 @@
status: proposed
date: 2026-03-23
contact: sergeymenshykh
deciders: rbarreto, westey-m, eavanvalkenburg
---
# Agent Skills: Multi-Source Architecture
## Context and Problem Statement
The Agent Framework needs a skills system that lets agents discover and use domain-specific knowledge, reference documents, and executable scripts. Skills can originate from different sources — filesystem directories (SKILL.md files), inline C# code, or reusable class libraries — and the framework must support all three uniformly while allowing extensibility, composition, and filtering.
## Decision Drivers
- Skills must be definable from multiple sources: filesystem, inline code, reusable classes, etc
- Common abstractions are needed so the provider and builder work uniformly regardless of skill origin
- File-based scripts must support user-defined executors, enabling custom runtimes and languages; code/class-based scripts execute in-process as C# delegates
- Skills must be filterable so consumers can include or exclude specific skills based on defined criteria
- Multiple skill sources must be composable into a single provider
- It must be possible to add custom skill sources (e.g., databases, REST APIs, package registries) by implementing a common abstraction
## Architecture
### Model-Facing Tools
Skills are presented to the model as up to three tools that progressively disclose skill content. The system prompt lists available skill names and descriptions; the model then calls these tools on demand:
- **`load_skill(skillName)`** — returns the full skill body (instructions, listed resources, listed scripts)
- **`read_skill_resource(skillName, resourceName)`** — reads a supplementary resource (file-based or code-defined) associated with a skill
- **`run_skill_script(skillName, scriptName, arguments?)`** — executes a script associated with a skill; only registered when at least one skill contains scripts
Each tool delegates to the corresponding method on the resolved `AgentSkill` — calling `Resource.ReadAsync()` or `Script.RunAsync()` respectively.
If skills have no scripts defined, the `run_skill_script` tool is **not advertised** to the model and instructions related to script execution are **not included** in the default skills instructions.
### Abstract Base Types
The architecture defines four abstract base types that all skill variants implement:
```csharp
public abstract class AgentSkill
{
public abstract AgentSkillFrontmatter Frontmatter { get; }
public abstract string Content { get; }
public abstract IReadOnlyList<AgentSkillResource>? Resources { get; }
public abstract IReadOnlyList<AgentSkillScript>? Scripts { get; }
}
public abstract class AgentSkillResource
{
public string Name { get; }
public string? Description { get; }
public abstract Task<object?> ReadAsync(IServiceProvider? serviceProvider = null, CancellationToken cancellationToken = default);
}
public abstract class AgentSkillScript
{
public string Name { get; }
public string? Description { get; }
public abstract Task<object?> RunAsync(AgentSkill skill, AIFunctionArguments arguments, CancellationToken cancellationToken = default);
}
public abstract class AgentSkillsSource
{
public abstract Task<IList<AgentSkill>> GetSkillsAsync(CancellationToken cancellationToken = default);
}
```
Skill metadata is captured via `AgentSkillFrontmatter`:
```csharp
public sealed class AgentSkillFrontmatter
{
public AgentSkillFrontmatter(string name, string description) { ... }
public string Name { get; }
public string Description { get; }
public string? License { get; set; }
public string? Compatibility { get; set; }
public string? AllowedTools { get; set; }
public AdditionalPropertiesDictionary? Metadata { get; set; }
}
```
The type hierarchy at a glance:
```
AgentSkill (abstract) AgentSkillsSource (abstract)
├── AgentFileSkill ├── AgentFileSkillsSource (public)
└── [Programmatic] ├── AgentInMemorySkillsSource (public)
├── AgentInlineSkill ├── AggregatingAgentSkillsSource (public)
└── AgentClassSkill (abstract) └── DelegatingAgentSkillsSource (abstract, public)
├── FilteringAgentSkillsSource (public)
AgentSkillResource (abstract) ├── CachingAgentSkillsSource (public)
├── AgentFileSkillResource └── DeduplicatingAgentSkillsSource (public)
└── AgentInlineSkillResource
AgentSkillScript (abstract)
├── AgentFileSkillScript
└── AgentInlineSkillScript
```
There are two top-level categories of skills:
1. **File-Based Skills** — discovered from `SKILL.md` files on the filesystem. Resources and scripts are files in subdirectories.
2. **Programmatic Skills** — defined in C# code. These are further divided into:
- **Inline Skills** — built at runtime via the `AgentInlineSkill` class and its fluent API. Ideal for quick, agent-specific skill definitions.
- **Class-Based Skills** — defined as reusable C# classes that subclass `AgentClassSkill`. Ideal for packaging skills as shared libraries or NuGet packages.
Both programmatic skill types use `AgentInlineSkillResource` and `AgentInlineSkillScript` for their resources and scripts. They are typically served by `AgentInMemorySkillsSource`, which accepts any `AgentSkill` and is not limited to programmatic skills.
### File-Based Skills
File-based skills are authored as `SKILL.md` files on disk. Resources and scripts are discovered from corresponding subfolders within the skill directory.
**`AgentFileSkill`** — A filesystem-based skill discovered from a directory containing a `SKILL.md` file. Parsed from YAML frontmatter; content is the raw markdown body. Resources and scripts are discovered from files in corresponding subfolders:
```csharp
public sealed class AgentFileSkill : AgentSkill
{
internal AgentFileSkill(
AgentSkillFrontmatter frontmatter, string content, string path,
IReadOnlyList<AgentSkillResource>? resources = null,
IReadOnlyList<AgentSkillScript>? scripts = null) { ... }
}
```
**`AgentFileSkillResource`** — A file-based skill resource. Reads content from a file on disk relative to the skill directory:
```csharp
internal sealed class AgentFileSkillResource : AgentSkillResource
{
public AgentFileSkillResource(string name, string fullPath) { ... }
public string FullPath { get; }
public override Task<object?> ReadAsync(IServiceProvider? serviceProvider = null, CancellationToken cancellationToken = default)
{
return File.ReadAllTextAsync(FullPath, Encoding.UTF8, cancellationToken);
}
}
```
**`AgentFileSkillScript`** — A file-based skill script that represents a script file on disk. Delegates execution to an external `AgentFileSkillScriptRunner` callback (e.g., runs Python/shell via `Process.Start`). Throws `NotSupportedException` if no executor is configured:
```csharp
public delegate Task<object?> AgentFileSkillScriptRunner(
AgentFileSkill skill, AgentFileSkillScript script,
AIFunctionArguments arguments, CancellationToken cancellationToken);
public sealed class AgentFileSkillScript : AgentSkillScript
{
private readonly AgentFileSkillScriptRunner _executor;
internal AgentFileSkillScript(string name, string fullPath, AgentFileSkillScriptRunner executor)
: base(name) { ... }
public override async Task<object?> RunAsync(AgentSkill skill, AIFunctionArguments arguments, ...)
{
return await _executor(fileSkill, this, arguments, cancellationToken);
}
}
```
The executor can be provided at the **provider level** via `AgentSkillsProviderBuilder.UseFileScriptRunner(executor)` and optionally overridden for a **particular file skill** or for a **set of skills** at the file skill source level, giving fine-grained control over how different scripts are executed.
**`AgentFileSkillsSource`** — A skill source that discovers skills from filesystem directories containing `SKILL.md` files. Recursively scans directories (max 2 levels), validates frontmatter, and enforces path traversal and symlink security checks:
```csharp
public sealed partial class AgentFileSkillsSource : AgentSkillsSource
{
public AgentFileSkillsSource(
IEnumerable<string> skillPaths,
AgentFileSkillScriptRunner scriptRunner,
AgentFileSkillsSourceOptions? options = null,
ILoggerFactory? loggerFactory = null) { ... }
}
```
**`AgentFileSkillsSourceOptions`** — Configuration options for `AgentFileSkillsSource`. Allows customizing the allowed file extensions for resources and scripts without adding constructor parameters:
```csharp
public sealed class AgentFileSkillsSourceOptions
{
public IEnumerable<string>? AllowedResourceExtensions { get; set; }
public IEnumerable<string>? AllowedScriptExtensions { get; set; }
}
```
**Example** — A file-based skill on disk and how it is added to a source:
```
skills/
└── unit-converter/
├── SKILL.md # frontmatter + instructions
├── resources/
│ └── conversion-table.csv # discovered as a resource
└── scripts/
└── convert.py # discovered as a script
```
```csharp
var source = new AgentFileSkillsSource(skillPaths: ["./skills"], scriptRunner: SubprocessScriptRunner.RunAsync);
var provider = new AgentSkillsProvider(source);
AIAgent agent = chatClient.AsAIAgent(new ChatClientAgentOptions
{
AIContextProviders = [provider],
});
```
### Programmatic Skills
Programmatic skills are defined in C# code rather than discovered from the filesystem. There are two kinds: **inline** and **class-based**. Both use `AgentInlineSkillResource` and `AgentInlineSkillScript` for resources and scripts, and are held by a single `AgentInMemorySkillsSource`.
**`AgentInMemorySkillsSource`** — A general-purpose skill source that holds any `AgentSkill` instances in memory. Although commonly used for programmatic skills (`AgentInlineSkill` and `AgentClassSkill`), it accepts any `AgentSkill` subclass and is not restricted to code-defined skills:
```csharp
public sealed class AgentInMemorySkillsSource : AgentSkillsSource
{
public AgentInMemorySkillsSource(
IEnumerable<AgentSkill> skills,
ILoggerFactory? loggerFactory = null) { ... }
}
```
#### Inline Skills
Inline skills are built at runtime via the `AgentInlineSkill` class and its fluent API. They are ideal for quick, agent-specific skill definitions where a full class hierarchy would be overkill.
**`AgentInlineSkill`** — A skill defined entirely in code. Resources can be static values or functions; scripts are always functions. Constructed with name, description, and instructions, then extended with resources and scripts:
```csharp
public sealed class AgentInlineSkill : AgentSkill
{
public AgentInlineSkill(string name, string description, string instructions, string? license = null, string? compatibility = null, ...) { ... }
public AgentInlineSkill(AgentSkillFrontmatter frontmatter, string instructions) { ... }
public AgentInlineSkill AddResource(object value, string name, string? description = null);
public AgentInlineSkill AddResource(Delegate handler, string name, string? description = null);
public AgentInlineSkill AddScript(Delegate handler, string name, string? description = null);
}
```
**`AgentInlineSkillResource`** — A skill resource that wraps a static value:
```csharp
public sealed class AgentInlineSkillResource : AgentSkillResource
{
public AgentInlineSkillResource(object value, string name, string? description = null)
: base(name, description)
{
_value = value;
}
public override Task<object?> ReadAsync(IServiceProvider? serviceProvider = null, CancellationToken cancellationToken = default)
{
return Task.FromResult<object?>(_value);
}
}
```
**`AgentInlineSkillResource`** — A skill resource backed by a delegate. The delegate is invoked via an `AIFunction` each time `ReadAsync` is called, producing a dynamic (computed) value:
```csharp
public sealed class AgentInlineSkillResource : AgentSkillResource
{
public AgentInlineSkillResource(Delegate handler, string name, string? description = null)
: base(name, description)
{
_function = AIFunctionFactory.Create(handler, name: name);
}
public override async Task<object?> ReadAsync(IServiceProvider? serviceProvider = null, CancellationToken cancellationToken = default)
{
return await _function.InvokeAsync(new AIFunctionArguments() { Services = serviceProvider }, cancellationToken);
}
}
```
**`AgentInlineSkillScript`** — A skill script backed by a delegate via an `AIFunction`:
```csharp
public sealed class AgentInlineSkillScript : AgentSkillScript
{
private readonly AIFunction _function;
public AgentInlineSkillScript(Delegate handler, string name, string? description = null)
: base(name, description)
{
_function = AIFunctionFactory.Create(handler, name: name);
}
public JsonElement? ParametersSchema => _function.JsonSchema;
public override async Task<object?> RunAsync(AgentSkill skill, AIFunctionArguments arguments, ...)
{
return await _function.InvokeAsync(arguments, cancellationToken);
}
}
```
**Example** — Creating an inline skill with a resource and script, then adding it to a source:
```csharp
var skill = new AgentInlineSkill(
name: "unit-converter",
description: "Converts between measurement units.",
instructions: """
Use this skill to convert values between metric and imperial units.
Refer to the conversion-table resource for supported unit pairs.
Run the convert script to perform conversions.
"""
)
.AddResource("kg=2.205lb, m=3.281ft, L=0.264gal", "conversion-table", "Supported unit pairs")
.AddScript(Convert, "convert", "Converts a value between units");
var source = new AgentInMemorySkillsSource([skill]);
var provider = new AgentSkillsProvider(source);
AIAgent agent = chatClient.AsAIAgent(new ChatClientAgentOptions
{
AIContextProviders = [provider],
});
static string Convert(double value, double factor)
=> JsonSerializer.Serialize(new { result = Math.Round(value * factor, 4) });
```
#### Class-Based Skills
Class-based skills are designed for packaging skills as reusable libraries. Users subclass `AgentClassSkill` and override properties. Unlike inline skills, class-based skills are self-contained, can live in shared libraries or NuGet packages, and are well-suited for dependency injection.
**`AgentClassSkill`** — An abstract base class for defining skills as reusable C# classes that bundle all skill components (frontmatter, instructions, resources, scripts) together. Designed for packaging skills as distributable libraries:
```csharp
public abstract class AgentClassSkill : AgentSkill
{
public abstract string Instructions { get; }
// Content is auto-synthesized from Frontmatter + Instructions + Resources + Scripts
public override string Content =>
SkillContentBuilder.BuildContent(Frontmatter.Name, Frontmatter.Description,
SkillContentBuilder.BuildBody(Instructions, Resources, Scripts));
}
```
**Example** — Defining a class-based skill and adding it to a source:
```csharp
public class UnitConverterSkill : AgentClassSkill
{
public override AgentSkillFrontmatter Frontmatter { get; } =
new("unit-converter", "Converts between measurement units.");
public override string Instructions => """
Use this skill to convert values between metric and imperial units.
Refer to the conversion-table resource for supported unit pairs.
Run the convert script to perform conversions.
""";
public override IReadOnlyList<AgentSkillResource>? Resources { get; } =
[
new AgentInlineSkillResource("kg=2.205lb, m=3.281ft", "conversion-table"),
];
public override IReadOnlyList<AgentSkillScript>? Scripts { get; } =
[
new AgentInlineSkillScript(Convert, "convert"),
];
private static string Convert(double value, double factor)
=> JsonSerializer.Serialize(new { result = Math.Round(value * factor, 4) });
}
var source = new AgentInMemorySkillsSource([new UnitConverterSkill()]);
var provider = new AgentSkillsProvider(source);
AIAgent agent = chatClient.AsAIAgent(new ChatClientAgentOptions
{
AIContextProviders = [provider],
});
```
## Filtering, Caching, and Deduplication
The following subsections present alternative approaches for handling filtering, caching, and deduplication of skills across multiple sources.
### Via Composition
In this approach, the `AgentSkillsProvider` accepts a **single** `AgentSkillsSource`. Multiple sources are composed externally via an aggregate source, and cross-cutting concerns like filtering, caching, and deduplication are implemented as **source decorators** — subclasses of `DelegatingAgentSkillsSource` that intercept `GetSkillsAsync()`.
**`FilteringAgentSkillsSource`** — A decorator that applies filter logic before returning results. The decorator pattern keeps filtering orthogonal to source implementations and allows composing multiple filters:
```csharp
public sealed class FilteringAgentSkillsSource : DelegatingAgentSkillsSource
{
private readonly Func<AgentSkill, bool> _predicate;
public FilteringAgentSkillsSource(AgentSkillsSource innerSource, Func<AgentSkill, bool> predicate)
: base(innerSource)
{
_predicate = predicate;
}
public override async Task<IList<AgentSkill>> GetSkillsAsync(CancellationToken cancellationToken = default)
{
var skills = await this.InnerSource.GetSkillsAsync(cancellationToken);
return skills.Where(_predicate).ToList();
}
}
```
**`CachingAgentSkillsSource`** — A decorator that caches skills after the first load, keeping the provider stateless and giving consumers control over caching granularity per source. For example, file-based skills (expensive to discover) can be cached while code-defined skills remain uncached:
```csharp
public sealed class CachingAgentSkillsSource : DelegatingAgentSkillsSource
{
private IList<AgentSkill>? _cached;
public CachingAgentSkillsSource(AgentSkillsSource innerSource)
: base(innerSource)
{
}
public override async Task<IList<AgentSkill>> GetSkillsAsync(CancellationToken cancellationToken = default)
{
return _cached ??= await this.InnerSource.GetSkillsAsync(cancellationToken);
}
}
```
**Deduplication** is similarly implemented as a decorator (`DeduplicatingAgentSkillsSource`) that deduplicates by name (case-insensitive, first-one-wins) and logs a warning for skipped duplicates.
**Example** — Combining file-based and code-defined sources with filtering and caching:
```csharp
var fileSource = new CachingAgentSkillsSource(new AgentFileSkillsSource(["./skills"]));
var codeSource = new AgentInMemorySkillsSource([myCodeSkill]);
var compositeSource = new FilteringAgentSkillsSource(
new AggregatingAgentSkillsSource([fileSource, codeSource]),
filter: s => s.Frontmatter.Name != "internal");
var provider = new AgentSkillsProvider(compositeSource);
AIAgent agent = chatClient.AsAIAgent(new ChatClientAgentOptions
{
AIContextProviders = [provider],
});
```
**Pros:**
- Clean single-responsibility: the provider serves skills, sources provide them.
- Caching, filtering, and deduplication are composable as source decorators — each concern is a separate, testable wrapper.
**Cons:**
- DI is less flexible: multiple `AgentSkillsSource` implementations registered in the container cannot be auto-injected into the provider. The consumer must manually compose them via an aggregate source.
- Increased public API surface: requires additional public classes (aggregate source, caching decorators, filtering decorators) that consumers need to learn and use.
### Via AgentSkillsProvider
In this approach, the `AgentSkillsProvider` accepts **`IEnumerable<AgentSkillsSource>`** and handles aggregation, filtering, caching, and deduplication internally.
The provider aggregates skills from all registered sources, deduplicates by name (case-insensitive, first-one-wins), caches the result after the first load, and optionally applies filtering via a predicate on `AgentSkillsProviderOptions`. Duplicate skill names are logged as warnings.
**Example** — Registering multiple sources directly with the provider:
```csharp
// Conceptual example — in practice, use AgentSkillsProviderBuilder
var fileSource = new AgentFileSkillsSource(["./skills"]);
var codeSource = new AgentInMemorySkillsSource([myCodeSkill]);
var provider = new AgentSkillsProvider(
sources: [fileSource, codeSource],
options: new AgentSkillsProviderOptions
{
Filter = s => s.Frontmatter.Name != "internal",
});
AIAgent agent = chatClient.AsAIAgent(new ChatClientAgentOptions
{
AIContextProviders = [provider],
});
```
**Pros:**
- DI-friendly: register multiple `AgentSkillsSource` implementations in the container, and they are all auto-injected into `AgentSkillsProvider` via `IEnumerable<AgentSkillsSource>`.
- Smaller public API surface: no need for aggregate source, caching decorators, or filtering decorator classes — these concerns are handled internally by the provider.
**Cons:**
- The provider takes on multiple responsibilities — aggregation, caching, deduplication, and filtering.
- Less granular caching control: caching is all-or-nothing across sources rather than per-source as with decorators.
- Less extensible: new behaviors (e.g., ordering, TTL expiration) require modifying the provider rather than adding a decorator.
### Builder Pattern
**`AgentSkillsProviderBuilder`** provides a fluent API for composing skills from multiple sources. The builder centralizes configuration — script executors, approval callbacks, prompt templates, and filtering — so consumers don't need to know the underlying source types.
The builder internally decides how to wire up the object graph: it creates the appropriate source instances, applies caching and filtering, and returns a fully configured `AgentSkillsProvider`. This keeps the setup code concise while still allowing fine-grained control when needed.
**Example** — Using the builder to combine multiple source types with configuration:
```csharp
var provider = new AgentSkillsProviderBuilder()
.UseFileSkill("./skills") // file-based source
.UseInlineSkills(codeSkill) // code-defined source
.UseClassSkills(new ClassSkill()) // class-based source
.UseFileScriptRunner(SubprocessScriptRunner.RunAsync) // script runner
.UseScriptApproval() // optional human-in-the-loop
.UsePromptTemplate(customTemplate) // optional prompt customization
.UseFilter(s => s.Frontmatter.Name != "internal") // optional skill filtering
.Build();
AIAgent agent = chatClient.AsAIAgent(new ChatClientAgentOptions
{
AIContextProviders = [provider],
});
```
## Adding a Custom Skill Type
The skills framework is designed for extensibility. While file-based and inline skills cover common
scenarios, you can introduce entirely new skill types by subclassing the four base classes:
| Base class | Purpose |
|-----------------------|-----------------------------------------------------|
| `AgentSkillsSource` | Discovers and loads skills from a particular origin |
| `AgentSkill` | Holds metadata, content, resources, and scripts |
| `AgentSkillResource` | Provides supplementary content to a skill |
| `AgentSkillScript` | Represents an executable action within a skill |
The example below implements a **cloud-based skill type** where skills, resources, and scripts are
all stored in and executed through a remote cloud service (e.g., Azure Blob Storage + Azure Functions).
### Step 1 — Define a custom resource
A `CloudSkillResource` reads resource content from a cloud storage endpoint instead of the local
filesystem:
```csharp
/// <summary>
/// A skill resource backed by a cloud storage endpoint.
/// </summary>
public sealed class CloudSkillResource : AgentSkillResource
{
private readonly HttpClient _httpClient;
public CloudSkillResource(string name, Uri blobUri, HttpClient httpClient, string? description = null)
: base(name, description)
{
BlobUri = blobUri ?? throw new ArgumentNullException(nameof(blobUri));
_httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient));
}
/// <summary>
/// Gets the URI of the cloud blob that holds this resource's content.
/// </summary>
public Uri BlobUri { get; }
/// <inheritdoc/>
public override async Task<object?> ReadAsync(
IServiceProvider? serviceProvider = null,
CancellationToken cancellationToken = default)
{
return await _httpClient.GetStringAsync(BlobUri, cancellationToken).ConfigureAwait(false);
}
}
```
### Step 2 — Define a custom script
A `CloudSkillScript` executes a script by calling a cloud function endpoint, passing arguments as
the request body:
```csharp
/// <summary>
/// A skill script executed via a cloud function endpoint.
/// </summary>
public sealed class CloudSkillScript : AgentSkillScript
{
private readonly HttpClient _httpClient;
public CloudSkillScript(string name, Uri functionUri, HttpClient httpClient, string? description = null)
: base(name, description)
{
FunctionUri = functionUri ?? throw new ArgumentNullException(nameof(functionUri));
_httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient));
}
/// <summary>
/// Gets the URI of the cloud function that runs this script.
/// </summary>
public Uri FunctionUri { get; }
/// <inheritdoc/>
public override async Task<object?> RunAsync(
AgentSkill skill,
AIFunctionArguments arguments,
CancellationToken cancellationToken = default)
{
var json = JsonSerializer.Serialize(arguments);
using var content = new StringContent(json, Encoding.UTF8, "application/json");
var response = await _httpClient.PostAsync(FunctionUri, content, cancellationToken)
.ConfigureAwait(false);
response.EnsureSuccessStatusCode();
return await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false);
}
}
```
### Step 3 — Define a custom skill
A `CloudSkill` bundles cloud-specific metadata (e.g., the base endpoint) with the standard skill
shape:
```csharp
/// <summary>
/// An <see cref="AgentSkill"/> whose content, resources, and scripts are stored in a cloud service.
/// </summary>
public sealed class CloudSkill : AgentSkill
{
public CloudSkill(
AgentSkillFrontmatter frontmatter,
string content,
Uri endpoint,
IReadOnlyList<AgentSkillResource>? resources = null,
IReadOnlyList<AgentSkillScript>? scripts = null)
{
Frontmatter = frontmatter ?? throw new ArgumentNullException(nameof(frontmatter));
Content = content ?? throw new ArgumentNullException(nameof(content));
Endpoint = endpoint ?? throw new ArgumentNullException(nameof(endpoint));
Resources = resources;
Scripts = scripts;
}
/// <inheritdoc/>
public override AgentSkillFrontmatter Frontmatter { get; }
/// <inheritdoc/>
public override string Content { get; }
/// <summary>
/// Gets the base cloud endpoint for this skill.
/// </summary>
public Uri Endpoint { get; }
/// <inheritdoc/>
public override IReadOnlyList<AgentSkillResource>? Resources { get; }
/// <inheritdoc/>
public override IReadOnlyList<AgentSkillScript>? Scripts { get; }
}
```
### Step 4 — Define a custom source
A `CloudSkillsSource` discovers skills from a cloud catalog API and constructs `CloudSkill`
instances with their associated resources and scripts:
```csharp
/// <summary>
/// A skill source that discovers and loads skills from a cloud catalog API.
/// </summary>
public sealed class CloudSkillsSource : AgentSkillsSource
{
private readonly Uri _catalogUri;
private readonly HttpClient _httpClient;
public CloudSkillsSource(Uri catalogUri, HttpClient httpClient)
{
_catalogUri = catalogUri ?? throw new ArgumentNullException(nameof(catalogUri));
_httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient));
}
/// <inheritdoc/>
public override async Task<IList<AgentSkill>> GetSkillsAsync(
CancellationToken cancellationToken = default)
{
// Fetch the skill catalog from the cloud service.
var json = await _httpClient.GetStringAsync(_catalogUri, cancellationToken)
.ConfigureAwait(false);
var catalog = JsonSerializer.Deserialize<CloudSkillCatalog>(json)!;
var skills = new List<AgentSkill>();
foreach (var entry in catalog.Skills)
{
var frontmatter = new AgentSkillFrontmatter(entry.Name, entry.Description);
// Build cloud-backed resources.
var resources = entry.Resources
.Select(r => new CloudSkillResource(r.Name, r.BlobUri, _httpClient, r.Description))
.ToList<AgentSkillResource>();
// Build cloud-backed scripts.
var scripts = entry.Scripts
.Select(s => new CloudSkillScript(s.Name, s.FunctionUri, _httpClient, s.Description))
.ToList<AgentSkillScript>();
skills.Add(new CloudSkill(frontmatter, entry.Content, entry.Endpoint, resources, scripts));
}
return skills;
}
}
```
### Step 5 — Register with the builder
Use `UseSource` to wire the custom source into the provider:
```csharp
var httpClient = new HttpClient();
var provider = new AgentSkillsProviderBuilder()
.UseSource(new CloudSkillsSource(
new Uri("https://my-service.example.com/skills/catalog"),
httpClient))
// Mix with other source types if needed:
.UseFileSkill("/local/skills", scriptRunner)
.UseInlineSkills(someInlineSkill)
.Build();
```
The `AgentSkillsProvider` handles all skill types uniformly — any combination of file-based, inline,
class-based, and custom skills can coexist in the same provider. Custom skills automatically
participate in the model-facing tools (`load_skill`, `read_skill_resource`, `run_skill_script`),
filtering, deduplication, and caching — no additional integration work is required.
## Script Representation: `AgentSkillScript` vs `AIFunction`
Two approaches were considered for representing executable scripts within skills:
### Option A — Custom `AgentSkillScript` abstract base class (original design)
Scripts are modeled as a custom `AgentSkillScript` abstract class with `Name`, `Description`, and
`RunAsync(AgentSkill, AIFunctionArguments, CancellationToken)`. Concrete implementations:
`AgentInlineSkillScript` (wraps a delegate/`AIFunction`) and `AgentFileSkillScript` (wraps a file path + executor delegate).
```csharp
// Base type
public abstract class AgentSkillScript
{
public string Name { get; }
public string? Description { get; }
public abstract Task<object?> RunAsync(AgentSkill skill, AIFunctionArguments arguments, CancellationToken cancellationToken = default);
}
// AgentSkill exposes scripts as:
public abstract IReadOnlyList<AgentSkillScript>? Scripts { get; }
// Inline script wraps an AIFunction internally
var script = new AgentInlineSkillScript(ConvertUnits, "convert");
// Pre-built AIFunction must be wrapped
var script = new AgentInlineSkillScript(myAIFunction);
// Class-based skill declares scripts as:
public override IReadOnlyList<AgentSkillScript>? Scripts { get; } =
[
new AgentInlineSkillScript(ConvertUnits, "convert"),
];
// Provider executes scripts by passing the owning skill:
await script.RunAsync(skill, arguments, cancellationToken);
```
**Pros:**
- **Explicit skill context at execution time.** `RunAsync` receives the owning `AgentSkill`, so any script can access skill metadata or resources during execution without requiring construction-time wiring.
- **Self-contained abstraction.** A dedicated type communicates clearly that scripts are a skills-framework concept, separate from general-purpose AI functions.
- **Easier extensibility for custom script types.** Third-party implementations can subclass `AgentSkillScript` and access the owning skill in `RunAsync` without special setup.
**Cons:**
- **Wrapper overhead.** `AgentInlineSkillScript` is a thin pass-through around `AIFunction` — it adds a class, a constructor, and an indirection layer for no behavioral difference.
- **Parallel abstraction.** `AgentSkillScript` and `AIFunction` serve overlapping purposes (named callable with arguments), creating two parallel hierarchies for the same concept.
- **Friction for consumers.** Users who already have `AIFunction` instances must wrap them in `AgentInlineSkillScript` to use them as scripts, adding ceremony.
### Option B — Reuse `AIFunction` directly
Scripts are represented as `AIFunction` (from `Microsoft.Extensions.AI`). `AgentSkill.Scripts` returns
`IReadOnlyList<AIFunction>?`. `AgentInlineSkillScript` is eliminated entirely — callers use
`AIFunctionFactory.Create(delegate, name: ...)` or pass `AIFunction` instances directly.
`AgentFileSkillScript` becomes an `AIFunction` subclass that captures its owning `AgentFileSkill` via
an internal back-reference set during construction.
```csharp
// AgentSkill exposes scripts as AIFunction directly:
public abstract IReadOnlyList<AIFunction>? Scripts { get; }
// Inline scripts use AIFunctionFactory — no wrapper class needed
var skill = new AgentInlineSkill("my-skill", "desc", "instructions");
skill.AddScript(ConvertUnits, "convert"); // delegate
skill.AddScript(myAIFunction); // pre-built AIFunction — no wrapping
// Class-based skill declares scripts as:
public override IReadOnlyList<AIFunction>? Scripts { get; } =
[
AIFunctionFactory.Create(ConvertUnits, name: "convert"),
];
// Provider executes scripts via standard AIFunction invocation:
await script.InvokeAsync(arguments, cancellationToken);
// File-based scripts extend AIFunction and capture the owning skill internally:
public sealed class AgentFileSkillScript : AIFunction
{
internal AgentFileSkill? Skill { get; set; } // set by AgentFileSkill constructor
protected override async ValueTask<object?> InvokeCoreAsync(
AIFunctionArguments arguments, CancellationToken cancellationToken)
{
return await _executor(Skill!, this, arguments, cancellationToken);
}
}
```
**Pros:**
- **Fewer types.** Eliminates `AgentSkillScript` and `AgentInlineSkillScript`, reducing the public API surface by two classes.
- **Seamless interop.** Any `AIFunction` — whether from `AIFunctionFactory`, a custom subclass, or an external library — can be used as a skill script with zero wrapping.
- **Consistent with `Microsoft.Extensions.AI` ecosystem.** Scripts share the same type as tool functions used by `IChatClient` and `FunctionInvokingChatClient`, reducing conceptual overhead for developers already familiar with the ecosystem.
**Cons:**
- **No owning-skill context in invocation signature.** `AIFunction.InvokeAsync` does not accept an `AgentSkill` parameter, so `AgentFileSkillScript` must capture its owning skill via an internal setter during construction. This adds a construction-order dependency: the skill must set the back-reference on its scripts.
- **Custom script types lose automatic skill access.** Third-party `AIFunction` subclasses that need the owning skill must implement their own mechanism (e.g., constructor injection, closure capture) instead of receiving it as a method parameter.
- **Semantic overloading.** `AIFunction` now means both "a tool the model can call" and "a script within a skill", which could blur the distinction for framework users.
## Resource Representation: `AgentSkillResource` vs `AIFunction`
Two approaches were considered for representing skill resources (supplementary content such as references, assets, or dynamic data):
### Option A — Custom `AgentSkillResource` abstract base class (original design)
Resources are modeled as a custom `AgentSkillResource` abstract class with `Name`, `Description`, and
`ReadAsync(IServiceProvider?, CancellationToken)`. Concrete implementations:
`AgentInlineSkillResource` (static value, delegate, or `AIFunction` wrapper) and `AgentFileSkillResource` (reads file content from disk).
```csharp
// Base type
public abstract class AgentSkillResource
{
public string Name { get; }
public string? Description { get; }
public abstract Task<object?> ReadAsync(IServiceProvider? serviceProvider = null, CancellationToken cancellationToken = default);
}
// AgentSkill exposes resources as:
public abstract IReadOnlyList<AgentSkillResource>? Resources { get; }
// Static resource
var resource = new AgentInlineSkillResource("static content", "my-resource");
// Dynamic resource (delegate)
var resource = new AgentInlineSkillResource((IServiceProvider sp) => GetData(sp), "my-resource");
// Pre-built AIFunction must be wrapped
var resource = new AgentInlineSkillResource(myAIFunction);
// Class-based skill declares resources as:
public override IReadOnlyList<AgentSkillResource>? Resources { get; } =
[
new AgentInlineSkillResource("# Conversion Tables\n...", "conversion-table"),
];
// Provider reads resources via:
await resource.ReadAsync(serviceProvider, cancellationToken);
```
**Pros:**
- **Clear semantic distinction.** A dedicated `AgentSkillResource` type distinguishes resources (data providers) from scripts (executable actions), making the API self-documenting.
- **Purpose-built API.** `ReadAsync` communicates intent better than `InvokeAsync` for a data-access operation.
**Cons:**
- **Wrapper overhead.** `AgentInlineSkillResource` wraps `AIFunction` internally for delegate/function cases — adding a class and indirection for no behavioral difference.
- **Parallel abstraction.** `AgentSkillResource` and `AIFunction` serve overlapping purposes (named callable that returns data), creating two parallel hierarchies.
- **Friction for consumers.** Users who already have `AIFunction` instances must wrap them in `AgentInlineSkillResource`, adding ceremony.
### Option B — Reuse `AIFunction` directly
Resources are represented as `AIFunction`. `AgentSkill.Resources` returns `IReadOnlyList<AIFunction>?`.
`AgentInlineSkillResource` becomes an `AIFunction` subclass (retained as a convenience for the static-value
pattern: `new AgentInlineSkillResource("data", "name")`). `AgentFileSkillResource` becomes an `AIFunction`
subclass that reads file content.
```csharp
// AgentSkill exposes resources as AIFunction directly:
public abstract IReadOnlyList<AIFunction>? Resources { get; }
// Static resource — AgentInlineSkillResource is retained as a convenience AIFunction subclass
var resource = new AgentInlineSkillResource("static content", "my-resource");
// Dynamic resource — AgentInlineSkillResource wraps delegate as AIFunction
var resource = new AgentInlineSkillResource((IServiceProvider sp) => GetData(sp), "my-resource");
// Pre-built AIFunction can be used directly — no wrapping needed
skill.AddResource(myAIFunction);
// Class-based skill declares resources as:
public override IReadOnlyList<AIFunction>? Resources { get; } =
[
new AgentInlineSkillResource("# Conversion Tables\n...", "conversion-table"),
];
// Provider reads resources via standard AIFunction invocation:
await resource.InvokeAsync(arguments, cancellationToken);
// File-based resources extend AIFunction directly:
internal sealed class AgentFileSkillResource : AIFunction
{
public string FullPath { get; }
protected override async ValueTask<object?> InvokeCoreAsync(
AIFunctionArguments arguments, CancellationToken cancellationToken)
{
return await File.ReadAllTextAsync(FullPath, Encoding.UTF8, cancellationToken);
}
}
```
**Pros:**
- **Fewer base types.** Eliminates the `AgentSkillResource` abstract class, reducing the public API surface.
- **Seamless interop.** Any `AIFunction` can be used as a skill resource with zero wrapping.
**Cons:**
- **Loss of semantic distinction.** Resources and scripts are now both `AIFunction`, which could make it less obvious which list a function belongs to when reading code.
- **Static values require a wrapper.** Unlike the original `ReadAsync` which could return a stored value directly, `AIFunction.InvokeAsync` implies invocation. `AgentInlineSkillResource` is retained as a convenience subclass to handle the static-value case, so this is not eliminated — just moved to a different class.
## Decision Outcome
### 1. Keep `AgentSkillResource` and `AgentSkillScript` (Option A for both sections)
We are staying with the custom `AgentSkillResource` and `AgentSkillScript` model classes instead of reusing `AIFunction`:
- **Resources have no parameters.** If a consumer provides an `AIFunction` with parameters, those parameters will never be advertised to the LLM, and the resulting call will fail.
- **Approval breaks for `AIFunction`-based representations.** When a resource or script represented by an `AIFunction` is configured with approval, the second approval invocation will not work correctly.
- **Injecting the owning skill into an `AIFunction`-based script is problematic.** Constructor injection would introduce a circular reference between the skill and the script. An internal property setter is possible but adds coupling.
### 2. Make all agent skill classes internal
All agent-skill-related classes are made `internal` to minimize the public API surface while the feature matures. We can reconsider and promote types to `public` later based on community signal.
This leaves two public entry points:
- **`AgentSkillsProvider`** — use directly when all skills come from a single source and filtering is not needed.
- **`AgentSkillsProviderBuilder`** — use when mixing skill types or when filtering support is required.
### 3. Caching at provider level
Caching of tools and instructions is implemented inside `AgentSkillsProvider` rather than as an external decorator. Recreating tools and instructions on every provider call is wasteful, and a caching decorator sitting outside the provider would not have the information needed to cache them effectively.
@@ -1,72 +0,0 @@
---
status: accepted
contact: eavanvalkenburg
date: 2026-03-20
deciders: eavanvalkenburg, sphenry, chetantoshnival
consulted: taochenosu, moonbox3, dmytrostruk, giles17, alliscode
---
# Provider-Leading Client Design & OpenAI Package Extraction
## Context and Problem Statement
The `agent-framework-core` package currently bundles OpenAI and Azure OpenAI client implementations along with their dependencies (`openai`, `azure-identity`, `azure-ai-projects`, `packaging`). This makes core heavier than necessary for users who don't use OpenAI, and it conflates the core abstractions with a specific provider implementation. Additionally, the current class naming (`OpenAIResponsesClient`, `OpenAIChatClient`) is based on the underlying OpenAI API names rather than what users actually want to do, making discoverability harder for newcomers.
## Decision Drivers
- **Lightweight core**: Core should only contain abstractions, middleware infrastructure, and telemetry — no provider-specific code or dependencies.
- **Discoverability-first**: Import namespaces should guide users to the right client. `from agent_framework.openai import ...` should surface all OpenAI-related clients; `from agent_framework.azure import ...` should surface Foundry, Azure AI, and other Azure-specific classes.
- **Provider-leading naming**: The primary client name should reflect the provider, not the underlying API. The Responses API is now the recommended default for OpenAI, so its client should be called `OpenAIChatClient` (not `OpenAIResponsesClient`).
- **Clean separation of concerns**: Azure-specific deprecated wrappers belong in the azure-ai package, not in the OpenAI package.
## Considered Options
- **Keep OpenAI in core**: Simpler but keeps core heavy; doesn't help discoverability.
- **Extract OpenAI with Azure wrappers in the OpenAI package**: Keeps Azure OpenAI wrappers alongside OpenAI code, but pollutes the OpenAI package with Azure concerns.
- **Extract OpenAI, place Azure wrappers in azure-ai**: Clean separation; the OpenAI package has zero Azure dependencies; deprecated Azure wrappers live in a single file in azure-ai for easy future deletion.
## Decision Outcome
Chosen option: "Extract OpenAI, place Azure wrappers in azure-ai", because it achieves the lightest core, cleanest OpenAI package, and the most maintainable deprecation path.
Key changes:
1. **New `agent-framework-openai` package** with dependencies on `agent-framework-core`, `openai`, and `packaging` only.
2. **Class renames**: `OpenAIResponsesClient``OpenAIChatClient` (Responses API), `OpenAIChatClient``OpenAIChatCompletionClient` (Chat Completions API). Old names remain as deprecated aliases.
3. **Deprecated classes**: `OpenAIAssistantsClient`, all `AzureOpenAI*Client` classes, `AzureAIClient`, `AzureAIAgentClient`, and `AzureAIProjectAgentProvider` are marked deprecated.
4. **New `FoundryChatClient`** in azure-ai for Azure AI Foundry Responses API access, built on `RawFoundryChatClient(RawOpenAIChatClient)`.
5. **All deprecated `AzureOpenAI*` classes** consolidated into a single file (`_deprecated_azure_openai.py`) in the azure-ai package for clean future deletion.
6. **Core's `agent_framework.openai` and `agent_framework.azure` namespaces** become lazy-loading gateways, preserving backward-compatible import paths while removing hard dependencies.
7. **Unified `model` parameter** replaces `model_id` (OpenAI), `deployment_name` (Azure OpenAI), and `model_deployment_name` (Azure AI) across all client constructors. The term `model` is intentionally generic: it naturally maps to an OpenAI model name *and* to an Azure OpenAI deployment name, making it straightforward to use `OpenAIChatClient` with either OpenAI or Azure OpenAI backends (via `AsyncAzureOpenAI`). Environment variables are similarly unified (e.g., `OPENAI_MODEL` instead of separate `OPENAI_CHAT_MODEL_ID` / `OPENAI_CHAT_COMPLETION_MODEL_ID`).
8. **`FoundryAgent`** replaces the pattern of `Agent(client=AzureAIClient(...))` for connecting to pre-configured agents in Azure AI Foundry (PromptAgents and HostedAgents). The underlying `RawFoundryAgentChatClient` is an implementation detail — most users interact only with `FoundryAgent`. `AzureAIAgentClient` is separately deprecated as it refers to the V1 Agents Service API. See below for design rationale.
### Foundry Agent Design: `FoundryAgentClient` vs `FoundryAgent`
The existing `AzureAIClient` combines two concerns: CRUD lifecycle management (creating/deleting agents on the service) and runtime communication (sending messages via the Responses API). The new design removes CRUD entirely — users connect to agents that already exist in Foundry.
**Two approaches were considered:**
**Option A — `FoundryAgentClient` only (public ChatClient):**
Users compose `Agent(client=FoundryAgentClient(...), tools=[...])`. This follows the universal `Agent(client=X)` pattern used by every other provider. However, a "client" that wraps a named remote agent (with `agent_name` as a constructor param) is semantically odd — clients typically wrap a model endpoint, not a specific agent.
**Option B — `FoundryAgent` (Agent subclass) + private `_FoundryAgentChatClient` and public `RawFoundryAgentChatClient`:**
Users write `FoundryAgent(agent_name="my-agent", ...)` for the common case. Internally, `FoundryAgent` creates a `_FoundryAgentChatClient` and passes it to the standard `Agent` base class. For advanced customization, users pass `client_type=RawFoundryAgentChatClient` (or a custom subclass) to control the client middleware layers. The `Agent(client=RawFoundryAgentChatClient(...))` composition pattern still works for users who prefer it.
**Chosen option: Option B**, because:
- The common case (`FoundryAgent(...)`) is a single object with no boilerplate.
- `client_type=` gives full control over client middleware without parameter duplication — the agent forwards connection params to the client internally.
- `RawFoundryAgent(RawAgent)` and `FoundryAgent(Agent)` mirror the established `RawAgent`/`Agent` pattern.
- Runtime validation (only `FunctionTool` allowed) lives in `RawFoundryAgentChatClient._prepare_options`, ensuring it applies regardless of how the client is used — through `FoundryAgent`, `Agent(client=...)`, or any custom composition.
**Public classes:**
- `RawFoundryAgentChatClient(RawOpenAIChatClient)` — Responses API client that injects agent reference and validates tools. Extension point for custom client middleware.
- `RawFoundryAgent(RawAgent)` — Agent without agent-level middleware/telemetry.
- `FoundryAgent(AgentTelemetryLayer, AgentMiddlewareLayer, RawFoundryAgent)` — Recommended production agent.
**Internal (private):**
- `_FoundryAgentChatClient` — Full client with function invocation, chat middleware, and telemetry layers. Created automatically by `FoundryAgent`; users customize via `client_type=RawFoundryAgentChatClient` or a custom subclass.
**Deprecated:**
- `AzureAIClient` — replaced by `FoundryAgent` (which uses `FoundryAgentClient` internally).
- `AzureAIAgentClient` — refers to V1 Agents Service API, no direct replacement.
- `AzureAIProjectAgentProvider` — replaced by `FoundryAgent`.
@@ -1,121 +0,0 @@
---
status: accepted
contact: westey-m
date: 2026-03-23
deciders: sergeymenshykh, markwallace, rbarreto, dmytrostruk, westey-m, eavanvalkenburg, stephentoub
consulted:
informed:
---
# Chat History Persistence Consistency
## Context and Problem Statement
When using `ChatClientAgent` with tools, the `FunctionInvokingChatClient` (FIC) loops multiple times — service call → tool execution → service call → … — before producing a final response. There are two points of discrepancy between how chat history is stored by the framework's `ChatHistoryProvider` and how the underlying AI service stores chat history (e.g., OpenAI Responses with `store=true`):
1. **Persistence timing**: The AI service persists messages after *each* service call within the FIC loop. The `ChatHistoryProvider` currently persists messages only once, at the *end* of the full agent run (after all FIC loop iterations complete).
2. **Trailing `FunctionResultContent` storage**: When tool calling is terminated mid-loop (e.g., via `FunctionInvokingChatClient` termination filters), the final response from the agent may contain `FunctionResultContent` that was never sent to a subsequent service call. The AI service never stores this trailing `FunctionResultContent`, but the `ChatHistoryProvider` currently stores all response content, including the trailing `FunctionResultContent`.
These discrepancies mean that a `ChatHistoryProvider`-managed conversation and a service-managed conversation can diverge in content and structure, even when processing the same interactions.
### Practical Impact: Resuming After Tool-Call Termination
Today, users of `AIAgent` get different behaviors depending on whether chat history is stored service-side or in a `ChatHistoryProvider`. This creates concrete challenges — for example, when the function call loop is terminated and the user wants to resume the conversation in a subsequent run. With service-stored history, the trailing `FunctionResultContent` is never persisted, so the last stored message is the `FunctionCallContent` from the service. With `ChatHistoryProvider`-stored history, the trailing `FunctionResultContent` *is* persisted. The user cannot know whether the last `FunctionResultContent` is in the chat history or not without inspecting the storage mechanism, making it difficult to write resumption logic that works correctly regardless of the storage backend.
### Relationship Between the Two Discrepancies
The persistence timing and `FunctionResultContent` trimming behaviors are interrelated:
- **Per-service-call persistence**: When messages are persisted after each individual service call, trailing `FunctionResultContent` trimming is unnecessary. If tool calling is terminated, the `FunctionResultContent` from the terminated call was never sent to a subsequent service call, so it is never persisted. The per-service-call approach naturally matches the service's behavior.
- **Per-run persistence**: When messages are batched and persisted at the end of the full run, trailing `FunctionResultContent` trimming becomes necessary to match the service's behavior. Without trimming, the stored history contains `FunctionResultContent` that the service would never have stored.
## Decision Drivers
- **A. Consistency**: The default behavior of `ChatHistoryProvider` should produce stored history that closely matches what the underlying AI service would store, minimizing surprise when switching between framework-managed and service-managed chat history.
- **B. Atomicity**: A run that fails mid-way through a multi-step tool-calling loop should not leave chat history in a partially-updated state, unless the user explicitly opts into that behavior.
- **C. Recoverability**: For long-running tool-calling loops, it should be possible to recover intermediate progress if the process is interrupted, rather than losing all work from the current run.
- **D. Simplicity**: The default behavior should be easy to understand and predict for most users, without requiring knowledge of the FIC loop internals.
- **E. Flexibility**: Regardless of the chosen default, users should be able to opt into the alternative behavior.
## Considered Options
- Option 1: Per-run persistence with opt-in FRC (FunctionResultContent) trimming
- Option 2: Opt-in per-service-call persistence (via `RequirePerServiceCallChatHistoryPersistence`)
## Pros and Cons of the Options
### Option 1: Per-run persistence with opt-in FRC trimming
Keep the current default behavior of persisting chat history only at the end of the full agent run. Add `FunctionResultContent` trimming as an opt-in behavior to improve consistency with service storage.
- Good, because runs are atomic — chat history is only updated when the full run succeeds, satisfying driver B.
- Good, because the mental model is simple: one run = one history update, satisfying driver D.
- Good, because trimming trailing `FunctionResultContent` improves consistency with service storage, partially satisfying driver A.
- Bad, because the default persistence timing still differs from the service's behavior (per-run vs. per-service-call), only partially satisfying driver A.
- Bad, because if the process crashes mid-loop, all intermediate progress from the current run is lost, not satisfying driver C.
- Bad, because this option alone does not provide a way for users to opt into per-service-call persistence, not satisfying driver E.
### Option 2: Opt-in per-service-call persistence (via `RequirePerServiceCallChatHistoryPersistence`)
Introduce an optional RequirePerServiceCallChatHistoryPersistence setting to persist chat history after each individual service call within the FIC loop, matching the AI service's behavior. Trailing `FunctionResultContent` trimming is unnecessary with this approach (it is naturally handled).
Settings:
- `RequirePerServiceCallChatHistoryPersistence` = `true`
- Good, because the stored history matches the service's behavior when opting in for both timing and content, fully satisfying driver A.
- Good, because intermediate progress is preserved if the process is interrupted, satisfying driver C.
- Good, because no separate `FunctionResultContent` trimming logic is needed, reducing complexity.
- Bad, because chat history may be left in an incomplete state if the run fails mid-loop (e.g., `FunctionCallContent` stored without corresponding `FunctionResultContent`), not satisfying driver B. A subsequent run cannot proceed without manually providing the missing `FunctionResultContent`.
- Bad, because the mental model is more complex: a single run may produce multiple history updates, partially failing driver D.
- Neutral, because users can opt out to per-run persistence if they prefer atomicity, satisfying driver E.
## Decision Outcome
Chosen option: **Option 2: Opt-in per-service-call persistence (via `RequirePerServiceCallChatHistoryPersistence`)**. The existing per-run persistence behavior is retained as-is, requiring no changes from users. Per-service-call persistence is available as an opt-in feature via the `RequirePerServiceCallChatHistoryPersistence` setting. This satisfies drivers B (atomicity) and D (simplicity) for the common case, while fully satisfying driver A (consistency) for users who opt into simulated service-stored behavior. Users who need per-service-call persistence for recoverability (driver C) can enable it explicitly.
### Configuration Matrix
The behavior depends on the combination of `UseProvidedChatClientAsIs` and `RequirePerServiceCallChatHistoryPersistence`:
| `UseProvidedChatClientAsIs` | `RequirePerServiceCallChatHistoryPersistence` | Behavior |
|---|---|---|
| `false` (default) | `false` (default) | **Per-run persistence.** Messages are persisted at the end of the full agent run via the `ChatHistoryProvider`. |
| `false` | `true` | **Per-service-call persistence (simulated).** A `PerServiceCallChatHistoryPersistingChatClient` middleware is automatically injected into the chat client pipeline between `FunctionInvokingChatClient` and the leaf `IChatClient`. Messages are persisted after each service call. A sentinel `ConversationId` causes FIC to treat the conversation as service-managed. |
| `true` | `false` | **Per-run persistence.** No middleware is injected because the user has provided a custom chat client stack. Messages are persisted at the end of the run. |
| `true` | `true` | **User responsibility.** The system checks whether the custom chat client stack includes a `PerServiceCallChatHistoryPersistingChatClient`. If not, a warning is emitted — the user is expected to have added their own per-service-call persistence mechanism. End-of-run persistence is skipped. |
### Consequences
- Good, because per-run persistence is atomic by default — chat history is only updated when the full run succeeds, satisfying driver B.
- Good, because the default mental model is simple: one run = one history update, satisfying driver D.
- Good, because users who opt into `RequirePerServiceCallChatHistoryPersistence` get stored history that matches the service's behavior for both timing and content, fully satisfying driver A.
- Good, because per-service-call persistence preserves intermediate progress if the process is interrupted, satisfying driver C when opted in.
- Good, because no separate `FunctionResultContent` trimming logic is needed when per-service-call persistence is active — it is naturally handled.
- Good, because conflict detection (configurable via `ThrowOnChatHistoryProviderConflict`, `WarnOnChatHistoryProviderConflict`, `ClearOnChatHistoryProviderConflict`) prevents misconfiguration when a service returns a `ConversationId` alongside a configured `ChatHistoryProvider`.
- Bad, because per-service-call persistence (when opted in) may leave chat history in an incomplete state if the run fails mid-loop (e.g., `FunctionCallContent` stored without corresponding `FunctionResultContent`), requiring manual recovery in rare cases.
- Neutral, because users who want per-service-call consistency can opt in via `RequirePerServiceCallChatHistoryPersistence = true`, satisfying driver E.
- Neutral, because increased write frequency from per-service-call persistence may impact performance for some storage backends; this can be mitigated with a caching decorator.
### Implementation Notes
#### Conversation ID Consistency
When `RequirePerServiceCallChatHistoryPersistence` is enabled, the `PerServiceCallChatHistoryPersistingChatClient`
decorator also updates `session.ConversationId` after each service call. This handles two scenarios:
1. **Framework-managed chat history** — the decorator sets a sentinel `ConversationId` on the response
so that `FunctionInvokingChatClient` treats the conversation as service-managed (clearing accumulated
history between iterations and not injecting duplicate `FunctionCallContent` during approval processing).
2. **Service-stored chat history** — when the service returns a real `ConversationId`, the decorator
updates `session.ConversationId` immediately after each service call, rather than deferring the update
to the end of the run. This ensures intermediate ConversationId changes are captured even if the
process is interrupted mid-loop.
For some service-stored scenarios (e.g., the Conversations API with the Responses API), there is only
one thread with one ID, so every service call returns the same ConversationId and this per-call update
makes no practical difference. Enabling `RequirePerServiceCallChatHistoryPersistence` ensures consistent
per-service-call behavior across all service types regardless of how they manage ConversationIds.
@@ -1,815 +0,0 @@
---
status: accepted
contact: bentho
date: 2026-02-27
deciders: bentho, markwallace-microsoft, westey-m
consulted: Pratyush Mishra, Shivam Shrivastava, Manni Arora (Centrica eval scenario)
informed: Agent Framework team, Foundry Evals team
---
# Agent Evaluation Architecture with Azure AI Foundry Integration
## Context and Problem Statement
Azure AI Foundry provides a rich evaluation service for AI agents — built-in evaluators for agent behavior (task adherence, intent resolution), tool usage (tool call accuracy, tool selection), quality (coherence, fluency, relevance), and safety (violence, self-harm, prohibited actions). Results are viewable in the Foundry portal with dashboards and comparison views.
However, using Foundry Evals with an agent-framework agent today requires significant manual effort. Developers must:
1. Transform agent-framework's `Message`/`Content` types into the OpenAI-style agent message schema that Foundry evaluators expect
2. Map tool definitions from agent-framework's `FunctionTool` format to evaluator-compatible schemas
3. Manually wire up the correct Foundry data source type (`azure_ai_traces`, `jsonl`, `azure_ai_target_completions`, etc.) depending on their scenario
4. Handle App Insights trace ID queries, response ID collection, and eval polling
Additionally, evaluation is a concern that extends beyond any single provider. Developers may want to use local evaluators (LLM-as-judge, regex, keyword matching), third-party evaluation libraries, or multiple providers in combination. The architecture must support this without creating a Foundry-specific lock-in at the API level.
### Functional Requirements for Agent Evaluation
- **Single agents and workflows.** Evaluate both individual agent responses and multi-agent workflow results, with per-agent breakdown to pinpoint underperformance.
- **One-shot and multi-turn conversations.** Capture full conversation trajectories — including tool calls and results — not just final query/response pairs.
- **Conversation factoring.** Support splitting conversations into query/response in multiple ways (last turn, full trajectory, per-turn) because different factorings measure different things.
- **Multiple providers, mix and match.** Run Foundry LLM-as-judge evaluators alongside fast local checks and custom evaluators on the same data, without restructuring code.
- **Third-party extensibility.** Any evaluation library can participate by implementing the `Evaluator` protocol (Python) or `IAgentEvaluator` interface (.NET). No predetermined list of supported libraries — the protocol is intentionally simple (`evaluate(items) → results`) so that wrappers for libraries like DeepEval, RAGAS, or Promptfoo are straightforward to write.
- **Bring your own evaluator.** Creating a custom evaluator should be as simple as writing a function.
- **Evaluate without re-running.** Evaluate existing responses from logs or previous runs without invoking the agent again.
## Decision Drivers
- **Zero-friction evaluation**: Developers should go from "I have an agent" to "I have eval results" with minimal code.
- **Provider-agnostic API**: Core evaluation capabilities must not be tied to any specific provider. Provider configuration should be separate from the evaluation call.
- **Lowest concept count**: Introduce the fewest possible new types, abstractions, and APIs for developers to learn.
- **Leverage existing knowledge**: The framework already knows which agents exist, what tools they have, and what conversations occurred. Evals should use this automatically rather than requiring the developer to re-specify it.
- **Foundry-native results**: When using Foundry, results should be viewable in the Foundry portal with dashboards and comparison views.
- **Progressive disclosure**: Simple scenarios should be near-zero code. Advanced scenarios should build on the same primitives.
- **Cross-language parity**: Design must be implementable in both Python and .NET.
## Considered Options
1. **Provider-specific functions** — Build Foundry-specific helper functions (`evaluate_agent()`, etc.) directly in the Azure package. All eval functions take Foundry connection parameters.
2. **Evaluator protocol with shared orchestration** — Define a provider-agnostic `Evaluator` protocol in the base agent library (`agent_framework` in Python, `Microsoft.Agents.AI` in .NET). Orchestration functions live alongside it. Providers implement the protocol.
3. **Full eval framework** — Build comprehensive eval infrastructure including custom evaluator definitions, scoring profiles, and reporting inside agent-framework.
## Decision Outcome
Proposed option: "Evaluator protocol with shared orchestration", because it delivers the low-friction developer experience, supports multiple providers without API changes, and keeps the concept count low.
### Usage Examples
#### Evaluate an agent
The agent is invoked once per query by default. For statistically meaningful evaluation, provide multiple diverse queries. For measuring **consistency** (does the same query produce reliable results?), use `num_repetitions` to run each query N times independently:
**Python:**
```python
evals = FoundryEvals(
project_client=client,
model_deployment="gpt-4o",
evaluators=[FoundryEvals.RELEVANCE, FoundryEvals.COHERENCE],
)
results = await evaluate_agent(
agent=my_agent,
queries=[
"What's the weather in Seattle?",
"Plan a weekend trip to Portland",
"What restaurants are near Pike Place?",
],
evaluators=evals,
)
for r in results:
r.assert_passed()
```
**C#:**
```csharp
var evals = new FoundryEvals(chatConfiguration, FoundryEvals.Relevance, FoundryEvals.Coherence);
AgentEvaluationResults results = await agent.EvaluateAsync(
new[] {
"What's the weather in Seattle?",
"Plan a weekend trip to Portland",
"What restaurants are near Pike Place?",
},
evals);
results.AssertAllPassed();
```
`evaluate_agent` returns one `EvalResults` per evaluator. Each result contains per-item scores with the evaluated response for auditing:
```
# results[0] (FoundryEvals)
EvalResults(status="completed", passed=3, failed=0, total=3)
items[0]: EvalItemResult(
query="What's the weather in Seattle?",
response="It's currently 72°F and sunny in Seattle.",
scores={"relevance": 5, "coherence": 5})
items[1]: EvalItemResult(
query="Plan a weekend trip to Portland",
response="Here's a 2-day Portland itinerary...",
scores={"relevance": 4, "coherence": 5})
items[2]: EvalItemResult(
query="What restaurants are near Pike Place?",
response="Top restaurants near Pike Place Market: ...",
scores={"relevance": 5, "coherence": 4})
```
#### Measure consistency with repetitions
Run each query multiple times to detect non-deterministic behavior:
**Python:**
```python
results = await evaluate_agent(
agent=my_agent,
queries=["What's the weather in Seattle?"],
evaluators=evals,
num_repetitions=3, # each query runs 3 times independently
)
# results contain 3 items (1 query × 3 repetitions)
```
**C#:**
```csharp
AgentEvaluationResults results = await agent.EvaluateAsync(
new[] { "What's the weather in Seattle?" },
evals,
numRepetitions: 3); // each query runs 3 times independently
// results contain 3 items (1 query × 3 repetitions)
```
#### Evaluate a response you already have
When you already have agent responses, pass them directly to skip re-running the agent. Each query is paired with its corresponding response:
**Python:**
```python
queries = ["What's the weather?", "What's the capital of France?"]
responses = [await agent.run([Message("user", [q])]) for q in queries]
results = await evaluate_agent(
responses=responses,
evaluators=evals,
)
```
**C#:**
```csharp
var queries = new[] { "What's the weather?" };
var responses = new List<AgentResponse>();
foreach (var q in queries)
responses.Add(await agent.RunAsync(new[] { new ChatMessage(ChatRole.User, q) }));
AgentEvaluationResults results = await agent.EvaluateAsync(
responses: responses,
evals);
```
Each `AgentResponse` already contains the conversation (query + response), so the evaluator extracts query/response from the conversation. When you pass `responses` without `queries`, the conversation is the source of truth.
#### Evaluate with conversation split strategies
By default, evaluators see only the last turn (final user message → final assistant response). For multi-turn conversations, you can control how the conversation is factored for evaluation:
**Python:**
```python
results = await evaluate_agent(
agent=agent,
queries=["Plan a 3-day trip to Paris"],
evaluators=evals,
conversation_split=ConversationSplit.FULL, # evaluate entire trajectory
)
# Or per-turn: each user→assistant exchange scored independently
results = await evaluate_agent(
agent=agent,
queries=["Plan a 3-day trip to Paris"],
evaluators=evals,
conversation_split=ConversationSplit.PER_TURN,
)
```
**C#:**
```csharp
// Full conversation as context
AgentEvaluationResults results = await agent.EvaluateAsync(
new[] { "Plan a 3-day trip to Paris" },
evals,
splitter: ConversationSplitters.Full);
// Per-turn splitting
var items = EvalItem.PerTurnItems(conversation); // one EvalItem per user turn
var results = await evals.EvaluateAsync(items);
```
With `PER_TURN`, a 3-turn conversation produces 3 scored items:
```
EvalResults(status="completed", passed=3, failed=0, total=3)
items[0]: query="Plan a 3-day trip to Paris" scores={"relevance": 5}
items[1]: query="What about restaurants?" scores={"relevance": 4}
items[2]: query="Make it budget-friendly" scores={"relevance": 5}
```
#### Evaluate a multi-agent workflow
**Python:**
```python
result = await workflow.run("Plan a trip to Paris")
eval_results = await evaluate_workflow(
workflow=workflow,
workflow_result=result,
evaluators=evals,
)
for r in eval_results:
print(f" overall: {r.passed}/{r.total}")
for name, sub in r.sub_results.items():
print(f" {name}: {sub.passed}/{sub.total}")
```
**C#:**
```csharp
WorkflowRunResult result = await workflow.RunAsync("Plan a trip to Paris");
IReadOnlyList<AgentEvaluationResults> evalResults = await result.EvaluateAsync(evals);
foreach (var r in evalResults)
{
Console.WriteLine($" overall: {r.Passed}/{r.Total}");
foreach (var (name, sub) in r.SubResults)
Console.WriteLine($" {name}: {sub.Passed}/{sub.Total}");
}
```
Workflows return one result per evaluator, with sub-results per agent in the workflow:
```
EvalResults(status="completed", passed=2, failed=0, total=2)
sub_results:
"planner": EvalResults(passed=1, total=1)
"researcher": EvalResults(passed=1, total=1)
```
#### Mix multiple providers
**Python:**
```python
@evaluator
def is_helpful(response: str) -> bool:
return len(response.split()) > 10
foundry = FoundryEvals(
project_client=client,
model_deployment="gpt-4o",
evaluators=[FoundryEvals.RELEVANCE, FoundryEvals.COHERENCE],
)
results = await evaluate_agent(
agent=agent,
queries=queries,
evaluators=[is_helpful, keyword_check("weather"), foundry],
)
```
**C#:**
```csharp
IReadOnlyList<AgentEvaluationResults> results = await agent.EvaluateAsync(
queries,
evaluators: new IAgentEvaluator[]
{
new LocalEvaluator(
EvalChecks.KeywordCheck("weather"),
FunctionEvaluator.Create("is_helpful", (string r) => r.Split(' ').Length > 10)),
new FoundryEvals(chatConfiguration, FoundryEvals.Relevance, FoundryEvals.Coherence),
});
```
Multiple evaluators return one result each — `results[0]` is the local evaluator, `results[1]` is Foundry.
#### Custom function evaluators
**Python:**
```python
@evaluator
def mentions_city(response: str, expected_output: str) -> bool:
return expected_output.lower() in response.lower()
@evaluator
def used_tools(conversation: list, tools: list) -> float:
# ... scoring logic
return score
local = LocalEvaluator(mentions_city, used_tools)
```
`@evaluator` uses **parameter name injection** — the function's parameter names determine what data it receives from the `EvalItem`. Supported names: `query`, `response`, `expected`, `expected_tool_calls`, `conversation`, `tools`, `context`. Any combination is valid.
**C#:**
```csharp
var local = new LocalEvaluator(
FunctionEvaluator.Create("mentions_city",
(EvalItem item) => item.ExpectedOutput != null
&& item.Response.Contains(item.ExpectedOutput, StringComparison.OrdinalIgnoreCase)),
FunctionEvaluator.Create("is_concise",
(string response) => response.Split(' ').Length < 500));
```
## What To Build
### Core: Evaluator Protocol
A runtime-checkable protocol that any evaluation provider implements:
```python
@runtime_checkable
class Evaluator(Protocol):
name: str
async def evaluate(
self, items: Sequence[EvalItem], *, eval_name: str = "Agent Framework Eval"
) -> EvalResults: ...
```
The protocol is minimal — just `name` and `evaluate()`.
### Core: EvalItem
Provider-agnostic data format for items to evaluate:
```python
@dataclass
class ExpectedToolCall:
name: str # Tool/function name
arguments: dict[str, Any] | None = None # None = don't check args
@dataclass
class EvalItem:
conversation: list[Message] # Single source of truth
tools: list[FunctionTool] | None = None # Agent's available tools
context: str | None = None
expected_output: str | None = None # Ground-truth for comparison
expected_tool_calls: list[ExpectedToolCall] | None = None
split_strategy: ConversationSplitter | None = None
query: str # property — derived from conversation split
response: str # property — derived from conversation split
```
`conversation` is the single source of truth. `query` and `response` are derived properties — splitting the conversation at the last user message (default) and extracting text from each side. Changing the `split_strategy` consistently changes all derived values.
`tools` provides typed `FunctionTool` objects — including MCP tools, which are automatically extracted after agent runs.
### Internal: AgentEvalConverter
Internal class that converts agent-framework types to `EvalItem`. Used by `evaluate_agent()` and `evaluate_workflow()` — not part of the public API:
| Agent Framework | Eval Format |
|---|---|
| `Content.function_call` | `tool_call` in OpenAI chat format |
| `Content.function_result` | `tool_result` in OpenAI chat format |
| `FunctionTool` | `{name, description, parameters}` schema |
| `Message` history | `conversation` list + `query`/`response` extraction |
### Core: EvalResults
Rich result type with convenience properties for CI integration:
```python
results.all_passed # bool: no failures or errors (recursive for workflow)
results.passed # int: passing count
results.failed # int: failure count
results.total # int: total = passed + failed + errored
results.items # list[EvalItemResult]: per-item detail with query, response, and scores
results.error # str | None: error details on failure
results.sub_results # dict: per-agent breakdown (workflow evals)
results.report_url # str | None: portal link (Foundry)
results.assert_passed() # raises AssertionError with details
```
### Core: Orchestration Functions
Provider-agnostic functions that extract data and delegate to evaluators:
| Function | What it does |
|---|---|
| `evaluate_agent()` | Runs agent against test queries (or evaluates pre-existing `responses=`), converts to `EvalItem`s, passes to evaluator. Accepts optional `expected_output=` for ground-truth comparison, `expected_tool_calls=` for tool-correctness evaluation, and `num_repetitions=` for consistency measurement |
| `evaluate_workflow()` | Extracts per-agent data from `WorkflowRunResult`, evaluates each agent and overall output. Per-agent breakdown in `sub_results`. Also accepts `num_repetitions=` |
### Core: Conversation Split Strategies
Multi-turn conversations must be split into query (input) and response (output) halves for evaluation. How you split determines *what you're evaluating*:
**Last-turn split** — split at the last user message. Everything up to and including it is the query context; the agent's subsequent actions are the response:
```
conversation: user1 → assistant1 → user2 → assistant2(tool) → tool_result → assistant3
query_messages: [user1, assistant1, user2]
response_messages: [assistant2(tool), tool_result, assistant3]
```
This evaluates: "Given all the context so far, did the agent answer the latest question well?" Best for response quality at a specific point in the conversation.
**Full-conversation split** — the first user message is the query; everything after is the response:
```
query_messages: [user1]
response_messages: [assistant1, user2, assistant2(tool), tool_result, assistant3]
```
This evaluates: "Given the original request, did the entire conversation trajectory serve the user?" Best for task completion and overall conversation quality.
**Per-turn split** — produces N eval items from an N-turn conversation. Each turn is evaluated with its cumulative context:
```
item 1: query = [user1], response = [assistant1]
item 2: query = [user1, assistant1, user2], response = [assistant2(tool), tool_result, assistant3]
```
This evaluates each response independently. Best for fine-grained analysis and pinpointing where a conversation goes wrong.
These factorings produce different scores for the same conversation. The framework ships all three as built-in strategies, defaulting to last-turn. Developers can also provide a custom splitter — a function (Python) or `IConversationSplitter` implementation (.NET) — and override the strategy at the call site or per evaluator.
### Azure AI: FoundryEvals
`Evaluator` implementation backed by Azure AI Foundry:
```python
class FoundryEvals:
def __init__(self, *, project_client=None, openai_client=None,
model_deployment: str, evaluators=None, ...)
async def evaluate(self, items, *, eval_name) -> EvalResults
```
**Smart auto-detection in `evaluate()`:**
- Default evaluators: relevance, coherence, task_adherence
- Auto-adds `tool_call_accuracy` when items have tools/`tool_definitions`
- Filters out tool evaluators for items without tools
### Azure AI: FoundryEvals Constants
```python
from agent_framework.foundry import FoundryEvals
evaluators = [FoundryEvals.RELEVANCE, FoundryEvals.TOOL_CALL_ACCURACY]
```
Categories: Agent behavior, Tool usage, Quality, Safety.
### Azure AI: Foundry-Specific Functions
| Function | What it does |
|---|---|
| `evaluate_traces()` | Evaluate from stored response IDs or OTel traces |
| `evaluate_foundry_target()` | Evaluate a Foundry-registered agent or deployment |
### Core: LocalEvaluator and Function Evaluators
`LocalEvaluator` implements the `Evaluator` protocol for fast, API-free evaluation. It runs check functions locally — useful for inner-loop development, CI smoke tests, and combining with cloud-based evaluators.
Built-in checks:
- `keyword_check(*keywords)` — response must contain specified keywords
- `tool_called_check(*tool_names)` — agent must have called specified tools
- `tool_calls_present` — all `expected_tool_calls` names appear in conversation (unordered, extras OK)
- `tool_call_args_match` — expected tool calls match on name + arguments (subset match on args)
Custom function evaluators use `@evaluator` to wrap plain Python functions. The function's **parameter names** determine what data it receives from the `EvalItem`:
```python
from agent_framework import evaluator, LocalEvaluator
# Tier 1: Simple check — just query + response
@evaluator
def is_concise(response: str) -> bool:
return len(response.split()) < 500
# Tier 2: Ground truth — compare against expected output
@evaluator
def mentions_city(response: str, expected_output: str) -> bool:
return expected_output.lower() in response.lower()
# Tier 3: Full context — inspect conversation and tools
@evaluator
def used_tools(conversation: list, tools: list) -> float:
# ... scoring logic
return score
local = LocalEvaluator(is_concise, mentions_city, used_tools)
```
Supported parameters: `query`, `response`, `expected`, `expected_tool_calls`, `conversation`, `tools`, `context`.
Return types: `bool`, `float` (≥0.5 = pass), `dict` with `score` or `passed` key, or `CheckResult`.
Async functions are handled automatically — `@evaluator` detects `async def` and produces the right wrapper.
### Example: GAIA Benchmark
[GAIA](https://huggingface.co/gaia-benchmark) tests real-world multi-step tasks with known expected answers. Each task has a question and a ground-truth answer, with optional file attachments. The framework accommodates GAIA's knobs (difficulty levels, file inputs, multi-step tool use) through the existing `EvalItem` fields:
```python
from datasets import load_dataset
from agent_framework import evaluate_agent, evaluator, LocalEvaluator
gaia = load_dataset("gaia-benchmark/GAIA", "2023_level1", split="test")
@evaluator
def exact_match(response: str, expected_output: str) -> bool:
return expected_output.strip().lower() in response.strip().lower()
# Simple path — evaluate_agent handles running + expected_output stamping
results = await evaluate_agent(
agent=agent,
queries=[task["Question"] for task in gaia],
expected_output=[task["Final answer"] for task in gaia],
evaluators=LocalEvaluator(exact_match),
)
```
### Package Location
- Core types and orchestration: `agent_framework._eval`, `agent_framework._local_eval` (Python), `Microsoft.Agents.AI` (.NET)
- Foundry provider: `agent_framework_azure_ai._foundry_evals` (Python), `Microsoft.Agents.AI.AzureAI` (.NET)
- Azure-AI re-exports core types for convenience (Python)
## Known Limitations
1. **Tool evaluators require query + agent**: Tool evaluators need tool definition schemas. When using these evaluators with `evaluate_agent(responses=...)`, provide `queries=` and pass an agent with tool definitions.
2. **`model_deployment` always required**: Could potentially be inferred from the Foundry project configuration.
## Open Questions
1. **Red teaming non-registered agents**: Requires Foundry API support for callback-based flows.
2. **Datasets with expected outputs**: A dataset abstraction for pre-populating `expected_output` values across eval runs is a natural next step but not yet designed.
3. **Multi-modal evaluation**: The `conversation` field on `EvalItem` already stores full `Message`/`Content` (Python) and `ChatMessage` (.NET) objects, which can represent multi-modal content (images, audio, structured data). Evaluators that accept the full `EvalItem` or `conversation` parameter can access this content today. However, the convenience shortcuts — `query`/`response` string projections and the `FunctionEvaluator` string overloads — are text-only. Multi-modal-aware evaluators should use the full-item path (`Func<EvalItem, CheckResult>` in .NET, `conversation: list` parameter in Python).
## .NET Implementation Design
### Key Difference: MEAI Ecosystem
Unlike Python, the .NET ecosystem already has `Microsoft.Extensions.AI.Evaluation` (v10.3.0) providing:
- `IEvaluator` — per-item evaluation of `(messages, chatResponse) → EvaluationResult`
- `CompositeEvaluator` — combines multiple evaluators
- Quality evaluators — `RelevanceEvaluator`, `CoherenceEvaluator`, `GroundednessEvaluator`
- Safety evaluators — `ContentHarmEvaluator`, `ProtectedMaterialEvaluator`
- Metric types — `NumericMetric`, `BooleanMetric`, `StringMetric`
The .NET integration uses MEAI's `IEvaluator` directly — no new evaluator interface. Our contribution is the **orchestration layer**: extension methods that run agents, extract data, call `IEvaluator` per item, and aggregate results.
### Architecture
```
┌──────────────────────────────────────────────────────────────┐
│ Developer Code │
│ agent.EvaluateAsync(queries, evaluator) │
│ run.EvaluateAsync(evaluator) │
└────────────────┬─────────────────────────────────────────────┘
┌────────────────▼─────────────────────────────────────────────┐
│ Orchestration Layer (Microsoft.Agents.AI) │
│ AgentEvaluationExtensions — runs agents, extracts data, │
│ calls IEvaluator per item, aggregates into │
│ AgentEvaluationResults │
└────────────────┬─────────────────────────────────────────────┘
│ IEvaluator (MEAI)
┌───────────┼────────────┐
│ │ │
┌───▼───-┐ ┌───▼────┐ ┌────▼──────────┐
│ MEAI │ │ Local │ │ Foundry │
│ Quality│ │ Checks │ │ (cloud batch) │
│ Safety │ │ Lambdas│ │ │
└────────┘ └────────┘ └───────────────┘
```
All evaluators implement MEAI's `IEvaluator`. The orchestration layer doesn't need to know which kind — it calls `EvaluateAsync(messages, chatResponse)` per item on all of them. `FoundryEvals` handles batching internally (buffers items, submits once, returns per-item results).
### .NET Core Types
**No new evaluator interface.** Use MEAI's `IEvaluator` directly.
**`AgentEvaluationResults`** — The only new type. Aggregates per-item MEAI `EvaluationResult`s across a batch of queries:
```csharp
public class AgentEvaluationResults
{
public string Provider { get; init; }
public string? ReportUrl { get; init; }
// Per-item — standard MEAI EvaluationResult, unchanged
public IReadOnlyList<EvaluationResult> Items { get; init; }
// Aggregate pass/fail derived from metric interpretations
public int Passed { get; }
public int Failed { get; }
public int Total { get; }
public bool AllPassed { get; }
// Workflow: per-agent breakdown
public IReadOnlyDictionary<string, AgentEvaluationResults>? SubResults { get; init; }
public void AssertAllPassed(string? message = null);
}
```
### .NET Evaluator Implementations
All implement MEAI's `IEvaluator`:
**`LocalEvaluator`** — Runs lambda checks locally, returns `BooleanMetric` per check:
```csharp
var local = new LocalEvaluator(
FunctionEvaluator.Create("is_concise",
(string response) => response.Split().Length < 500),
EvalChecks.KeywordCheck("weather"),
EvalChecks.ToolCalledCheck("get_weather"));
```
**MEAI evaluators** — Used directly, no adapter needed:
```csharp
var quality = new CompositeEvaluator(
new RelevanceEvaluator(),
new CoherenceEvaluator());
```
**`FoundryEvals`** — Implements `IEvaluator` but batches internally. On first call, buffers the item. On the last item (or when explicitly flushed), submits the batch to Foundry and distributes per-item results:
```csharp
var foundry = new FoundryEvals(projectClient, "gpt-4o");
```
### .NET Orchestration: Extension Methods
```csharp
public static class AgentEvaluationExtensions
{
// Evaluate an agent against test queries
public static Task<AgentEvaluationResults> EvaluateAsync(
this AIAgent agent,
IEnumerable<string> queries,
IEvaluator evaluator,
ChatConfiguration? chatConfiguration = null,
IEnumerable<string>? expectedOutput = null,
CancellationToken cancellationToken = default);
// Evaluate pre-existing responses (without re-running the agent)
public static Task<AgentEvaluationResults> EvaluateAsync(
this AIAgent agent,
AgentResponse responses,
IEvaluator evaluator,
IEnumerable<string>? queries = null,
ChatConfiguration? chatConfiguration = null,
IEnumerable<string>? expectedOutput = null,
CancellationToken cancellationToken = default);
// Evaluate with multiple evaluators (one result per evaluator)
public static Task<IReadOnlyList<AgentEvaluationResults>> EvaluateAsync(
this AIAgent agent,
IEnumerable<string> queries,
IEnumerable<IEvaluator> evaluators,
ChatConfiguration? chatConfiguration = null,
IEnumerable<string>? expectedOutput = null,
CancellationToken cancellationToken = default);
// Evaluate a workflow run with per-agent breakdown
public static Task<AgentEvaluationResults> EvaluateAsync(
this Run run,
IEvaluator evaluator,
ChatConfiguration? chatConfiguration = null,
bool includeOverall = true,
bool includePerAgent = true,
CancellationToken cancellationToken = default);
}
```
**Usage:**
```csharp
// MEAI evaluators — just works
var results = await agent.EvaluateAsync(
queries: ["What's the weather?"],
evaluator: new RelevanceEvaluator(),
chatConfiguration: new ChatConfiguration(evalClient));
// Local checks
var results = await agent.EvaluateAsync(
queries: ["What's the weather?"],
evaluator: new LocalEvaluator(
EvalChecks.KeywordCheck("weather")));
// Foundry cloud
var results = await agent.EvaluateAsync(
queries: ["What's the weather?"],
evaluator: new FoundryEvals(projectClient, "gpt-4o"));
// Evaluate existing response (without re-running the agent)
var response = await agent.RunAsync("What's the weather?");
var results = await agent.EvaluateAsync(
responses: response,
queries: ["What's the weather?"],
evaluator: new FoundryEvals(projectClient, "gpt-4o"));
// Mixed — one result per evaluator
var results = await agent.EvaluateAsync(
queries: ["What's the weather?"],
evaluators: [
new LocalEvaluator(EvalChecks.KeywordCheck("weather")),
new RelevanceEvaluator(),
new FoundryEvals(projectClient, "gpt-4o")
],
chatConfiguration: new ChatConfiguration(evalClient));
// Workflow with per-agent breakdown
Run run = await workflowRunner.RunAsync(workflow, "Plan a trip");
var results = await run.EvaluateAsync(
evaluator: new FoundryEvals(projectClient, "gpt-4o"));
```
### .NET Function Evaluators
Typed factory overloads (C# equivalent of Python's `@evaluator`):
```csharp
public static class FunctionEvaluator
{
public static EvalCheck Create(string name, Func<string, bool> check); // response only
public static EvalCheck Create(string name, Func<string, string?, bool> check); // expectedOutput
public static EvalCheck Create(string name, Func<EvalItem, bool> check); // full item
public static EvalCheck Create(string name, Func<EvalItem, CheckResult> check); // full control
public static EvalCheck Create(string name, Func<string, Task<bool>> check); // async
}
```
`EvalItem` is a lightweight record used only by `FunctionEvaluator` and `LocalEvaluator` to pass context to check functions. It is not part of the `IEvaluator` interface:
```csharp
public record ExpectedToolCall(string Name, IReadOnlyDictionary<string, object>? Arguments = null);
public sealed class EvalItem
{
public EvalItem(string query, string response, IReadOnlyList<ChatMessage> conversation);
public string Query { get; }
public string Response { get; }
public IReadOnlyList<ChatMessage> Conversation { get; }
public IReadOnlyList<AITool>? Tools { get; set; }
public string? ExpectedOutput { get; set; }
public IReadOnlyList<ExpectedToolCall>? ExpectedToolCalls { get; set; }
public string? Context { get; set; }
public IConversationSplitter? Splitter { get; set; }
}
```
### Workflow Data Extraction (.NET)
`run.EvaluateAsync()` walks `Run.OutgoingEvents` via LINQ:
1. Pair `ExecutorInvokedEvent` / `ExecutorCompletedEvent` by `ExecutorId`
2. Extract `AgentResponseEvent` for per-agent `ChatResponse`
3. Call `evaluator.EvaluateAsync()` per invocation
4. Group by `ExecutorId` for per-agent `SubResults`
5. Use final workflow output for overall eval
### .NET Package Structure
| Package | Contents |
|---------|----------|
| `Microsoft.Agents.AI` | `IAgentEvaluator`, `AgentEvaluationResults`, `LocalEvaluator`, `FunctionEvaluator`, `EvalChecks`, `EvalItem`, `ExpectedToolCall`, `AgentEvaluationExtensions` |
| `Microsoft.Agents.AI.AzureAI` | `FoundryEvals` (provider + constants) |
### Python ↔ .NET Mapping
| Python | .NET |
|--------|------|
| `Evaluator` protocol | `IAgentEvaluator` (our interface; MEAI provides `IEvaluator` for per-item scoring) |
| `EvalItem` dataclass | `EvalItem` class |
| `EvalResults` | `AgentEvaluationResults` |
| `EvalItemResult` / `EvalScoreResult` | MEAI `EvaluationResult` / `EvaluationMetric` (reused) |
| `LocalEvaluator` | `LocalEvaluator` (implements `IAgentEvaluator`) |
| `@evaluator` | `FunctionEvaluator.Create()` overloads |
| `keyword_check()` / `tool_called_check()` | `EvalChecks.KeywordCheck()` / `EvalChecks.ToolCalledCheck()` |
| `tool_calls_present` / `tool_call_args_match` | (custom `FunctionEvaluator` — same pattern) |
| `ExpectedToolCall` dataclass | `ExpectedToolCall` record |
| `FoundryEvals` | `FoundryEvals` (implements `IAgentEvaluator`, includes evaluator name constants) |
| `evaluate_agent()` | `agent.EvaluateAsync(queries, evaluator)` extension method |
| `evaluate_agent(responses=)` | `agent.EvaluateAsync(responses, evaluator)` extension method |
| `evaluate_workflow()` | `run.EvaluateAsync()` extension method |
## More Information
- [Foundry Evals documentation](https://learn.microsoft.com/azure/ai-foundry/concepts/evaluation-approach-gen-ai) — Azure AI Foundry evaluation overview
@@ -177,7 +177,7 @@ This feature ports the vector store abstractions, embedding generator abstractio
**Goal:** Add embedding generators to all existing AF provider packages that have chat clients.
**Mergeable:** Yes — each is independent, added to existing provider packages.
#### 2.1 — Foundry inference embedding (in `packages/foundry/`)
#### 2.1 — Azure AI Inference embedding (in `packages/azure-ai/`)
#### 2.2 — Ollama embedding (in `packages/ollama/`)
#### 2.3 — Anthropic embedding (in `packages/anthropic/`)
#### 2.4 — Bedrock embedding (in `packages/bedrock/`)
+2 -2
View File
@@ -12,8 +12,8 @@ dotnet/
│ ├── Microsoft.Agents.AI.Abstractions/ # Core AI agent abstractions
│ ├── Microsoft.Agents.AI.A2A/ # Agent-to-Agent (A2A) provider
│ ├── Microsoft.Agents.AI.OpenAI/ # OpenAI provider
│ ├── Microsoft.Agents.AI.Foundry/ # Microsoft Foundry Agents (v2) provider
│ ├── Microsoft.Agents.AI.AzureAI.Persistent/ # Legacy Microsoft Foundry Agents (v1) provider
│ ├── Microsoft.Agents.AI.AzureAI/ # Azure AI Foundry Agents (v2) provider
│ ├── Microsoft.Agents.AI.AzureAI.Persistent/ # Legacy Azure AI Foundry Agents (v1) provider
│ ├── Microsoft.Agents.AI.Anthropic/ # Anthropic provider
│ ├── Microsoft.Agents.AI.Workflows/ # Workflow orchestration
│ └── ... # Other packages
-225
View File
@@ -1,225 +0,0 @@
---
name: verify-samples-tool
description: How to use the verify-samples tool to run, verify, and manage sample definitions in the Agent Framework repository. Use this when adding, updating, or running sample verification.
---
# verify-samples Tool
The `verify-samples` project (`dotnet/eng/verify-samples/`) is an automated tool that runs sample projects and verifies their output using deterministic checks and AI-powered verification.
## Running verify-samples
**Important:** By default, samples must be pre-built before running verify-samples. Build the solution first, or pass `--build` to build samples during the run:
```bash
cd dotnet
dotnet build agent-framework-dotnet.slnx -f net10.0
```
Then run verify-samples:
```bash
# Run all samples across all categories
dotnet run --project eng/verify-samples -- --log results.log --csv results.csv
# Run a specific category
dotnet run --project eng/verify-samples -- --category 02-agents --log results.log
# Run specific samples by name
dotnet run --project eng/verify-samples -- Agent_Step02_StructuredOutput Agent_Step09_AsFunctionTool
# Control parallelism (default 8)
dotnet run --project eng/verify-samples -- --parallel 8 --log results.log
# Build samples during run (skips the need for a prior build step)
# This may cause build conflicts as multiple samples are built in parallel, so use with caution
dotnet run --project eng/verify-samples -- --build --log results.log
# Combine options
dotnet run --project eng/verify-samples -- --category 03-workflows --parallel 4 --log results.log --csv results.csv --md results.md
```
### Required Environment Variables
The tool itself needs:
- `AZURE_OPENAI_ENDPOINT` — for the AI verification agent
- `AZURE_OPENAI_DEPLOYMENT_NAME` (optional, defaults to `gpt-5-mini`)
Individual samples require their own env vars (e.g., `AZURE_AI_PROJECT_ENDPOINT`). The tool automatically checks and skips samples with missing env vars.
### Output Files
- `--log results.log` — detailed per-sample log with stdout/stderr, AI reasoning, and a summary
- `--csv results.csv` — tabular summary with Sample, ProjectPath, Status, FailedChecks, and Failures columns
- `--md results.md` — Markdown summary with results table and collapsible failure details (suitable for GitHub PR comments)
## Sample Categories
Definitions are in the `dotnet/eng/verify-samples/` directory:
| Category | Config File | Registered Key |
|----------|-------------|----------------|
| 01-get-started | `GetStartedSamples.cs` | `01-get-started` |
| 02-agents | `AgentsSamples.cs` | `02-agents` |
| 03-workflows | `WorkflowSamples.cs` | `03-workflows` |
Categories are registered in `VerifyOptions.cs` in the `s_sampleSets` dictionary.
## SampleDefinition Properties
Each sample is defined as a `SampleDefinition` in the appropriate config file. Key properties:
```csharp
new SampleDefinition
{
// Required: Display name for the sample
Name = "Agent_Step02_StructuredOutput",
// Required: Relative path from dotnet/ to the sample project directory
ProjectPath = "samples/02-agents/Agents/Agent_Step02_StructuredOutput",
// Environment variables the sample requires (throws if missing)
RequiredEnvironmentVariables = ["AZURE_OPENAI_ENDPOINT"],
// Environment variables with defaults that would prompt on console if unset
OptionalEnvironmentVariables = ["AZURE_OPENAI_DEPLOYMENT_NAME"],
// Skip this sample with a reason (for structural issues only)
SkipReason = null, // or "Requires external service X."
// Deterministic checks: substrings that must appear in stdout
MustContain = ["=== Section Header ==="],
// Substrings that must NOT appear in stdout
MustNotContain = [],
// If true, only MustContain checks are used (no AI verification)
IsDeterministic = false,
// AI verification: natural-language descriptions of expected output
// Each entry describes one aspect to verify independently
ExpectedOutputDescription =
[
"The output should show structured person information with Name, Age, and Occupation fields.",
"The output should not contain error messages or stack traces.",
],
// Stdin inputs to feed to the sample (for interactive samples)
Inputs = ["Y", "Y", "Y"],
// Delay between stdin inputs in ms (default 2000, increase for LLM calls between inputs)
InputDelayMs = 3000,
}
```
## How to Add a New Sample Definition
1. **Check the sample's Program.cs** to understand:
- What environment variables it reads (look for `GetEnvironmentVariable`)
- Whether it needs stdin input (look for `Console.ReadLine`, `Application.GetInput`)
- Whether it has an external loop (look for `EXIT` patterns in YAML workflows)
- What output it produces (section headers, markers, expected behavior)
- Whether it exits on its own or runs as a server
2. **Choose the right verification strategy:**
- **Deterministic** (`IsDeterministic = true`): Use `MustContain` for samples with fixed output strings. No AI verification.
- **AI-verified** (default): Use `ExpectedOutputDescription` with semantic descriptions. Write expectations that are flexible enough for non-deterministic LLM output.
- **Both**: Use `MustContain` for fixed markers AND `ExpectedOutputDescription` for LLM-generated content.
3. **Set `SkipReason` only for structural issues:**
- Web servers that don't exit
- Multi-process client/server architectures
- Samples requiring external infrastructure (MCP servers you can't reach, Docker, etc.)
- Do NOT skip for missing env vars — the tool checks those dynamically.
4. **For interactive samples, provide `Inputs`:**
- Samples using `Application.GetInput(args)` need one initial input
- Samples with `Console.ReadLine()` approval loops need `"Y"` inputs
- YAML workflows with `externalLoop` need `"EXIT"` as the last input
- Set `InputDelayMs` to 3000-8000ms for samples with LLM calls between inputs
5. **Add the definition** to the appropriate config file (e.g., `AgentsSamples.cs`) in the `All` list.
6. **Register new categories** (if needed) in `VerifyOptions.cs` `s_sampleSets` dictionary.
### Writing Good ExpectedOutputDescription
- Write descriptions that are **semantically flexible** — LLM output varies between runs
- Each array entry should describe **one independent aspect** to verify
- Always include `"The output should not contain error messages or stack traces."` as the last entry
- Avoid exact wording expectations — use "should mention", "should contain information about", "should show"
- Bad: `"The output should say 'The weather in Amsterdam is cloudy with a high of 15°C'"`
- Good: `"The output should contain weather information about Amsterdam mentioning cloudy weather with a high of 15°C."`
### Example: Simple LLM Sample
```csharp
new SampleDefinition
{
Name = "Agent_With_AzureOpenAIChatCompletion",
ProjectPath = "samples/02-agents/AgentProviders/Agent_With_AzureOpenAIChatCompletion",
RequiredEnvironmentVariables = ["AZURE_OPENAI_ENDPOINT"],
OptionalEnvironmentVariables = ["AZURE_OPENAI_DEPLOYMENT_NAME"],
ExpectedOutputDescription =
[
"The output should contain a joke about a pirate.",
"The output should not contain error messages or stack traces.",
],
},
```
### Example: Deterministic Sample
```csharp
new SampleDefinition
{
Name = "Workflow_Declarative_GenerateCode",
ProjectPath = "samples/03-workflows/Declarative/GenerateCode",
IsDeterministic = true,
MustContain = ["WORKFLOW: Parsing", "WORKFLOW: Defined"],
ExpectedOutputDescription = ["The output should show a YAML workflow being parsed and C# code being generated from it."],
},
```
### Example: Interactive Sample with Approval Loop
```csharp
new SampleDefinition
{
Name = "FoundryAgent_Hosted_MCP",
ProjectPath = "samples/02-agents/ModelContextProtocol/FoundryAgent_Hosted_MCP",
RequiredEnvironmentVariables = ["AZURE_AI_PROJECT_ENDPOINT"],
OptionalEnvironmentVariables = ["AZURE_AI_MODEL_DEPLOYMENT_NAME"],
Inputs = ["Y", "Y", "Y", "Y", "Y"],
InputDelayMs = 5000,
ExpectedOutputDescription = ["The output should show an agent using the Microsoft Learn MCP tool with approval prompts."],
},
```
### Example: Declarative Workflow with External Loop
```csharp
new SampleDefinition
{
Name = "Workflow_Declarative_FunctionTools",
ProjectPath = "samples/03-workflows/Declarative/FunctionTools",
RequiredEnvironmentVariables = ["AZURE_AI_PROJECT_ENDPOINT"],
OptionalEnvironmentVariables = ["AZURE_AI_MODEL_DEPLOYMENT_NAME"],
Inputs = ["What are today's specials?", "EXIT"],
InputDelayMs = 8000,
ExpectedOutputDescription = ["The output should show a workflow calling function tools to answer a question about restaurant specials."],
},
```
### Example: Skipped Sample
```csharp
new SampleDefinition
{
Name = "Agent_MCP_Server",
ProjectPath = "samples/02-agents/ModelContextProtocol/Agent_MCP_Server",
RequiredEnvironmentVariables = ["AZURE_OPENAI_ENDPOINT"],
OptionalEnvironmentVariables = ["AZURE_OPENAI_DEPLOYMENT_NAME"],
SkipReason = "Runs as an MCP stdio server that does not exit on its own.",
},
```
+2 -3
View File
@@ -29,14 +29,13 @@ using types like `IChatClient`, `FunctionInvokingChatClient`, `AITool`, `AIFunct
## Key Conventions
- **Command output capture**: When running `dotnet build`, `dotnet test`, `dotnet format`, or similar commands, redirect output to a temp file first (e.g., `dotnet build --tl:off 2>&1 | Out-File $env:TEMP\build.log`), then analyze the file as needed. This avoids re-running expensive commands when the initial analysis misses something.
- **Encoding**: All new files must be saved with UTF-8 encoding with BOM (Byte Order Mark). This is required for `dotnet format` to work correctly. When using PowerShell `Set-Content`, always pass `-Encoding UTF8BOM` to preserve the BOM (e.g., `Set-Content $file $content -NoNewline -Encoding UTF8BOM`).
- **Encoding**: All new files must be saved with UTF-8 encoding with BOM (Byte Order Mark). This is required for `dotnet format` to work correctly.
- **Copyright header**: `// Copyright (c) Microsoft. All rights reserved.` at top of all `.cs` files
- **XML docs**: Required for all public methods and classes
- **Async**: Use `Async` suffix for methods returning `Task`/`ValueTask`
- **Private classes**: Should be `sealed` unless subclassed
- **Config**: Read from environment variables with `UPPER_SNAKE_CASE` naming
- **Tests**: Add Arrange/Act/Assert comments; use Moq for mocking; test methods returning `Task`/`ValueTask` must use the `Async` suffix.
- **Tests**: Add Arrange/Act/Assert comments; use Moq for mocking
## Key Design Principles
-1
View File
@@ -17,7 +17,6 @@
<PropertyGroup>
<IsReleaseCandidate>false</IsReleaseCandidate>
<IsReleased>false</IsReleased>
</PropertyGroup>
<PropertyGroup>
+2 -3
View File
@@ -4,9 +4,8 @@
<!-- https://learn.microsoft.com/en-us/nuget/consume-packages/Central-Package-Management -->
<Sdk Name="Microsoft.Build.CentralPackageVersions" Version="2.1.3" />
<!-- Only run 'dotnet format' on dev machines, Release builds. Skip on GitHub Actions -->
<!-- as this runs in its own Actions job. Only run for net10.0 target frameworks since the dotnet format command -->
<!-- already formats all target frameworks in project. Otherwise it will run format x times x where x is the number of target frameworks -->
<Target Name="DotnetFormatOnBuild" BeforeTargets="Build" Condition=" '$(Configuration)' == 'Release' AND '$(GITHUB_ACTIONS)' == '' AND '$(TargetFramework)' == 'net10.0' ">
<!-- as this runs in its own Actions job. -->
<Target Name="DotnetFormatOnBuild" BeforeTargets="Build" Condition=" '$(Configuration)' == 'Release' AND '$(GITHUB_ACTIONS)' == '' ">
<Message Text="Running dotnet format" Importance="high" />
<Exec Command="dotnet format --no-restore -v diag $(ProjectFileName)" />
</Target>
+33 -23
View File
@@ -11,40 +11,41 @@
</PropertyGroup>
<ItemGroup>
<!-- Aspire.* -->
<PackageVersion Include="Anthropic" Version="12.13.0" />
<PackageVersion Include="Anthropic.Foundry" Version="0.5.0" />
<PackageVersion Include="Anthropic" Version="12.8.0" />
<PackageVersion Include="Anthropic.Foundry" Version="0.4.2" />
<PackageVersion Include="Aspire.Azure.AI.OpenAI" Version="13.0.0-preview.1.25560.3" />
<PackageVersion Include="Aspire.Hosting.AppHost" Version="$(AspireAppHostSdkVersion)" />
<PackageVersion Include="Aspire.Hosting.Azure.CognitiveServices" Version="$(AspireAppHostSdkVersion)" />
<PackageVersion Include="Aspire.Microsoft.Azure.Cosmos" Version="$(AspireAppHostSdkVersion)" />
<PackageVersion Include="CommunityToolkit.Aspire.OllamaSharp" Version="13.0.0" />
<!-- Azure.* -->
<PackageVersion Include="Azure.AI.Projects" Version="2.0.0" />
<PackageVersion Include="Azure.AI.Agents.Persistent" Version="1.2.0-beta.10" />
<PackageVersion Include="Azure.AI.OpenAI" Version="2.9.0-beta.1" />
<PackageVersion Include="Azure.Identity" Version="1.20.0" />
<PackageVersion Include="Azure.AI.Projects" Version="2.0.0-beta.1" />
<PackageVersion Include="Azure.AI.Projects.OpenAI" Version="2.0.0-beta.1" />
<PackageVersion Include="Azure.AI.Agents.Persistent" Version="1.2.0-beta.8" />
<PackageVersion Include="Azure.AI.OpenAI" Version="2.8.0-beta.1" />
<PackageVersion Include="Azure.Identity" Version="1.17.1" />
<PackageVersion Include="Azure.Monitor.OpenTelemetry.Exporter" Version="1.4.0" />
<!-- Google Gemini -->
<PackageVersion Include="Google.GenAI" Version="1.6.0" />
<PackageVersion Include="Google.GenAI" Version="0.11.0" />
<PackageVersion Include="Mscc.GenerativeAI.Microsoft" Version="2.9.3" />
<!-- Microsoft.Azure.* -->
<PackageVersion Include="Microsoft.Azure.Cosmos" Version="3.54.0" />
<!-- Newtonsoft.Json -->
<PackageVersion Include="Newtonsoft.Json" Version="13.0.4" />
<!-- System.* -->
<PackageVersion Include="Microsoft.Bcl.AsyncInterfaces" Version="10.0.6" />
<PackageVersion Include="Microsoft.Bcl.AsyncInterfaces" Version="10.0.4" />
<PackageVersion Include="Microsoft.Bcl.HashCode" Version="6.0.0" />
<PackageVersion Include="Microsoft.Bcl.Memory" Version="10.0.4" />
<PackageVersion Include="System.ClientModel" Version="1.10.0" />
<PackageVersion Include="System.ClientModel" Version="1.9.0" />
<PackageVersion Include="System.CodeDom" Version="10.0.0" />
<PackageVersion Include="System.Collections.Immutable" Version="10.0.1" />
<PackageVersion Include="System.CommandLine" Version="2.0.0-rc.2.25502.107" />
<PackageVersion Include="System.Diagnostics.DiagnosticSource" Version="10.0.6" />
<PackageVersion Include="System.Diagnostics.DiagnosticSource" Version="10.0.3" />
<PackageVersion Include="System.Linq.AsyncEnumerable" Version="10.0.4" />
<PackageVersion Include="System.Net.Http.Json" Version="10.0.0" />
<PackageVersion Include="System.Net.ServerSentEvents" Version="10.0.4" />
<PackageVersion Include="System.Text.Json" Version="10.0.6" />
<PackageVersion Include="System.Threading.Channels" Version="10.0.6" />
<PackageVersion Include="System.Net.ServerSentEvents" Version="10.0.3" />
<PackageVersion Include="System.Text.Json" Version="10.0.3" />
<PackageVersion Include="System.Threading.Channels" Version="10.0.3" />
<PackageVersion Include="System.Threading.Tasks.Extensions" Version="4.6.3" />
<PackageVersion Include="System.Net.Security" Version="4.3.2" />
<!-- OpenTelemetry -->
@@ -63,28 +64,36 @@
<PackageVersion Include="Microsoft.AspNetCore.OpenApi" Version="10.0.0" />
<PackageVersion Include="Swashbuckle.AspNetCore.SwaggerUI" Version="10.0.0" />
<!-- Microsoft.Extensions.* -->
<PackageVersion Include="Microsoft.Extensions.AI" Version="10.5.0" />
<PackageVersion Include="Microsoft.Extensions.AI.Abstractions" Version="10.5.0" />
<PackageVersion Include="Microsoft.Extensions.AI.OpenAI" Version="10.5.0" />
<PackageVersion Include="Microsoft.Extensions.AI" Version="10.3.0" />
<PackageVersion Include="Microsoft.Extensions.AI.Abstractions" Version="10.3.0" />
<PackageVersion Include="Microsoft.Extensions.AI.Evaluation" Version="10.3.0" />
<PackageVersion Include="Microsoft.Extensions.AI.Evaluation.Quality" Version="10.3.0" />
<PackageVersion Include="Microsoft.Extensions.AI.Evaluation.Safety" Version="10.3.0-preview.1.26109.11" />
<PackageVersion Include="Microsoft.Extensions.AI.OpenAI" Version="10.3.0" />
<PackageVersion Include="Microsoft.Extensions.Caching.Memory" Version="10.0.0" />
<PackageVersion Include="Microsoft.Extensions.Compliance.Abstractions" Version="10.5.0" />
<PackageVersion Include="Microsoft.Extensions.Configuration" Version="10.0.0" />
<PackageVersion Include="Microsoft.Extensions.Configuration.Binder" Version="10.0.0" />
<PackageVersion Include="Microsoft.Extensions.Configuration.EnvironmentVariables" Version="10.0.0" />
<PackageVersion Include="Microsoft.Extensions.Configuration.Json" Version="10.0.0" />
<PackageVersion Include="Microsoft.Extensions.Configuration.UserSecrets" Version="10.0.0" />
<PackageVersion Include="Microsoft.Extensions.DependencyInjection" Version="10.0.0" />
<PackageVersion Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="10.0.6" />
<PackageVersion Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="10.0.3" />
<PackageVersion Include="Microsoft.Extensions.Hosting" Version="10.0.0" />
<PackageVersion Include="Microsoft.Extensions.Http.Resilience" Version="10.0.0" />
<PackageVersion Include="Microsoft.Extensions.Logging" Version="10.0.0" />
<PackageVersion Include="Microsoft.Extensions.Logging.Abstractions" Version="10.0.6" />
<PackageVersion Include="Microsoft.Extensions.Logging.Abstractions" Version="10.0.3" />
<PackageVersion Include="Microsoft.Extensions.Logging.Console" Version="10.0.0" />
<PackageVersion Include="Microsoft.Extensions.ServiceDiscovery" Version="10.0.0" />
<PackageVersion Include="Microsoft.Extensions.VectorData.Abstractions" Version="9.7.0" />
<!-- Vector Stores -->
<PackageVersion Include="Microsoft.SemanticKernel.Connectors.InMemory" Version="1.67.0-preview" />
<PackageVersion Include="Microsoft.SemanticKernel.Connectors.Qdrant" Version="1.67.0-preview" />
<!-- Semantic Kernel -->
<PackageVersion Include="Microsoft.SemanticKernel" Version="1.67.0" />
<PackageVersion Include="Microsoft.SemanticKernel.Agents.Core" Version="1.67.0" />
<PackageVersion Include="Microsoft.SemanticKernel.Agents.OpenAI" Version="1.67.0-preview" />
<PackageVersion Include="Microsoft.SemanticKernel.Agents.AzureAI" Version="1.67.0-preview" />
<PackageVersion Include="Microsoft.SemanticKernel.Plugins.OpenApi" Version="1.67.0" />
<!-- Agent SDKs -->
<PackageVersion Include="GitHub.Copilot.SDK" Version="0.1.29" />
<PackageVersion Include="Microsoft.Agents.CopilotStudio.Client" Version="1.3.171-beta" />
@@ -98,12 +107,13 @@
<!-- MCP -->
<PackageVersion Include="ModelContextProtocol" Version="1.1.0" />
<!-- Inference SDKs -->
<PackageVersion Include="AWSSDK.Extensions.Bedrock.MEAI" Version="4.0.5.1" />
<PackageVersion Include="Microsoft.ML.OnnxRuntimeGenAI" Version="0.10.0" />
<PackageVersion Include="Microsoft.ML.Tokenizers" Version="2.0.0" />
<PackageVersion Include="OllamaSharp" Version="5.4.8" />
<PackageVersion Include="OpenAI" Version="2.10.0" />
<PackageVersion Include="OpenAI" Version="2.8.0" />
<!-- Identity -->
<PackageVersion Include="Microsoft.Identity.Client.Extensions.Msal" Version="4.83.1" />
<PackageVersion Include="Microsoft.Identity.Client.Extensions.Msal" Version="4.78.0" />
<!-- Workflows -->
<PackageVersion Include="Microsoft.Agents.ObjectModel" Version="2026.2.4.1" />
<PackageVersion Include="Microsoft.Agents.ObjectModel.Json" Version="2026.2.4.1" />
@@ -116,7 +126,8 @@
<PackageVersion Include="Microsoft.DurableTask.Worker.AzureManaged" Version="1.18.0" />
<!-- Azure Functions -->
<PackageVersion Include="Microsoft.Azure.Functions.Worker" Version="2.50.0" />
<PackageVersion Include="Microsoft.Azure.Functions.Worker.Extensions.DurableTask" Version="1.12.1" />
<PackageVersion Include="Microsoft.Azure.Functions.Worker.ApplicationInsights" Version="2.50.0" />
<PackageVersion Include="Microsoft.Azure.Functions.Worker.Extensions.DurableTask" Version="1.11.0" />
<PackageVersion Include="Microsoft.Azure.Functions.Worker.Extensions.DurableTask.AzureManaged" Version="1.0.1" />
<PackageVersion Include="Microsoft.Azure.Functions.Worker.Extensions.Http" Version="3.3.0" />
<PackageVersion Include="Microsoft.Azure.Functions.Worker.Extensions.Http.AspNetCore" Version="2.1.0" />
@@ -138,7 +149,6 @@
<!-- Symbols -->
<PackageVersion Include="Microsoft.SourceLink.GitHub" Version="8.0.0" />
<!-- Toolset -->
<PackageVersion Include="ReferenceTrimmer" Version="3.4.5" />
<PackageVersion Include="Microsoft.CodeAnalysis.Analyzers" Version="3.11.0" />
<PackageVersion Include="Microsoft.CodeAnalysis.CSharp" Version="4.14.0" />
<PackageVersion Include="Microsoft.CodeAnalysis.NetAnalyzers" Version="10.0.100" />
+49 -81
View File
@@ -1,4 +1,4 @@
<Solution>
<Solution>
<Configurations>
<BuildType Name="Debug" />
<BuildType Name="Publish" />
@@ -7,7 +7,6 @@
<Folder Name="/Samples/">
<File Path="samples/AGENTS.md" />
<File Path="samples/README.md" />
<Project Path="eng/verify-samples/verify-samples.csproj" />
</Folder>
<Folder Name="/Samples/01-get-started/">
<Project Path="samples/01-get-started/01_hello_agent/01_hello_agent.csproj" />
@@ -34,6 +33,7 @@
<Project Path="samples/02-agents/AgentProviders/Agent_With_GoogleGemini/Agent_With_GoogleGemini.csproj" />
<Project Path="samples/02-agents/AgentProviders/Agent_With_Ollama/Agent_With_Ollama.csproj" />
<Project Path="samples/02-agents/AgentProviders/Agent_With_ONNX/Agent_With_ONNX.csproj" />
<Project Path="samples/02-agents/AgentProviders/Agent_With_OpenAIAssistants/Agent_With_OpenAIAssistants.csproj" />
<Project Path="samples/02-agents/AgentProviders/Agent_With_OpenAIChatCompletion/Agent_With_OpenAIChatCompletion.csproj" />
<Project Path="samples/02-agents/AgentProviders/Agent_With_OpenAIResponses/Agent_With_OpenAIResponses.csproj" />
</Folder>
@@ -57,32 +57,10 @@
<Project Path="samples/02-agents/Agents/Agent_Step16_Declarative/Agent_Step16_Declarative.csproj" />
<Project Path="samples/02-agents/Agents/Agent_Step17_AdditionalAIContext/Agent_Step17_AdditionalAIContext.csproj" />
<Project Path="samples/02-agents/Agents/Agent_Step18_CompactionPipeline/Agent_Step18_CompactionPipeline.csproj" />
<Project Path="samples/02-agents/Agents/Agent_Step19_InFunctionLoopCheckpointing/Agent_Step19_InFunctionLoopCheckpointing.csproj" />
</Folder>
<Folder Name="/Samples/02-agents/DeclarativeAgents/">
<Project Path="samples/02-agents/DeclarativeAgents/ChatClient/DeclarativeChatClientAgents.csproj" />
</Folder>
<Folder Name="/Samples/04-hosting/DurableWorkflows/" />
<Folder Name="/Samples/04-hosting/DurableWorkflows/ConsoleApps/">
<Project Path="samples/04-hosting/DurableWorkflows/ConsoleApps/01_SequentialWorkflow/01_SequentialWorkflow.csproj" />
<Project Path="samples/04-hosting/DurableWorkflows/ConsoleApps/02_ConcurrentWorkflow/02_ConcurrentWorkflow.csproj" />
<Project Path="samples/04-hosting/DurableWorkflows/ConsoleApps/03_ConditionalEdges/03_ConditionalEdges.csproj" />
<Project Path="samples/04-hosting/DurableWorkflows/ConsoleApps/04_WorkflowAndAgents/04_WorkflowAndAgents.csproj" />
<Project Path="samples/04-hosting/DurableWorkflows/ConsoleApps/05_WorkflowEvents/05_WorkflowEvents.csproj" />
<Project Path="samples/04-hosting/DurableWorkflows/ConsoleApps/06_WorkflowSharedState/06_WorkflowSharedState.csproj" />
<Project Path="samples/04-hosting/DurableWorkflows/ConsoleApps/07_SubWorkflows/07_SubWorkflows.csproj" />
<Project Path="samples/04-hosting/DurableWorkflows/ConsoleApps/08_WorkflowHITL/08_WorkflowHITL.csproj" />
</Folder>
<Folder Name="/Samples/04-hosting/DurableWorkflows/AzureFunctions/">
<Project Path="samples/04-hosting/DurableWorkflows/AzureFunctions/01_SequentialWorkflow/01_SequentialWorkflow.csproj" />
<Project Path="samples/04-hosting/DurableWorkflows/AzureFunctions/02_ConcurrentWorkflow/02_ConcurrentWorkflow.csproj" />
<Project Path="samples/04-hosting/DurableWorkflows/AzureFunctions/03_WorkflowHITL/03_WorkflowHITL.csproj" />
<Project Path="samples/04-hosting/DurableWorkflows/AzureFunctions/04_WorkflowMcpTool/04_WorkflowMcpTool.csproj" />
<Project Path="samples/04-hosting/DurableWorkflows/AzureFunctions/05_WorkflowAndAgents/05_WorkflowAndAgents.csproj" />
</Folder>
<Folder Name="/Samples/GettingStarted/">
<File Path="samples/GettingStarted/README.md" />
</Folder>
<Folder Name="/Samples/02-agents/AGUI/">
<File Path="samples/02-agents/AGUI/README.md" />
</Folder>
@@ -104,11 +82,7 @@
</Folder>
<Folder Name="/Samples/02-agents/AgentSkills/">
<File Path="samples/02-agents/AgentSkills/README.md" />
<Project Path="samples/02-agents/AgentSkills/Agent_Step01_FileBasedSkills/Agent_Step01_FileBasedSkills.csproj" />
<Project Path="samples/02-agents/AgentSkills/Agent_Step02_CodeDefinedSkills/Agent_Step02_CodeDefinedSkills.csproj" />
<Project Path="samples/02-agents/AgentSkills/Agent_Step03_ClassBasedSkills/Agent_Step03_ClassBasedSkills.csproj" />
<Project Path="samples/02-agents/AgentSkills/Agent_Step04_MixedSkills/Agent_Step04_MixedSkills.csproj" />
<Project Path="samples/02-agents/AgentSkills/Agent_Step05_SkillsWithDI/Agent_Step05_SkillsWithDI.csproj" />
<Project Path="samples/02-agents/AgentSkills/Agent_Step01_BasicSkills/Agent_Step01_BasicSkills.csproj" />
</Folder>
<Folder Name="/Samples/02-agents/AGUI/Step05_StateManagement/">
<Project Path="samples/02-agents/AGUI/Step05_StateManagement/Client/Client.csproj" />
@@ -125,34 +99,6 @@
<Project Path="samples/02-agents/AgentWithAnthropic/Agent_Anthropic_Step03_UsingFunctionTools/Agent_Anthropic_Step03_UsingFunctionTools.csproj" />
<Project Path="samples/02-agents/AgentWithAnthropic/Agent_Anthropic_Step04_UsingSkills/Agent_Anthropic_Step04_UsingSkills.csproj" />
</Folder>
<Folder Name="/Samples/02-agents/AgentsWithFoundry/">
<File Path="samples/02-agents/AgentsWithFoundry/README.md" />
<Project Path="samples/02-agents/AgentsWithFoundry/Agent_Step00_FoundryAgentLifecycle/Agent_Step00_FoundryAgentLifecycle.csproj" />
<Project Path="samples/02-agents/AgentsWithFoundry/Agent_Step01_Basics/Agent_Step01_Basics.csproj" />
<Project Path="samples/02-agents/AgentsWithFoundry/Agent_Step02.1_MultiturnConversation/Agent_Step02.1_MultiturnConversation.csproj" />
<Project Path="samples/02-agents/AgentsWithFoundry/Agent_Step02.2_MultiturnWithServerConversations/Agent_Step02.2_MultiturnWithServerConversations.csproj" />
<Project Path="samples/02-agents/AgentsWithFoundry/Agent_Step03_UsingFunctionTools/Agent_Step03_UsingFunctionTools.csproj" />
<Project Path="samples/02-agents/AgentsWithFoundry/Agent_Step04_UsingFunctionToolsWithApprovals/Agent_Step04_UsingFunctionToolsWithApprovals.csproj" />
<Project Path="samples/02-agents/AgentsWithFoundry/Agent_Step05_StructuredOutput/Agent_Step05_StructuredOutput.csproj" />
<Project Path="samples/02-agents/AgentsWithFoundry/Agent_Step06_PersistedConversations/Agent_Step06_PersistedConversations.csproj" />
<Project Path="samples/02-agents/AgentsWithFoundry/Agent_Step07_Observability/Agent_Step07_Observability.csproj" />
<Project Path="samples/02-agents/AgentsWithFoundry/Agent_Step08_DependencyInjection/Agent_Step08_DependencyInjection.csproj" />
<Project Path="samples/02-agents/AgentsWithFoundry/Agent_Step09_UsingMcpClientAsTools/Agent_Step09_UsingMcpClientAsTools.csproj" />
<Project Path="samples/02-agents/AgentsWithFoundry/Agent_Step10_UsingImages/Agent_Step10_UsingImages.csproj" />
<Project Path="samples/02-agents/AgentsWithFoundry/Agent_Step11_AsFunctionTool/Agent_Step11_AsFunctionTool.csproj" />
<Project Path="samples/02-agents/AgentsWithFoundry/Agent_Step12_Middleware/Agent_Step12_Middleware.csproj" />
<Project Path="samples/02-agents/AgentsWithFoundry/Agent_Step13_Plugins/Agent_Step13_Plugins.csproj" />
<Project Path="samples/02-agents/AgentsWithFoundry/Agent_Step14_CodeInterpreter/Agent_Step14_CodeInterpreter.csproj" />
<Project Path="samples/02-agents/AgentsWithFoundry/Agent_Step15_ComputerUse/Agent_Step15_ComputerUse.csproj" />
<Project Path="samples/02-agents/AgentsWithFoundry/Agent_Step16_FileSearch/Agent_Step16_FileSearch.csproj" />
<Project Path="samples/02-agents/AgentsWithFoundry/Agent_Step17_OpenAPITools/Agent_Step17_OpenAPITools.csproj" />
<Project Path="samples/02-agents/AgentsWithFoundry/Agent_Step18_BingCustomSearch/Agent_Step18_BingCustomSearch.csproj" />
<Project Path="samples/02-agents/AgentsWithFoundry/Agent_Step19_SharePoint/Agent_Step19_SharePoint.csproj" />
<Project Path="samples/02-agents/AgentsWithFoundry/Agent_Step20_MicrosoftFabric/Agent_Step20_MicrosoftFabric.csproj" />
<Project Path="samples/02-agents/AgentsWithFoundry/Agent_Step21_WebSearch/Agent_Step21_WebSearch.csproj" />
<Project Path="samples/02-agents/AgentsWithFoundry/Agent_Step22_MemorySearch/Agent_Step22_MemorySearch.csproj" />
<Project Path="samples/02-agents/AgentsWithFoundry/Agent_Step23_LocalMCP/Agent_Step23_LocalMCP.csproj" />
</Folder>
<Folder Name="/Samples/02-agents/AgentWithMemory/">
<File Path="samples/02-agents/AgentWithMemory/README.md" />
<Project Path="samples/02-agents/AgentWithMemory/AgentWithMemory_Step01_ChatHistoryMemory/AgentWithMemory_Step01_ChatHistoryMemory.csproj" />
@@ -174,7 +120,35 @@
<Project Path="samples/02-agents/AgentWithRAG/AgentWithRAG_Step02_CustomVectorStoreRAG/AgentWithRAG_Step02_CustomVectorStoreRAG.csproj" />
<Project Path="samples/02-agents/AgentWithRAG/AgentWithRAG_Step03_CustomRAGDataSource/AgentWithRAG_Step03_CustomRAGDataSource.csproj" />
<Project Path="samples/02-agents/AgentWithRAG/AgentWithRAG_Step04_FoundryServiceRAG/AgentWithRAG_Step04_FoundryServiceRAG.csproj" />
<Project Path="samples/02-agents/AgentWithRAG/AgentWithRAG_Step05_Neo4jGraphRAG/AgentWithRAG_Step05_Neo4jGraphRAG.csproj" />
</Folder>
<Folder Name="/Samples/02-agents/FoundryAgents/">
<File Path="samples/02-agents/FoundryAgents/README.md" />
<Project Path="samples/02-agents/FoundryAgents/FoundryAgents_Evaluations_Step01_RedTeaming/FoundryAgents_Evaluations_Step01_RedTeaming.csproj" />
<Project Path="samples/02-agents/FoundryAgents/FoundryAgents_Evaluations_Step02_SelfReflection/FoundryAgents_Evaluations_Step02_SelfReflection.csproj" />
<Project Path="samples/02-agents/FoundryAgents/FoundryAgents_Step01.1_Basics/FoundryAgents_Step01.1_Basics.csproj" />
<Project Path="samples/02-agents/FoundryAgents/FoundryAgents_Step01.2_Running/FoundryAgents_Step01.2_Running.csproj" />
<Project Path="samples/02-agents/FoundryAgents/FoundryAgents_Step02_MultiturnConversation/FoundryAgents_Step02_MultiturnConversation.csproj" />
<Project Path="samples/02-agents/FoundryAgents/FoundryAgents_Step03_UsingFunctionTools/FoundryAgents_Step03_UsingFunctionTools.csproj" />
<Project Path="samples/02-agents/FoundryAgents/FoundryAgents_Step04_UsingFunctionToolsWithApprovals/FoundryAgents_Step04_UsingFunctionToolsWithApprovals.csproj" />
<Project Path="samples/02-agents/FoundryAgents/FoundryAgents_Step05_StructuredOutput/FoundryAgents_Step05_StructuredOutput.csproj" />
<Project Path="samples/02-agents/FoundryAgents/FoundryAgents_Step06_PersistedConversations/FoundryAgents_Step06_PersistedConversations.csproj" />
<Project Path="samples/02-agents/FoundryAgents/FoundryAgents_Step07_Observability/FoundryAgents_Step07_Observability.csproj" />
<Project Path="samples/02-agents/FoundryAgents/FoundryAgents_Step08_DependencyInjection/FoundryAgents_Step08_DependencyInjection.csproj" />
<Project Path="samples/02-agents/FoundryAgents/FoundryAgents_Step09_UsingMcpClientAsTools/FoundryAgents_Step09_UsingMcpClientAsTools.csproj" />
<Project Path="samples/02-agents/FoundryAgents/FoundryAgents_Step10_UsingImages/FoundryAgents_Step10_UsingImages.csproj" />
<Project Path="samples/02-agents/FoundryAgents/FoundryAgents_Step11_AsFunctionTool/FoundryAgents_Step11_AsFunctionTool.csproj" />
<Project Path="samples/02-agents/FoundryAgents/FoundryAgents_Step12_Middleware/FoundryAgents_Step12_Middleware.csproj" />
<Project Path="samples/02-agents/FoundryAgents/FoundryAgents_Step13_Plugins/FoundryAgents_Step13_Plugins.csproj" />
<Project Path="samples/02-agents/FoundryAgents/FoundryAgents_Step14_CodeInterpreter/FoundryAgents_Step14_CodeInterpreter.csproj" />
<Project Path="samples/02-agents/FoundryAgents/FoundryAgents_Step15_ComputerUse/FoundryAgents_Step15_ComputerUse.csproj" />
<Project Path="samples/02-agents/FoundryAgents/FoundryAgents_Step16_FileSearch/FoundryAgents_Step16_FileSearch.csproj" />
<Project Path="samples/02-agents/FoundryAgents/FoundryAgents_Step17_OpenAPITools/FoundryAgents_Step17_OpenAPITools.csproj" />
<Project Path="samples/02-agents/FoundryAgents/FoundryAgents_Step18_BingCustomSearch/FoundryAgents_Step18_BingCustomSearch.csproj" />
<Project Path="samples/02-agents/FoundryAgents/FoundryAgents_Step19_SharePoint/FoundryAgents_Step19_SharePoint.csproj" />
<Project Path="samples/02-agents/FoundryAgents/FoundryAgents_Step20_MicrosoftFabric/FoundryAgents_Step20_MicrosoftFabric.csproj" />
<Project Path="samples/02-agents/FoundryAgents/FoundryAgents_Step21_WebSearch/FoundryAgents_Step21_WebSearch.csproj" />
<Project Path="samples/02-agents/FoundryAgents/FoundryAgents_Step22_MemorySearch/FoundryAgents_Step22_MemorySearch.csproj" />
<Project Path="samples/02-agents/FoundryAgents/FoundryAgents_Step23_LocalMCP/FoundryAgents_Step23_LocalMCP.csproj" />
</Folder>
<Folder Name="/Samples/02-agents/ModelContextProtocol/">
<File Path="samples/02-agents/ModelContextProtocol/README.md" />
@@ -216,12 +190,12 @@
<Project Path="samples/03-workflows/Declarative/ToolApproval/ToolApproval.csproj" />
</Folder>
<Folder Name="/Samples/03-workflows/Declarative/Examples/">
<File Path="../declarative-agents/workflow-samples/CustomerSupport.yaml" />
<File Path="../declarative-agents/workflow-samples/DeepResearch.yaml" />
<File Path="../declarative-agents/workflow-samples/Marketing.yaml" />
<File Path="../declarative-agents/workflow-samples/MathChat.yaml" />
<File Path="../declarative-agents/workflow-samples/README.md" />
<File Path="../declarative-agents/workflow-samples/wttr.json" />
<File Path="../workflow-samples/CustomerSupport.yaml" />
<File Path="../workflow-samples/DeepResearch.yaml" />
<File Path="../workflow-samples/Marketing.yaml" />
<File Path="../workflow-samples/MathChat.yaml" />
<File Path="../workflow-samples/README.md" />
<File Path="../workflow-samples/wttr.json" />
</Folder>
<Folder Name="/Samples/03-workflows/SharedStates/">
<Project Path="samples/03-workflows/SharedStates/SharedStates.csproj" />
@@ -243,9 +217,6 @@
<Folder Name="/Samples/03-workflows/HumanInTheLoop/">
<Project Path="samples/03-workflows/HumanInTheLoop/HumanInTheLoopBasic/HumanInTheLoopBasic.csproj" />
</Folder>
<Folder Name="/Samples/03-workflows/Orchestration/">
<Project Path="samples/03-workflows/Orchestration/Handoff/Handoff.csproj" />
</Folder>
<Folder Name="/Samples/03-workflows/Observability/">
<Project Path="samples/03-workflows/Observability/ApplicationInsights/ApplicationInsights.csproj" />
<Project Path="samples/03-workflows/Observability/AspireDashboard/AspireDashboard.csproj" />
@@ -291,7 +262,7 @@
<File Path="samples/04-hosting/A2A/README.md" />
<Project Path="samples/04-hosting/A2A/A2AAgent_AsFunctionTools/A2AAgent_AsFunctionTools.csproj" />
<Project Path="samples/04-hosting/A2A/A2AAgent_PollingForTaskCompletion/A2AAgent_PollingForTaskCompletion.csproj" />
</Folder>
</Folder>
<Folder Name="/Samples/05-end-to-end/">
<Project Path="samples/05-end-to-end/AgentWithPurview/AgentWithPurview.csproj" />
<Project Path="samples/05-end-to-end/M365Agent/M365Agent.csproj" />
@@ -319,14 +290,15 @@
<Project Path="samples/05-end-to-end/HostedAgents/AgentWithHostedMCP/AgentWithHostedMCP.csproj" />
<Project Path="samples/05-end-to-end/HostedAgents/AgentWithLocalTools/AgentWithLocalTools.csproj" />
<Project Path="samples/05-end-to-end/HostedAgents/AgentWithTextSearchRag/AgentWithTextSearchRag.csproj" />
<Project Path="samples/05-end-to-end/HostedAgents/AgentWithTools/AgentWithTools.csproj" />
<Project Path="samples/05-end-to-end/HostedAgents/FoundryMultiAgent/FoundryMultiAgent.csproj" />
<Project Path="samples/05-end-to-end/HostedAgents/FoundrySingleAgent/FoundrySingleAgent.csproj" />
</Folder>
<Folder Name="/Samples/05-end-to-end/AspNetAgentAuthorization/">
<File Path="samples/05-end-to-end/AspNetAgentAuthorization/docker-compose.yml" />
<File Path="samples/05-end-to-end/AspNetAgentAuthorization/README.md" />
<Project Path="samples/05-end-to-end/AspNetAgentAuthorization/RazorWebClient/RazorWebClient.csproj" />
<Project Path="samples/05-end-to-end/AspNetAgentAuthorization/Service/Service.csproj" />
<Project Path="samples/05-end-to-end/AspNetAgentAuthorization/RazorWebClient/RazorWebClient.csproj" />
</Folder>
<Folder Name="/Solution Items/">
<File Path=".editorconfig" />
@@ -462,10 +434,6 @@
<File Path="src/Shared/Samples/TextOutputHelperExtensions.cs" />
<File Path="src/Shared/Samples/XunitLogger.cs" />
</Folder>
<Folder Name="/Solution Items/src/Shared/Redaction/">
<File Path="src/Shared/Redaction/README.md" />
<File Path="src/Shared/Redaction/ReplacingRedactor.cs" />
</Folder>
<Folder Name="/Solution Items/src/Shared/Throw/">
<File Path="src/Shared/Throw/README.md" />
<File Path="src/Shared/Throw/Throw.cs" />
@@ -483,13 +451,13 @@
<Project Path="src/Microsoft.Agents.AI.AGUI/Microsoft.Agents.AI.AGUI.csproj" />
<Project Path="src/Microsoft.Agents.AI.Anthropic/Microsoft.Agents.AI.Anthropic.csproj" />
<Project Path="src/Microsoft.Agents.AI.AzureAI.Persistent/Microsoft.Agents.AI.AzureAI.Persistent.csproj" />
<Project Path="src/Microsoft.Agents.AI.Foundry/Microsoft.Agents.AI.Foundry.csproj" />
<Project Path="src/Microsoft.Agents.AI.AzureAI/Microsoft.Agents.AI.AzureAI.csproj" />
<Project Path="src/Microsoft.Agents.AI.CopilotStudio/Microsoft.Agents.AI.CopilotStudio.csproj" />
<Project Path="src/Microsoft.Agents.AI.CosmosNoSql/Microsoft.Agents.AI.CosmosNoSql.csproj" />
<Project Path="src/Microsoft.Agents.AI.Declarative/Microsoft.Agents.AI.Declarative.csproj" />
<Project Path="src/Microsoft.Agents.AI.DevUI/Microsoft.Agents.AI.DevUI.csproj" />
<Project Path="src/Microsoft.Agents.AI.DurableTask/Microsoft.Agents.AI.DurableTask.csproj" />
<Project Path="src/Microsoft.Agents.AI.FoundryMemory/Microsoft.Agents.AI.FoundryMemory.csproj" />
<Project Path="src/Microsoft.Agents.AI.GitHub.Copilot/Microsoft.Agents.AI.GitHub.Copilot.csproj" />
<Project Path="src/Microsoft.Agents.AI.Hosting.A2A.AspNetCore/Microsoft.Agents.AI.Hosting.A2A.AspNetCore.csproj" />
<Project Path="src/Microsoft.Agents.AI.Hosting.A2A/Microsoft.Agents.AI.Hosting.A2A.csproj" />
@@ -500,7 +468,7 @@
<Project Path="src/Microsoft.Agents.AI.Mem0/Microsoft.Agents.AI.Mem0.csproj" />
<Project Path="src/Microsoft.Agents.AI.OpenAI/Microsoft.Agents.AI.OpenAI.csproj" />
<Project Path="src/Microsoft.Agents.AI.Purview/Microsoft.Agents.AI.Purview.csproj" />
<Project Path="src/Microsoft.Agents.AI.Workflows.Declarative.Foundry/Microsoft.Agents.AI.Workflows.Declarative.Foundry.csproj" />
<Project Path="src/Microsoft.Agents.AI.Workflows.Declarative.AzureAI/Microsoft.Agents.AI.Workflows.Declarative.AzureAI.csproj" />
<Project Path="src/Microsoft.Agents.AI.Workflows.Declarative.Mcp/Microsoft.Agents.AI.Workflows.Declarative.Mcp.csproj" />
<Project Path="src/Microsoft.Agents.AI.Workflows.Declarative/Microsoft.Agents.AI.Workflows.Declarative.csproj" />
<Project Path="src/Microsoft.Agents.AI.Workflows.Generators/Microsoft.Agents.AI.Workflows.Generators.csproj" />
@@ -511,11 +479,11 @@
<Folder Name="/Tests/IntegrationTests/">
<Project Path="tests/AgentConformance.IntegrationTests/AgentConformance.IntegrationTests.csproj" />
<Project Path="tests/AnthropicChatCompletion.IntegrationTests/AnthropicChatCompletion.IntegrationTests.csproj" />
<Project Path="tests/Foundry.IntegrationTests/Foundry.IntegrationTests.csproj" />
<Project Path="tests/AzureAI.IntegrationTests/AzureAI.IntegrationTests.csproj" />
<Project Path="tests/AzureAIAgentsPersistent.IntegrationTests/AzureAIAgentsPersistent.IntegrationTests.csproj" />
<Project Path="tests/CopilotStudio.IntegrationTests/CopilotStudio.IntegrationTests.csproj" />
<Project Path="tests/Microsoft.Agents.AI.DurableTask.IntegrationTests/Microsoft.Agents.AI.DurableTask.IntegrationTests.csproj" />
<Project Path="tests/Microsoft.Agents.AI.FoundryMemory.IntegrationTests/Microsoft.Agents.AI.FoundryMemory.IntegrationTests.csproj" />
<Project Path="tests/Microsoft.Agents.AI.GitHub.Copilot.IntegrationTests/Microsoft.Agents.AI.GitHub.Copilot.IntegrationTests.csproj" />
<Project Path="tests/Microsoft.Agents.AI.Hosting.AGUI.AspNetCore.IntegrationTests/Microsoft.Agents.AI.Hosting.AGUI.AspNetCore.IntegrationTests.csproj" />
<Project Path="tests/Microsoft.Agents.AI.Hosting.AzureFunctions.IntegrationTests/Microsoft.Agents.AI.Hosting.AzureFunctions.IntegrationTests.csproj" />
@@ -531,12 +499,12 @@
<Project Path="tests/Microsoft.Agents.AI.AGUI.UnitTests/Microsoft.Agents.AI.AGUI.UnitTests.csproj" />
<Project Path="tests/Microsoft.Agents.AI.Anthropic.UnitTests/Microsoft.Agents.AI.Anthropic.UnitTests.csproj" />
<Project Path="tests/Microsoft.Agents.AI.AzureAI.Persistent.UnitTests/Microsoft.Agents.AI.AzureAI.Persistent.UnitTests.csproj" />
<Project Path="tests/Microsoft.Agents.AI.Foundry.UnitTests/Microsoft.Agents.AI.Foundry.UnitTests.csproj" />
<Project Path="tests/Microsoft.Agents.AI.AzureAI.UnitTests/Microsoft.Agents.AI.AzureAI.UnitTests.csproj" />
<Project Path="tests/Microsoft.Agents.AI.CosmosNoSql.UnitTests/Microsoft.Agents.AI.CosmosNoSql.UnitTests.csproj" />
<Project Path="tests/Microsoft.Agents.AI.Declarative.UnitTests/Microsoft.Agents.AI.Declarative.UnitTests.csproj" />
<Project Path="tests/Microsoft.Agents.AI.DevUI.UnitTests/Microsoft.Agents.AI.DevUI.UnitTests.csproj" />
<Project Path="tests/Microsoft.Agents.AI.DurableTask.UnitTests/Microsoft.Agents.AI.DurableTask.UnitTests.csproj" />
<Project Path="tests/Microsoft.Agents.AI.FoundryMemory.UnitTests/Microsoft.Agents.AI.FoundryMemory.UnitTests.csproj" />
<Project Path="tests/Microsoft.Agents.AI.GitHub.Copilot.UnitTests/Microsoft.Agents.AI.GitHub.Copilot.UnitTests.csproj" />
<Project Path="tests/Microsoft.Agents.AI.Hosting.A2A.UnitTests/Microsoft.Agents.AI.Hosting.A2A.UnitTests.csproj" />
<Project Path="tests/Microsoft.Agents.AI.Hosting.AGUI.AspNetCore.UnitTests/Microsoft.Agents.AI.Hosting.AGUI.AspNetCore.UnitTests.csproj" />
@@ -552,4 +520,4 @@
<Project Path="tests/Microsoft.Agents.AI.Workflows.Generators.UnitTests/Microsoft.Agents.AI.Workflows.Generators.UnitTests.csproj" />
<Project Path="tests/Microsoft.Agents.AI.Workflows.UnitTests/Microsoft.Agents.AI.Workflows.UnitTests.csproj" />
</Folder>
</Solution>
</Solution>
+3 -3
View File
@@ -8,13 +8,13 @@
"src\\Microsoft.Agents.AI.Anthropic\\Microsoft.Agents.AI.Anthropic.csproj",
"src\\Microsoft.Agents.AI.GitHub.Copilot\\Microsoft.Agents.AI.GitHub.Copilot.csproj",
"src\\Microsoft.Agents.AI.AzureAI.Persistent\\Microsoft.Agents.AI.AzureAI.Persistent.csproj",
"src\\Microsoft.Agents.AI.Foundry\\Microsoft.Agents.AI.Foundry.csproj",
"src\\Microsoft.Agents.AI.AzureAI\\Microsoft.Agents.AI.AzureAI.csproj",
"src\\Microsoft.Agents.AI.CopilotStudio\\Microsoft.Agents.AI.CopilotStudio.csproj",
"src\\Microsoft.Agents.AI.CosmosNoSql\\Microsoft.Agents.AI.CosmosNoSql.csproj",
"src\\Microsoft.Agents.AI.Declarative\\Microsoft.Agents.AI.Declarative.csproj",
"src\\Microsoft.Agents.AI.DevUI\\Microsoft.Agents.AI.DevUI.csproj",
"src\\Microsoft.Agents.AI.DurableTask\\Microsoft.Agents.AI.DurableTask.csproj",
"src\\Microsoft.Agents.AI.FoundryMemory\\Microsoft.Agents.AI.FoundryMemory.csproj",
"src\\Microsoft.Agents.AI.Hosting.A2A.AspNetCore\\Microsoft.Agents.AI.Hosting.A2A.AspNetCore.csproj",
"src\\Microsoft.Agents.AI.Hosting.A2A\\Microsoft.Agents.AI.Hosting.A2A.csproj",
"src\\Microsoft.Agents.AI.Hosting.AGUI.AspNetCore\\Microsoft.Agents.AI.Hosting.AGUI.AspNetCore.csproj",
@@ -24,7 +24,7 @@
"src\\Microsoft.Agents.AI.Mem0\\Microsoft.Agents.AI.Mem0.csproj",
"src\\Microsoft.Agents.AI.OpenAI\\Microsoft.Agents.AI.OpenAI.csproj",
"src\\Microsoft.Agents.AI.Purview\\Microsoft.Agents.AI.Purview.csproj",
"src\\Microsoft.Agents.AI.Workflows.Declarative.Foundry\\Microsoft.Agents.AI.Workflows.Declarative.Foundry.csproj",
"src\\Microsoft.Agents.AI.Workflows.Declarative.AzureAI\\Microsoft.Agents.AI.Workflows.Declarative.AzureAI.csproj",
"src\\Microsoft.Agents.AI.Workflows.Declarative\\Microsoft.Agents.AI.Workflows.Declarative.csproj",
"src\\Microsoft.Agents.AI.Workflows.Generators\\Microsoft.Agents.AI.Workflows.Generators.csproj",
"src\\Microsoft.Agents.AI.Workflows\\Microsoft.Agents.AI.Workflows.csproj",
-3
View File
@@ -29,7 +29,4 @@
<ItemGroup Condition="'$(InjectSharedDiagnosticIds)' == 'true'">
<Compile Include="$(MSBuildThisFileDirectory)\..\..\src\Shared\DiagnosticIds\*.cs" LinkBase="Shared\DiagnosticIds" />
</ItemGroup>
<ItemGroup Condition="'$(InjectSharedRedaction)' == 'true'">
<Compile Include="$(MSBuildThisFileDirectory)\..\..\src\Shared\Redaction\*.cs" LinkBase="Shared\Redaction" />
</ItemGroup>
</Project>
File diff suppressed because it is too large Load Diff
@@ -1,95 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
namespace VerifySamples;
/// <summary>
/// Thread-safe console output with sample-name prefixes and colored status.
/// </summary>
internal sealed class ConsoleReporter
{
private readonly object _lock = new();
/// <summary>
/// Writes a complete prefixed line atomically to the console.
/// </summary>
public void WriteLineWithPrefix(string sampleName, string message, ConsoleColor? color = null)
{
lock (this._lock)
{
Console.ForegroundColor = ConsoleColor.Cyan;
Console.Write($"[{sampleName}] ");
if (color.HasValue)
{
Console.ForegroundColor = color.Value;
}
else
{
Console.ResetColor();
}
Console.WriteLine(message);
Console.ResetColor();
}
}
/// <summary>
/// Prints the final summary table and elapsed time to the console.
/// </summary>
public void PrintSummary(
IReadOnlyList<VerificationResult> orderedResults,
IReadOnlyList<(string Name, string Reason)> skipped,
TimeSpan elapsed)
{
var passCount = orderedResults.Count(r => r.Passed);
var failCount = orderedResults.Count(r => !r.Passed);
Console.WriteLine();
Console.WriteLine(new string('─', 60));
Console.ForegroundColor = ConsoleColor.White;
Console.WriteLine("SUMMARY");
Console.ResetColor();
foreach (var result in orderedResults)
{
Console.ForegroundColor = result.Passed ? ConsoleColor.Green : ConsoleColor.Red;
Console.Write(result.Passed ? " ✓ " : " ✗ ");
Console.ResetColor();
Console.WriteLine($"{result.SampleName}: {result.Summary}");
}
foreach (var (name, reason) in skipped)
{
Console.ForegroundColor = ConsoleColor.Yellow;
Console.Write(" ○ ");
Console.ResetColor();
Console.WriteLine($"{name}: Skipped — {reason}");
}
Console.WriteLine();
Console.Write("Results: ");
Console.ForegroundColor = ConsoleColor.Green;
Console.Write($"{passCount} passed");
Console.ResetColor();
if (failCount > 0)
{
Console.Write(", ");
Console.ForegroundColor = ConsoleColor.Red;
Console.Write($"{failCount} failed");
Console.ResetColor();
}
if (skipped.Count > 0)
{
Console.Write(", ");
Console.ForegroundColor = ConsoleColor.Yellow;
Console.Write($"{skipped.Count} skipped");
Console.ResetColor();
}
Console.WriteLine();
Console.ForegroundColor = ConsoleColor.DarkGray;
Console.WriteLine($"Elapsed: {elapsed.Hours:D2}:{elapsed.Minutes:D2}:{elapsed.Seconds:D2}");
Console.ResetColor();
}
}
@@ -1,56 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
using System.Text;
namespace VerifySamples;
/// <summary>
/// Writes a CSV summary of sample verification results.
/// </summary>
internal static class CsvResultWriter
{
/// <summary>
/// Writes the results to a CSV file at the specified path.
/// </summary>
public static async Task WriteAsync(
string path,
IReadOnlyList<VerificationResult> orderedResults,
IReadOnlyList<(string Name, string Reason)> skipped,
IReadOnlyList<SampleDefinition> samples)
{
var pathLookup = samples.ToDictionary(s => s.Name, s => s.ProjectPath);
var sb = new StringBuilder();
sb.AppendLine("Sample,ProjectPath,Status,FailedChecks,Failures");
foreach (var result in orderedResults)
{
var status = result.Passed ? "PASSED" : "FAILED";
var failedChecks = result.Failures.Count;
var failures = string.Join("; ", result.Failures);
pathLookup.TryGetValue(result.SampleName, out var projectPath);
sb.AppendLine($"{CsvEscape(result.SampleName)},{CsvEscape(projectPath ?? "")},{status},{failedChecks},{CsvEscape(failures)}");
}
foreach (var (name, reason) in skipped)
{
pathLookup.TryGetValue(name, out var projectPath);
sb.AppendLine($"{CsvEscape(name)},{CsvEscape(projectPath ?? "")},SKIPPED,0,{CsvEscape(reason)}");
}
await File.WriteAllTextAsync(path, sb.ToString());
}
/// <summary>
/// Escapes a value for CSV: wraps in quotes if it contains commas, quotes, or newlines.
/// </summary>
private static string CsvEscape(string value)
{
if (value.Contains('"') || value.Contains(',') || value.Contains('\n') || value.Contains('\r'))
{
return $"\"{value.Replace("\"", "\"\"")}\"";
}
return value;
}
}
@@ -1,105 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
namespace VerifySamples;
/// <summary>
/// Defines the expected behavior for each sample in 01-get-started.
/// </summary>
internal static class GetStartedSamples
{
public static IReadOnlyList<SampleDefinition> All { get; } =
[
new SampleDefinition
{
Name = "05_first_workflow",
ProjectPath = "samples/01-get-started/05_first_workflow",
RequiredEnvironmentVariables = [],
IsDeterministic = true,
MustContain =
[
"UppercaseExecutor: HELLO, WORLD!",
"ReverseTextExecutor: !DLROW ,OLLEH",
],
},
new SampleDefinition
{
Name = "01_hello_agent",
ProjectPath = "samples/01-get-started/01_hello_agent",
RequiredEnvironmentVariables = ["AZURE_OPENAI_ENDPOINT"],
OptionalEnvironmentVariables = ["AZURE_OPENAI_DEPLOYMENT_NAME"],
ExpectedOutputDescription =
[
"The output should contain a joke about a pirate.",
"There should be two separate joke responses — one from a non-streaming call and one from a streaming call.",
"The output should not contain error messages or stack traces.",
],
},
new SampleDefinition
{
Name = "02_add_tools",
ProjectPath = "samples/01-get-started/02_add_tools",
RequiredEnvironmentVariables = ["AZURE_OPENAI_ENDPOINT"],
OptionalEnvironmentVariables = ["AZURE_OPENAI_DEPLOYMENT_NAME"],
MustContain = [],
ExpectedOutputDescription =
[
"The output should contain information about the weather in Amsterdam.",
"The response should mention that it is cloudy with a high of 15°C (or equivalent), since this comes from a tool that returns a canned response.",
"There should be two responses — one from a non-streaming call and one from a streaming call.",
"The output should not contain error messages or stack traces.",
],
},
new SampleDefinition
{
Name = "03_multi_turn",
ProjectPath = "samples/01-get-started/03_multi_turn",
RequiredEnvironmentVariables = ["AZURE_OPENAI_ENDPOINT"],
OptionalEnvironmentVariables = ["AZURE_OPENAI_DEPLOYMENT_NAME"],
ExpectedOutputDescription =
[
"The output should contain a joke about a pirate.",
"After the initial joke, there should be a modified version that includes emojis and is told in the voice of a pirate's parrot.",
"The pattern repeats: first a non-streaming pirate joke + parrot version, then a streaming pirate joke + parrot version.",
"The output should not contain error messages or stack traces.",
],
},
new SampleDefinition
{
Name = "04_memory",
ProjectPath = "samples/01-get-started/04_memory",
RequiredEnvironmentVariables = ["AZURE_OPENAI_ENDPOINT"],
OptionalEnvironmentVariables = ["AZURE_OPENAI_DEPLOYMENT_NAME"],
MustContain =
[
">> Use session with blank memory",
">> Use deserialized session with previously created memories",
">> Read memories using memory component",
"MEMORY - User Name:",
"MEMORY - User Age:",
">> Use new session with previously created memories",
],
ExpectedOutputDescription =
[
"In the 'Use session with blank memory' section, the agent should respond to the user's messages. It may ask for the user's name or age if not yet known.",
"In the 'Use deserialized session with previously created memories' section, the agent should correctly recall that the user's name is Ruaidhrí and age is 20.",
"The 'MEMORY - User Name:' line should show 'Ruaidhrí' (or a close transliteration).",
"The 'MEMORY - User Age:' line should show '20'.",
"In the 'Use new session with previously created memories' section, the agent should know the user's name and age from the transferred memory.",
"The output should not contain error messages or stack traces.",
],
},
new SampleDefinition
{
Name = "06_host_your_agent",
ProjectPath = "samples/01-get-started/06_host_your_agent",
RequiredEnvironmentVariables = ["AZURE_OPENAI_ENDPOINT"],
OptionalEnvironmentVariables = ["AZURE_OPENAI_DEPLOYMENT_NAME"],
SkipReason = "Requires Azure Functions Core Tools runtime and starts a web server.",
},
];
}
-153
View File
@@ -1,153 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
using System.Text;
namespace VerifySamples;
/// <summary>
/// Incrementally writes a sequential (non-interleaved) log file, appending after each sample completes.
/// Thread-safe: multiple parallel tasks may call write methods concurrently.
/// </summary>
internal sealed class LogFileWriter : IDisposable
{
private readonly string _path;
private readonly SemaphoreSlim _writeLock = new(1, 1);
public LogFileWriter(string path)
{
this._path = path;
}
/// <inheritdoc />
public void Dispose()
{
this._writeLock.Dispose();
}
/// <summary>
/// Writes the log file header. Call once at the start of the run.
/// </summary>
public async Task WriteHeaderAsync()
{
var sb = new StringBuilder();
sb.AppendLine($"Sample Verification Log — {DateTime.UtcNow:yyyy-MM-dd HH:mm:ss} UTC");
sb.AppendLine(new string('═', 72));
sb.AppendLine();
await File.WriteAllTextAsync(this._path, sb.ToString());
}
/// <summary>
/// Appends a skipped-sample entry to the log file.
/// </summary>
public async Task WriteSkippedAsync(string name, string reason)
{
var sb = new StringBuilder();
sb.AppendLine($"── {name} ──");
sb.AppendLine($"Status: SKIPPED — {reason}");
sb.AppendLine();
await this.AppendAsync(sb.ToString());
}
/// <summary>
/// Appends a completed sample's full output section to the log file.
/// </summary>
public async Task WriteSampleResultAsync(VerificationResult result)
{
var sb = new StringBuilder();
sb.AppendLine(new string('─', 72));
sb.AppendLine($"── {result.SampleName} ──");
sb.AppendLine($"Status: {(result.Passed ? "PASSED" : "FAILED")}");
sb.AppendLine();
foreach (var line in result.LogLines)
{
sb.AppendLine(line);
}
sb.AppendLine();
if (!string.IsNullOrWhiteSpace(result.Stdout))
{
sb.AppendLine("--- stdout ---");
sb.AppendLine(result.Stdout.TrimEnd());
sb.AppendLine("--- end stdout ---");
sb.AppendLine();
}
if (!string.IsNullOrWhiteSpace(result.Stderr))
{
sb.AppendLine("--- stderr ---");
sb.AppendLine(result.Stderr.TrimEnd());
sb.AppendLine("--- end stderr ---");
sb.AppendLine();
}
if (result.Failures.Count > 0)
{
sb.AppendLine("Failures:");
foreach (var failure in result.Failures)
{
sb.AppendLine($" ✗ {failure}");
}
sb.AppendLine();
}
if (result.AIReasoning is not null)
{
sb.AppendLine("AI Reasoning:");
sb.AppendLine(result.AIReasoning);
sb.AppendLine();
}
await this.AppendAsync(sb.ToString());
}
/// <summary>
/// Appends the final summary section and elapsed time to the log file.
/// </summary>
public async Task WriteSummaryAsync(
IReadOnlyList<VerificationResult> orderedResults,
IReadOnlyList<(string Name, string Reason)> skipped,
TimeSpan elapsed)
{
var passCount = orderedResults.Count(r => r.Passed);
var failCount = orderedResults.Count(r => !r.Passed);
var sb = new StringBuilder();
sb.AppendLine(new string('═', 72));
sb.AppendLine("SUMMARY");
sb.AppendLine();
foreach (var result in orderedResults)
{
sb.AppendLine($" {(result.Passed ? "" : "")} {result.SampleName}: {result.Summary}");
}
foreach (var (name, reason) in skipped)
{
sb.AppendLine($" ○ {name}: Skipped — {reason}");
}
sb.AppendLine();
sb.AppendLine($"Results: {passCount} passed{(failCount > 0 ? $", {failCount} failed" : "")}{(skipped.Count > 0 ? $", {skipped.Count} skipped" : "")}");
sb.AppendLine($"Elapsed: {elapsed.Hours:D2}:{elapsed.Minutes:D2}:{elapsed.Seconds:D2}");
await this.AppendAsync(sb.ToString());
}
private async Task AppendAsync(string text)
{
await this._writeLock.WaitAsync();
try
{
await File.AppendAllTextAsync(this._path, text);
}
finally
{
this._writeLock.Release();
}
}
}
@@ -1,98 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
using System.Text;
namespace VerifySamples;
/// <summary>
/// Writes a Markdown summary of sample verification results.
/// </summary>
internal static class MarkdownResultWriter
{
/// <summary>
/// Writes the results to a Markdown file at the specified path.
/// </summary>
public static async Task WriteAsync(
string path,
IReadOnlyList<VerificationResult> orderedResults,
IReadOnlyList<(string Name, string Reason)> skipped,
TimeSpan elapsed)
{
var passCount = orderedResults.Count(r => r.Passed);
var failCount = orderedResults.Count(r => !r.Passed);
var sb = new StringBuilder();
sb.AppendLine("# Sample Verification Results");
sb.AppendLine();
sb.AppendLine($"**{passCount} passed, {failCount} failed, {skipped.Count} skipped** | Elapsed: {elapsed.Hours:D2}:{elapsed.Minutes:D2}:{elapsed.Seconds:D2}");
sb.AppendLine();
// Results table
sb.AppendLine("| Sample | Status | Failed Checks | Failures |");
sb.AppendLine("|--------|--------|---------------|----------|");
foreach (var result in orderedResults)
{
var status = result.Passed ? "✅ PASSED" : "❌ FAILED";
var failedChecks = result.Failures.Count;
var failures = MdEscape(string.Join("; ", result.Failures));
sb.AppendLine($"| {MdEscape(result.SampleName)} | {status} | {failedChecks} | {failures} |");
}
foreach (var (name, reason) in skipped)
{
sb.AppendLine($"| {MdEscape(name)} | ⏭️ SKIPPED | 0 | {MdEscape(reason)} |");
}
// Collapsible AI reasoning details for failures
var failures2 = orderedResults.Where(r => !r.Passed && !string.IsNullOrEmpty(r.AIReasoning)).ToList();
if (failures2.Count > 0)
{
sb.AppendLine();
sb.AppendLine("## Failure Details");
sb.AppendLine();
foreach (var result in failures2)
{
sb.AppendLine($"<details><summary><strong>{HtmlEscape(result.SampleName)}</strong></summary>");
sb.AppendLine();
if (result.Failures.Count > 0)
{
foreach (var failure in result.Failures)
{
sb.AppendLine($"- {MdEscape(failure)}");
}
sb.AppendLine();
}
sb.AppendLine("**AI Reasoning:**");
sb.AppendLine();
sb.AppendLine("```");
sb.AppendLine(result.AIReasoning);
sb.AppendLine("```");
sb.AppendLine();
sb.AppendLine("</details>");
sb.AppendLine();
}
}
await File.WriteAllTextAsync(path, sb.ToString());
}
/// <summary>
/// Escapes pipe characters and newlines for use inside Markdown table cells.
/// </summary>
private static string MdEscape(string value)
{
return value.Replace("|", "\\|").Replace("\n", " ").Replace("\r", "");
}
/// <summary>
/// Escapes HTML special characters for use inside HTML tags.
/// </summary>
private static string HtmlEscape(string value)
{
return value.Replace("&", "&amp;").Replace("<", "&lt;").Replace(">", "&gt;").Replace("\"", "&quot;");
}
}
-109
View File
@@ -1,109 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
// This tool runs the 01-get-started, 02-agents, and 03-workflows samples and verifies their output.
// Deterministic samples are verified with exact string matching.
// Non-deterministic (LLM) samples are verified using an agent-framework agent.
//
// Usage:
// dotnet run # Run all samples
// dotnet run -- 01_hello_agent 05_first_workflow # Run specific samples by name
// dotnet run -- --category 01-get-started # Run the 01-get-started category
// dotnet run -- --category 02-agents # Run the 02-agents category
// dotnet run -- --category 03-workflows # Run the 03-workflows category
// dotnet run -- --parallel 16 # Run up to 16 samples concurrently
// dotnet run -- --log results.log # Write sequential log to file
// dotnet run -- --csv results.csv # Write CSV summary to file
// dotnet run -- --md results.md # Write Markdown summary to file
// dotnet run -- --build # Build samples during run (default: --no-build)
// Note: By default, this tool expects sample build outputs to already exist.
// Pre-build the solution before running, or pass --build to avoid missing build output failures.
//
// Required environment variables (for AI-powered samples):
// AZURE_OPENAI_ENDPOINT
// AZURE_OPENAI_DEPLOYMENT_NAME (optional, defaults to gpt-5-mini)
using System.Diagnostics;
using Azure.AI.OpenAI;
using Azure.Identity;
using VerifySamples;
var options = VerifyOptions.Parse(args);
if (options is null)
{
return 1;
}
var stopwatch = Stopwatch.StartNew();
// Resolve the dotnet/ root directory (verify-samples is at dotnet/eng/verify-samples/)
var dotnetRoot = Path.GetFullPath(Path.Combine(AppContext.BaseDirectory, "..", "..", "..", "..", ".."));
if (!File.Exists(Path.Combine(dotnetRoot, "agent-framework-dotnet.slnx")))
{
dotnetRoot = Path.GetFullPath(Path.Combine(Directory.GetCurrentDirectory(), "..", ".."));
}
// Set up the AI verifier
var endpoint = Environment.GetEnvironmentVariable("AZURE_OPENAI_ENDPOINT");
var deploymentName = Environment.GetEnvironmentVariable("AZURE_OPENAI_DEPLOYMENT_NAME") ?? "gpt-5-mini";
OpenAI.Chat.ChatClient? chatClient = null;
if (!string.IsNullOrEmpty(endpoint))
{
chatClient = new AzureOpenAIClient(new Uri(endpoint), new DefaultAzureCredential())
.GetChatClient(deploymentName);
}
// Set up optional log file writer
LogFileWriter? logWriter = null;
if (options.LogFilePath is not null)
{
logWriter = new LogFileWriter(options.LogFilePath);
await logWriter.WriteHeaderAsync();
}
try
{
// Run all samples
var reporter = new ConsoleReporter();
var verifier = new SampleVerifier(chatClient);
var orchestrator = new VerificationOrchestrator(verifier, reporter, dotnetRoot, TimeSpan.FromMinutes(3), logWriter, buildSamples: options.BuildSamples);
var run = await orchestrator.RunAllAsync(options.Samples, options.MaxParallelism);
stopwatch.Stop();
// Print summary
var orderedResults = run.SampleOrder
.Where(run.Results.ContainsKey)
.Select(name => run.Results[name])
.ToList();
reporter.PrintSummary(orderedResults, run.Skipped, stopwatch.Elapsed);
// Write log file summary
if (logWriter is not null)
{
await logWriter.WriteSummaryAsync(orderedResults, run.Skipped, stopwatch.Elapsed);
Console.WriteLine($"Log written to: {options.LogFilePath}");
}
// Write CSV summary
if (options.CsvFilePath is not null)
{
await CsvResultWriter.WriteAsync(options.CsvFilePath, orderedResults, run.Skipped, options.Samples);
Console.WriteLine($"CSV written to: {options.CsvFilePath}");
}
// Write Markdown summary
if (options.MarkdownFilePath is not null)
{
await MarkdownResultWriter.WriteAsync(options.MarkdownFilePath, orderedResults, run.Skipped, stopwatch.Elapsed);
Console.WriteLine($"Markdown written to: {options.MarkdownFilePath}");
}
return orderedResults.Any(r => !r.Passed) ? 1 : 0;
}
finally
{
logWriter?.Dispose();
}
@@ -1,79 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
namespace VerifySamples;
/// <summary>
/// Describes a sample to verify, including its expected output.
/// </summary>
internal sealed class SampleDefinition
{
/// <summary>
/// Display name for the sample (e.g., "01_hello_agent").
/// </summary>
public required string Name { get; init; }
/// <summary>
/// Relative path from the dotnet/ directory to the sample project directory.
/// </summary>
public required string ProjectPath { get; init; }
/// <summary>
/// Environment variables that the sample requires for a meaningful run.
/// The runner checks these before running and will skip the sample if any are unset,
/// recording a skip reason that indicates which required variables are missing.
/// </summary>
public string[] RequiredEnvironmentVariables { get; init; } = [];
/// <summary>
/// Environment variables that the sample can use but typically has fallbacks or defaults for.
/// If these are not set, the sample might prompt or behave interactively, which could cause
/// automated verification to hang. The runner checks these and skips the sample if they are unset
/// to avoid non-deterministic or blocking behavior in automated runs.
/// </summary>
public string[] OptionalEnvironmentVariables { get; init; } = [];
/// <summary>
/// If set, the sample is skipped with this reason.
/// Use only for structural reasons (e.g., web server, multi-process, needs external service).
/// Do NOT use for missing environment variables — those are checked dynamically.
/// </summary>
public string? SkipReason { get; init; }
/// <summary>
/// Substrings that must appear in stdout for the sample to pass.
/// Used for deterministic verification.
/// </summary>
public string[] MustContain { get; init; } = [];
/// <summary>
/// Substrings that must not appear in stdout for the sample to pass.
/// </summary>
public string[] MustNotContain { get; init; } = [];
/// <summary>
/// If true, <see cref="MustContain"/> entries cover the entire expected output —
/// no AI verification is needed.
/// </summary>
public bool IsDeterministic { get; init; }
/// <summary>
/// Natural-language description of what the sample output should look like.
/// Used by the AI verifier for non-deterministic samples.
/// Each entry describes one aspect of the expected output that should be verified.
/// </summary>
public string[] ExpectedOutputDescription { get; init; } = [];
/// <summary>
/// Sequence of stdin inputs to feed to the sample process.
/// Each entry is written as a line (followed by newline) to the process stdin.
/// A <c>null</c> entry inserts a delay without writing anything.
/// Inputs are sent with a short delay between each to allow the process to prompt.
/// </summary>
public string?[] Inputs { get; init; } = [];
/// <summary>
/// Delay in milliseconds between each input line. Default is 2000ms.
/// Increase for samples that need more time between prompts (e.g., LLM calls between inputs).
/// </summary>
public int InputDelayMs { get; init; } = 2000;
}
-141
View File
@@ -1,141 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
using System.Diagnostics;
namespace VerifySamples;
/// <summary>
/// Result of running a sample process.
/// </summary>
internal sealed record SampleRunResult(
string Stdout,
string Stderr,
int ExitCode,
TimeSpan Elapsed);
/// <summary>
/// Runs a sample project via <c>dotnet run</c> and captures its output.
/// </summary>
internal static class SampleRunner
{
/// <summary>
/// Runs <c>dotnet run --framework net10.0</c> in the given project directory.
/// When <paramref name="build"/> is false (the default), <c>--no-build</c> is passed
/// to skip building, assuming the project was pre-built.
/// </summary>
public static Task<SampleRunResult> RunAsync(
string projectPath,
TimeSpan timeout,
bool build = false,
CancellationToken cancellationToken = default)
=> RunAsync(projectPath, DotnetRunArgs(build), timeout, inputs: null, inputDelayMs: 0, cancellationToken: cancellationToken);
/// <summary>
/// Runs <c>dotnet run --framework net10.0</c> with stdin inputs.
/// When <paramref name="build"/> is false (the default), <c>--no-build</c> is passed
/// to skip building, assuming the project was pre-built.
/// </summary>
public static Task<SampleRunResult> RunAsync(
string projectPath,
TimeSpan timeout,
string?[]? inputs,
int inputDelayMs = 2000,
bool build = false,
CancellationToken cancellationToken = default)
=> RunAsync(projectPath, DotnetRunArgs(build), timeout, inputs, inputDelayMs, cancellationToken);
private static string DotnetRunArgs(bool build) =>
$"run {(build ? "" : "--no-build")} --framework net10.0";
/// <summary>
/// Runs an arbitrary <c>dotnet</c> command in the given working directory.
/// </summary>
public static async Task<SampleRunResult> RunAsync(
string workingDirectory,
string dotnetArgs,
TimeSpan timeout,
string?[]? inputs = null,
int inputDelayMs = 0,
CancellationToken cancellationToken = default)
{
var psi = new ProcessStartInfo
{
FileName = "dotnet",
Arguments = dotnetArgs,
WorkingDirectory = workingDirectory,
RedirectStandardOutput = true,
RedirectStandardError = true,
RedirectStandardInput = inputs is { Length: > 0 },
UseShellExecute = false,
CreateNoWindow = true,
};
var sw = Stopwatch.StartNew();
using var process = new Process { StartInfo = psi };
process.Start();
var stdoutTask = process.StandardOutput.ReadToEndAsync(cancellationToken);
var stderrTask = process.StandardError.ReadToEndAsync(cancellationToken);
// Feed stdin inputs with delays if configured
if (inputs is { Length: > 0 })
{
_ = Task.Run(async () =>
{
try
{
foreach (var input in inputs)
{
await Task.Delay(inputDelayMs, cancellationToken);
if (input is not null)
{
await process.StandardInput.WriteLineAsync(input.AsMemory(), cancellationToken);
await process.StandardInput.FlushAsync(cancellationToken);
}
}
process.StandardInput.Close();
}
catch (Exception ex) when (ex is IOException or ObjectDisposedException or OperationCanceledException)
{
// Process may have exited before all inputs were sent
}
}, cancellationToken);
}
using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
cts.CancelAfter(timeout);
try
{
await process.WaitForExitAsync(cts.Token);
}
catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested)
{
// Timeout — kill the process
try
{
process.Kill(entireProcessTree: true);
}
catch
{
// Best effort
}
sw.Stop();
return new SampleRunResult(
Stdout: await stdoutTask,
Stderr: $"TIMEOUT: Sample did not complete within {timeout.TotalSeconds}s.\n{await stderrTask}",
ExitCode: -1,
Elapsed: sw.Elapsed);
}
sw.Stop();
return new SampleRunResult(
Stdout: await stdoutTask,
Stderr: await stderrTask,
ExitCode: process.ExitCode,
Elapsed: sw.Elapsed);
}
}
-229
View File
@@ -1,229 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
using System.ComponentModel;
using System.Text.Json.Serialization;
using Microsoft.Agents.AI;
using Microsoft.Extensions.AI;
using OpenAI.Chat;
namespace VerifySamples;
/// <summary>
/// Verifies sample output using deterministic checks and an AI agent
/// for non-deterministic output validation.
/// </summary>
internal sealed class SampleVerifier
{
private readonly AIAgent? _verifierAgent;
/// <summary>
/// Creates a verifier. If <paramref name="chatClient"/> is provided,
/// AI-based verification is available for non-deterministic samples.
/// </summary>
public SampleVerifier(ChatClient? chatClient = null)
{
if (chatClient is not null)
{
this._verifierAgent = chatClient.AsAIAgent(
instructions: """
You are a test output verifier. You will be given:
1. The actual stdout output of a program
2. The stderr output (if any)
3. A list of expectations about what the output should contain or demonstrate
Your job is to determine whether the actual output satisfies each expectation.
Be reasonable the output comes from an LLM so exact wording won't match, but the
semantic intent should be clearly satisfied.
In your response, you MUST:
- Always provide ai_reasoning with a brief overall assessment.
- Always provide exactly one entry in expectation_results for each expectation,
in the same order as the input list.
- For each expectation_results entry, echo the expectation text in the expectation
field and explain your assessment in the detail field, citing evidence from the output.
""",
name: "OutputVerifier");
}
}
/// <summary>
/// Verifies the output of a sample run against its definition.
/// </summary>
public async Task<VerificationResult> VerifyAsync(SampleDefinition sample, SampleRunResult run)
{
var failures = new List<string>();
// 1. Exit code check
if (run.ExitCode != 0)
{
failures.Add($"Exit code was {run.ExitCode}, expected 0. Stderr: {Truncate(run.Stderr, 500)}");
}
// 2. Must-contain checks
foreach (var expected in sample.MustContain)
{
if (!run.Stdout.Contains(expected, StringComparison.Ordinal))
{
failures.Add($"Output missing expected substring: \"{expected}\"");
}
}
// 3. Must-not-contain checks
foreach (var unexpected in sample.MustNotContain)
{
if (run.Stdout.Contains(unexpected, StringComparison.Ordinal))
{
failures.Add($"Output contains unexpected substring: \"{unexpected}\"");
}
}
// 4. AI verification for non-deterministic samples
string? aiReasoning = null;
if (!sample.IsDeterministic && sample.ExpectedOutputDescription.Length > 0)
{
if (this._verifierAgent is null)
{
failures.Add("AI verification required but no AI agent configured (missing AZURE_OPENAI_ENDPOINT).");
}
else
{
var aiResult = await this.VerifyWithAIAsync(run.Stdout, run.Stderr, sample.ExpectedOutputDescription);
aiReasoning = aiResult.Reasoning;
foreach (var unmet in aiResult.UnmetExpectations)
{
failures.Add($"AI expectation not met: {unmet}");
}
}
}
bool passed = failures.Count == 0;
return new VerificationResult
{
SampleName = sample.Name,
Passed = passed,
Summary = passed ? "All checks passed" : $"{failures.Count} check(s) failed",
Failures = failures,
AIReasoning = aiReasoning,
};
}
private async Task<(string Reasoning, List<string> UnmetExpectations)> VerifyWithAIAsync(
string stdout,
string stderr,
string[] expectations)
{
var expectationList = string.Join("\n", expectations.Select((e, i) => $" {i + 1}. {e}"));
var stderrSection = string.IsNullOrWhiteSpace(stderr)
? ""
: $"""
Stderr output:
---
{Truncate(stderr, 2000)}
---
""";
var prompt = $"""
Actual program output:
---
{Truncate(stdout, 4000)}
---
{stderrSection}
Expectations to verify:
{expectationList}
Does the output satisfy all expectations?
""";
try
{
var response = await this._verifierAgent!.RunAsync<AIVerificationResponse>(prompt);
var result = response.Result;
if (result is null)
{
return ($"AI verification returned null result. Raw: {response.Text}", ["AI verification returned null result."]);
}
var reasoning = string.IsNullOrWhiteSpace(result.AIReasoning)
? "(no reasoning provided)"
: result.AIReasoning;
// Collect unmet expectations as individual failures
var unmet = new List<string>();
if (result.ExpectationResults is { Count: > 0 })
{
foreach (var er in result.ExpectationResults.Where(er => !er.Met))
{
var detail = string.IsNullOrWhiteSpace(er.Detail) ? er.Expectation : $"{er.Expectation} — {er.Detail}";
unmet.Add(detail ?? "Unknown expectation");
}
// If the model flagged overall failure but all individual expectations were met,
// still treat as failure using the overall reasoning.
if (unmet.Count == 0 && !result.Pass)
{
unmet.Add(reasoning);
}
}
else if (!result.Pass)
{
// Fallback: no per-expectation detail but overall pass is false
unmet.Add(reasoning);
}
return (reasoning, unmet);
}
catch (Exception ex)
{
return ($"AI verification error: {ex.Message}", [$"AI verification error: {ex.Message}"]);
}
}
private static string Truncate(string text, int maxLength)
=> text.Length <= maxLength ? text : text[..maxLength] + "... (truncated)";
}
/// <summary>
/// Structured response from the AI verification agent.
/// </summary>
[System.Diagnostics.CodeAnalysis.SuppressMessage("Performance", "CA1812:Avoid uninstantiated internal classes", Justification = "Instantiated by JSON deserialization via RunAsync<T>.")]
internal sealed class AIVerificationResponse
{
/// <summary>Whether all expectations were met.</summary>
[JsonPropertyName("pass")]
public bool Pass { get; set; }
/// <summary>Brief explanation of the overall assessment.</summary>
[JsonPropertyName("ai_reasoning")]
[Description("Always required. Brief explanation of the overall assessment, covering all expectations.")]
public string AIReasoning { get; set; } = string.Empty;
/// <summary>Per-expectation results.</summary>
[JsonPropertyName("expectation_results")]
[Description("Always required. One entry per expectation, in the same order as the input list.")]
public List<ExpectationResult> ExpectationResults { get; set; } = [];
}
/// <summary>
/// Result for an individual expectation check.
/// </summary>
[System.Diagnostics.CodeAnalysis.SuppressMessage("Performance", "CA1812:Avoid uninstantiated internal classes", Justification = "Instantiated by JSON deserialization via RunAsync<T>.")]
internal sealed class ExpectationResult
{
/// <summary>The expectation text that was evaluated.</summary>
[JsonPropertyName("expectation")]
[Description("Echo back the expectation text being evaluated.")]
public string Expectation { get; set; } = string.Empty;
/// <summary>Whether this expectation was met.</summary>
[JsonPropertyName("met")]
public bool Met { get; set; }
/// <summary>Detail about how the expectation was or was not met.</summary>
[JsonPropertyName("detail")]
[Description("Explain how the expectation was or was not met, citing specific evidence from the output.")]
public string Detail { get; set; } = string.Empty;
}
@@ -1,200 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
using System.Collections.Concurrent;
namespace VerifySamples;
/// <summary>
/// Orchestrates sample verification: filters, runs in parallel, and collects results.
/// </summary>
internal sealed class VerificationOrchestrator
{
private readonly SampleVerifier _verifier;
private readonly ConsoleReporter _reporter;
private readonly LogFileWriter? _logWriter;
private readonly string _dotnetRoot;
private readonly TimeSpan _timeout;
private readonly bool _buildSamples;
public VerificationOrchestrator(
SampleVerifier verifier,
ConsoleReporter reporter,
string dotnetRoot,
TimeSpan timeout,
LogFileWriter? logWriter = null,
bool buildSamples = false)
{
this._verifier = verifier;
this._reporter = reporter;
this._logWriter = logWriter;
this._dotnetRoot = dotnetRoot;
this._timeout = timeout;
this._buildSamples = buildSamples;
}
/// <summary>
/// The result of running all samples through the orchestrator.
/// </summary>
internal sealed record RunAllResult(
ConcurrentDictionary<string, VerificationResult> Results,
List<(string Name, string Reason)> Skipped,
List<string> SampleOrder);
/// <summary>
/// Filters samples, runs the runnable ones in parallel, and returns all results.
/// </summary>
public async Task<RunAllResult> RunAllAsync(
IReadOnlyList<SampleDefinition> samples,
int maxParallelism)
{
var skipped = new List<(string Name, string Reason)>();
var runnableSamples = new List<SampleDefinition>();
var sampleOrder = new List<string>();
// Separate samples into skipped and runnable
foreach (var sample in samples)
{
sampleOrder.Add(sample.Name);
if (sample.SkipReason is not null)
{
skipped.Add((sample.Name, sample.SkipReason));
this._reporter.WriteLineWithPrefix(sample.Name, $"SKIPPED — {sample.SkipReason}", ConsoleColor.Yellow);
if (this._logWriter is not null)
{
await this._logWriter.WriteSkippedAsync(sample.Name, sample.SkipReason);
}
continue;
}
var missingRequired = sample.RequiredEnvironmentVariables
.Where(v => string.IsNullOrEmpty(Environment.GetEnvironmentVariable(v)))
.ToList();
var missingOptional = sample.OptionalEnvironmentVariables
.Where(v => string.IsNullOrEmpty(Environment.GetEnvironmentVariable(v)))
.ToList();
if (missingRequired.Count > 0 || missingOptional.Count > 0)
{
var reasons = new List<string>();
if (missingRequired.Count > 0)
{
reasons.Add($"Missing required: {string.Join(", ", missingRequired)}");
}
if (missingOptional.Count > 0)
{
reasons.Add($"Missing optional (would cause console prompt hang): {string.Join(", ", missingOptional)}");
}
var skipReason = string.Join("; ", reasons);
skipped.Add((sample.Name, skipReason));
this._reporter.WriteLineWithPrefix(sample.Name, $"SKIPPED — {skipReason}", ConsoleColor.Yellow);
if (this._logWriter is not null)
{
await this._logWriter.WriteSkippedAsync(sample.Name, skipReason);
}
continue;
}
runnableSamples.Add(sample);
}
// Run samples in parallel
var results = new ConcurrentDictionary<string, VerificationResult>();
var semaphore = new SemaphoreSlim(maxParallelism);
this._reporter.WriteLineWithPrefix(
"runner", $"Running {runnableSamples.Count} samples (max {maxParallelism} parallel)...");
try
{
var tasks = runnableSamples.Select(sample => this.RunSingleAsync(sample, results, semaphore)).ToArray();
await Task.WhenAll(tasks);
}
finally
{
semaphore.Dispose();
}
return new RunAllResult(results, skipped, sampleOrder);
}
private async Task RunSingleAsync(
SampleDefinition sample,
ConcurrentDictionary<string, VerificationResult> results,
SemaphoreSlim semaphore)
{
await semaphore.WaitAsync();
try
{
var log = new List<string>();
log.Add($"[{sample.Name}] Running...");
this._reporter.WriteLineWithPrefix(sample.Name, "Running...");
var projectPath = Path.Combine(this._dotnetRoot, sample.ProjectPath);
var run = sample.Inputs.Length > 0
? await SampleRunner.RunAsync(projectPath, this._timeout, sample.Inputs, sample.InputDelayMs, build: this._buildSamples)
: await SampleRunner.RunAsync(projectPath, this._timeout, build: this._buildSamples);
log.Add($"[{sample.Name}] Completed ({run.Elapsed.TotalSeconds:F1}s, exit={run.ExitCode})");
this._reporter.WriteLineWithPrefix(
sample.Name, $"Completed ({run.Elapsed.TotalSeconds:F1}s, exit={run.ExitCode}). Verifying...");
var result = await this._verifier.VerifyAsync(sample, run);
if (result.Passed)
{
log.Add($"[{sample.Name}] PASSED");
this._reporter.WriteLineWithPrefix(sample.Name, "PASSED", ConsoleColor.Green);
}
else
{
log.Add($"[{sample.Name}] FAILED");
this._reporter.WriteLineWithPrefix(sample.Name, "FAILED", ConsoleColor.Red);
foreach (var failure in result.Failures)
{
log.Add($"[{sample.Name}] ✗ {failure}");
this._reporter.WriteLineWithPrefix(sample.Name, $" ✗ {failure}", ConsoleColor.Red);
}
}
if (result.AIReasoning is not null)
{
log.Add($"[{sample.Name}] AI: {result.AIReasoning}");
this._reporter.WriteLineWithPrefix(
sample.Name, $" AI: {Truncate(result.AIReasoning, 300)}", ConsoleColor.DarkGray);
}
var verificationResult = new VerificationResult
{
SampleName = result.SampleName,
Passed = result.Passed,
Summary = result.Summary,
Failures = result.Failures,
AIReasoning = result.AIReasoning,
Stdout = run.Stdout,
Stderr = run.Stderr,
LogLines = log,
};
results[sample.Name] = verificationResult;
if (this._logWriter is not null)
{
await this._logWriter.WriteSampleResultAsync(verificationResult);
}
}
finally
{
semaphore.Release();
}
}
private static string Truncate(string text, int maxLength)
=> text.Length <= maxLength ? text : text[..maxLength] + "...";
}
@@ -1,31 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
namespace VerifySamples;
/// <summary>
/// The result of verifying a single sample.
/// </summary>
internal sealed class VerificationResult
{
public required string SampleName { get; init; }
public required bool Passed { get; init; }
public required string Summary { get; init; }
public List<string> Failures { get; init; } = [];
public string? AIReasoning { get; init; }
/// <summary>
/// The sample's stdout output, captured for log file output.
/// </summary>
public string? Stdout { get; init; }
/// <summary>
/// The sample's stderr output, captured for log file output.
/// </summary>
public string? Stderr { get; init; }
/// <summary>
/// Per-sample log lines, buffered during parallel execution
/// and written sequentially to the log file.
/// </summary>
public List<string> LogLines { get; init; } = [];
}
-151
View File
@@ -1,151 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
namespace VerifySamples;
/// <summary>
/// Parsed command-line options for the sample verification tool.
/// </summary>
internal sealed class VerifyOptions
{
/// <summary>
/// Maximum number of samples to run concurrently.
/// </summary>
public int MaxParallelism { get; init; } = 8;
/// <summary>
/// Path to write a CSV summary file, or <c>null</c> to skip.
/// </summary>
public string? CsvFilePath { get; init; }
/// <summary>
/// Path to write a Markdown summary file, or <c>null</c> to skip.
/// </summary>
public string? MarkdownFilePath { get; init; }
/// <summary>
/// Path to write a sequential log file, or <c>null</c> to skip.
/// </summary>
public string? LogFilePath { get; init; }
/// <summary>
/// When true, samples are built as part of <c>dotnet run</c>.
/// When false (the default), <c>--no-build</c> is passed, assuming a prior build step.
/// </summary>
public bool BuildSamples { get; init; }
/// <summary>
/// The filtered list of samples to process.
/// </summary>
public required IReadOnlyList<SampleDefinition> Samples { get; init; }
/// <summary>
/// All known sample set registries, keyed by category name.
/// </summary>
private static readonly Dictionary<string, IReadOnlyList<SampleDefinition>> s_sampleSets =
new(StringComparer.OrdinalIgnoreCase)
{
["01-get-started"] = GetStartedSamples.All,
["02-agents"] = AgentsSamples.All,
["03-workflows"] = WorkflowSamples.All,
};
/// <summary>
/// Parses command-line arguments and resolves the sample list.
/// Returns <c>null</c> and writes to stderr if the arguments are invalid.
/// </summary>
public static VerifyOptions? Parse(string[] args)
{
var argList = args.ToList();
var categoryFilter = ExtractArg(argList, "--category");
var logFilePath = ExtractArg(argList, "--log");
var csvFilePath = ExtractArg(argList, "--csv");
var markdownFilePath = ExtractArg(argList, "--md");
var buildSamples = ExtractFlag(argList, "--build");
int maxParallelism = 8;
var parallelArg = ExtractArg(argList, "--parallel");
if (parallelArg is not null && int.TryParse(parallelArg, out var p) && p > 0)
{
maxParallelism = p;
}
HashSet<string>? nameFilter = null;
if (argList.Count > 0)
{
nameFilter = argList.ToHashSet(StringComparer.OrdinalIgnoreCase);
}
// Build the sample list
IReadOnlyList<SampleDefinition> samples;
if (categoryFilter is not null)
{
if (!s_sampleSets.TryGetValue(categoryFilter, out var categoryList))
{
Console.Error.WriteLine(
$"Unknown category '{categoryFilter}'. Available: {string.Join(", ", s_sampleSets.Keys)}");
return null;
}
samples = categoryList;
}
else
{
samples = s_sampleSets.Values.SelectMany(s => s).ToList();
}
if (nameFilter is not null)
{
samples = samples.Where(s => nameFilter.Contains(s.Name)).ToList();
}
if (samples.Count == 0)
{
var allNames = s_sampleSets.Values.SelectMany(s => s).Select(s => s.Name);
Console.Error.WriteLine($"No matching samples found. Available: {string.Join(", ", allNames)}");
return null;
}
return new VerifyOptions
{
MaxParallelism = maxParallelism,
LogFilePath = logFilePath,
CsvFilePath = csvFilePath,
MarkdownFilePath = markdownFilePath,
BuildSamples = buildSamples,
Samples = samples,
};
}
private static string? ExtractArg(List<string> list, string flag)
{
var idx = list.IndexOf(flag);
if (idx < 0)
{
return null;
}
if (idx + 1 >= list.Count)
{
Console.Error.WriteLine($"Missing value for {flag}.");
list.RemoveAt(idx);
return null;
}
var value = list[idx + 1];
list.RemoveRange(idx, 2);
return value;
}
private static bool ExtractFlag(List<string> list, string flag)
{
var idx = list.IndexOf(flag);
if (idx < 0)
{
return false;
}
list.RemoveAt(idx);
return true;
}
}
@@ -1,525 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
namespace VerifySamples;
/// <summary>
/// Defines the expected behavior for each sample in 03-workflows.
/// </summary>
internal static class WorkflowSamples
{
public static IReadOnlyList<SampleDefinition> All { get; } =
[
// ───────────────────────────────────────────────────────────────────
// _StartHere
// ───────────────────────────────────────────────────────────────────
new SampleDefinition
{
Name = "Workflow_StartHere_01_Streaming",
ProjectPath = "samples/03-workflows/_StartHere/01_Streaming",
RequiredEnvironmentVariables = [],
IsDeterministic = true,
MustContain =
[
"UppercaseExecutor: HELLO, WORLD!",
"ReverseTextExecutor: !DLROW ,OLLEH",
],
},
new SampleDefinition
{
Name = "Workflow_StartHere_02_AgentsInWorkflows",
ProjectPath = "samples/03-workflows/_StartHere/02_AgentsInWorkflows",
RequiredEnvironmentVariables = ["AZURE_OPENAI_ENDPOINT"],
OptionalEnvironmentVariables = ["AZURE_OPENAI_DEPLOYMENT_NAME"],
ExpectedOutputDescription =
[
"The output should show agent responses from a translation workflow.",
"The output should not contain error messages or stack traces.",
],
},
new SampleDefinition
{
Name = "Workflow_StartHere_03_AgentWorkflowPatterns",
ProjectPath = "samples/03-workflows/_StartHere/03_AgentWorkflowPatterns",
RequiredEnvironmentVariables = ["AZURE_OPENAI_ENDPOINT"],
OptionalEnvironmentVariables = ["AZURE_OPENAI_DEPLOYMENT_NAME"],
Inputs = ["sequential"],
InputDelayMs = 3000,
ExpectedOutputDescription =
[
"The output should show a sequential workflow pattern with multiple agents executing tasks in order.",
"The output should not contain error messages or stack traces.",
],
},
new SampleDefinition
{
Name = "Workflow_StartHere_04_MultiModelService",
ProjectPath = "samples/03-workflows/_StartHere/04_MultiModelService",
RequiredEnvironmentVariables = ["BEDROCK_ACCESS_KEY", "BEDROCK_SECRET_KEY", "ANTHROPIC_API_KEY", "OPENAI_API_KEY"],
SkipReason = "Requires multiple external provider API keys (Bedrock, Anthropic, OpenAI).",
},
new SampleDefinition
{
Name = "Workflow_StartHere_05_SubWorkflows",
ProjectPath = "samples/03-workflows/_StartHere/05_SubWorkflows",
RequiredEnvironmentVariables = [],
IsDeterministic = true,
MustContain =
[
"=== Sub-Workflow Demonstration ===",
"Final Output:",
"=== Main Workflow Completed ===",
"Sample Complete: Workflows can be composed hierarchically using sub-workflows",
],
},
new SampleDefinition
{
Name = "Workflow_StartHere_06_MixedWorkflowAgentsAndExecutors",
ProjectPath = "samples/03-workflows/_StartHere/06_MixedWorkflowAgentsAndExecutors",
RequiredEnvironmentVariables = ["AZURE_OPENAI_ENDPOINT"],
OptionalEnvironmentVariables = ["AZURE_OPENAI_DEPLOYMENT_NAME"],
Inputs = ["What is 2 plus 2?"],
InputDelayMs = 3000,
ExpectedOutputDescription =
[
"The output should show agents and executors working together to process a user question.",
"The output should not contain error messages or stack traces.",
],
},
new SampleDefinition
{
Name = "Workflow_StartHere_07_WriterCriticWorkflow",
ProjectPath = "samples/03-workflows/_StartHere/07_WriterCriticWorkflow",
RequiredEnvironmentVariables = ["AZURE_OPENAI_ENDPOINT"],
OptionalEnvironmentVariables = ["AZURE_OPENAI_DEPLOYMENT_NAME"],
MustContain = ["=== Writer-Critic Iteration Workflow ==="],
ExpectedOutputDescription =
[
"The output should show a writer-critic iteration workflow with writer and critic sections.",
"The critic should either approve or request revisions.",
"The output should not contain error messages or stack traces.",
],
},
// ───────────────────────────────────────────────────────────────────
// Agents
// ───────────────────────────────────────────────────────────────────
new SampleDefinition
{
Name = "Workflow_Agents_CustomAgentExecutors",
ProjectPath = "samples/03-workflows/Agents/CustomAgentExecutors",
RequiredEnvironmentVariables = ["AZURE_OPENAI_ENDPOINT"],
OptionalEnvironmentVariables = ["AZURE_OPENAI_DEPLOYMENT_NAME"],
ExpectedOutputDescription =
[
"The output should show custom workflow events including slogan generation and feedback.",
"The output should not contain error messages or stack traces.",
],
},
new SampleDefinition
{
Name = "Workflow_Agents_FoundryAgent",
ProjectPath = "samples/03-workflows/Agents/FoundryAgent",
RequiredEnvironmentVariables = ["AZURE_AI_PROJECT_ENDPOINT"],
OptionalEnvironmentVariables = ["AZURE_AI_MODEL_DEPLOYMENT_NAME"],
SkipReason = "Requires Azure AI Foundry project endpoint.",
},
new SampleDefinition
{
Name = "Workflow_Agents_GroupChatToolApproval",
ProjectPath = "samples/03-workflows/Agents/GroupChatToolApproval",
RequiredEnvironmentVariables = ["AZURE_OPENAI_ENDPOINT"],
OptionalEnvironmentVariables = ["AZURE_OPENAI_DEPLOYMENT_NAME"],
MustContain = ["Starting group chat workflow for software deployment..."],
ExpectedOutputDescription =
[
"The output should show a group chat workflow with QA and DevOps agents for software deployment.",
"There should be approval requests for tool calls.",
"The workflow should show interaction between QA and DevOps agents toward deployment.",
"The output should not contain error messages or stack traces.",
],
},
new SampleDefinition
{
Name = "Workflow_Agents_WorkflowAsAnAgent",
ProjectPath = "samples/03-workflows/Agents/WorkflowAsAnAgent",
RequiredEnvironmentVariables = ["AZURE_OPENAI_ENDPOINT"],
OptionalEnvironmentVariables = ["AZURE_OPENAI_DEPLOYMENT_NAME"],
Inputs = ["hello", "exit"],
InputDelayMs = 5000,
ExpectedOutputDescription =
[
"The output should show a conversational workflow responding to the user's hello message.",
"The output should not contain error messages or stack traces.",
],
},
// ───────────────────────────────────────────────────────────────────
// Checkpoint
// ───────────────────────────────────────────────────────────────────
new SampleDefinition
{
Name = "Workflow_Checkpoint_CheckpointAndRehydrate",
ProjectPath = "samples/03-workflows/Checkpoint/CheckpointAndRehydrate",
RequiredEnvironmentVariables = [],
IsDeterministic = true,
MustContain =
[
"Workflow completed with result:",
"Number of checkpoints created:",
"Hydrating a new workflow instance from the 6th checkpoint.",
],
},
new SampleDefinition
{
Name = "Workflow_Checkpoint_CheckpointAndResume",
ProjectPath = "samples/03-workflows/Checkpoint/CheckpointAndResume",
RequiredEnvironmentVariables = [],
IsDeterministic = true,
MustContain =
[
"Workflow completed with result:",
"Number of checkpoints created:",
"Restoring from the 6th checkpoint.",
],
},
new SampleDefinition
{
Name = "Workflow_Checkpoint_CheckpointWithHumanInTheLoop",
ProjectPath = "samples/03-workflows/Checkpoint/CheckpointWithHumanInTheLoop",
RequiredEnvironmentVariables = [],
Inputs = ["50", "25", "40", "45", "42", "50", "25", "40", "45", "42"],
InputDelayMs = 1000,
MustContain = ["found in"],
ExpectedOutputDescription =
[
"The output should show a number guessing game with higher/lower hints that eventually reaches the correct number.",
"The output should demonstrate checkpoint save and restore behavior.",
],
},
// ───────────────────────────────────────────────────────────────────
// Concurrent
// ───────────────────────────────────────────────────────────────────
new SampleDefinition
{
Name = "Workflow_Concurrent_Concurrent",
ProjectPath = "samples/03-workflows/Concurrent/Concurrent",
RequiredEnvironmentVariables = ["AZURE_OPENAI_ENDPOINT"],
OptionalEnvironmentVariables = ["AZURE_OPENAI_DEPLOYMENT_NAME"],
ExpectedOutputDescription =
[
"The output should show results from concurrent agent processing.",
"The output should not contain error messages or stack traces.",
],
},
new SampleDefinition
{
Name = "Workflow_Concurrent_MapReduce",
ProjectPath = "samples/03-workflows/Concurrent/MapReduce",
RequiredEnvironmentVariables = [],
MustContain =
[
"=== RUNNING WORKFLOW ===",
],
},
// ───────────────────────────────────────────────────────────────────
// ConditionalEdges
// ───────────────────────────────────────────────────────────────────
new SampleDefinition
{
Name = "Workflow_ConditionalEdges_01_EdgeCondition",
ProjectPath = "samples/03-workflows/ConditionalEdges/01_EdgeCondition",
RequiredEnvironmentVariables = ["AZURE_OPENAI_ENDPOINT"],
OptionalEnvironmentVariables = ["AZURE_OPENAI_DEPLOYMENT_NAME"],
ExpectedOutputDescription =
[
"The output should show an email being classified as spam or not spam and processed accordingly.",
"The output should not contain error messages or stack traces.",
],
},
new SampleDefinition
{
Name = "Workflow_ConditionalEdges_02_SwitchCase",
ProjectPath = "samples/03-workflows/ConditionalEdges/02_SwitchCase",
RequiredEnvironmentVariables = ["AZURE_OPENAI_ENDPOINT"],
OptionalEnvironmentVariables = ["AZURE_OPENAI_DEPLOYMENT_NAME"],
ExpectedOutputDescription =
[
"The output should show an ambiguous email being classified as spam, not spam, or uncertain.",
"The output should not contain error messages or stack traces.",
],
},
new SampleDefinition
{
Name = "Workflow_ConditionalEdges_03_MultiSelection",
ProjectPath = "samples/03-workflows/ConditionalEdges/03_MultiSelection",
RequiredEnvironmentVariables = ["AZURE_OPENAI_ENDPOINT"],
OptionalEnvironmentVariables = ["AZURE_OPENAI_DEPLOYMENT_NAME"],
ExpectedOutputDescription =
[
"The output should show an email being classified and potentially routed to multiple handlers.",
"The output should not contain error messages or stack traces.",
],
},
// ───────────────────────────────────────────────────────────────────
// HumanInTheLoop
// ───────────────────────────────────────────────────────────────────
new SampleDefinition
{
Name = "Workflow_HumanInTheLoop_Basic",
ProjectPath = "samples/03-workflows/HumanInTheLoop/HumanInTheLoopBasic",
RequiredEnvironmentVariables = [],
Inputs = ["50", "25", "40", "45", "42"],
InputDelayMs = 1000,
MustContain = ["found in"],
ExpectedOutputDescription =
[
"The output should show a number guessing game with higher/lower hints that eventually reaches the correct number 42.",
],
},
// ───────────────────────────────────────────────────────────────────
// Loop
// ───────────────────────────────────────────────────────────────────
new SampleDefinition
{
Name = "Workflow_Loop",
ProjectPath = "samples/03-workflows/Loop",
RequiredEnvironmentVariables = [],
MustContain = ["Result:"],
},
// ───────────────────────────────────────────────────────────────────
// SharedStates
// ───────────────────────────────────────────────────────────────────
new SampleDefinition
{
Name = "Workflow_SharedStates",
ProjectPath = "samples/03-workflows/SharedStates",
RequiredEnvironmentVariables = [],
IsDeterministic = true,
MustContain =
[
"Total Paragraphs:",
"Total Words:",
],
},
// ───────────────────────────────────────────────────────────────────
// Visualization
// ───────────────────────────────────────────────────────────────────
new SampleDefinition
{
Name = "Workflow_Visualization",
ProjectPath = "samples/03-workflows/Visualization",
RequiredEnvironmentVariables = [],
IsDeterministic = true,
MustContain =
[
"Generating workflow visualization...",
"Mermaid string:",
"DiGraph string:",
],
},
// ───────────────────────────────────────────────────────────────────
// Observability
// ───────────────────────────────────────────────────────────────────
new SampleDefinition
{
Name = "Workflow_Observability_ApplicationInsights",
ProjectPath = "samples/03-workflows/Observability/ApplicationInsights",
RequiredEnvironmentVariables = ["APPLICATIONINSIGHTS_CONNECTION_STRING"],
SkipReason = "Requires Application Insights connection string.",
},
new SampleDefinition
{
Name = "Workflow_Observability_AspireDashboard",
ProjectPath = "samples/03-workflows/Observability/AspireDashboard",
RequiredEnvironmentVariables = [],
SkipReason = "Requires Aspire Dashboard / OTLP endpoint.",
},
new SampleDefinition
{
Name = "Workflow_Observability_WorkflowAsAnAgent",
ProjectPath = "samples/03-workflows/Observability/WorkflowAsAnAgent",
RequiredEnvironmentVariables = ["AZURE_OPENAI_ENDPOINT"],
OptionalEnvironmentVariables = ["AZURE_OPENAI_DEPLOYMENT_NAME"],
SkipReason = "Interactive console with ReadLine loop; requires OTLP endpoint.",
},
// ───────────────────────────────────────────────────────────────────
// Declarative
// ───────────────────────────────────────────────────────────────────
new SampleDefinition
{
Name = "Workflow_Declarative_ConfirmInput",
ProjectPath = "samples/03-workflows/Declarative/ConfirmInput",
RequiredEnvironmentVariables = ["AZURE_AI_PROJECT_ENDPOINT"],
Inputs = ["hello", "hello"],
InputDelayMs = 8000,
ExpectedOutputDescription = ["The output should show a confirmation prompt and a user response."],
},
new SampleDefinition
{
Name = "Workflow_Declarative_CustomerSupport",
ProjectPath = "samples/03-workflows/Declarative/CustomerSupport",
RequiredEnvironmentVariables = ["AZURE_AI_PROJECT_ENDPOINT"],
OptionalEnvironmentVariables = ["AZURE_AI_MODEL_DEPLOYMENT_NAME"],
Inputs = ["My laptop won't start"],
InputDelayMs = 3000,
ExpectedOutputDescription = ["The output should show a customer support workflow processing a laptop issue, with agent responses providing troubleshooting or support."],
},
new SampleDefinition
{
Name = "Workflow_Declarative_DeepResearch",
ProjectPath = "samples/03-workflows/Declarative/DeepResearch",
RequiredEnvironmentVariables = ["AZURE_AI_PROJECT_ENDPOINT"],
OptionalEnvironmentVariables = ["AZURE_AI_MODEL_DEPLOYMENT_NAME"],
SkipReason = "Requires external weather API (wttr.in).",
},
new SampleDefinition
{
Name = "Workflow_Declarative_ExecuteCode",
ProjectPath = "samples/03-workflows/Declarative/ExecuteCode",
RequiredEnvironmentVariables = ["AZURE_AI_PROJECT_ENDPOINT"],
Inputs = ["What is 12 * 34?"],
InputDelayMs = 5000,
ExpectedOutputDescription = ["The output should show a declarative workflow executing generated code, processing a math question and producing a result."],
},
new SampleDefinition
{
Name = "Workflow_Declarative_ExecuteWorkflow",
ProjectPath = "samples/03-workflows/Declarative/ExecuteWorkflow",
RequiredEnvironmentVariables = ["AZURE_AI_PROJECT_ENDPOINT"],
SkipReason = "Requires a workflow file path as a CLI argument.",
},
new SampleDefinition
{
Name = "Workflow_Declarative_FunctionTools",
ProjectPath = "samples/03-workflows/Declarative/FunctionTools",
RequiredEnvironmentVariables = ["AZURE_AI_PROJECT_ENDPOINT"],
OptionalEnvironmentVariables = ["AZURE_AI_MODEL_DEPLOYMENT_NAME"],
Inputs = ["What are today's specials?", "EXIT"],
InputDelayMs = 8000,
ExpectedOutputDescription = ["The output should show a workflow calling function tools (e.g. a menu plugin) to answer a question about restaurant specials."],
},
new SampleDefinition
{
Name = "Workflow_Declarative_GenerateCode",
ProjectPath = "samples/03-workflows/Declarative/GenerateCode",
IsDeterministic = true,
MustContain = ["WORKFLOW: Parsing", "WORKFLOW: Defined"],
ExpectedOutputDescription = ["The output should show a YAML workflow being parsed and C# code being generated from it."],
},
new SampleDefinition
{
Name = "Workflow_Declarative_HostedWorkflow",
ProjectPath = "samples/03-workflows/Declarative/HostedWorkflow",
RequiredEnvironmentVariables = ["AZURE_AI_PROJECT_ENDPOINT"],
SkipReason = "Hosts a persistent workflow server that does not exit.",
},
new SampleDefinition
{
Name = "Workflow_Declarative_InputArguments",
ProjectPath = "samples/03-workflows/Declarative/InputArguments",
RequiredEnvironmentVariables = ["AZURE_AI_PROJECT_ENDPOINT"],
OptionalEnvironmentVariables = ["AZURE_AI_MODEL_DEPLOYMENT_NAME"],
Inputs = ["I'd like to visit Seattle", "EXIT"],
InputDelayMs = 8000,
ExpectedOutputDescription = ["The output should show a workflow capturing location input and providing travel-related information about Seattle."],
},
new SampleDefinition
{
Name = "Workflow_Declarative_InvokeFunctionTool",
ProjectPath = "samples/03-workflows/Declarative/InvokeFunctionTool",
RequiredEnvironmentVariables = ["AZURE_AI_PROJECT_ENDPOINT"],
OptionalEnvironmentVariables = ["AZURE_AI_MODEL_DEPLOYMENT_NAME"],
Inputs = ["What's the soup of the day?", "EXIT"],
InputDelayMs = 8000,
ExpectedOutputDescription = ["The output should show a workflow invoking a function tool (e.g. a menu plugin) to answer a question about the soup of the day."],
},
new SampleDefinition
{
Name = "Workflow_Declarative_InvokeMcpTool",
ProjectPath = "samples/03-workflows/Declarative/InvokeMcpTool",
RequiredEnvironmentVariables = ["AZURE_AI_PROJECT_ENDPOINT"],
OptionalEnvironmentVariables = ["AZURE_AI_MODEL_DEPLOYMENT_NAME"],
Inputs = ["Search for .NET tutorials on Microsoft Learn"],
InputDelayMs = 3000,
ExpectedOutputDescription = ["The output should show a workflow using MCP tools to search Microsoft Learn documentation and provide a summary of results."],
},
new SampleDefinition
{
Name = "Workflow_Declarative_Marketing",
ProjectPath = "samples/03-workflows/Declarative/Marketing",
RequiredEnvironmentVariables = ["AZURE_AI_PROJECT_ENDPOINT"],
OptionalEnvironmentVariables = ["AZURE_AI_MODEL_DEPLOYMENT_NAME"],
Inputs = ["A smart water bottle that tracks hydration"],
InputDelayMs = 3000,
ExpectedOutputDescription = ["The output should show a marketing workflow generating content about a smart water bottle product."],
},
new SampleDefinition
{
Name = "Workflow_Declarative_StudentTeacher",
ProjectPath = "samples/03-workflows/Declarative/StudentTeacher",
RequiredEnvironmentVariables = ["AZURE_AI_PROJECT_ENDPOINT"],
OptionalEnvironmentVariables = ["AZURE_AI_MODEL_DEPLOYMENT_NAME"],
Inputs = ["What is 18 + 27?"],
InputDelayMs = 3000,
ExpectedOutputDescription = ["The output should show a student-teacher workflow where a student asks a math question and a teacher provides the answer."],
},
new SampleDefinition
{
Name = "Workflow_Declarative_ToolApproval",
ProjectPath = "samples/03-workflows/Declarative/ToolApproval",
RequiredEnvironmentVariables = ["AZURE_AI_PROJECT_ENDPOINT"],
OptionalEnvironmentVariables = ["AZURE_AI_MODEL_DEPLOYMENT_NAME"],
Inputs = ["Search for .NET tutorials", "EXIT"],
InputDelayMs = 8000,
ExpectedOutputDescription = ["The output should show a workflow using an MCP tool with approval to search Microsoft Learn, followed by an exit from the input loop."],
},
];
}
@@ -1,24 +0,0 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFrameworks>net10.0</TargetFrameworks>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
<IsPackable>false</IsPackable>
<IsAotCompatible>false</IsAotCompatible>
<!-- This is a top-level console app; ConfigureAwait is unnecessary -->
<NoWarn>$(NoWarn);CA2007</NoWarn>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Azure.AI.OpenAI" />
<PackageReference Include="Azure.Identity" />
<PackageReference Include="Microsoft.Extensions.AI.OpenAI" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\..\src\Microsoft.Agents.AI.OpenAI\Microsoft.Agents.AI.OpenAI.csproj" />
</ItemGroup>
</Project>
+6 -9
View File
@@ -1,21 +1,18 @@
<Project>
<PropertyGroup>
<!-- Central version prefix - applies to all nuget packages. -->
<VersionPrefix>1.1.0</VersionPrefix>
<RCNumber>1</RCNumber>
<VersionPrefix>1.0.0</VersionPrefix>
<RCNumber>4</RCNumber>
<PackageVersion Condition="'$(IsReleaseCandidate)' == 'true'">$(VersionPrefix)-rc$(RCNumber)</PackageVersion>
<PackageVersion Condition="'$(IsReleaseCandidate)' != 'true' AND '$(VersionSuffix)' != ''">$(VersionPrefix)-$(VersionSuffix).260410.1</PackageVersion>
<PackageVersion Condition="'$(IsReleaseCandidate)' != 'true' AND '$(VersionSuffix)' == ''">$(VersionPrefix)-preview.260410.1</PackageVersion>
<PackageVersion Condition="'$(IsReleased)' == 'true'">$(VersionPrefix)</PackageVersion>
<GitTag>1.1.0</GitTag>
<PackageVersion Condition="'$(IsReleaseCandidate)' != 'true' AND '$(VersionSuffix)' != ''">$(VersionPrefix)-$(VersionSuffix).260311.1</PackageVersion>
<PackageVersion Condition="'$(IsReleaseCandidate)' != 'true' AND '$(VersionSuffix)' == ''">$(VersionPrefix)-preview.260311.1</PackageVersion>
<GitTag>1.0.0-rc4</GitTag>
<Configurations>Debug;Release;Publish</Configurations>
<IsPackable>true</IsPackable>
<!-- Package validation. Baseline Version should be the latest version available on NuGet. -->
<PackageValidationBaselineVersion>1.0.0</PackageValidationBaselineVersion>
<!-- Enable validation for GA packages -->
<EnablePackageValidation Condition="'$(IsReleased)' == 'true'">true</EnablePackageValidation>
<PackageValidationBaselineVersion>0.0.1</PackageValidationBaselineVersion>
<!-- Validate assembly attributes only for Publish builds -->
<NoWarn Condition="'$(Configuration)' != 'Publish'">$(NoWarn);CP0003</NoWarn>
<!-- Do not validate reference assemblies -->
@@ -8,7 +8,7 @@ using Microsoft.Agents.AI;
using OpenAI.Chat;
var endpoint = Environment.GetEnvironmentVariable("AZURE_OPENAI_ENDPOINT") ?? throw new InvalidOperationException("AZURE_OPENAI_ENDPOINT is not set.");
var deploymentName = Environment.GetEnvironmentVariable("AZURE_OPENAI_DEPLOYMENT_NAME") ?? "gpt-5.4-mini";
var deploymentName = Environment.GetEnvironmentVariable("AZURE_OPENAI_DEPLOYMENT_NAME") ?? "gpt-4o-mini";
// WARNING: DefaultAzureCredential is convenient for development but requires careful consideration in production.
// In production, consider using a specific credential (e.g., ManagedIdentityCredential) to avoid
@@ -11,7 +11,7 @@ using Microsoft.Extensions.AI;
using OpenAI.Chat;
var endpoint = Environment.GetEnvironmentVariable("AZURE_OPENAI_ENDPOINT") ?? throw new InvalidOperationException("AZURE_OPENAI_ENDPOINT is not set.");
var deploymentName = Environment.GetEnvironmentVariable("AZURE_OPENAI_DEPLOYMENT_NAME") ?? "gpt-5.4-mini";
var deploymentName = Environment.GetEnvironmentVariable("AZURE_OPENAI_DEPLOYMENT_NAME") ?? "gpt-4o-mini";
[Description("Get the weather for a given location.")]
static string GetWeather([Description("The location to get the weather for.")] string location)
@@ -8,7 +8,7 @@ using Microsoft.Agents.AI;
using OpenAI.Chat;
var endpoint = Environment.GetEnvironmentVariable("AZURE_OPENAI_ENDPOINT") ?? throw new InvalidOperationException("AZURE_OPENAI_ENDPOINT is not set.");
var deploymentName = Environment.GetEnvironmentVariable("AZURE_OPENAI_DEPLOYMENT_NAME") ?? "gpt-5.4-mini";
var deploymentName = Environment.GetEnvironmentVariable("AZURE_OPENAI_DEPLOYMENT_NAME") ?? "gpt-4o-mini";
// WARNING: DefaultAzureCredential is convenient for development but requires careful consideration in production.
// In production, consider using a specific credential (e.g., ManagedIdentityCredential) to avoid
@@ -16,7 +16,7 @@ using OpenAI.Chat;
using SampleApp;
var endpoint = Environment.GetEnvironmentVariable("AZURE_OPENAI_ENDPOINT") ?? throw new InvalidOperationException("AZURE_OPENAI_ENDPOINT is not set.");
var deploymentName = Environment.GetEnvironmentVariable("AZURE_OPENAI_DEPLOYMENT_NAME") ?? "gpt-5.4-mini";
var deploymentName = Environment.GetEnvironmentVariable("AZURE_OPENAI_DEPLOYMENT_NAME") ?? "gpt-4o-mini";
// WARNING: DefaultAzureCredential is convenient for development but requires careful consideration in production.
// In production, consider using a specific credential (e.g., ManagedIdentityCredential) to avoid
@@ -8,7 +8,7 @@
//
// Environment variables:
// AZURE_OPENAI_ENDPOINT
// AZURE_OPENAI_DEPLOYMENT_NAME (defaults to "gpt-5.4-mini")
// AZURE_OPENAI_DEPLOYMENT_NAME (defaults to "gpt-4o-mini")
//
// Run with: func start
// Then call: POST http://localhost:7071/api/agents/HostedAgent/run
@@ -23,7 +23,7 @@ using OpenAI.Chat;
var endpoint = Environment.GetEnvironmentVariable("AZURE_OPENAI_ENDPOINT")
?? throw new InvalidOperationException("AZURE_OPENAI_ENDPOINT is not set.");
var deploymentName = Environment.GetEnvironmentVariable("AZURE_OPENAI_DEPLOYMENT_NAME") ?? "gpt-5.4-mini";
var deploymentName = Environment.GetEnvironmentVariable("AZURE_OPENAI_DEPLOYMENT_NAME") ?? "gpt-4o-mini";
// Set up an AI agent following the standard Microsoft Agent Framework pattern.
// WARNING: DefaultAzureCredential is convenient for development but requires careful consideration in production.
+1 -1
View File
@@ -15,7 +15,7 @@ All samples require the following environment variables:
```bash
export AZURE_OPENAI_ENDPOINT="https://your-resource.openai.azure.com/"
export AZURE_OPENAI_DEPLOYMENT_NAME="gpt-5.4-mini"
export AZURE_OPENAI_DEPLOYMENT_NAME="gpt-4o-mini"
```
For the client samples, you can optionally set:
@@ -59,18 +59,18 @@ while ((input = Console.ReadLine()) != null && !input.Equals("exit", StringCompa
{
switch (content)
{
case ToolApprovalRequestContent approvalRequest when approvalRequest.ToolCall is FunctionCallContent fcc:
DisplayApprovalRequest(approvalRequest, fcc);
case FunctionApprovalRequestContent approvalRequest:
DisplayApprovalRequest(approvalRequest);
Console.Write($"\nApprove '{fcc.Name}'? (yes/no): ");
Console.Write($"\nApprove '{approvalRequest.FunctionCall.Name}'? (yes/no): ");
string? userInput = Console.ReadLine();
bool approved = userInput?.ToUpperInvariant() is "YES" or "Y";
ToolApprovalResponseContent approvalResponse = approvalRequest.CreateResponse(approved);
FunctionApprovalResponseContent approvalResponse = approvalRequest.CreateResponse(approved);
if (approvalRequest.AdditionalProperties != null)
{
approvalResponse.AdditionalProperties = [];
approvalResponse.AdditionalProperties = new AdditionalPropertiesDictionary();
foreach (var kvp in approvalRequest.AdditionalProperties)
{
approvalResponse.AdditionalProperties[kvp.Key] = kvp.Value;
@@ -128,19 +128,19 @@ while ((input = Console.ReadLine()) != null && !input.Equals("exit", StringCompa
}
#pragma warning disable MEAI001
static void DisplayApprovalRequest(ToolApprovalRequestContent approvalRequest, FunctionCallContent fcc)
static void DisplayApprovalRequest(FunctionApprovalRequestContent approvalRequest)
{
Console.ForegroundColor = ConsoleColor.Yellow;
Console.WriteLine();
Console.WriteLine("============================================================");
Console.WriteLine("APPROVAL REQUIRED");
Console.WriteLine("============================================================");
Console.WriteLine($"Function: {fcc.Name}");
Console.WriteLine($"Function: {approvalRequest.FunctionCall.Name}");
if (fcc.Arguments != null)
if (approvalRequest.FunctionCall.Arguments != null)
{
Console.WriteLine("Arguments:");
foreach (var arg in fcc.Arguments)
foreach (var arg in approvalRequest.FunctionCall.Arguments)
{
Console.WriteLine($" {arg.Key} = {arg.Value}");
}
@@ -9,7 +9,7 @@ using ServerFunctionApproval;
/// <summary>
/// A delegating agent that handles server function approval requests and responses.
/// Transforms between ToolApprovalRequestContent/ToolApprovalResponseContent
/// Transforms between FunctionApprovalRequestContent/FunctionApprovalResponseContent
/// and the server's request_approval tool call pattern.
/// </summary>
internal sealed class ServerFunctionApprovalClientAgent : DelegatingAIAgent
@@ -50,14 +50,14 @@ internal sealed class ServerFunctionApprovalClientAgent : DelegatingAIAgent
}
#pragma warning disable MEAI001 // Type is for evaluation purposes only
private static FunctionResultContent ConvertApprovalResponseToToolResult(ToolApprovalResponseContent approvalResponse, JsonSerializerOptions jsonOptions)
private static FunctionResultContent ConvertApprovalResponseToToolResult(FunctionApprovalResponseContent approvalResponse, JsonSerializerOptions jsonOptions)
{
return new FunctionResultContent(
callId: approvalResponse.RequestId,
callId: approvalResponse.Id,
result: JsonSerializer.SerializeToElement(
new ApprovalResponse
{
ApprovalId = approvalResponse.RequestId,
ApprovalId = approvalResponse.Id,
Approved = approvalResponse.Approved
},
jsonOptions));
@@ -89,7 +89,7 @@ internal sealed class ServerFunctionApprovalClientAgent : DelegatingAIAgent
{
List<ChatMessage>? result = null;
Dictionary<string, ToolApprovalRequestContent> approvalRequests = [];
Dictionary<string, FunctionApprovalRequestContent> approvalRequests = [];
for (var messageIndex = 0; messageIndex < messages.Count; messageIndex++)
{
var message = messages[messageIndex];
@@ -102,21 +102,21 @@ internal sealed class ServerFunctionApprovalClientAgent : DelegatingAIAgent
var content = message.Contents[contentIndex];
// Handle pending approval requests (transform to tool call)
if (content is ToolApprovalRequestContent approvalRequest &&
if (content is FunctionApprovalRequestContent approvalRequest &&
approvalRequest.AdditionalProperties?.TryGetValue("original_function", out var originalFunction) == true &&
originalFunction is FunctionCallContent original)
{
approvalRequests[approvalRequest.RequestId] = approvalRequest;
approvalRequests[approvalRequest.Id] = approvalRequest;
transformedContents ??= CopyContentsUpToIndex(message.Contents, contentIndex);
transformedContents.Add(original);
}
// Handle pending approval responses (transform to tool result)
else if (content is ToolApprovalResponseContent approvalResponse &&
approvalRequests.TryGetValue(approvalResponse.RequestId, out var correspondingRequest))
else if (content is FunctionApprovalResponseContent approvalResponse &&
approvalRequests.TryGetValue(approvalResponse.Id, out var correspondingRequest))
{
transformedContents ??= CopyContentsUpToIndex(message.Contents, contentIndex);
transformedContents.Add(ConvertApprovalResponseToToolResult(approvalResponse, jsonSerializerOptions));
approvalRequests.Remove(approvalResponse.RequestId);
approvalRequests.Remove(approvalResponse.Id);
correspondingRequest.AdditionalProperties?.Remove("original_function");
}
// Skip historical approval content
@@ -131,9 +131,9 @@ internal sealed class ServerFunctionApprovalClientAgent : DelegatingAIAgent
transformedContents ??= CopyContentsUpToIndex(message.Contents, contentIndex);
approvalCalls.Remove(functionResult.CallId);
}
else
else if (transformedContents != null)
{
transformedContents?.Add(content);
transformedContents.Add(content);
}
}
@@ -155,10 +155,10 @@ internal sealed class ServerFunctionApprovalClientAgent : DelegatingAIAgent
result ??= CopyMessagesUpToIndex(messages, messageIndex);
result.Add(newMessage);
}
else
else if (result != null)
{
// We're already copying messages, so copy this unchanged message too
result?.Add(message);
result.Add(message);
}
// If result is null, we haven't made any changes yet, so keep processing
}
@@ -198,8 +198,8 @@ internal sealed class ServerFunctionApprovalClientAgent : DelegatingAIAgent
var functionCallArgs = (Dictionary<string, object?>?)approvalRequest.FunctionArguments?
.Deserialize(jsonSerializerOptions.GetTypeInfo(typeof(Dictionary<string, object?>)));
var approvalRequestContent = new ToolApprovalRequestContent(
requestId: approvalRequest.ApprovalId,
var approvalRequestContent = new FunctionApprovalRequestContent(
id: approvalRequest.ApprovalId,
new FunctionCallContent(
callId: approvalRequest.ApprovalId,
name: approvalRequest.FunctionName,
@@ -9,7 +9,7 @@ using ServerFunctionApproval;
/// <summary>
/// A delegating agent that handles function approval requests on the server side.
/// Transforms between ToolApprovalRequestContent/ToolApprovalResponseContent
/// Transforms between FunctionApprovalRequestContent/FunctionApprovalResponseContent
/// and the request_approval tool call pattern for client communication.
/// </summary>
internal sealed class ServerFunctionApprovalAgent : DelegatingAIAgent
@@ -50,32 +50,44 @@ internal sealed class ServerFunctionApprovalAgent : DelegatingAIAgent
}
#pragma warning disable MEAI001 // Type is for evaluation purposes only
private static ToolApprovalRequestContent ConvertToolCallToApprovalRequest(FunctionCallContent toolCall, JsonSerializerOptions jsonSerializerOptions)
private static FunctionApprovalRequestContent ConvertToolCallToApprovalRequest(FunctionCallContent toolCall, JsonSerializerOptions jsonSerializerOptions)
{
if (toolCall.Name != "request_approval" || toolCall.Arguments == null)
{
throw new InvalidOperationException("Invalid request_approval tool call");
}
var request = (toolCall.Arguments.TryGetValue("request", out var reqObj) &&
var request = toolCall.Arguments.TryGetValue("request", out var reqObj) &&
reqObj is JsonElement argsElement &&
argsElement.Deserialize(jsonSerializerOptions.GetTypeInfo(typeof(ApprovalRequest))) is ApprovalRequest approvalRequest &&
approvalRequest != null ? approvalRequest : null) ?? throw new InvalidOperationException("Failed to deserialize approval request from tool call");
return new ToolApprovalRequestContent(
requestId: request.ApprovalId,
approvalRequest != null ? approvalRequest : null;
if (request == null)
{
throw new InvalidOperationException("Failed to deserialize approval request from tool call");
}
return new FunctionApprovalRequestContent(
id: request.ApprovalId,
new FunctionCallContent(
callId: request.ApprovalId,
name: request.FunctionName,
arguments: request.FunctionArguments));
}
private static ToolApprovalResponseContent ConvertToolResultToApprovalResponse(FunctionResultContent result, ToolApprovalRequestContent approval, JsonSerializerOptions jsonSerializerOptions)
private static FunctionApprovalResponseContent ConvertToolResultToApprovalResponse(FunctionResultContent result, FunctionApprovalRequestContent approval, JsonSerializerOptions jsonSerializerOptions)
{
var approvalResponse = (result.Result is JsonElement je ?
var approvalResponse = result.Result is JsonElement je ?
(ApprovalResponse?)je.Deserialize(jsonSerializerOptions.GetTypeInfo(typeof(ApprovalResponse))) :
result.Result is string str ?
(ApprovalResponse?)JsonSerializer.Deserialize(str, jsonSerializerOptions.GetTypeInfo(typeof(ApprovalResponse))) :
result.Result as ApprovalResponse) ?? throw new InvalidOperationException("Failed to deserialize approval response from tool result");
result.Result as ApprovalResponse;
if (approvalResponse == null)
{
throw new InvalidOperationException("Failed to deserialize approval response from tool result");
}
return approval.CreateResponse(approvalResponse.Approved);
}
#pragma warning restore MEAI001
@@ -109,7 +121,7 @@ internal sealed class ServerFunctionApprovalAgent : DelegatingAIAgent
// Track approval ID to original call ID mapping
_ = new Dictionary<string, string>();
#pragma warning disable MEAI001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed.
Dictionary<string, ToolApprovalRequestContent> trackedRequestApprovalToolCalls = []; // Remote approvals
Dictionary<string, FunctionApprovalRequestContent> trackedRequestApprovalToolCalls = new(); // Remote approvals
for (int messageIndex = 0; messageIndex < messages.Count; messageIndex++)
{
var message = messages[messageIndex];
@@ -134,7 +146,7 @@ internal sealed class ServerFunctionApprovalAgent : DelegatingAIAgent
});
}
else if (content is FunctionResultContent toolResult &&
trackedRequestApprovalToolCalls.TryGetValue(toolResult.CallId, out var approval))
trackedRequestApprovalToolCalls.TryGetValue(toolResult.CallId, out var approval) == true)
{
result ??= CopyMessagesUpToIndex(messages, messageIndex);
transformedContents ??= CopyContentsUpToIndex(message.Contents, j);
@@ -149,9 +161,9 @@ internal sealed class ServerFunctionApprovalAgent : DelegatingAIAgent
AdditionalProperties = message.AdditionalProperties
});
}
else
else if (result != null)
{
result?.Add(message);
result.Add(message);
}
}
}
@@ -169,10 +181,11 @@ internal sealed class ServerFunctionApprovalAgent : DelegatingAIAgent
{
var content = update.Contents[i];
#pragma warning disable MEAI001 // Type is for evaluation purposes only
if (content is ToolApprovalRequestContent request && request.ToolCall is FunctionCallContent functionCall)
if (content is FunctionApprovalRequestContent request)
{
updatedContents ??= [.. update.Contents];
var approvalId = request.RequestId;
var functionCall = request.FunctionCall;
var approvalId = request.Id;
var approvalData = new ApprovalRequest
{
@@ -72,9 +72,10 @@ internal sealed class StatefulAgent<TState> : DelegatingAIAgent
if (content is DataContent dataContent && dataContent.MediaType == "application/json")
{
// Deserialize the state
if (JsonSerializer.Deserialize(
TState? newState = JsonSerializer.Deserialize(
dataContent.Data.Span,
this._jsonSerializerOptions.GetTypeInfo(typeof(TState))) is TState newState)
this._jsonSerializerOptions.GetTypeInfo(typeof(TState))) as TState;
if (newState != null)
{
this.State = newState;
}
@@ -18,12 +18,11 @@ using OpenTelemetry.Trace;
#region Setup Telemetry
// Source name for this sample's custom ActivitySource and Meter; other instrumentation uses their own sources/categories.
const string SourceName = "OpenTelemetryAspire.ConsoleApp";
const string ServiceName = "AgentOpenTelemetry";
// Configure OpenTelemetry for Aspire dashboard
var otlpEndpoint = Environment.GetEnvironmentVariable("OTEL_EXPORTER_OTLP_ENDPOINT") ?? "http://localhost:4317";
var otlpEndpoint = Environment.GetEnvironmentVariable("OTEL_EXPORTER_OTLP_ENDPOINT") ?? "http://localhost:4318";
var applicationInsightsConnectionString = Environment.GetEnvironmentVariable("APPLICATIONINSIGHTS_CONNECTION_STRING");
@@ -41,6 +40,7 @@ var resource = ResourceBuilder.CreateDefault()
var tracerProviderBuilder = Sdk.CreateTracerProviderBuilder()
.SetResourceBuilder(ResourceBuilder.CreateDefault().AddService(ServiceName, serviceVersion: "1.0.0"))
.AddSource(SourceName) // Our custom activity source
.AddSource("*Microsoft.Agents.AI") // Agent Framework telemetry
.AddHttpClientInstrumentation() // Capture HTTP calls to OpenAI
.AddOtlpExporter(options => options.Endpoint = new Uri(otlpEndpoint));
@@ -54,7 +54,8 @@ using var tracerProvider = tracerProviderBuilder.Build();
// Setup metrics with resource and instrument name filtering
using var meterProvider = Sdk.CreateMeterProviderBuilder()
.SetResourceBuilder(ResourceBuilder.CreateDefault().AddService(ServiceName, serviceVersion: "1.0.0"))
.AddMeter(SourceName) // Our custom meter source
.AddMeter(SourceName) // Our custom meter
.AddMeter("*Microsoft.Agents.AI") // Agent Framework metrics
.AddHttpClientInstrumentation() // HTTP client metrics
.AddRuntimeInstrumentation() // .NET runtime metrics
.AddOtlpExporter(options => options.Endpoint = new Uri(otlpEndpoint))
@@ -97,7 +98,7 @@ Console.WriteLine("""
""");
var endpoint = Environment.GetEnvironmentVariable("AZURE_OPENAI_ENDPOINT") ?? throw new InvalidOperationException("AZURE_OPENAI_ENDPOINT environment variable is not set.");
var deploymentName = Environment.GetEnvironmentVariable("AZURE_OPENAI_DEPLOYMENT_NAME") ?? "gpt-5.4-mini";
var deploymentName = Environment.GetEnvironmentVariable("AZURE_OPENAI_DEPLOYMENT_NAME") ?? "gpt-4o-mini";
// Log application startup
appLogger.LogInformation("OpenTelemetry Aspire Demo application started");
@@ -127,7 +128,7 @@ var agent = new ChatClientAgent(instrumentedChatClient,
instructions: "You are a helpful assistant that provides concise and informative responses.",
tools: [AIFunctionFactory.Create(GetWeatherAsync)])
.AsBuilder()
.UseOpenTelemetry(sourceName: SourceName, configure: (cfg) => cfg.EnableSensitiveData = true) // enable telemetry at the agent level
.UseOpenTelemetry(SourceName, configure: (cfg) => cfg.EnableSensitiveData = true) // enable telemetry at the agent level
.Build();
var session = await agent.CreateSessionAsync();
@@ -34,7 +34,7 @@ graph TD
Set the following environment variables:
```powershell
$env:AZURE_OPENAI_ENDPOINT="https://your-resource.openai.azure.com/"
$env:AZURE_OPENAI_DEPLOYMENT_NAME="gpt-5.4-mini" # Optional, defaults to gpt-5.4-mini
$env:AZURE_OPENAI_DEPLOYMENT_NAME="gpt-4o-mini" # Optional, defaults to gpt-4o-mini
```
**Note**: This demo uses Azure CLI credentials for authentication. Make sure you're logged in with `az login` and have access to the Azure OpenAI resource.
@@ -5,8 +5,8 @@ This sample demonstrates how to create an AIAgent using Anthropic Claude models
The sample supports three deployment scenarios:
1. **Anthropic Public API** - Direct connection to Anthropic's public API
2. **Microsoft Foundry with API Key** - Anthropic models deployed through Microsoft Foundry using API key authentication
3. **Microsoft Foundry with Azure CLI** - Anthropic models deployed through Microsoft Foundry using Azure CLI credentials
2. **Azure Foundry with API Key** - Anthropic models deployed through Azure Foundry using API key authentication
3. **Azure Foundry with Azure CLI** - Anthropic models deployed through Azure Foundry using Azure CLI credentials
## Prerequisites
@@ -25,29 +25,29 @@ $env:ANTHROPIC_API_KEY="your-anthropic-api-key" # Replace with your Anthropic A
$env:ANTHROPIC_CHAT_MODEL_NAME="claude-haiku-4-5" # Optional, defaults to claude-haiku-4-5
```
### For Microsoft Foundry with API Key
### For Azure Foundry with API Key
- Microsoft Foundry service endpoint and deployment configured
- Azure Foundry service endpoint and deployment configured
- Anthropic API key
Set the following environment variables:
```powershell
$env:ANTHROPIC_RESOURCE="your-foundry-resource-name" # Replace with your Microsoft Foundry resource name (subdomain before .services.ai.azure.com)
$env:ANTHROPIC_RESOURCE="your-foundry-resource-name" # Replace with your Azure Foundry resource name (subdomain before .services.ai.azure.com)
$env:ANTHROPIC_API_KEY="your-anthropic-api-key" # Replace with your Anthropic API key
$env:ANTHROPIC_CHAT_MODEL_NAME="claude-haiku-4-5" # Optional, defaults to claude-haiku-4-5
```
### For Microsoft Foundry with Azure CLI
### For Azure Foundry with Azure CLI
- Microsoft Foundry service endpoint and deployment configured
- Azure Foundry service endpoint and deployment configured
- Azure CLI installed and authenticated (for Azure credential authentication)
Set the following environment variables:
```powershell
$env:ANTHROPIC_RESOURCE="your-foundry-resource-name" # Replace with your Microsoft Foundry resource name (subdomain before .services.ai.azure.com)
$env:ANTHROPIC_RESOURCE="your-foundry-resource-name" # Replace with your Azure Foundry resource name (subdomain before .services.ai.azure.com)
$env:ANTHROPIC_CHAT_MODEL_NAME="claude-haiku-4-5" # Optional, defaults to claude-haiku-4-5
```
**Note**: When using Microsoft Foundry with Azure CLI, make sure you're logged in with `az login` and have access to the Microsoft Foundry resource. For more information, see the [Azure CLI documentation](https://learn.microsoft.com/cli/azure/authenticate-azure-cli-interactively).
**Note**: When using Azure Foundry with Azure CLI, make sure you're logged in with `az login` and have access to the Azure Foundry resource. For more information, see the [Azure CLI documentation](https://learn.microsoft.com/cli/azure/authenticate-azure-cli-interactively).
@@ -1,15 +1,13 @@
// Copyright (c) Microsoft. All rights reserved.
#pragma warning disable CS0618 // Type or member is obsolete - sample uses deprecated PersistentAgentsClientExtensions
// This sample shows how to create and use a simple AI agent with Microsoft Foundry Agents as the backend.
// This sample shows how to create and use a simple AI agent with Azure Foundry Agents as the backend.
using Azure.AI.Agents.Persistent;
using Azure.Identity;
using Microsoft.Agents.AI;
var endpoint = Environment.GetEnvironmentVariable("AZURE_AI_PROJECT_ENDPOINT") ?? throw new InvalidOperationException("AZURE_AI_PROJECT_ENDPOINT is not set.");
var deploymentName = Environment.GetEnvironmentVariable("AZURE_AI_MODEL_DEPLOYMENT_NAME") ?? "gpt-5.4-mini";
var deploymentName = Environment.GetEnvironmentVariable("AZURE_AI_MODEL_DEPLOYMENT_NAME") ?? "gpt-4o-mini";
const string JokerName = "Joker";
const string JokerInstructions = "You are good at telling jokes.";
@@ -13,14 +13,14 @@ Below is a comparison between the classic and new Foundry Agents approaches:
Before you begin, ensure you have the following prerequisites:
- .NET 10 SDK or later
- Microsoft Foundry service endpoint and deployment configured
- Azure Foundry service endpoint and deployment configured
- Azure CLI installed and authenticated (for Azure credential authentication)
**Note**: This demo uses Azure CLI credentials for authentication. Make sure you're logged in with `az login` and have access to the Microsoft Foundry resource. For more information, see the [Azure CLI documentation](https://learn.microsoft.com/cli/azure/authenticate-azure-cli-interactively).
**Note**: This demo uses Azure CLI credentials for authentication. Make sure you're logged in with `az login` and have access to the Azure Foundry resource. For more information, see the [Azure CLI documentation](https://learn.microsoft.com/cli/azure/authenticate-azure-cli-interactively).
Set the following environment variables:
```powershell
$env:AZURE_AI_PROJECT_ENDPOINT="https://your-foundry-service.services.ai.azure.com/api/projects/your-foundry-project" # Replace with your Microsoft Foundry resource endpoint
$env:AZURE_AI_MODEL_DEPLOYMENT_NAME="gpt-5.4-mini" # Optional, defaults to gpt-5.4-mini
$env:AZURE_AI_PROJECT_ENDPOINT="https://your-foundry-service.services.ai.azure.com/api/projects/your-foundry-project" # Replace with your Azure Foundry resource endpoint
$env:AZURE_AI_MODEL_DEPLOYMENT_NAME="gpt-4o-mini" # Optional, defaults to gpt-4o-mini
```
@@ -15,7 +15,7 @@
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\..\..\..\src\Microsoft.Agents.AI.Foundry\Microsoft.Agents.AI.Foundry.csproj" />
<ProjectReference Include="..\..\..\..\src\Microsoft.Agents.AI.AzureAI\Microsoft.Agents.AI.AzureAI.csproj" />
</ItemGroup>
</Project>
@@ -1,29 +1,28 @@
// Copyright (c) Microsoft. All rights reserved.
// This sample shows how to create and use AI agents with Microsoft Foundry Agents as the backend.
// This sample shows how to create and use a AI agents with Azure Foundry Agents as the backend.
using Azure.AI.Projects;
using Azure.AI.Projects.Agents;
using Azure.AI.Projects.OpenAI;
using Azure.Identity;
using Microsoft.Agents.AI;
using Microsoft.Agents.AI.Foundry;
var endpoint = Environment.GetEnvironmentVariable("AZURE_AI_PROJECT_ENDPOINT") ?? throw new InvalidOperationException("AZURE_AI_PROJECT_ENDPOINT is not set.");
var deploymentName = Environment.GetEnvironmentVariable("AZURE_AI_MODEL_DEPLOYMENT_NAME") ?? "gpt-5.4-mini";
var deploymentName = Environment.GetEnvironmentVariable("AZURE_AI_MODEL_DEPLOYMENT_NAME") ?? "gpt-4o-mini";
const string JokerName = "JokerAgent";
// Get a client to create/retrieve/delete server side agents with Microsoft Foundry Agents.
// Get a client to create/retrieve/delete server side agents with Azure Foundry Agents.
// WARNING: DefaultAzureCredential is convenient for development but requires careful consideration in production.
// In production, consider using a specific credential (e.g., ManagedIdentityCredential) to avoid
// latency issues, unintended credential probing, and potential security risks from fallback mechanisms.
var aiProjectClient = new AIProjectClient(new Uri(endpoint), new DefaultAzureCredential());
// Define the agent you want to create. (Prompt Agent in this case)
var agentVersionCreationOptions = new ProjectsAgentVersionCreationOptions(new DeclarativeAgentDefinition(model: deploymentName) { Instructions = "You are good at telling jokes." });
var agentVersionCreationOptions = new AgentVersionCreationOptions(new PromptAgentDefinition(model: deploymentName) { Instructions = "You are good at telling jokes." });
// Azure.AI.Agents SDK creates and manages agent by name and versions.
// You can create a server side agent version with the Azure.AI.Agents SDK client below.
var createdAgentVersion = aiProjectClient.AgentAdministrationClient.CreateAgentVersion(agentName: JokerName, options: agentVersionCreationOptions);
var createdAgentVersion = aiProjectClient.Agents.CreateAgentVersion(agentName: JokerName, options: agentVersionCreationOptions);
// Note:
// agentVersion.Id = "<agentName>:<versionNumber>",
@@ -31,18 +30,14 @@ var createdAgentVersion = aiProjectClient.AgentAdministrationClient.CreateAgentV
// agentVersion.Name = <agentName>
// You can use an AIAgent with an already created server side agent version.
FoundryAgent existingJokerAgent = aiProjectClient.AsAIAgent(createdAgentVersion);
AIAgent existingJokerAgent = aiProjectClient.AsAIAgent(createdAgentVersion);
// You can also create another AIAgent version by providing the same name with a different definition.
ProjectsAgentVersion newJokerAgentVersion = await aiProjectClient.AgentAdministrationClient.CreateAgentVersionAsync(
JokerName,
new ProjectsAgentVersionCreationOptions(new DeclarativeAgentDefinition(model: deploymentName) { Instructions = "You are extremely hilarious at telling jokes." }));
FoundryAgent newJokerAgent = aiProjectClient.AsAIAgent(newJokerAgentVersion);
AIAgent newJokerAgent = await aiProjectClient.CreateAIAgentAsync(name: JokerName, model: deploymentName, instructions: "You are extremely hilarious at telling jokes.");
// You can also get the AIAgent latest version just providing its name.
ProjectsAgentRecord jokerAgentRecord = await aiProjectClient.AgentAdministrationClient.GetAgentAsync(JokerName);
FoundryAgent jokerAgentLatest = aiProjectClient.AsAIAgent(jokerAgentRecord);
ProjectsAgentVersion latestAgentVersion = jokerAgentRecord.GetLatestVersion();
AIAgent jokerAgentLatest = await aiProjectClient.GetAIAgentAsync(name: JokerName);
var latestAgentVersion = jokerAgentLatest.GetService<AgentVersion>()!;
// The AIAgent version can be accessed via the GetService method.
Console.WriteLine($"Latest agent version id: {latestAgentVersion.Id}");
@@ -55,4 +50,4 @@ Console.WriteLine(await jokerAgentLatest.RunAsync("Tell me a joke about a pirate
Console.WriteLine(await jokerAgentLatest.RunAsync("Now tell me a joke about a cat and a dog using last joke as the anchor.", session));
// Cleanup by agent name removes both agent versions created.
aiProjectClient.AgentAdministrationClient.DeleteAgent(existingJokerAgent.Name);
aiProjectClient.Agents.DeleteAgent(existingJokerAgent.Name);
@@ -13,14 +13,14 @@ Below is a comparison between the classic and new Foundry Agents approaches:
Before you begin, ensure you have the following prerequisites:
- .NET 10 SDK or later
- Microsoft Foundry service endpoint and deployment configured
- Azure Foundry service endpoint and deployment configured
- Azure CLI installed and authenticated (for Azure credential authentication)
**Note**: This demo uses Azure CLI credentials for authentication. Make sure you're logged in with `az login` and have access to the Microsoft Foundry resource. For more information, see the [Azure CLI documentation](https://learn.microsoft.com/cli/azure/authenticate-azure-cli-interactively).
**Note**: This demo uses Azure CLI credentials for authentication. Make sure you're logged in with `az login` and have access to the Azure Foundry resource. For more information, see the [Azure CLI documentation](https://learn.microsoft.com/cli/azure/authenticate-azure-cli-interactively).
Set the following environment variables:
```powershell
$env:AZURE_AI_PROJECT_ENDPOINT="https://your-foundry-service.services.ai.azure.com/api/projects/your-foundry-project" # Replace with your Microsoft Foundry resource endpoint
$env:AZURE_AI_MODEL_DEPLOYMENT_NAME="gpt-5.4-mini" # Optional, defaults to gpt-5.4-mini
$env:AZURE_AI_PROJECT_ENDPOINT="https://your-foundry-service.services.ai.azure.com/api/projects/your-foundry-project" # Replace with your Azure Foundry resource endpoint
$env:AZURE_AI_MODEL_DEPLOYMENT_NAME="gpt-4o-mini" # Optional, defaults to gpt-4o-mini
```
@@ -1,7 +1,7 @@
// Copyright (c) Microsoft. All rights reserved.
// This sample shows how to use the OpenAI SDK to create and use a simple AI agent with any model hosted in Microsoft Foundry.
// You could use models from Microsoft, OpenAI, DeepSeek, Hugging Face, Meta, xAI or any other model you have deployed in your Microsoft Foundry resource.
// This sample shows how to use the OpenAI SDK to create and use a simple AI agent with any model hosted in Azure AI Foundry.
// You could use models from Microsoft, OpenAI, DeepSeek, Hugging Face, Meta, xAI or any other model you have deployed in your Azure AI Foundry resource.
// Note: Ensure that you pick a model that suits your needs. For example, if you want to use function calling, ensure that the model you pick supports function calling.
using System.ClientModel;
@@ -15,7 +15,7 @@ var endpoint = Environment.GetEnvironmentVariable("AZURE_OPENAI_ENDPOINT") ?? th
var apiKey = Environment.GetEnvironmentVariable("AZURE_OPENAI_API_KEY");
var model = Environment.GetEnvironmentVariable("AZURE_AI_MODEL_DEPLOYMENT_NAME") ?? "Phi-4-mini-instruct";
// Since we are using the OpenAI Client SDK, we need to override the default endpoint to point to Microsoft Foundry.
// Since we are using the OpenAI Client SDK, we need to override the default endpoint to point to Azure Foundry.
var clientOptions = new OpenAIClientOptions() { Endpoint = new Uri(endpoint) };
// Create the OpenAI client with either an API key or Azure CLI credential.
@@ -1,8 +1,8 @@
## Overview
This sample shows how to use the OpenAI SDK to create and use a simple AI agent with any model hosted in Microsoft Foundry.
This sample shows how to use the OpenAI SDK to create and use a simple AI agent with any model hosted in Azure AI Foundry.
You could use models from Microsoft, OpenAI, DeepSeek, Hugging Face, Meta, xAI or any other model you have deployed in Microsoft Foundry.
You could use models from Microsoft, OpenAI, DeepSeek, Hugging Face, Meta, xAI or any other model you have deployed in Azure AI Foundry.
**Note**: Ensure that you pick a model that suits your needs. For example, if you want to use function calling, ensure that the model you pick supports function calling.
@@ -11,19 +11,19 @@ You could use models from Microsoft, OpenAI, DeepSeek, Hugging Face, Meta, xAI o
Before you begin, ensure you have the following prerequisites:
- .NET 10 SDK or later
- Microsoft Foundry resource
- A model deployment in your Microsoft Foundry resource. This example defaults to using the `Phi-4-mini-instruct` model,
- Azure AI Foundry resource
- A model deployment in your Azure AI Foundry resource. This example defaults to using the `Phi-4-mini-instruct` model,
so if you want to use a different model, ensure that you set your `AZURE_AI_MODEL_DEPLOYMENT_NAME` environment
variable to the name of your deployed model.
- An API key or role based authentication to access the Microsoft Foundry resource
- An API key or role based authentication to access the Azure AI Foundry resource
See [here](https://learn.microsoft.com/en-us/azure/ai-foundry/quickstarts/get-started-code?tabs=csharp) for more info on setting up these prerequisites
Set the following environment variables:
```powershell
# Replace with your Microsoft Foundry resource endpoint
# Ensure that you have the "/openai/v1/" path in the URL, since this is required when using the OpenAI SDK to access Microsoft Foundry models.
# Replace with your Azure AI Foundry resource endpoint
# Ensure that you have the "/openai/v1/" path in the URL, since this is required when using the OpenAI SDK to access Azure Foundry models.
$env:AZURE_OPENAI_ENDPOINT="https://ai-foundry-<myresourcename>.services.ai.azure.com/openai/v1/"
# Optional, defaults to using Azure CLI for authentication if not provided
@@ -8,7 +8,7 @@ using Microsoft.Agents.AI;
using OpenAI.Chat;
var endpoint = Environment.GetEnvironmentVariable("AZURE_OPENAI_ENDPOINT") ?? throw new InvalidOperationException("AZURE_OPENAI_ENDPOINT is not set.");
var deploymentName = Environment.GetEnvironmentVariable("AZURE_OPENAI_DEPLOYMENT_NAME") ?? "gpt-5.4-mini";
var deploymentName = Environment.GetEnvironmentVariable("AZURE_OPENAI_DEPLOYMENT_NAME") ?? "gpt-4o-mini";
// WARNING: DefaultAzureCredential is convenient for development but requires careful consideration in production.
// In production, consider using a specific credential (e.g., ManagedIdentityCredential) to avoid
@@ -12,5 +12,5 @@ Set the following environment variables:
```powershell
$env:AZURE_OPENAI_ENDPOINT="https://your-resource.openai.azure.com/" # Replace with your Azure OpenAI resource endpoint
$env:AZURE_OPENAI_DEPLOYMENT_NAME="gpt-5.4-mini" # Optional, defaults to gpt-5.4-mini
$env:AZURE_OPENAI_DEPLOYMENT_NAME="gpt-4o-mini" # Optional, defaults to gpt-4o-mini
```
@@ -9,7 +9,7 @@ using Microsoft.Extensions.AI;
using OpenAI.Responses;
var endpoint = Environment.GetEnvironmentVariable("AZURE_OPENAI_ENDPOINT") ?? throw new InvalidOperationException("AZURE_OPENAI_ENDPOINT is not set.");
var deploymentName = Environment.GetEnvironmentVariable("AZURE_OPENAI_DEPLOYMENT_NAME") ?? "gpt-5.4-mini";
var deploymentName = Environment.GetEnvironmentVariable("AZURE_OPENAI_DEPLOYMENT_NAME") ?? "gpt-4o-mini";
// WARNING: DefaultAzureCredential is convenient for development but requires careful consideration in production.
// In production, consider using a specific credential (e.g., ManagedIdentityCredential) to avoid
@@ -17,8 +17,8 @@ var deploymentName = Environment.GetEnvironmentVariable("AZURE_OPENAI_DEPLOYMENT
AIAgent agent = new AzureOpenAIClient(
new Uri(endpoint),
new DefaultAzureCredential())
.GetResponsesClient()
.AsAIAgent(model: deploymentName, instructions: "You are good at telling jokes.", name: "Joker");
.GetResponsesClient(deploymentName)
.AsAIAgent(instructions: "You are good at telling jokes.", name: "Joker");
// Invoke the agent and output the text result.
Console.WriteLine(await agent.RunAsync("Tell me a joke about a pirate."));
@@ -29,8 +29,8 @@ Console.WriteLine(await agent.RunAsync("Tell me a joke about a pirate."));
AIAgent agentStoreFalse = new AzureOpenAIClient(
new Uri(endpoint),
new DefaultAzureCredential())
.GetResponsesClient()
.AsIChatClientWithStoredOutputDisabled(model: deploymentName)
.GetResponsesClient(deploymentName)
.AsIChatClientWithStoredOutputDisabled()
.AsAIAgent(instructions: "You are good at telling jokes.", name: "Joker");
// Invoke the agent and output the text result.
@@ -12,5 +12,5 @@ Set the following environment variables:
```powershell
$env:AZURE_OPENAI_ENDPOINT="https://your-resource.openai.azure.com/" # Replace with your Azure OpenAI resource endpoint
$env:AZURE_OPENAI_DEPLOYMENT_NAME="gpt-5.4-mini" # Optional, defaults to gpt-5.4-mini
$env:AZURE_OPENAI_DEPLOYMENT_NAME="gpt-4o-mini" # Optional, defaults to gpt-4o-mini
```
@@ -9,7 +9,7 @@
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\..\..\..\src\Microsoft.Agents.AI.Abstractions\Microsoft.Agents.AI.Abstractions.csproj" />
<ProjectReference Include="..\..\..\..\src\Microsoft.Agents.AI\Microsoft.Agents.AI.csproj" />
</ItemGroup>
</Project>
@@ -9,7 +9,7 @@
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\..\..\..\src\Microsoft.Agents.AI.Foundry\Microsoft.Agents.AI.Foundry.csproj" />
<ProjectReference Include="..\..\..\..\src\Microsoft.Agents.AI.OpenAI\Microsoft.Agents.AI.OpenAI.csproj" />
</ItemGroup>
</Project>
@@ -0,0 +1,41 @@
// Copyright (c) Microsoft. All rights reserved.
// This sample shows how to create and use a simple AI agent with OpenAI Assistants as the backend.
// WARNING: The Assistants API is deprecated and will be shut down.
// For more information see the OpenAI documentation: https://platform.openai.com/docs/assistants/migration
#pragma warning disable CS0618 // Type or member is obsolete - OpenAI Assistants API is deprecated but still used in this sample
using Microsoft.Agents.AI;
using OpenAI;
using OpenAI.Assistants;
var apiKey = Environment.GetEnvironmentVariable("OPENAI_API_KEY") ?? throw new InvalidOperationException("OPENAI_API_KEY is not set.");
var model = Environment.GetEnvironmentVariable("OPENAI_CHAT_MODEL_NAME") ?? "gpt-4o-mini";
const string JokerName = "Joker";
const string JokerInstructions = "You are good at telling jokes.";
// Get a client to create/retrieve server side agents with.
var assistantClient = new OpenAIClient(apiKey).GetAssistantClient();
// You can create a server side assistant with the OpenAI SDK.
var createResult = await assistantClient.CreateAssistantAsync(model, new() { Name = JokerName, Instructions = JokerInstructions });
// You can retrieve an already created server side assistant as an AIAgent.
AIAgent agent1 = await assistantClient.GetAIAgentAsync(createResult.Value.Id);
// You can also create a server side assistant and return it as an AIAgent directly.
AIAgent agent2 = await assistantClient.CreateAIAgentAsync(
model: model,
name: JokerName,
instructions: JokerInstructions);
// You can invoke the agent like any other AIAgent.
AgentSession session = await agent1.CreateSessionAsync();
Console.WriteLine(await agent1.RunAsync("Tell me a joke about a pirate.", session));
// Cleanup for sample purposes.
await assistantClient.DeleteAssistantAsync(agent1.Id);
await assistantClient.DeleteAssistantAsync(agent2.Id);
@@ -0,0 +1,16 @@
# Prerequisites
WARNING: The Assistants API is deprecated and will be shut down.
For more information see the OpenAI documentation: https://platform.openai.com/docs/assistants/migration
Before you begin, ensure you have the following prerequisites:
- .NET 10 SDK or later
- OpenAI API key
Set the following environment variables:
```powershell
$env:OPENAI_API_KEY="*****" # Replace with your OpenAI API key
$env:OPENAI_CHAT_MODEL_NAME="gpt-4o-mini" # Optional, defaults to gpt-4o-mini
```
@@ -8,7 +8,7 @@ using OpenAI;
using OpenAI.Chat;
var apiKey = Environment.GetEnvironmentVariable("OPENAI_API_KEY") ?? throw new InvalidOperationException("OPENAI_API_KEY is not set.");
var model = Environment.GetEnvironmentVariable("OPENAI_CHAT_MODEL_NAME") ?? "gpt-5.4-mini";
var model = Environment.GetEnvironmentVariable("OPENAI_CHAT_MODEL_NAME") ?? "gpt-4o-mini";
AIAgent agent = new OpenAIClient(
apiKey)
@@ -9,5 +9,5 @@ Set the following environment variables:
```powershell
$env:OPENAI_API_KEY="*****" # Replace with your OpenAI api key
$env:OPENAI_CHAT_MODEL_NAME="gpt-5.4-mini" # Optional, defaults to gpt-5.4-mini
$env:OPENAI_CHAT_MODEL_NAME="gpt-4o-mini" # Optional, defaults to gpt-4o-mini
```
@@ -7,12 +7,12 @@ using OpenAI;
using OpenAI.Responses;
var apiKey = Environment.GetEnvironmentVariable("OPENAI_API_KEY") ?? throw new InvalidOperationException("OPENAI_API_KEY is not set.");
var model = Environment.GetEnvironmentVariable("OPENAI_CHAT_MODEL_NAME") ?? "gpt-5.4-mini";
var model = Environment.GetEnvironmentVariable("OPENAI_CHAT_MODEL_NAME") ?? "gpt-4o-mini";
AIAgent agent = new OpenAIClient(
apiKey)
.GetResponsesClient()
.AsAIAgent(model: model, instructions: "You are good at telling jokes.", name: "Joker");
.GetResponsesClient(model)
.AsAIAgent(instructions: "You are good at telling jokes.", name: "Joker");
// Invoke the agent and output the text result.
Console.WriteLine(await agent.RunAsync("Tell me a joke about a pirate."));
@@ -9,5 +9,5 @@ Set the following environment variables:
```powershell
$env:OPENAI_API_KEY="*****" # Replace with your OpenAI api key
$env:OPENAI_CHAT_MODEL_NAME="gpt-5.4-mini" # Optional, defaults to gpt-5.4-mini
$env:OPENAI_CHAT_MODEL_NAME="gpt-4o-mini" # Optional, defaults to gpt-4o-mini
```
@@ -18,13 +18,14 @@ See the README.md for each sample for the prerequisites for that sample.
|[Creating an AIAgent with Anthropic](./Agent_With_Anthropic/)|This sample demonstrates how to create an AIAgent using Anthropic Claude models as the underlying inference service|
|[Creating an AIAgent with Foundry Agents using Azure.AI.Agents.Persistent](./Agent_With_AzureAIAgentsPersistent/)|This sample demonstrates how to create a Foundry Persistent agent and expose it as an AIAgent using the Azure.AI.Agents.Persistent SDK|
|[Creating an AIAgent with Foundry Agents using Azure.AI.Project](./Agent_With_AzureAIProject/)|This sample demonstrates how to create an Foundry Project agent and expose it as an AIAgent using the Azure.AI.Project SDK|
|[Creating an AIAgent with Foundry Model](./Agent_With_AzureFoundryModel/)|This sample demonstrates how to use any model deployed to Microsoft Foundry to create an AIAgent|
|[Creating an AIAgent with AzureFoundry Model](./Agent_With_AzureFoundryModel/)|This sample demonstrates how to use any model deployed to Azure Foundry to create an AIAgent|
|[Creating an AIAgent with Azure OpenAI ChatCompletion](./Agent_With_AzureOpenAIChatCompletion/)|This sample demonstrates how to create an AIAgent using Azure OpenAI ChatCompletion as the underlying inference service|
|[Creating an AIAgent with Azure OpenAI Responses](./Agent_With_AzureOpenAIResponses/)|This sample demonstrates how to create an AIAgent using Azure OpenAI Responses as the underlying inference service|
|[Creating an AIAgent with a custom implementation](./Agent_With_CustomImplementation/)|This sample demonstrates how to create an AIAgent with a custom implementation|
|[Creating an AIAgent with GitHub Copilot](./Agent_With_GitHubCopilot/)|This sample demonstrates how to create an AIAgent using GitHub Copilot SDK as the underlying inference service|
|[Creating an AIAgent with Ollama](./Agent_With_Ollama/)|This sample demonstrates how to create an AIAgent using Ollama as the underlying inference service|
|[Creating an AIAgent with ONNX](./Agent_With_ONNX/)|This sample demonstrates how to create an AIAgent using ONNX as the underlying inference service|
|[Creating an AIAgent with OpenAI Assistants](./Agent_With_OpenAIAssistants/)|This sample demonstrates how to create an AIAgent using OpenAI Assistants as the underlying inference service.</br>WARNING: The Assistants API is deprecated and will be shut down. For more information see the OpenAI documentation: https://platform.openai.com/docs/assistants/migration|
|[Creating an AIAgent with OpenAI ChatCompletion](./Agent_With_OpenAIChatCompletion/)|This sample demonstrates how to create an AIAgent using OpenAI ChatCompletion as the underlying inference service|
|[Creating an AIAgent with OpenAI Responses](./Agent_With_OpenAIResponses/)|This sample demonstrates how to create an AIAgent using OpenAI Responses as the underlying inference service|

Some files were not shown because too many files have changed in this diff Show More