mirror of
https://github.com/microsoft/agent-framework.git
synced 2026-06-16 21:04:09 +08:00
Compare commits
76
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
2b0dc2f8c5 | ||
|
|
26eb2f3bf6 | ||
|
|
90c0e2ce37 | ||
|
|
2c000b032d | ||
|
|
cc85bbc2dc | ||
|
|
01aaf2baea | ||
|
|
cb96347c95 | ||
|
|
5070c67d0e | ||
|
|
9a47620f64 | ||
|
|
e11633a2c8 | ||
|
|
7e6d87e7ec | ||
|
|
9dfe7c40ca | ||
|
|
6803058e36 | ||
|
|
7645ec4e07 | ||
|
|
51828abed4 | ||
|
|
88ea9d08c7 | ||
|
|
8edcb282f4 | ||
|
|
81e2336d47 | ||
|
|
8fc19a3437 | ||
|
|
26cd5cc1bf | ||
|
|
b4c4f5094e | ||
|
|
0cd40f8354 | ||
|
|
cefda44283 | ||
|
|
4afc088f01 | ||
|
|
1272ec5adf | ||
|
|
47ead84753 | ||
|
|
4c287c2424 | ||
|
|
100086a276 | ||
|
|
fc6721ca8e | ||
|
|
4b21f38650 | ||
|
|
bf8d9672e1 | ||
|
|
5374dd47c5 | ||
|
|
29dfcbb584 | ||
|
|
c9321b9028 | ||
|
|
f48c4512d3 | ||
|
|
d3d0100822 | ||
|
|
acaf6b7054 | ||
|
|
c2fec6b51c | ||
|
|
705ed47a0b | ||
|
|
192a283c9a | ||
|
|
c74b1b08eb | ||
|
|
7c85f98c27 | ||
|
|
1e6f8909ec | ||
|
|
008fe23585 | ||
|
|
6af0511e2b | ||
|
|
6dbb0a5bb4 | ||
|
|
21af304c7d | ||
|
|
94af83680e | ||
|
|
cdb51e6a41 | ||
|
|
cbcdb2d29e | ||
|
|
0fdcfd0f4c | ||
|
|
414496dda7 | ||
|
|
55011b7258 | ||
|
|
bf0af178bd | ||
|
|
1b7940c91e | ||
|
|
2f4c4aa614 | ||
|
|
052ba7be07 | ||
|
|
c67d3523ae | ||
|
|
83ce6a9602 | ||
|
|
50fdcbaf57 | ||
|
|
67b0282813 | ||
|
|
0009e330af | ||
|
|
a4b9539b62 | ||
|
|
b7990908fe | ||
|
|
84bae0f42a | ||
|
|
f696ac9b57 | ||
|
|
5e33deff45 | ||
|
|
b6a1315386 | ||
|
|
ed2fb3b9dd | ||
|
|
aa2ff672fb | ||
|
|
bcb55b4a98 | ||
|
|
921c5f9c17 | ||
|
|
fcdaaff9cd | ||
|
|
384291ba27 | ||
|
|
378bee577e | ||
|
|
18e433fc6d |
@@ -3,6 +3,7 @@
|
||||
"image": "mcr.microsoft.com/devcontainers/dotnet",
|
||||
"features": {
|
||||
"ghcr.io/devcontainers/features/azure-cli:1.2.9": {},
|
||||
"ghcr.io/devcontainers/features/docker-in-docker:2": {},
|
||||
"ghcr.io/devcontainers/features/github-cli:1": {
|
||||
"version": "2"
|
||||
},
|
||||
|
||||
@@ -0,0 +1,216 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
"""Scan open issues and PRs labeled 'waiting-for-author' for stale follow-ups.
|
||||
|
||||
Team members manually add the 'waiting-for-author' label when they need a
|
||||
response from the external author. If the author hasn't replied within
|
||||
DAYS_THRESHOLD days of the last team comment, post a reminder and add the
|
||||
'requested-info' label to prevent duplicate pings.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from github import Auth, Github, GithubException
|
||||
from github.Issue import Issue
|
||||
from github.IssueComment import IssueComment
|
||||
|
||||
|
||||
PING_COMMENT = (
|
||||
"@{author}, friendly reminder — this issue is waiting on your response. "
|
||||
"Please share any updates when you get a chance. (This is an automated message.)"
|
||||
)
|
||||
TRIGGER_LABEL = "waiting-for-author"
|
||||
PINGED_LABEL = "requested-info"
|
||||
|
||||
|
||||
def get_team_members(g: Github, org: str, team_slug: str) -> set[str]:
|
||||
"""Fetch active team member usernames."""
|
||||
try:
|
||||
org_obj = g.get_organization(org)
|
||||
team = org_obj.get_team_by_slug(team_slug)
|
||||
return {m.login for m in team.get_members()}
|
||||
except GithubException as exc:
|
||||
if exc.status in (403, 404):
|
||||
print(
|
||||
f"ERROR: Failed to fetch team members for {org}/{team_slug} "
|
||||
f"(HTTP {exc.status}). Check that the token has the 'read:org' "
|
||||
f"scope and that the team slug '{team_slug}' is correct."
|
||||
)
|
||||
else:
|
||||
print(f"ERROR: Failed to fetch team members for {org}/{team_slug}: {exc}")
|
||||
sys.exit(1)
|
||||
except Exception as exc:
|
||||
print(f"ERROR: Failed to fetch team members for {org}/{team_slug}: {exc}")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def find_last_team_comment(
|
||||
comments: list[IssueComment], team_members: set[str]
|
||||
) -> IssueComment | None:
|
||||
"""Return the most recent comment from a team member, or None."""
|
||||
for comment in reversed(comments):
|
||||
if comment.user and comment.user.login in team_members:
|
||||
return comment
|
||||
return None
|
||||
|
||||
|
||||
def author_replied_after(
|
||||
comments: list[IssueComment], author: str, after: datetime
|
||||
) -> bool:
|
||||
"""Check if the issue author commented after the given timestamp."""
|
||||
for comment in comments:
|
||||
if (
|
||||
comment.user
|
||||
and comment.user.login == author
|
||||
and comment.created_at > after
|
||||
):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def should_ping(
|
||||
issue: Issue,
|
||||
team_members: set[str],
|
||||
days_threshold: int,
|
||||
now: datetime,
|
||||
) -> bool:
|
||||
"""Determine whether this issue/PR should be pinged.
|
||||
|
||||
Only issues/PRs carrying the 'waiting-for-author' label are candidates.
|
||||
"""
|
||||
author = issue.user.login
|
||||
|
||||
# Skip if the trigger label is not present
|
||||
if not any(label.name == TRIGGER_LABEL for label in issue.labels):
|
||||
return False
|
||||
# Skip if author is a team member
|
||||
if author in team_members:
|
||||
return False
|
||||
|
||||
# Skip if already pinged
|
||||
if any(label.name == PINGED_LABEL for label in issue.labels):
|
||||
return False
|
||||
|
||||
# Skip if no comments at all
|
||||
if issue.comments == 0:
|
||||
return False
|
||||
|
||||
# Fetch comments once for both lookups
|
||||
comments = list(issue.get_comments())
|
||||
|
||||
# Find last team member comment
|
||||
last_team_comment = find_last_team_comment(comments, team_members)
|
||||
if last_team_comment is None:
|
||||
return False
|
||||
|
||||
# Skip if author replied after the last team comment
|
||||
if author_replied_after(comments, author, last_team_comment.created_at):
|
||||
return False
|
||||
|
||||
# Check if enough days have passed
|
||||
days_since = (now - last_team_comment.created_at.astimezone(timezone.utc)).days
|
||||
if days_since < days_threshold:
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def ping(issue: Issue, dry_run: bool) -> bool:
|
||||
"""Post a reminder comment and add the 'requested-info' label. Returns True on success."""
|
||||
author = issue.user.login
|
||||
kind = "PR" if issue.pull_request else "Issue"
|
||||
|
||||
if dry_run:
|
||||
print(f" [DRY RUN] Would ping {kind} #{issue.number} (@{author})")
|
||||
return True
|
||||
|
||||
max_retries = 3
|
||||
commented = False
|
||||
labeled = False
|
||||
for attempt in range(1, max_retries + 1):
|
||||
try:
|
||||
if not commented:
|
||||
issue.create_comment(PING_COMMENT.format(author=author))
|
||||
commented = True
|
||||
if not labeled:
|
||||
issue.add_to_labels(PINGED_LABEL)
|
||||
labeled = True
|
||||
print(f" Pinged {kind} #{issue.number} (@{author})")
|
||||
return True
|
||||
except Exception as exc:
|
||||
if attempt < max_retries:
|
||||
wait = 2 ** attempt # 2s, 4s
|
||||
print(f" WARN: Attempt {attempt}/{max_retries} failed for {kind} #{issue.number}: {exc}. Retrying in {wait}s...")
|
||||
time.sleep(wait)
|
||||
else:
|
||||
print(f" ERROR: Failed to ping {kind} #{issue.number} after {max_retries} attempts: {exc}")
|
||||
return False
|
||||
|
||||
|
||||
def main() -> None:
|
||||
token = os.environ.get("GITHUB_TOKEN")
|
||||
if not token:
|
||||
print("ERROR: GITHUB_TOKEN environment variable is required")
|
||||
sys.exit(1)
|
||||
|
||||
repository = os.environ.get("GITHUB_REPOSITORY")
|
||||
if not repository:
|
||||
print("ERROR: GITHUB_REPOSITORY environment variable is required")
|
||||
sys.exit(1)
|
||||
|
||||
team_slug = os.environ.get("TEAM_SLUG")
|
||||
if not team_slug:
|
||||
print("ERROR: TEAM_SLUG environment variable is required")
|
||||
sys.exit(1)
|
||||
|
||||
days_threshold_raw = os.environ.get("DAYS_THRESHOLD", "4")
|
||||
try:
|
||||
days_threshold = int(days_threshold_raw)
|
||||
except ValueError:
|
||||
print(f"ERROR: DAYS_THRESHOLD must be a numeric value, got '{days_threshold_raw}'")
|
||||
sys.exit(1)
|
||||
dry_run = os.environ.get("DRY_RUN", "false").lower() == "true"
|
||||
|
||||
org = repository.split("/")[0]
|
||||
|
||||
if dry_run:
|
||||
print("Running in DRY RUN mode — no comments or labels will be applied.\n")
|
||||
|
||||
g = Github(auth=Auth.Token(token))
|
||||
repo = g.get_repo(repository)
|
||||
|
||||
print(f"Fetching team members for {org}/{team_slug}...")
|
||||
team_members = get_team_members(g, org, team_slug)
|
||||
print(f"Found {len(team_members)} team members.\n")
|
||||
|
||||
now = datetime.now(timezone.utc)
|
||||
pinged = []
|
||||
failed = []
|
||||
scanned = 0
|
||||
|
||||
print(f"Scanning open issues and PRs labeled '{TRIGGER_LABEL}' (threshold: {days_threshold} days)...\n")
|
||||
|
||||
for issue in repo.get_issues(state="open", labels=[TRIGGER_LABEL]):
|
||||
scanned += 1
|
||||
|
||||
if should_ping(issue, team_members, days_threshold, now):
|
||||
if ping(issue, dry_run):
|
||||
pinged.append(issue.number)
|
||||
else:
|
||||
failed.append(issue.number)
|
||||
|
||||
print(f"\nDone. Scanned {scanned} items, pinged {len(pinged)}, failed {len(failed)}.")
|
||||
if pinged:
|
||||
print(f"Pinged: {', '.join(f'#{n}' for n in pinged)}")
|
||||
if failed:
|
||||
print(f"Failed: {', '.join(f'#{n}' for n in failed)}")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,297 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
"""Tests for stale_issue_pr_ping.py."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
from datetime import datetime, timezone, timedelta
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
# Ensure the script directory is importable
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "scripts"))
|
||||
|
||||
from stale_issue_pr_ping import (
|
||||
PINGED_LABEL,
|
||||
PING_COMMENT,
|
||||
TRIGGER_LABEL,
|
||||
author_replied_after,
|
||||
find_last_team_comment,
|
||||
get_team_members,
|
||||
main,
|
||||
ping,
|
||||
should_ping,
|
||||
)
|
||||
|
||||
TEAM = {"alice", "bob"}
|
||||
NOW = datetime(2026, 3, 15, 12, 0, 0, tzinfo=timezone.utc)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _make_comment(login: str | None, created_at: datetime) -> MagicMock:
|
||||
"""Create a mock IssueComment."""
|
||||
c = MagicMock()
|
||||
if login is None:
|
||||
c.user = None
|
||||
else:
|
||||
c.user = MagicMock()
|
||||
c.user.login = login
|
||||
c.created_at = created_at
|
||||
return c
|
||||
|
||||
|
||||
def _make_label(name: str) -> MagicMock:
|
||||
lbl = MagicMock()
|
||||
lbl.name = name
|
||||
return lbl
|
||||
|
||||
|
||||
def _make_issue(
|
||||
author: str = "external",
|
||||
labels: list[str] | None = None,
|
||||
comment_count: int = 1,
|
||||
comments: list[MagicMock] | None = None,
|
||||
pull_request: bool = False,
|
||||
number: int = 42,
|
||||
) -> MagicMock:
|
||||
issue = MagicMock()
|
||||
issue.user = MagicMock()
|
||||
issue.user.login = author
|
||||
issue.number = number
|
||||
# Default to having the trigger label, since the API query pre-filters.
|
||||
if labels is None:
|
||||
labels = [TRIGGER_LABEL]
|
||||
issue.labels = [_make_label(n) for n in labels]
|
||||
issue.comments = comment_count
|
||||
issue.pull_request = MagicMock() if pull_request else None
|
||||
if comments is not None:
|
||||
issue.get_comments.return_value = comments
|
||||
return issue
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# find_last_team_comment
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestFindLastTeamComment:
|
||||
def test_returns_last_team_comment(self):
|
||||
c1 = _make_comment("alice", datetime(2026, 3, 1, tzinfo=timezone.utc))
|
||||
c2 = _make_comment("external", datetime(2026, 3, 2, tzinfo=timezone.utc))
|
||||
c3 = _make_comment("bob", datetime(2026, 3, 3, tzinfo=timezone.utc))
|
||||
assert find_last_team_comment([c1, c2, c3], TEAM) is c3
|
||||
|
||||
def test_returns_none_when_no_team_comments(self):
|
||||
c1 = _make_comment("external", datetime(2026, 3, 1, tzinfo=timezone.utc))
|
||||
assert find_last_team_comment([c1], TEAM) is None
|
||||
|
||||
def test_returns_none_for_empty_list(self):
|
||||
assert find_last_team_comment([], TEAM) is None
|
||||
|
||||
def test_skips_deleted_user(self):
|
||||
c1 = _make_comment(None, datetime(2026, 3, 1, tzinfo=timezone.utc))
|
||||
c2 = _make_comment("alice", datetime(2026, 3, 2, tzinfo=timezone.utc))
|
||||
assert find_last_team_comment([c1, c2], TEAM) is c2
|
||||
|
||||
def test_only_deleted_users(self):
|
||||
c1 = _make_comment(None, datetime(2026, 3, 1, tzinfo=timezone.utc))
|
||||
assert find_last_team_comment([c1], TEAM) is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# author_replied_after
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestAuthorRepliedAfter:
|
||||
def test_author_replied(self):
|
||||
after = datetime(2026, 3, 1, tzinfo=timezone.utc)
|
||||
c1 = _make_comment("external", datetime(2026, 3, 2, tzinfo=timezone.utc))
|
||||
assert author_replied_after([c1], "external", after) is True
|
||||
|
||||
def test_author_not_replied(self):
|
||||
after = datetime(2026, 3, 5, tzinfo=timezone.utc)
|
||||
c1 = _make_comment("external", datetime(2026, 3, 2, tzinfo=timezone.utc))
|
||||
assert author_replied_after([c1], "external", after) is False
|
||||
|
||||
def test_different_user_replied(self):
|
||||
after = datetime(2026, 3, 1, tzinfo=timezone.utc)
|
||||
c1 = _make_comment("someone_else", datetime(2026, 3, 2, tzinfo=timezone.utc))
|
||||
assert author_replied_after([c1], "external", after) is False
|
||||
|
||||
def test_deleted_user_comment(self):
|
||||
after = datetime(2026, 3, 1, tzinfo=timezone.utc)
|
||||
c1 = _make_comment(None, datetime(2026, 3, 2, tzinfo=timezone.utc))
|
||||
assert author_replied_after([c1], "external", after) is False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# should_ping
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestShouldPing:
|
||||
def test_should_ping_stale_issue(self):
|
||||
team_comment = _make_comment("alice", NOW - timedelta(days=5))
|
||||
issue = _make_issue(comments=[team_comment], comment_count=1)
|
||||
assert should_ping(issue, TEAM, 4, NOW) is True
|
||||
|
||||
def test_skip_team_member_author(self):
|
||||
issue = _make_issue(author="alice", labels=[TRIGGER_LABEL], comment_count=1)
|
||||
assert should_ping(issue, TEAM, 4, NOW) is False
|
||||
|
||||
def test_skip_already_pinged(self):
|
||||
issue = _make_issue(labels=[TRIGGER_LABEL, PINGED_LABEL], comment_count=1)
|
||||
assert should_ping(issue, TEAM, 4, NOW) is False
|
||||
|
||||
def test_skip_no_comments(self):
|
||||
issue = _make_issue(comment_count=0)
|
||||
assert should_ping(issue, TEAM, 4, NOW) is False
|
||||
|
||||
def test_skip_no_team_comment(self):
|
||||
c = _make_comment("external", NOW - timedelta(days=5))
|
||||
issue = _make_issue(comments=[c], comment_count=1)
|
||||
assert should_ping(issue, TEAM, 4, NOW) is False
|
||||
|
||||
def test_skip_author_replied(self):
|
||||
team_c = _make_comment("alice", NOW - timedelta(days=5))
|
||||
author_c = _make_comment("external", NOW - timedelta(days=3))
|
||||
issue = _make_issue(comments=[team_c, author_c], comment_count=2)
|
||||
assert should_ping(issue, TEAM, 4, NOW) is False
|
||||
|
||||
def test_skip_not_enough_days(self):
|
||||
team_comment = _make_comment("alice", NOW - timedelta(days=2))
|
||||
issue = _make_issue(comments=[team_comment], comment_count=1)
|
||||
assert should_ping(issue, TEAM, 4, NOW) is False
|
||||
|
||||
def test_aware_datetime_handled(self):
|
||||
"""Timezone-aware datetimes should not be mangled by astimezone."""
|
||||
aware_dt = (NOW - timedelta(days=5)).replace(tzinfo=timezone.utc)
|
||||
team_comment = _make_comment("alice", aware_dt)
|
||||
issue = _make_issue(comments=[team_comment], comment_count=1)
|
||||
assert should_ping(issue, TEAM, 4, NOW) is True
|
||||
|
||||
def test_naive_datetime_handled(self):
|
||||
"""Naive datetimes (pre-PyGithub 2.x) should be handled by astimezone."""
|
||||
naive_dt = (NOW - timedelta(days=5)).replace(tzinfo=None)
|
||||
team_comment = _make_comment("alice", naive_dt)
|
||||
issue = _make_issue(comments=[team_comment], comment_count=1)
|
||||
# astimezone on naive datetime treats it as local time; just verify no crash
|
||||
should_ping(issue, TEAM, 4, NOW)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ping
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestPing:
|
||||
def test_dry_run(self, capsys):
|
||||
issue = _make_issue()
|
||||
assert ping(issue, dry_run=True) is True
|
||||
issue.create_comment.assert_not_called()
|
||||
assert "DRY RUN" in capsys.readouterr().out
|
||||
|
||||
def test_success(self, capsys):
|
||||
issue = _make_issue()
|
||||
assert ping(issue, dry_run=False) is True
|
||||
issue.create_comment.assert_called_once()
|
||||
issue.add_to_labels.assert_called_once_with(PINGED_LABEL)
|
||||
|
||||
@patch("stale_issue_pr_ping.time.sleep")
|
||||
def test_retry_on_failure(self, mock_sleep):
|
||||
issue = _make_issue()
|
||||
issue.create_comment.side_effect = [Exception("net error"), None]
|
||||
assert ping(issue, dry_run=False) is True
|
||||
assert issue.create_comment.call_count == 2
|
||||
mock_sleep.assert_called_once()
|
||||
|
||||
@patch("stale_issue_pr_ping.time.sleep")
|
||||
def test_idempotent_retry_skips_comment_on_label_failure(self, mock_sleep):
|
||||
"""If create_comment succeeds but add_to_labels fails, retry should not re-comment."""
|
||||
issue = _make_issue()
|
||||
issue.add_to_labels.side_effect = [Exception("label error"), None]
|
||||
assert ping(issue, dry_run=False) is True
|
||||
# Comment should only be created once even though there were 2 attempts
|
||||
assert issue.create_comment.call_count == 1
|
||||
assert issue.add_to_labels.call_count == 2
|
||||
|
||||
@patch("stale_issue_pr_ping.time.sleep")
|
||||
def test_all_retries_fail(self, mock_sleep):
|
||||
issue = _make_issue()
|
||||
issue.create_comment.side_effect = Exception("permanent error")
|
||||
assert ping(issue, dry_run=False) is False
|
||||
assert issue.create_comment.call_count == 3
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# get_team_members
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestGetTeamMembers:
|
||||
def test_success(self):
|
||||
g = MagicMock()
|
||||
member = MagicMock()
|
||||
member.login = "alice"
|
||||
g.get_organization.return_value.get_team_by_slug.return_value.get_members.return_value = [member]
|
||||
assert get_team_members(g, "org", "my-team") == {"alice"}
|
||||
|
||||
def test_403_error_message(self, capsys):
|
||||
from github import GithubException
|
||||
|
||||
g = MagicMock()
|
||||
g.get_organization.return_value.get_team_by_slug.side_effect = GithubException(
|
||||
403, {"message": "Forbidden"}, None
|
||||
)
|
||||
with pytest.raises(SystemExit):
|
||||
get_team_members(g, "org", "my-team")
|
||||
out = capsys.readouterr().out
|
||||
assert "read:org" in out
|
||||
assert "403" in out
|
||||
|
||||
def test_404_error_message(self, capsys):
|
||||
from github import GithubException
|
||||
|
||||
g = MagicMock()
|
||||
g.get_organization.return_value.get_team_by_slug.side_effect = GithubException(
|
||||
404, {"message": "Not Found"}, None
|
||||
)
|
||||
with pytest.raises(SystemExit):
|
||||
get_team_members(g, "org", "bad-slug")
|
||||
out = capsys.readouterr().out
|
||||
assert "read:org" in out
|
||||
assert "bad-slug" in out
|
||||
|
||||
def test_generic_error(self, capsys):
|
||||
g = MagicMock()
|
||||
g.get_organization.side_effect = RuntimeError("boom")
|
||||
with pytest.raises(SystemExit):
|
||||
get_team_members(g, "org", "team")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# main – env var validation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestMain:
|
||||
@patch.dict(os.environ, {
|
||||
"GITHUB_TOKEN": "tok",
|
||||
"GITHUB_REPOSITORY": "org/repo",
|
||||
"TEAM_SLUG": "my-team",
|
||||
"DAYS_THRESHOLD": "abc",
|
||||
}, clear=True)
|
||||
def test_invalid_days_threshold(self, capsys):
|
||||
with pytest.raises(SystemExit):
|
||||
main()
|
||||
assert "numeric" in capsys.readouterr().out
|
||||
|
||||
@patch.dict(os.environ, {
|
||||
"GITHUB_TOKEN": "tok",
|
||||
"GITHUB_REPOSITORY": "org/repo",
|
||||
}, clear=True)
|
||||
def test_missing_team_slug(self, capsys):
|
||||
with pytest.raises(SystemExit):
|
||||
main()
|
||||
assert "TEAM_SLUG" in capsys.readouterr().out
|
||||
@@ -85,7 +85,7 @@ jobs:
|
||||
workflow-samples
|
||||
|
||||
- name: Setup dotnet
|
||||
uses: actions/setup-dotnet@v5.1.0
|
||||
uses: actions/setup-dotnet@v5.2.0
|
||||
with:
|
||||
global-json-file: ${{ github.workspace }}/dotnet/global.json
|
||||
- name: Build dotnet solutions
|
||||
@@ -165,7 +165,7 @@ jobs:
|
||||
echo "COSMOSDB_EMULATOR_AVAILABLE=true" >> $env:GITHUB_ENV
|
||||
|
||||
- name: Setup dotnet
|
||||
uses: actions/setup-dotnet@v5.1.0
|
||||
uses: actions/setup-dotnet@v5.2.0
|
||||
with:
|
||||
global-json-file: ${{ github.workspace }}/dotnet/global.json
|
||||
|
||||
@@ -281,7 +281,7 @@ jobs:
|
||||
# Generate test reports and check coverage
|
||||
- name: Generate test reports
|
||||
if: matrix.targetFramework == env.COVERAGE_FRAMEWORK
|
||||
uses: danielpalme/ReportGenerator-GitHub-Action@5.5.1
|
||||
uses: danielpalme/ReportGenerator-GitHub-Action@5.5.3
|
||||
with:
|
||||
reports: "./TestResults/Coverage/**/*.cobertura.xml"
|
||||
targetdir: "./TestResults/Reports"
|
||||
@@ -289,7 +289,7 @@ jobs:
|
||||
|
||||
- name: Upload coverage report artifact
|
||||
if: matrix.targetFramework == env.COVERAGE_FRAMEWORK
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: CoverageReport-${{ matrix.os }}-${{ matrix.targetFramework }}-${{ matrix.configuration }} # Artifact name
|
||||
path: ./TestResults/Reports # Directory containing files to upload
|
||||
|
||||
@@ -50,7 +50,7 @@ jobs:
|
||||
echo "COSMOS_EMULATOR_AVAILABLE=true" >> $env:GITHUB_ENV
|
||||
|
||||
- name: Setup dotnet
|
||||
uses: actions/setup-dotnet@v5.1.0
|
||||
uses: actions/setup-dotnet@v5.2.0
|
||||
with:
|
||||
global-json-file: ${{ github.workspace }}/dotnet/global.json
|
||||
|
||||
|
||||
@@ -75,7 +75,7 @@ jobs:
|
||||
os: ${{ runner.os }}
|
||||
env:
|
||||
UV_CACHE_DIR: /tmp/.uv-cache
|
||||
- name: Run fmt, lint, pyright in parallel across packages
|
||||
- name: Run syntax and pyright across packages
|
||||
run: uv run poe check-packages
|
||||
|
||||
samples-markdown:
|
||||
@@ -104,10 +104,8 @@ jobs:
|
||||
os: ${{ runner.os }}
|
||||
env:
|
||||
UV_CACHE_DIR: /tmp/.uv-cache
|
||||
- name: Run samples lint
|
||||
run: uv run poe samples-lint
|
||||
- name: Run samples syntax check
|
||||
run: uv run poe samples-syntax
|
||||
- name: Run samples checks
|
||||
run: uv run poe check -S
|
||||
- name: Run markdown code lint
|
||||
run: uv run poe markdown-code-lint
|
||||
|
||||
@@ -140,4 +138,4 @@ jobs:
|
||||
- name: Run Mypy
|
||||
env:
|
||||
GITHUB_BASE_REF: ${{ github.event.pull_request.base.ref || github.base_ref || 'main' }}
|
||||
run: uv run poe ci-mypy
|
||||
run: uv run python scripts/workspace_poe_tasks.py ci-mypy
|
||||
|
||||
@@ -0,0 +1,216 @@
|
||||
# Probe the highest allowed dependency versions, then open issues/PRs from the passing updates.
|
||||
name: Python - Dependency Range Validation
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
issues: write
|
||||
pull-requests: write
|
||||
|
||||
env:
|
||||
UV_CACHE_DIR: /tmp/.uv-cache
|
||||
|
||||
jobs:
|
||||
dependency-range-validation:
|
||||
name: Dependency Range Validation
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
# For now only run 3.13, if we do encounter situations where there are mismatches between packages and python versions (other then 3.10 and 3.14 which are known to not be able to install everything)
|
||||
# then we will have to reevaluate.
|
||||
UV_PYTHON: "3.13"
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Set up python and install the project
|
||||
uses: ./.github/actions/python-setup
|
||||
with:
|
||||
python-version: ${{ env.UV_PYTHON }}
|
||||
os: ${{ runner.os }}
|
||||
env:
|
||||
UV_CACHE_DIR: /tmp/.uv-cache
|
||||
|
||||
- name: Run dependency range validation
|
||||
id: validate_ranges
|
||||
# Keep workflow running so we can still publish diagnostics from this run.
|
||||
continue-on-error: true
|
||||
run: uv run poe validate-dependency-bounds-project --mode upper --package "*"
|
||||
working-directory: ./python
|
||||
|
||||
- name: Upload dependency range report
|
||||
# Always publish the report so failures are inspectable even when validation fails.
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: dependency-range-results
|
||||
path: python/scripts/dependencies/dependency-range-results.json
|
||||
if-no-files-found: warn
|
||||
|
||||
- name: Create issues for failed dependency candidates
|
||||
# Always process the report so failed candidates create actionable tracking issues.
|
||||
if: always()
|
||||
uses: actions/github-script@v8
|
||||
with:
|
||||
script: |
|
||||
const fs = require("fs")
|
||||
const reportPath = "python/scripts/dependencies/dependency-range-results.json"
|
||||
|
||||
if (!fs.existsSync(reportPath)) {
|
||||
core.warning(`No dependency range report found at ${reportPath}`)
|
||||
return
|
||||
}
|
||||
|
||||
const report = JSON.parse(fs.readFileSync(reportPath, "utf8"))
|
||||
const dependencyFailures = []
|
||||
|
||||
for (const packageResult of report.packages ?? []) {
|
||||
for (const dependency of packageResult.dependencies ?? []) {
|
||||
const candidateVersions = new Set(dependency.candidate_versions ?? [])
|
||||
const failedAttempts = (dependency.attempts ?? []).filter(
|
||||
(attempt) => attempt.status === "failed" && candidateVersions.has(attempt.trial_upper)
|
||||
)
|
||||
if (!failedAttempts.length) {
|
||||
continue
|
||||
}
|
||||
|
||||
const failuresByVersion = new Map()
|
||||
for (const attempt of failedAttempts) {
|
||||
const version = attempt.trial_upper || "unknown"
|
||||
if (!failuresByVersion.has(version)) {
|
||||
failuresByVersion.set(version, attempt.error || "No error output captured.")
|
||||
}
|
||||
}
|
||||
|
||||
dependencyFailures.push({
|
||||
packageName: packageResult.package_name,
|
||||
projectPath: packageResult.project_path,
|
||||
dependencyName: dependency.name,
|
||||
originalRequirements: dependency.original_requirements ?? [],
|
||||
finalRequirements: dependency.final_requirements ?? [],
|
||||
failedVersions: [...failuresByVersion.entries()].map(([version, error]) => ({ version, error })),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
if (!dependencyFailures.length) {
|
||||
core.info("No failing dependency candidates found.")
|
||||
return
|
||||
}
|
||||
|
||||
const owner = context.repo.owner
|
||||
const repo = context.repo.repo
|
||||
const openIssues = await github.paginate(github.rest.issues.listForRepo, {
|
||||
owner,
|
||||
repo,
|
||||
state: "open",
|
||||
per_page: 100,
|
||||
})
|
||||
const openIssueTitles = new Set(
|
||||
openIssues.filter((issue) => !issue.pull_request).map((issue) => issue.title)
|
||||
)
|
||||
|
||||
const formatError = (message) => String(message || "No error output captured.").replace(/```/g, "'''")
|
||||
|
||||
for (const failure of dependencyFailures) {
|
||||
const title = `Dependency validation failed: ${failure.dependencyName} (${failure.packageName})`
|
||||
if (openIssueTitles.has(title)) {
|
||||
core.info(`Issue already exists: ${title}`)
|
||||
continue
|
||||
}
|
||||
|
||||
const visibleFailures = failure.failedVersions.slice(0, 5)
|
||||
const omittedCount = failure.failedVersions.length - visibleFailures.length
|
||||
const failureDetails = visibleFailures
|
||||
.map(
|
||||
(entry) =>
|
||||
`- \`${entry.version}\`\n\n\`\`\`\n${formatError(entry.error).slice(0, 3500)}\n\`\`\``
|
||||
)
|
||||
.join("\n\n")
|
||||
|
||||
const body = [
|
||||
"Automated dependency range validation found candidate versions that failed checks.",
|
||||
"",
|
||||
`- Package: \`${failure.packageName}\``,
|
||||
`- Project path: \`${failure.projectPath}\``,
|
||||
`- Dependency: \`${failure.dependencyName}\``,
|
||||
`- Original requirements: ${
|
||||
failure.originalRequirements.length
|
||||
? failure.originalRequirements.map((value) => `\`${value}\``).join(", ")
|
||||
: "_none_"
|
||||
}`,
|
||||
`- Final requirements after run: ${
|
||||
failure.finalRequirements.length
|
||||
? failure.finalRequirements.map((value) => `\`${value}\``).join(", ")
|
||||
: "_none_"
|
||||
}`,
|
||||
"",
|
||||
"### Failed versions and errors",
|
||||
failureDetails,
|
||||
omittedCount > 0 ? `\n_Additional failed versions omitted: ${omittedCount}_` : "",
|
||||
"",
|
||||
`Workflow run: ${context.serverUrl}/${owner}/${repo}/actions/runs/${context.runId}`,
|
||||
].join("\n")
|
||||
|
||||
await github.rest.issues.create({
|
||||
owner,
|
||||
repo,
|
||||
title,
|
||||
body,
|
||||
})
|
||||
openIssueTitles.add(title)
|
||||
core.info(`Created issue: ${title}`)
|
||||
}
|
||||
|
||||
- name: Refresh lockfile
|
||||
# Only refresh lockfile after a clean validation to avoid committing known-bad ranges.
|
||||
if: steps.validate_ranges.outcome == 'success'
|
||||
run: uv lock --upgrade
|
||||
working-directory: ./python
|
||||
|
||||
- name: Commit and push dependency updates
|
||||
id: commit_updates
|
||||
if: steps.validate_ranges.outcome == 'success'
|
||||
run: |
|
||||
BRANCH="automation/python-dependency-range-updates"
|
||||
|
||||
git config user.name "github-actions[bot]"
|
||||
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
|
||||
git checkout -B "${BRANCH}"
|
||||
|
||||
git add python/packages/*/pyproject.toml python/uv.lock
|
||||
if git diff --cached --quiet; then
|
||||
echo "has_changes=false" >> "$GITHUB_OUTPUT"
|
||||
echo "No dependency updates to commit."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
git commit -m "chore: update dependency ranges"
|
||||
git push --force-with-lease --set-upstream origin "${BRANCH}"
|
||||
echo "has_changes=true" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Create or update pull request with GitHub CLI
|
||||
# Only open/update PRs for validated updates to keep automation branches trustworthy.
|
||||
if: steps.validate_ranges.outcome == 'success' && steps.commit_updates.outputs.has_changes == 'true'
|
||||
run: |
|
||||
BRANCH="automation/python-dependency-range-updates"
|
||||
PR_TITLE="Python: chore: update dependency ranges"
|
||||
PR_BODY_FILE="$(mktemp)"
|
||||
|
||||
cat > "${PR_BODY_FILE}" <<'EOF'
|
||||
This PR was generated by the dependency range validation workflow.
|
||||
|
||||
- Ran `uv run poe validate-dependency-bounds-project --mode upper --package "*"`
|
||||
- Updated package dependency bounds
|
||||
- Refreshed `python/uv.lock` with `uv lock --upgrade`
|
||||
EOF
|
||||
|
||||
PR_NUMBER="$(gh pr list --head "${BRANCH}" --base main --state open --json number --jq '.[0].number')"
|
||||
if [ -n "${PR_NUMBER}" ]; then
|
||||
gh pr edit "${PR_NUMBER}" --title "${PR_TITLE}" --body-file "${PR_BODY_FILE}"
|
||||
else
|
||||
gh pr create --base main --head "${BRANCH}" --title "${PR_TITLE}" --body-file "${PR_BODY_FILE}"
|
||||
fi
|
||||
@@ -0,0 +1,91 @@
|
||||
name: Python - Dev Dependency Upgrade
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
pull-requests: write
|
||||
|
||||
env:
|
||||
UV_CACHE_DIR: /tmp/.uv-cache
|
||||
|
||||
jobs:
|
||||
upgrade-dev-dependencies:
|
||||
name: Upgrade Dev Dependencies
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
UV_PYTHON: "3.13"
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Set up python and install the project
|
||||
uses: ./.github/actions/python-setup
|
||||
with:
|
||||
python-version: ${{ env.UV_PYTHON }}
|
||||
os: ${{ runner.os }}
|
||||
env:
|
||||
UV_CACHE_DIR: /tmp/.uv-cache
|
||||
|
||||
- name: Upgrade dev dependencies and validate workspace
|
||||
run: uv run poe upgrade-dev-dependencies
|
||||
working-directory: ./python
|
||||
|
||||
- name: Commit and push dev dependency updates
|
||||
id: commit_updates
|
||||
run: |
|
||||
BRANCH="automation/python-dev-dependency-updates"
|
||||
|
||||
git config user.name "github-actions[bot]"
|
||||
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
|
||||
git checkout -B "${BRANCH}"
|
||||
|
||||
git add python/pyproject.toml python/packages/*/pyproject.toml python/uv.lock
|
||||
if git diff --cached --quiet; then
|
||||
echo "has_changes=false" >> "$GITHUB_OUTPUT"
|
||||
echo "No dev dependency updates to commit."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
git commit -F- <<'EOF'
|
||||
Python: chore: upgrade dev dependencies
|
||||
EOF
|
||||
git push --force-with-lease --set-upstream origin "${BRANCH}"
|
||||
echo "has_changes=true" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Create or update pull request with GitHub CLI
|
||||
if: steps.commit_updates.outputs.has_changes == 'true'
|
||||
run: |
|
||||
BRANCH="automation/python-dev-dependency-updates"
|
||||
PR_TITLE="Python: chore: upgrade dev dependencies"
|
||||
PR_BODY_FILE="$(mktemp)"
|
||||
|
||||
cat > "${PR_BODY_FILE}" <<'EOF'
|
||||
### Motivation and Context
|
||||
|
||||
This automated update refreshes Python dev dependency pins across the workspace and reruns the repo validation gates before opening a pull request.
|
||||
|
||||
### Description
|
||||
|
||||
- Ran `uv run poe upgrade-dev-dependencies`
|
||||
- Refreshed dev dependency pins in workspace `pyproject.toml` files
|
||||
- Refreshed `python/uv.lock` with `uv lock --upgrade`
|
||||
- Reinstalled from the frozen lockfile and reran `check`, `typing`, and `test`
|
||||
|
||||
### Contribution Checklist
|
||||
|
||||
- [x] The code builds clean without any errors or warnings
|
||||
- [x] The PR follows the [Contribution Guidelines](https://github.com/microsoft/agent-framework/blob/main/CONTRIBUTING.md)
|
||||
- [x] All unit tests pass, and I have added new tests where possible
|
||||
- [ ] **Is this a breaking change?** If yes, add "[BREAKING]" prefix to the title of the PR.
|
||||
EOF
|
||||
|
||||
PR_NUMBER="$(gh pr list --head "${BRANCH}" --base main --state open --json number --jq '.[0].number')"
|
||||
if [ -n "${PR_NUMBER}" ]; then
|
||||
gh pr edit "${PR_NUMBER}" --title "${PR_TITLE}" --body-file "${PR_BODY_FILE}"
|
||||
else
|
||||
gh pr create --base main --head "${BRANCH}" --title "${PR_TITLE}" --body-file "${PR_BODY_FILE}"
|
||||
fi
|
||||
@@ -48,9 +48,8 @@ jobs:
|
||||
os: ${{ runner.os }}
|
||||
- name: Test with pytest (unit tests only)
|
||||
run: >
|
||||
uv run poe all-tests
|
||||
uv run poe test -A
|
||||
-m "not integration"
|
||||
-n logical --dist worksteal
|
||||
--timeout=120 --session-timeout=900 --timeout_method thread
|
||||
--retries 2 --retry-delay 5
|
||||
|
||||
|
||||
@@ -76,6 +76,9 @@ jobs:
|
||||
- name: Run lab tests
|
||||
run: cd packages/lab && uv run poe test
|
||||
|
||||
- name: Run resource-intensive lab tests
|
||||
run: cd packages/lab && uv run pytest -m "resource_intensive and not integration" --junitxml=test-results-resource-intensive.xml
|
||||
|
||||
- name: Run lab lint
|
||||
run: cd packages/lab && uv run poe lint
|
||||
|
||||
|
||||
@@ -100,9 +100,8 @@ jobs:
|
||||
os: ${{ runner.os }}
|
||||
- name: Test with pytest (unit tests only)
|
||||
run: >
|
||||
uv run poe all-tests
|
||||
uv run poe test -A
|
||||
-m "not integration"
|
||||
-n logical --dist worksteal
|
||||
--timeout=120 --session-timeout=900 --timeout_method thread
|
||||
--retries 2 --retry-delay 5
|
||||
working-directory: ./python
|
||||
|
||||
@@ -46,7 +46,7 @@ jobs:
|
||||
cd scripts && uv run python -m sample_validation --subdir 01-get-started --save-report --report-name 01-get-started
|
||||
|
||||
- name: Upload validation report
|
||||
uses: actions/upload-artifact@v4
|
||||
uses: actions/upload-artifact@v7
|
||||
if: always()
|
||||
with:
|
||||
name: validation-report-01-get-started
|
||||
@@ -89,7 +89,7 @@ jobs:
|
||||
cd scripts && uv run python -m sample_validation --subdir 02-agents --save-report --report-name 02-agents
|
||||
|
||||
- name: Upload validation report
|
||||
uses: actions/upload-artifact@v4
|
||||
uses: actions/upload-artifact@v7
|
||||
if: always()
|
||||
with:
|
||||
name: validation-report-02-agents
|
||||
@@ -126,7 +126,7 @@ jobs:
|
||||
cd scripts && uv run python -m sample_validation --subdir 03-workflows --save-report --report-name 03-workflows
|
||||
|
||||
- name: Upload validation report
|
||||
uses: actions/upload-artifact@v4
|
||||
uses: actions/upload-artifact@v7
|
||||
if: always()
|
||||
with:
|
||||
name: validation-report-03-workflows
|
||||
@@ -165,7 +165,7 @@ jobs:
|
||||
cd scripts && uv run python -m sample_validation --subdir 04-hosting --save-report --report-name 04-hosting
|
||||
|
||||
- name: Upload validation report
|
||||
uses: actions/upload-artifact@v4
|
||||
uses: actions/upload-artifact@v7
|
||||
if: always()
|
||||
with:
|
||||
name: validation-report-04-hosting
|
||||
@@ -209,7 +209,7 @@ jobs:
|
||||
cd scripts && uv run python -m sample_validation --subdir 05-end-to-end --save-report --report-name 05-end-to-end
|
||||
|
||||
- name: Upload validation report
|
||||
uses: actions/upload-artifact@v4
|
||||
uses: actions/upload-artifact@v7
|
||||
if: always()
|
||||
with:
|
||||
name: validation-report-05-end-to-end
|
||||
@@ -249,7 +249,7 @@ jobs:
|
||||
cd scripts && uv run python -m sample_validation --subdir autogen-migration --save-report --report-name autogen-migration
|
||||
|
||||
- name: Upload validation report
|
||||
uses: actions/upload-artifact@v4
|
||||
uses: actions/upload-artifact@v7
|
||||
if: always()
|
||||
with:
|
||||
name: validation-report-autogen-migration
|
||||
@@ -295,7 +295,7 @@ jobs:
|
||||
cd scripts && uv run python -m sample_validation --subdir semantic-kernel-migration --save-report --report-name semantic-kernel-migration
|
||||
|
||||
- name: Upload validation report
|
||||
uses: actions/upload-artifact@v4
|
||||
uses: actions/upload-artifact@v7
|
||||
if: always()
|
||||
with:
|
||||
name: validation-report-semantic-kernel-migration
|
||||
|
||||
@@ -21,7 +21,7 @@ jobs:
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- name: Download coverage report
|
||||
uses: actions/download-artifact@v7
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
github-token: ${{ secrets.GH_ACTIONS_PR_WRITE }}
|
||||
run-id: ${{ github.event.workflow_run.id }}
|
||||
@@ -46,7 +46,7 @@ jobs:
|
||||
echo "PR_NUMBER=$PR_NUMBER" >> "$GITHUB_ENV"
|
||||
- name: Pytest coverage comment
|
||||
id: coverageComment
|
||||
uses: MishaKav/pytest-coverage-comment@v1.2.0
|
||||
uses: MishaKav/pytest-coverage-comment@v1.6.0
|
||||
with:
|
||||
github-token: ${{ secrets.GH_ACTIONS_PR_WRITE }}
|
||||
issue-number: ${{ env.PR_NUMBER }}
|
||||
|
||||
@@ -32,17 +32,17 @@ jobs:
|
||||
id: python-setup
|
||||
uses: ./.github/actions/python-setup
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
python-version: ${{ env.UV_PYTHON }}
|
||||
os: ${{ runner.os }}
|
||||
env:
|
||||
# Configure a constant location for the uv cache
|
||||
UV_CACHE_DIR: /tmp/.uv-cache
|
||||
- name: Run all tests with coverage report
|
||||
run: uv run poe all-tests-cov --cov-report=xml:python-coverage.xml -q --junitxml=pytest.xml
|
||||
run: uv run poe test -A -C --cov-report=xml:python-coverage.xml -q --junitxml=pytest.xml
|
||||
- name: Check coverage threshold
|
||||
run: python ${{ github.workspace }}/.github/workflows/python-check-coverage.py python-coverage.xml ${{ env.COVERAGE_THRESHOLD }}
|
||||
- name: Upload coverage report
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
path: |
|
||||
python/python-coverage.xml
|
||||
|
||||
@@ -40,7 +40,7 @@ jobs:
|
||||
UV_CACHE_DIR: /tmp/.uv-cache
|
||||
# Unit tests
|
||||
- name: Run all tests
|
||||
run: uv run poe all-tests ${{ matrix.python-version == '3.10' && '--ignore-glob=packages/github_copilot/**' || '' }}
|
||||
run: uv run poe test -A
|
||||
working-directory: ./python
|
||||
|
||||
# Surface failing tests
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
name: Stale issue and PR ping
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: '0 0 * * *' # Midnight UTC daily
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
days_threshold:
|
||||
description: 'Days of silence before pinging the author'
|
||||
required: false
|
||||
default: '4'
|
||||
dry_run:
|
||||
description: 'Log what would be pinged without taking action'
|
||||
required: false
|
||||
default: 'false'
|
||||
type: choice
|
||||
options:
|
||||
- 'false'
|
||||
- 'true'
|
||||
|
||||
concurrency:
|
||||
group: stale-issue-pr-ping
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
ping_stale:
|
||||
name: "Ping stale issues and PRs"
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
issues: write
|
||||
pull-requests: write
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.13'
|
||||
|
||||
- name: Install dependencies
|
||||
run: pip install PyGithub==2.6.0
|
||||
|
||||
- name: Run stale issue/PR ping
|
||||
run: python .github/scripts/stale_issue_pr_ping.py
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GH_ACTIONS_PR_WRITE }}
|
||||
TEAM_SLUG: ${{ secrets.DEVELOPER_TEAM }}
|
||||
DAYS_THRESHOLD: ${{ github.event.inputs.days_threshold || '4' }}
|
||||
DRY_RUN: ${{ github.event.inputs.dry_run || 'false' }}
|
||||
@@ -205,6 +205,9 @@ WARP.md
|
||||
**/memory-bank/
|
||||
**/projectBrief.md
|
||||
**/tmpclaude*
|
||||
# Dependency-bound validation reports
|
||||
python/scripts/dependency-*-results.json
|
||||
python/scripts/dependencies/dependency-*-results.json
|
||||
|
||||
# Azurite storage emulator files
|
||||
*/__azurite_db_blob__.json*
|
||||
|
||||
@@ -4,8 +4,8 @@ status: accepted
|
||||
contact: westey-m
|
||||
date: 2025-07-10 {YYYY-MM-DD when the decision was last updated}
|
||||
deciders: sergeymenshykh, markwallace, rbarreto, dmytrostruk, westey-m, eavanvalkenburg, stephentoub
|
||||
consulted:
|
||||
informed:
|
||||
consulted:
|
||||
informed:
|
||||
---
|
||||
|
||||
# Agent Run Responses Design
|
||||
@@ -64,7 +64,7 @@ Approaches observed from the compared SDKs:
|
||||
| AutoGen | **Approach 1** Separates messages into Agent-Agent (maps to Primary) and Internal (maps to Secondary) and these are returned as separate properties on the agent response object. See [types of messages](https://microsoft.github.io/autogen/stable/user-guide/agentchat-user-guide/tutorial/messages.html#types-of-messages) and [Response](https://microsoft.github.io/autogen/stable/reference/python/autogen_agentchat.base.html#autogen_agentchat.base.Response) | **Approach 2** Returns a stream of internal events and the last item is a Response object. See [ChatAgent.on_messages_stream](https://microsoft.github.io/autogen/stable/reference/python/autogen_agentchat.base.html#autogen_agentchat.base.ChatAgent.on_messages_stream) |
|
||||
| OpenAI Agent SDK | **Approach 1** Separates new_items (Primary+Secondary) from final output (Primary) as separate properties on the [RunResult](https://github.com/openai/openai-agents-python/blob/main/src/agents/result.py#L39) | **Approach 1** Similar to non-streaming, has a way of streaming updates via a method on the response object which includes all data, and then a separate final output property on the response object which is populated only when the run is complete. See [RunResultStreaming](https://github.com/openai/openai-agents-python/blob/main/src/agents/result.py#L136) |
|
||||
| Google ADK | **Approach 2** [Emits events](https://google.github.io/adk-docs/runtime/#step-by-step-breakdown) with [FinalResponse](https://github.com/google/adk-java/blob/main/core/src/main/java/com/google/adk/events/Event.java#L232) true (Primary) / false (Secondary) and callers have to filter out those with false to get just the final response message | **Approach 2** Similar to non-streaming except [events](https://google.github.io/adk-docs/runtime/#streaming-vs-non-streaming-output-partialtrue) are emitted with [Partial](https://github.com/google/adk-java/blob/main/core/src/main/java/com/google/adk/events/Event.java#L133) true to indicate that they are streaming messages. A final non partial event is also emitted. |
|
||||
| AWS (Strands) | **Approach 3** Returns an [AgentResult](https://strandsagents.com/docs/api/python/strands.agent.agent_result/#agentresult) (Primary) with messages and a reason for the run's completion. | **Approach 2** [Streams events](https://strandsagents.com/docs/user-guide/concepts/streaming/) (Primary+Secondary) including, response text, current_tool_use, even data from "callbacks" (strands plugins) |
|
||||
| AWS (Strands) | **Approach 3** Returns an [AgentResult](https://strandsagents.com/docs/api/python/strands.agent.agent_result/) (Primary) with messages and a reason for the run's completion. | **Approach 2** [Streams events](https://strandsagents.com/docs/api/python/strands.agent.agent/) (Primary+Secondary) including, response text, current_tool_use, even data from "callbacks" (strands plugins) |
|
||||
| LangGraph | **Approach 2** A mixed list of all [messages](https://langchain-ai.github.io/langgraph/agents/run_agents/#output-format) | **Approach 2** A mixed list of all [messages](https://langchain-ai.github.io/langgraph/agents/run_agents/#output-format) |
|
||||
| Agno | **Combination of various approaches** Returns a [RunResponse](https://docs.agno.com/reference/agents/run-response) object with text content, messages (essentially chat history including inputs and instructions), reasoning and thinking text properties. Secondary events could potentially be extracted from messages. | **Approach 2** Returns [RunResponseEvent](https://docs.agno.com/reference/agents/run-response#runresponseevent-types-and-attributes) objects including tool call, memory update, etc, information, where the [RunResponseCompletedEvent](https://docs.agno.com/reference/agents/run-response#runresponsecompletedevent) has similar properties to RunResponse|
|
||||
| A2A | **Approach 3** Returns a [Task or Message](https://a2aproject.github.io/A2A/latest/specification/#71-messagesend) where the message is the final result (Primary) and task is a reference to a long running process. | **Approach 2** Returns a [stream](https://a2aproject.github.io/A2A/latest/specification/#72-messagestream) that contains task updates (Secondary) and a final message (Primary) |
|
||||
@@ -496,7 +496,7 @@ We need to decide what AIContent types, each agent response type will be mapped
|
||||
|-|-|
|
||||
| AutoGen | **Approach 1** Supports [configuring an agent](https://microsoft.github.io/autogen/stable/user-guide/agentchat-user-guide/tutorial/agents.html#structured-output) at agent creation. |
|
||||
| Google ADK | **Approach 1** Both [input and output schemas can be specified for LLM Agents](https://google.github.io/adk-docs/agents/llm-agents/#structuring-data-input_schema-output_schema-output_key) at construction time. This option is specific to this agent type and other agent types do not necessarily support |
|
||||
| AWS (Strands) | **Approach 2** Supports a special invocation method called [structured_output](https://strandsagents.com/docs/user-guide/concepts/agents/structured-output/) |
|
||||
| AWS (Strands) | **Approach 2** Supports a special invocation method called [structured_output](https://strandsagents.com/docs/api/python/strands.agent.agent/) |
|
||||
| LangGraph | **Approach 1** Supports [configuring an agent](https://langchain-ai.github.io/langgraph/agents/agents/?h=structured#6-configure-structured-output) at agent construction time, and a [structured response](https://langchain-ai.github.io/langgraph/agents/run_agents/#output-format) can be retrieved as a special property on the agent response |
|
||||
| Agno | **Approach 1** Supports [configuring an agent](https://docs.agno.com/input-output/structured-output/agent) at agent construction time |
|
||||
| A2A | **Informal Approach 2** Doesn't formally support schema negotiation, but [hints can be provided via metadata](https://a2a-protocol.org/latest/specification/#97-structured-data-exchange-requesting-and-providing-json) at invocation time |
|
||||
@@ -508,7 +508,7 @@ We need to decide what AIContent types, each agent response type will be mapped
|
||||
|-|-|
|
||||
| AutoGen | Supports a [stop reason](https://microsoft.github.io/autogen/stable/reference/python/autogen_agentchat.base.html#autogen_agentchat.base.TaskResult.stop_reason) which is a freeform text string |
|
||||
| Google ADK | [No equivalent present](https://github.com/google/adk-python/blob/main/src/google/adk/events/event.py) |
|
||||
| AWS (Strands) | Exposes a `stop_reason` property on the [AgentResult](https://strandsagents.com/docs/api/python/strands.agent.agent_result/#agentresult) class with options that are tied closely to LLM operations. |
|
||||
| AWS (Strands) | Exposes a [stop_reason](https://strandsagents.com/docs/api/python/strands.types.event_loop/) property on the [AgentResult](https://strandsagents.com/docs/api/python/strands.agent.agent_result/) class with options that are tied closely to LLM operations. |
|
||||
| LangGraph | No equivalent present, output contains only [messages](https://langchain-ai.github.io/langgraph/agents/run_agents/#output-format) |
|
||||
| Agno | [No equivalent present](https://docs.agno.com/reference/agents/run-response) |
|
||||
| A2A | No equivalent present, response only contains a [message](https://a2a-protocol.org/latest/specification/#64-message-object) or [task](https://a2a-protocol.org/latest/specification/#61-task-object). |
|
||||
|
||||
@@ -0,0 +1,815 @@
|
||||
---
|
||||
status: accepted
|
||||
contact: bentho
|
||||
date: 2026-02-27
|
||||
deciders: bentho, markwallace-microsoft, westey-m
|
||||
consulted: Pratyush Mishra, Shivam Shrivastava, Manni Arora (Centrica eval scenario)
|
||||
informed: Agent Framework team, Foundry Evals team
|
||||
---
|
||||
|
||||
# Agent Evaluation Architecture with Azure AI Foundry Integration
|
||||
|
||||
## Context and Problem Statement
|
||||
|
||||
Azure AI Foundry provides a rich evaluation service for AI agents — built-in evaluators for agent behavior (task adherence, intent resolution), tool usage (tool call accuracy, tool selection), quality (coherence, fluency, relevance), and safety (violence, self-harm, prohibited actions). Results are viewable in the Foundry portal with dashboards and comparison views.
|
||||
|
||||
However, using Foundry Evals with an agent-framework agent today requires significant manual effort. Developers must:
|
||||
|
||||
1. Transform agent-framework's `Message`/`Content` types into the OpenAI-style agent message schema that Foundry evaluators expect
|
||||
2. Map tool definitions from agent-framework's `FunctionTool` format to evaluator-compatible schemas
|
||||
3. Manually wire up the correct Foundry data source type (`azure_ai_traces`, `jsonl`, `azure_ai_target_completions`, etc.) depending on their scenario
|
||||
4. Handle App Insights trace ID queries, response ID collection, and eval polling
|
||||
|
||||
Additionally, evaluation is a concern that extends beyond any single provider. Developers may want to use local evaluators (LLM-as-judge, regex, keyword matching), third-party evaluation libraries, or multiple providers in combination. The architecture must support this without creating a Foundry-specific lock-in at the API level.
|
||||
|
||||
### Functional Requirements for Agent Evaluation
|
||||
|
||||
- **Single agents and workflows.** Evaluate both individual agent responses and multi-agent workflow results, with per-agent breakdown to pinpoint underperformance.
|
||||
- **One-shot and multi-turn conversations.** Capture full conversation trajectories — including tool calls and results — not just final query/response pairs.
|
||||
- **Conversation factoring.** Support splitting conversations into query/response in multiple ways (last turn, full trajectory, per-turn) because different factorings measure different things.
|
||||
- **Multiple providers, mix and match.** Run Foundry LLM-as-judge evaluators alongside fast local checks and custom evaluators on the same data, without restructuring code.
|
||||
- **Third-party extensibility.** Any evaluation library can participate by implementing the `Evaluator` protocol (Python) or `IAgentEvaluator` interface (.NET). No predetermined list of supported libraries — the protocol is intentionally simple (`evaluate(items) → results`) so that wrappers for libraries like DeepEval, RAGAS, or Promptfoo are straightforward to write.
|
||||
- **Bring your own evaluator.** Creating a custom evaluator should be as simple as writing a function.
|
||||
- **Evaluate without re-running.** Evaluate existing responses from logs or previous runs without invoking the agent again.
|
||||
|
||||
## Decision Drivers
|
||||
|
||||
- **Zero-friction evaluation**: Developers should go from "I have an agent" to "I have eval results" with minimal code.
|
||||
- **Provider-agnostic API**: Core evaluation capabilities must not be tied to any specific provider. Provider configuration should be separate from the evaluation call.
|
||||
- **Lowest concept count**: Introduce the fewest possible new types, abstractions, and APIs for developers to learn.
|
||||
- **Leverage existing knowledge**: The framework already knows which agents exist, what tools they have, and what conversations occurred. Evals should use this automatically rather than requiring the developer to re-specify it.
|
||||
- **Foundry-native results**: When using Foundry, results should be viewable in the Foundry portal with dashboards and comparison views.
|
||||
- **Progressive disclosure**: Simple scenarios should be near-zero code. Advanced scenarios should build on the same primitives.
|
||||
- **Cross-language parity**: Design must be implementable in both Python and .NET.
|
||||
|
||||
## Considered Options
|
||||
|
||||
1. **Provider-specific functions** — Build Foundry-specific helper functions (`evaluate_agent()`, etc.) directly in the Azure package. All eval functions take Foundry connection parameters.
|
||||
2. **Evaluator protocol with shared orchestration** — Define a provider-agnostic `Evaluator` protocol in the base agent library (`agent_framework` in Python, `Microsoft.Agents.AI` in .NET). Orchestration functions live alongside it. Providers implement the protocol.
|
||||
3. **Full eval framework** — Build comprehensive eval infrastructure including custom evaluator definitions, scoring profiles, and reporting inside agent-framework.
|
||||
|
||||
## Decision Outcome
|
||||
|
||||
Proposed option: "Evaluator protocol with shared orchestration", because it delivers the low-friction developer experience, supports multiple providers without API changes, and keeps the concept count low.
|
||||
|
||||
### Usage Examples
|
||||
|
||||
#### Evaluate an agent
|
||||
|
||||
The agent is invoked once per query by default. For statistically meaningful evaluation, provide multiple diverse queries. For measuring **consistency** (does the same query produce reliable results?), use `num_repetitions` to run each query N times independently:
|
||||
|
||||
**Python:**
|
||||
|
||||
```python
|
||||
evals = FoundryEvals(
|
||||
project_client=client,
|
||||
model_deployment="gpt-4o",
|
||||
evaluators=[FoundryEvals.RELEVANCE, FoundryEvals.COHERENCE],
|
||||
)
|
||||
|
||||
results = await evaluate_agent(
|
||||
agent=my_agent,
|
||||
queries=[
|
||||
"What's the weather in Seattle?",
|
||||
"Plan a weekend trip to Portland",
|
||||
"What restaurants are near Pike Place?",
|
||||
],
|
||||
evaluators=evals,
|
||||
)
|
||||
for r in results:
|
||||
r.assert_passed()
|
||||
```
|
||||
|
||||
**C#:**
|
||||
|
||||
```csharp
|
||||
var evals = new FoundryEvals(chatConfiguration, FoundryEvals.Relevance, FoundryEvals.Coherence);
|
||||
|
||||
AgentEvaluationResults results = await agent.EvaluateAsync(
|
||||
new[] {
|
||||
"What's the weather in Seattle?",
|
||||
"Plan a weekend trip to Portland",
|
||||
"What restaurants are near Pike Place?",
|
||||
},
|
||||
evals);
|
||||
|
||||
results.AssertAllPassed();
|
||||
```
|
||||
|
||||
`evaluate_agent` returns one `EvalResults` per evaluator. Each result contains per-item scores with the evaluated response for auditing:
|
||||
|
||||
```
|
||||
# results[0] (FoundryEvals)
|
||||
EvalResults(status="completed", passed=3, failed=0, total=3)
|
||||
items[0]: EvalItemResult(
|
||||
query="What's the weather in Seattle?",
|
||||
response="It's currently 72°F and sunny in Seattle.",
|
||||
scores={"relevance": 5, "coherence": 5})
|
||||
items[1]: EvalItemResult(
|
||||
query="Plan a weekend trip to Portland",
|
||||
response="Here's a 2-day Portland itinerary...",
|
||||
scores={"relevance": 4, "coherence": 5})
|
||||
items[2]: EvalItemResult(
|
||||
query="What restaurants are near Pike Place?",
|
||||
response="Top restaurants near Pike Place Market: ...",
|
||||
scores={"relevance": 5, "coherence": 4})
|
||||
```
|
||||
|
||||
#### Measure consistency with repetitions
|
||||
|
||||
Run each query multiple times to detect non-deterministic behavior:
|
||||
|
||||
**Python:**
|
||||
|
||||
```python
|
||||
results = await evaluate_agent(
|
||||
agent=my_agent,
|
||||
queries=["What's the weather in Seattle?"],
|
||||
evaluators=evals,
|
||||
num_repetitions=3, # each query runs 3 times independently
|
||||
)
|
||||
# results contain 3 items (1 query × 3 repetitions)
|
||||
```
|
||||
|
||||
**C#:**
|
||||
|
||||
```csharp
|
||||
AgentEvaluationResults results = await agent.EvaluateAsync(
|
||||
new[] { "What's the weather in Seattle?" },
|
||||
evals,
|
||||
numRepetitions: 3); // each query runs 3 times independently
|
||||
// results contain 3 items (1 query × 3 repetitions)
|
||||
```
|
||||
|
||||
#### Evaluate a response you already have
|
||||
|
||||
When you already have agent responses, pass them directly to skip re-running the agent. Each query is paired with its corresponding response:
|
||||
|
||||
**Python:**
|
||||
|
||||
```python
|
||||
queries = ["What's the weather?", "What's the capital of France?"]
|
||||
responses = [await agent.run([Message("user", [q])]) for q in queries]
|
||||
|
||||
results = await evaluate_agent(
|
||||
responses=responses,
|
||||
evaluators=evals,
|
||||
)
|
||||
```
|
||||
|
||||
**C#:**
|
||||
|
||||
```csharp
|
||||
var queries = new[] { "What's the weather?" };
|
||||
var responses = new List<AgentResponse>();
|
||||
foreach (var q in queries)
|
||||
responses.Add(await agent.RunAsync(new[] { new ChatMessage(ChatRole.User, q) }));
|
||||
|
||||
AgentEvaluationResults results = await agent.EvaluateAsync(
|
||||
responses: responses,
|
||||
evals);
|
||||
```
|
||||
|
||||
Each `AgentResponse` already contains the conversation (query + response), so the evaluator extracts query/response from the conversation. When you pass `responses` without `queries`, the conversation is the source of truth.
|
||||
|
||||
#### Evaluate with conversation split strategies
|
||||
|
||||
By default, evaluators see only the last turn (final user message → final assistant response). For multi-turn conversations, you can control how the conversation is factored for evaluation:
|
||||
|
||||
**Python:**
|
||||
|
||||
```python
|
||||
results = await evaluate_agent(
|
||||
agent=agent,
|
||||
queries=["Plan a 3-day trip to Paris"],
|
||||
evaluators=evals,
|
||||
conversation_split=ConversationSplit.FULL, # evaluate entire trajectory
|
||||
)
|
||||
|
||||
# Or per-turn: each user→assistant exchange scored independently
|
||||
results = await evaluate_agent(
|
||||
agent=agent,
|
||||
queries=["Plan a 3-day trip to Paris"],
|
||||
evaluators=evals,
|
||||
conversation_split=ConversationSplit.PER_TURN,
|
||||
)
|
||||
```
|
||||
|
||||
**C#:**
|
||||
|
||||
```csharp
|
||||
// Full conversation as context
|
||||
AgentEvaluationResults results = await agent.EvaluateAsync(
|
||||
new[] { "Plan a 3-day trip to Paris" },
|
||||
evals,
|
||||
splitter: ConversationSplitters.Full);
|
||||
|
||||
// Per-turn splitting
|
||||
var items = EvalItem.PerTurnItems(conversation); // one EvalItem per user turn
|
||||
var results = await evals.EvaluateAsync(items);
|
||||
```
|
||||
|
||||
With `PER_TURN`, a 3-turn conversation produces 3 scored items:
|
||||
|
||||
```
|
||||
EvalResults(status="completed", passed=3, failed=0, total=3)
|
||||
items[0]: query="Plan a 3-day trip to Paris" scores={"relevance": 5}
|
||||
items[1]: query="What about restaurants?" scores={"relevance": 4}
|
||||
items[2]: query="Make it budget-friendly" scores={"relevance": 5}
|
||||
```
|
||||
|
||||
#### Evaluate a multi-agent workflow
|
||||
|
||||
**Python:**
|
||||
|
||||
```python
|
||||
result = await workflow.run("Plan a trip to Paris")
|
||||
eval_results = await evaluate_workflow(
|
||||
workflow=workflow,
|
||||
workflow_result=result,
|
||||
evaluators=evals,
|
||||
)
|
||||
|
||||
for r in eval_results:
|
||||
print(f" overall: {r.passed}/{r.total}")
|
||||
for name, sub in r.sub_results.items():
|
||||
print(f" {name}: {sub.passed}/{sub.total}")
|
||||
```
|
||||
|
||||
**C#:**
|
||||
|
||||
```csharp
|
||||
WorkflowRunResult result = await workflow.RunAsync("Plan a trip to Paris");
|
||||
|
||||
IReadOnlyList<AgentEvaluationResults> evalResults = await result.EvaluateAsync(evals);
|
||||
|
||||
foreach (var r in evalResults)
|
||||
{
|
||||
Console.WriteLine($" overall: {r.Passed}/{r.Total}");
|
||||
foreach (var (name, sub) in r.SubResults)
|
||||
Console.WriteLine($" {name}: {sub.Passed}/{sub.Total}");
|
||||
}
|
||||
```
|
||||
|
||||
Workflows return one result per evaluator, with sub-results per agent in the workflow:
|
||||
|
||||
```
|
||||
EvalResults(status="completed", passed=2, failed=0, total=2)
|
||||
sub_results:
|
||||
"planner": EvalResults(passed=1, total=1)
|
||||
"researcher": EvalResults(passed=1, total=1)
|
||||
```
|
||||
|
||||
#### Mix multiple providers
|
||||
|
||||
**Python:**
|
||||
|
||||
```python
|
||||
@evaluator
|
||||
def is_helpful(response: str) -> bool:
|
||||
return len(response.split()) > 10
|
||||
|
||||
foundry = FoundryEvals(
|
||||
project_client=client,
|
||||
model_deployment="gpt-4o",
|
||||
evaluators=[FoundryEvals.RELEVANCE, FoundryEvals.COHERENCE],
|
||||
)
|
||||
|
||||
results = await evaluate_agent(
|
||||
agent=agent,
|
||||
queries=queries,
|
||||
evaluators=[is_helpful, keyword_check("weather"), foundry],
|
||||
)
|
||||
```
|
||||
|
||||
**C#:**
|
||||
|
||||
```csharp
|
||||
IReadOnlyList<AgentEvaluationResults> results = await agent.EvaluateAsync(
|
||||
queries,
|
||||
evaluators: new IAgentEvaluator[]
|
||||
{
|
||||
new LocalEvaluator(
|
||||
EvalChecks.KeywordCheck("weather"),
|
||||
FunctionEvaluator.Create("is_helpful", (string r) => r.Split(' ').Length > 10)),
|
||||
new FoundryEvals(chatConfiguration, FoundryEvals.Relevance, FoundryEvals.Coherence),
|
||||
});
|
||||
```
|
||||
|
||||
Multiple evaluators return one result each — `results[0]` is the local evaluator, `results[1]` is Foundry.
|
||||
|
||||
#### Custom function evaluators
|
||||
|
||||
**Python:**
|
||||
|
||||
```python
|
||||
@evaluator
|
||||
def mentions_city(response: str, expected_output: str) -> bool:
|
||||
return expected_output.lower() in response.lower()
|
||||
|
||||
@evaluator
|
||||
def used_tools(conversation: list, tools: list) -> float:
|
||||
# ... scoring logic
|
||||
return score
|
||||
|
||||
local = LocalEvaluator(mentions_city, used_tools)
|
||||
```
|
||||
|
||||
`@evaluator` uses **parameter name injection** — the function's parameter names determine what data it receives from the `EvalItem`. Supported names: `query`, `response`, `expected`, `expected_tool_calls`, `conversation`, `tools`, `context`. Any combination is valid.
|
||||
|
||||
**C#:**
|
||||
|
||||
```csharp
|
||||
var local = new LocalEvaluator(
|
||||
FunctionEvaluator.Create("mentions_city",
|
||||
(EvalItem item) => item.ExpectedOutput != null
|
||||
&& item.Response.Contains(item.ExpectedOutput, StringComparison.OrdinalIgnoreCase)),
|
||||
FunctionEvaluator.Create("is_concise",
|
||||
(string response) => response.Split(' ').Length < 500));
|
||||
```
|
||||
|
||||
## What To Build
|
||||
|
||||
### Core: Evaluator Protocol
|
||||
|
||||
A runtime-checkable protocol that any evaluation provider implements:
|
||||
|
||||
```python
|
||||
@runtime_checkable
|
||||
class Evaluator(Protocol):
|
||||
name: str
|
||||
|
||||
async def evaluate(
|
||||
self, items: Sequence[EvalItem], *, eval_name: str = "Agent Framework Eval"
|
||||
) -> EvalResults: ...
|
||||
```
|
||||
|
||||
The protocol is minimal — just `name` and `evaluate()`.
|
||||
|
||||
### Core: EvalItem
|
||||
|
||||
Provider-agnostic data format for items to evaluate:
|
||||
|
||||
```python
|
||||
@dataclass
|
||||
class ExpectedToolCall:
|
||||
name: str # Tool/function name
|
||||
arguments: dict[str, Any] | None = None # None = don't check args
|
||||
|
||||
@dataclass
|
||||
class EvalItem:
|
||||
conversation: list[Message] # Single source of truth
|
||||
tools: list[FunctionTool] | None = None # Agent's available tools
|
||||
context: str | None = None
|
||||
expected_output: str | None = None # Ground-truth for comparison
|
||||
expected_tool_calls: list[ExpectedToolCall] | None = None
|
||||
split_strategy: ConversationSplitter | None = None
|
||||
|
||||
query: str # property — derived from conversation split
|
||||
response: str # property — derived from conversation split
|
||||
```
|
||||
|
||||
`conversation` is the single source of truth. `query` and `response` are derived properties — splitting the conversation at the last user message (default) and extracting text from each side. Changing the `split_strategy` consistently changes all derived values.
|
||||
|
||||
`tools` provides typed `FunctionTool` objects — including MCP tools, which are automatically extracted after agent runs.
|
||||
|
||||
### Internal: AgentEvalConverter
|
||||
|
||||
Internal class that converts agent-framework types to `EvalItem`. Used by `evaluate_agent()` and `evaluate_workflow()` — not part of the public API:
|
||||
|
||||
| Agent Framework | Eval Format |
|
||||
|---|---|
|
||||
| `Content.function_call` | `tool_call` in OpenAI chat format |
|
||||
| `Content.function_result` | `tool_result` in OpenAI chat format |
|
||||
| `FunctionTool` | `{name, description, parameters}` schema |
|
||||
| `Message` history | `conversation` list + `query`/`response` extraction |
|
||||
|
||||
### Core: EvalResults
|
||||
|
||||
Rich result type with convenience properties for CI integration:
|
||||
|
||||
```python
|
||||
results.all_passed # bool: no failures or errors (recursive for workflow)
|
||||
results.passed # int: passing count
|
||||
results.failed # int: failure count
|
||||
results.total # int: total = passed + failed + errored
|
||||
results.items # list[EvalItemResult]: per-item detail with query, response, and scores
|
||||
results.error # str | None: error details on failure
|
||||
results.sub_results # dict: per-agent breakdown (workflow evals)
|
||||
results.report_url # str | None: portal link (Foundry)
|
||||
results.assert_passed() # raises AssertionError with details
|
||||
```
|
||||
|
||||
### Core: Orchestration Functions
|
||||
|
||||
Provider-agnostic functions that extract data and delegate to evaluators:
|
||||
|
||||
| Function | What it does |
|
||||
|---|---|
|
||||
| `evaluate_agent()` | Runs agent against test queries (or evaluates pre-existing `responses=`), converts to `EvalItem`s, passes to evaluator. Accepts optional `expected_output=` for ground-truth comparison, `expected_tool_calls=` for tool-correctness evaluation, and `num_repetitions=` for consistency measurement |
|
||||
| `evaluate_workflow()` | Extracts per-agent data from `WorkflowRunResult`, evaluates each agent and overall output. Per-agent breakdown in `sub_results`. Also accepts `num_repetitions=` |
|
||||
|
||||
### Core: Conversation Split Strategies
|
||||
|
||||
Multi-turn conversations must be split into query (input) and response (output) halves for evaluation. How you split determines *what you're evaluating*:
|
||||
|
||||
**Last-turn split** — split at the last user message. Everything up to and including it is the query context; the agent's subsequent actions are the response:
|
||||
|
||||
```
|
||||
conversation: user1 → assistant1 → user2 → assistant2(tool) → tool_result → assistant3
|
||||
query_messages: [user1, assistant1, user2]
|
||||
response_messages: [assistant2(tool), tool_result, assistant3]
|
||||
```
|
||||
|
||||
This evaluates: "Given all the context so far, did the agent answer the latest question well?" Best for response quality at a specific point in the conversation.
|
||||
|
||||
**Full-conversation split** — the first user message is the query; everything after is the response:
|
||||
|
||||
```
|
||||
query_messages: [user1]
|
||||
response_messages: [assistant1, user2, assistant2(tool), tool_result, assistant3]
|
||||
```
|
||||
|
||||
This evaluates: "Given the original request, did the entire conversation trajectory serve the user?" Best for task completion and overall conversation quality.
|
||||
|
||||
**Per-turn split** — produces N eval items from an N-turn conversation. Each turn is evaluated with its cumulative context:
|
||||
|
||||
```
|
||||
item 1: query = [user1], response = [assistant1]
|
||||
item 2: query = [user1, assistant1, user2], response = [assistant2(tool), tool_result, assistant3]
|
||||
```
|
||||
|
||||
This evaluates each response independently. Best for fine-grained analysis and pinpointing where a conversation goes wrong.
|
||||
|
||||
These factorings produce different scores for the same conversation. The framework ships all three as built-in strategies, defaulting to last-turn. Developers can also provide a custom splitter — a function (Python) or `IConversationSplitter` implementation (.NET) — and override the strategy at the call site or per evaluator.
|
||||
|
||||
### Azure AI: FoundryEvals
|
||||
|
||||
`Evaluator` implementation backed by Azure AI Foundry:
|
||||
|
||||
```python
|
||||
class FoundryEvals:
|
||||
def __init__(self, *, project_client=None, openai_client=None,
|
||||
model_deployment: str, evaluators=None, ...)
|
||||
async def evaluate(self, items, *, eval_name) -> EvalResults
|
||||
```
|
||||
|
||||
**Smart auto-detection in `evaluate()`:**
|
||||
- Default evaluators: relevance, coherence, task_adherence
|
||||
- Auto-adds `tool_call_accuracy` when items have tools/`tool_definitions`
|
||||
- Filters out tool evaluators for items without tools
|
||||
|
||||
### Azure AI: FoundryEvals Constants
|
||||
|
||||
```python
|
||||
from agent_framework_azure_ai import FoundryEvals
|
||||
|
||||
evaluators = [FoundryEvals.RELEVANCE, FoundryEvals.TOOL_CALL_ACCURACY]
|
||||
```
|
||||
|
||||
Categories: Agent behavior, Tool usage, Quality, Safety.
|
||||
|
||||
### Azure AI: Foundry-Specific Functions
|
||||
|
||||
| Function | What it does |
|
||||
|---|---|
|
||||
| `evaluate_traces()` | Evaluate from stored response IDs or OTel traces |
|
||||
| `evaluate_foundry_target()` | Evaluate a Foundry-registered agent or deployment |
|
||||
|
||||
### Core: LocalEvaluator and Function Evaluators
|
||||
|
||||
`LocalEvaluator` implements the `Evaluator` protocol for fast, API-free evaluation. It runs check functions locally — useful for inner-loop development, CI smoke tests, and combining with cloud-based evaluators.
|
||||
|
||||
Built-in checks:
|
||||
- `keyword_check(*keywords)` — response must contain specified keywords
|
||||
- `tool_called_check(*tool_names)` — agent must have called specified tools
|
||||
- `tool_calls_present` — all `expected_tool_calls` names appear in conversation (unordered, extras OK)
|
||||
- `tool_call_args_match` — expected tool calls match on name + arguments (subset match on args)
|
||||
|
||||
Custom function evaluators use `@evaluator` to wrap plain Python functions. The function's **parameter names** determine what data it receives from the `EvalItem`:
|
||||
|
||||
```python
|
||||
from agent_framework import evaluator, LocalEvaluator
|
||||
|
||||
# Tier 1: Simple check — just query + response
|
||||
@evaluator
|
||||
def is_concise(response: str) -> bool:
|
||||
return len(response.split()) < 500
|
||||
|
||||
# Tier 2: Ground truth — compare against expected output
|
||||
@evaluator
|
||||
def mentions_city(response: str, expected_output: str) -> bool:
|
||||
return expected_output.lower() in response.lower()
|
||||
|
||||
# Tier 3: Full context — inspect conversation and tools
|
||||
@evaluator
|
||||
def used_tools(conversation: list, tools: list) -> float:
|
||||
# ... scoring logic
|
||||
return score
|
||||
|
||||
local = LocalEvaluator(is_concise, mentions_city, used_tools)
|
||||
```
|
||||
|
||||
Supported parameters: `query`, `response`, `expected`, `expected_tool_calls`, `conversation`, `tools`, `context`.
|
||||
Return types: `bool`, `float` (≥0.5 = pass), `dict` with `score` or `passed` key, or `CheckResult`.
|
||||
|
||||
Async functions are handled automatically — `@evaluator` detects `async def` and produces the right wrapper.
|
||||
|
||||
### Example: GAIA Benchmark
|
||||
|
||||
[GAIA](https://huggingface.co/gaia-benchmark) tests real-world multi-step tasks with known expected answers. Each task has a question and a ground-truth answer, with optional file attachments. The framework accommodates GAIA's knobs (difficulty levels, file inputs, multi-step tool use) through the existing `EvalItem` fields:
|
||||
|
||||
```python
|
||||
from datasets import load_dataset
|
||||
from agent_framework import evaluate_agent, evaluator, LocalEvaluator
|
||||
|
||||
gaia = load_dataset("gaia-benchmark/GAIA", "2023_level1", split="test")
|
||||
|
||||
@evaluator
|
||||
def exact_match(response: str, expected_output: str) -> bool:
|
||||
return expected_output.strip().lower() in response.strip().lower()
|
||||
|
||||
# Simple path — evaluate_agent handles running + expected_output stamping
|
||||
results = await evaluate_agent(
|
||||
agent=agent,
|
||||
queries=[task["Question"] for task in gaia],
|
||||
expected_output=[task["Final answer"] for task in gaia],
|
||||
evaluators=LocalEvaluator(exact_match),
|
||||
)
|
||||
```
|
||||
|
||||
### Package Location
|
||||
|
||||
- Core types and orchestration: `agent_framework._eval`, `agent_framework._local_eval` (Python), `Microsoft.Agents.AI` (.NET)
|
||||
- Foundry provider: `agent_framework_azure_ai._foundry_evals` (Python), `Microsoft.Agents.AI.AzureAI` (.NET)
|
||||
- Azure-AI re-exports core types for convenience (Python)
|
||||
|
||||
## Known Limitations
|
||||
|
||||
1. **Tool evaluators require query + agent**: Tool evaluators need tool definition schemas. When using these evaluators with `evaluate_agent(responses=...)`, provide `queries=` and pass an agent with tool definitions.
|
||||
2. **`model_deployment` always required**: Could potentially be inferred from the Foundry project configuration.
|
||||
|
||||
## Open Questions
|
||||
|
||||
1. **Red teaming non-registered agents**: Requires Foundry API support for callback-based flows.
|
||||
2. **Datasets with expected outputs**: A dataset abstraction for pre-populating `expected_output` values across eval runs is a natural next step but not yet designed.
|
||||
3. **Multi-modal evaluation**: The `conversation` field on `EvalItem` already stores full `Message`/`Content` (Python) and `ChatMessage` (.NET) objects, which can represent multi-modal content (images, audio, structured data). Evaluators that accept the full `EvalItem` or `conversation` parameter can access this content today. However, the convenience shortcuts — `query`/`response` string projections and the `FunctionEvaluator` string overloads — are text-only. Multi-modal-aware evaluators should use the full-item path (`Func<EvalItem, CheckResult>` in .NET, `conversation: list` parameter in Python).
|
||||
|
||||
## .NET Implementation Design
|
||||
|
||||
### Key Difference: MEAI Ecosystem
|
||||
|
||||
Unlike Python, the .NET ecosystem already has `Microsoft.Extensions.AI.Evaluation` (v10.3.0) providing:
|
||||
|
||||
- `IEvaluator` — per-item evaluation of `(messages, chatResponse) → EvaluationResult`
|
||||
- `CompositeEvaluator` — combines multiple evaluators
|
||||
- Quality evaluators — `RelevanceEvaluator`, `CoherenceEvaluator`, `GroundednessEvaluator`
|
||||
- Safety evaluators — `ContentHarmEvaluator`, `ProtectedMaterialEvaluator`
|
||||
- Metric types — `NumericMetric`, `BooleanMetric`, `StringMetric`
|
||||
|
||||
The .NET integration uses MEAI's `IEvaluator` directly — no new evaluator interface. Our contribution is the **orchestration layer**: extension methods that run agents, extract data, call `IEvaluator` per item, and aggregate results.
|
||||
|
||||
### Architecture
|
||||
|
||||
```
|
||||
┌──────────────────────────────────────────────────────────────┐
|
||||
│ Developer Code │
|
||||
│ agent.EvaluateAsync(queries, evaluator) │
|
||||
│ run.EvaluateAsync(evaluator) │
|
||||
└────────────────┬─────────────────────────────────────────────┘
|
||||
│
|
||||
┌────────────────▼─────────────────────────────────────────────┐
|
||||
│ Orchestration Layer (Microsoft.Agents.AI) │
|
||||
│ AgentEvaluationExtensions — runs agents, extracts data, │
|
||||
│ calls IEvaluator per item, aggregates into │
|
||||
│ AgentEvaluationResults │
|
||||
└────────────────┬─────────────────────────────────────────────┘
|
||||
│ IEvaluator (MEAI)
|
||||
│
|
||||
┌───────────┼────────────┐
|
||||
│ │ │
|
||||
┌───▼───-┐ ┌───▼────┐ ┌────▼──────────┐
|
||||
│ MEAI │ │ Local │ │ Foundry │
|
||||
│ Quality│ │ Checks │ │ (cloud batch) │
|
||||
│ Safety │ │ Lambdas│ │ │
|
||||
└────────┘ └────────┘ └───────────────┘
|
||||
```
|
||||
|
||||
All evaluators implement MEAI's `IEvaluator`. The orchestration layer doesn't need to know which kind — it calls `EvaluateAsync(messages, chatResponse)` per item on all of them. `FoundryEvals` handles batching internally (buffers items, submits once, returns per-item results).
|
||||
|
||||
### .NET Core Types
|
||||
|
||||
**No new evaluator interface.** Use MEAI's `IEvaluator` directly.
|
||||
|
||||
**`AgentEvaluationResults`** — The only new type. Aggregates per-item MEAI `EvaluationResult`s across a batch of queries:
|
||||
|
||||
```csharp
|
||||
public class AgentEvaluationResults
|
||||
{
|
||||
public string Provider { get; init; }
|
||||
public string? ReportUrl { get; init; }
|
||||
|
||||
// Per-item — standard MEAI EvaluationResult, unchanged
|
||||
public IReadOnlyList<EvaluationResult> Items { get; init; }
|
||||
|
||||
// Aggregate pass/fail derived from metric interpretations
|
||||
public int Passed { get; }
|
||||
public int Failed { get; }
|
||||
public int Total { get; }
|
||||
public bool AllPassed { get; }
|
||||
|
||||
// Workflow: per-agent breakdown
|
||||
public IReadOnlyDictionary<string, AgentEvaluationResults>? SubResults { get; init; }
|
||||
|
||||
public void AssertAllPassed(string? message = null);
|
||||
}
|
||||
```
|
||||
|
||||
### .NET Evaluator Implementations
|
||||
|
||||
All implement MEAI's `IEvaluator`:
|
||||
|
||||
**`LocalEvaluator`** — Runs lambda checks locally, returns `BooleanMetric` per check:
|
||||
|
||||
```csharp
|
||||
var local = new LocalEvaluator(
|
||||
FunctionEvaluator.Create("is_concise",
|
||||
(string response) => response.Split().Length < 500),
|
||||
EvalChecks.KeywordCheck("weather"),
|
||||
EvalChecks.ToolCalledCheck("get_weather"));
|
||||
```
|
||||
|
||||
**MEAI evaluators** — Used directly, no adapter needed:
|
||||
|
||||
```csharp
|
||||
var quality = new CompositeEvaluator(
|
||||
new RelevanceEvaluator(),
|
||||
new CoherenceEvaluator());
|
||||
```
|
||||
|
||||
**`FoundryEvals`** — Implements `IEvaluator` but batches internally. On first call, buffers the item. On the last item (or when explicitly flushed), submits the batch to Foundry and distributes per-item results:
|
||||
|
||||
```csharp
|
||||
var foundry = new FoundryEvals(projectClient, "gpt-4o");
|
||||
```
|
||||
|
||||
### .NET Orchestration: Extension Methods
|
||||
|
||||
```csharp
|
||||
public static class AgentEvaluationExtensions
|
||||
{
|
||||
// Evaluate an agent against test queries
|
||||
public static Task<AgentEvaluationResults> EvaluateAsync(
|
||||
this AIAgent agent,
|
||||
IEnumerable<string> queries,
|
||||
IEvaluator evaluator,
|
||||
ChatConfiguration? chatConfiguration = null,
|
||||
IEnumerable<string>? expectedOutput = null,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
// Evaluate pre-existing responses (without re-running the agent)
|
||||
public static Task<AgentEvaluationResults> EvaluateAsync(
|
||||
this AIAgent agent,
|
||||
AgentResponse responses,
|
||||
IEvaluator evaluator,
|
||||
IEnumerable<string>? queries = null,
|
||||
ChatConfiguration? chatConfiguration = null,
|
||||
IEnumerable<string>? expectedOutput = null,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
// Evaluate with multiple evaluators (one result per evaluator)
|
||||
public static Task<IReadOnlyList<AgentEvaluationResults>> EvaluateAsync(
|
||||
this AIAgent agent,
|
||||
IEnumerable<string> queries,
|
||||
IEnumerable<IEvaluator> evaluators,
|
||||
ChatConfiguration? chatConfiguration = null,
|
||||
IEnumerable<string>? expectedOutput = null,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
// Evaluate a workflow run with per-agent breakdown
|
||||
public static Task<AgentEvaluationResults> EvaluateAsync(
|
||||
this Run run,
|
||||
IEvaluator evaluator,
|
||||
ChatConfiguration? chatConfiguration = null,
|
||||
bool includeOverall = true,
|
||||
bool includePerAgent = true,
|
||||
CancellationToken cancellationToken = default);
|
||||
}
|
||||
```
|
||||
|
||||
**Usage:**
|
||||
|
||||
```csharp
|
||||
// MEAI evaluators — just works
|
||||
var results = await agent.EvaluateAsync(
|
||||
queries: ["What's the weather?"],
|
||||
evaluator: new RelevanceEvaluator(),
|
||||
chatConfiguration: new ChatConfiguration(evalClient));
|
||||
|
||||
// Local checks
|
||||
var results = await agent.EvaluateAsync(
|
||||
queries: ["What's the weather?"],
|
||||
evaluator: new LocalEvaluator(
|
||||
EvalChecks.KeywordCheck("weather")));
|
||||
|
||||
// Foundry cloud
|
||||
var results = await agent.EvaluateAsync(
|
||||
queries: ["What's the weather?"],
|
||||
evaluator: new FoundryEvals(projectClient, "gpt-4o"));
|
||||
|
||||
// Evaluate existing response (without re-running the agent)
|
||||
var response = await agent.RunAsync("What's the weather?");
|
||||
var results = await agent.EvaluateAsync(
|
||||
responses: response,
|
||||
queries: ["What's the weather?"],
|
||||
evaluator: new FoundryEvals(projectClient, "gpt-4o"));
|
||||
|
||||
// Mixed — one result per evaluator
|
||||
var results = await agent.EvaluateAsync(
|
||||
queries: ["What's the weather?"],
|
||||
evaluators: [
|
||||
new LocalEvaluator(EvalChecks.KeywordCheck("weather")),
|
||||
new RelevanceEvaluator(),
|
||||
new FoundryEvals(projectClient, "gpt-4o")
|
||||
],
|
||||
chatConfiguration: new ChatConfiguration(evalClient));
|
||||
|
||||
// Workflow with per-agent breakdown
|
||||
Run run = await workflowRunner.RunAsync(workflow, "Plan a trip");
|
||||
var results = await run.EvaluateAsync(
|
||||
evaluator: new FoundryEvals(projectClient, "gpt-4o"));
|
||||
```
|
||||
|
||||
### .NET Function Evaluators
|
||||
|
||||
Typed factory overloads (C# equivalent of Python's `@evaluator`):
|
||||
|
||||
```csharp
|
||||
public static class FunctionEvaluator
|
||||
{
|
||||
public static EvalCheck Create(string name, Func<string, bool> check); // response only
|
||||
public static EvalCheck Create(string name, Func<string, string?, bool> check); // expectedOutput
|
||||
public static EvalCheck Create(string name, Func<EvalItem, bool> check); // full item
|
||||
public static EvalCheck Create(string name, Func<EvalItem, CheckResult> check); // full control
|
||||
public static EvalCheck Create(string name, Func<string, Task<bool>> check); // async
|
||||
}
|
||||
```
|
||||
|
||||
`EvalItem` is a lightweight record used only by `FunctionEvaluator` and `LocalEvaluator` to pass context to check functions. It is not part of the `IEvaluator` interface:
|
||||
|
||||
```csharp
|
||||
public record ExpectedToolCall(string Name, IReadOnlyDictionary<string, object>? Arguments = null);
|
||||
|
||||
public sealed class EvalItem
|
||||
{
|
||||
public EvalItem(string query, string response, IReadOnlyList<ChatMessage> conversation);
|
||||
|
||||
public string Query { get; }
|
||||
public string Response { get; }
|
||||
public IReadOnlyList<ChatMessage> Conversation { get; }
|
||||
public IReadOnlyList<AITool>? Tools { get; set; }
|
||||
public string? ExpectedOutput { get; set; }
|
||||
public IReadOnlyList<ExpectedToolCall>? ExpectedToolCalls { get; set; }
|
||||
public string? Context { get; set; }
|
||||
public IConversationSplitter? Splitter { get; set; }
|
||||
}
|
||||
```
|
||||
|
||||
### Workflow Data Extraction (.NET)
|
||||
|
||||
`run.EvaluateAsync()` walks `Run.OutgoingEvents` via LINQ:
|
||||
|
||||
1. Pair `ExecutorInvokedEvent` / `ExecutorCompletedEvent` by `ExecutorId`
|
||||
2. Extract `AgentResponseEvent` for per-agent `ChatResponse`
|
||||
3. Call `evaluator.EvaluateAsync()` per invocation
|
||||
4. Group by `ExecutorId` for per-agent `SubResults`
|
||||
5. Use final workflow output for overall eval
|
||||
|
||||
### .NET Package Structure
|
||||
|
||||
| Package | Contents |
|
||||
|---------|----------|
|
||||
| `Microsoft.Agents.AI` | `IAgentEvaluator`, `AgentEvaluationResults`, `LocalEvaluator`, `FunctionEvaluator`, `EvalChecks`, `EvalItem`, `ExpectedToolCall`, `AgentEvaluationExtensions` |
|
||||
| `Microsoft.Agents.AI.AzureAI` | `FoundryEvals` (provider + constants) |
|
||||
|
||||
### Python ↔ .NET Mapping
|
||||
|
||||
| Python | .NET |
|
||||
|--------|------|
|
||||
| `Evaluator` protocol | `IAgentEvaluator` (our interface; MEAI provides `IEvaluator` for per-item scoring) |
|
||||
| `EvalItem` dataclass | `EvalItem` class |
|
||||
| `EvalResults` | `AgentEvaluationResults` |
|
||||
| `EvalItemResult` / `EvalScoreResult` | MEAI `EvaluationResult` / `EvaluationMetric` (reused) |
|
||||
| `LocalEvaluator` | `LocalEvaluator` (implements `IAgentEvaluator`) |
|
||||
| `@evaluator` | `FunctionEvaluator.Create()` overloads |
|
||||
| `keyword_check()` / `tool_called_check()` | `EvalChecks.KeywordCheck()` / `EvalChecks.ToolCalledCheck()` |
|
||||
| `tool_calls_present` / `tool_call_args_match` | (custom `FunctionEvaluator` — same pattern) |
|
||||
| `ExpectedToolCall` dataclass | `ExpectedToolCall` record |
|
||||
| `FoundryEvals` | `FoundryEvals` (implements `IAgentEvaluator`, includes evaluator name constants) |
|
||||
| `evaluate_agent()` | `agent.EvaluateAsync(queries, evaluator)` extension method |
|
||||
| `evaluate_agent(responses=)` | `agent.EvaluateAsync(responses, evaluator)` extension method |
|
||||
| `evaluate_workflow()` | `run.EvaluateAsync()` extension method |
|
||||
|
||||
## More Information
|
||||
|
||||
- [Foundry Evals documentation](https://learn.microsoft.com/azure/ai-foundry/concepts/evaluation-approach-gen-ai) — Azure AI Foundry evaluation overview
|
||||
@@ -19,11 +19,10 @@
|
||||
<PackageVersion Include="Aspire.Microsoft.Azure.Cosmos" Version="$(AspireAppHostSdkVersion)" />
|
||||
<PackageVersion Include="CommunityToolkit.Aspire.OllamaSharp" Version="13.0.0" />
|
||||
<!-- Azure.* -->
|
||||
<PackageVersion Include="Azure.AI.Projects" Version="2.0.0-beta.1" />
|
||||
<PackageVersion Include="Azure.AI.Projects.OpenAI" Version="2.0.0-beta.1" />
|
||||
<PackageVersion Include="Azure.AI.Agents.Persistent" Version="1.2.0-beta.8" />
|
||||
<PackageVersion Include="Azure.AI.OpenAI" Version="2.8.0-beta.1" />
|
||||
<PackageVersion Include="Azure.Identity" Version="1.17.1" />
|
||||
<PackageVersion Include="Azure.AI.Projects" Version="2.0.0-beta.2" />
|
||||
<PackageVersion Include="Azure.AI.Agents.Persistent" Version="1.2.0-beta.10" />
|
||||
<PackageVersion Include="Azure.AI.OpenAI" Version="2.9.0-beta.1" />
|
||||
<PackageVersion Include="Azure.Identity" Version="1.19.0" />
|
||||
<PackageVersion Include="Azure.Monitor.OpenTelemetry.Exporter" Version="1.4.0" />
|
||||
<!-- Google Gemini -->
|
||||
<PackageVersion Include="Google.GenAI" Version="0.11.0" />
|
||||
@@ -33,18 +32,19 @@
|
||||
<!-- Newtonsoft.Json -->
|
||||
<PackageVersion Include="Newtonsoft.Json" Version="13.0.4" />
|
||||
<!-- System.* -->
|
||||
<PackageVersion Include="Microsoft.Bcl.AsyncInterfaces" Version="10.0.3" />
|
||||
<PackageVersion Include="Microsoft.Bcl.AsyncInterfaces" Version="10.0.4" />
|
||||
<PackageVersion Include="Microsoft.Bcl.HashCode" Version="6.0.0" />
|
||||
<PackageVersion Include="Microsoft.Bcl.Memory" Version="10.0.4" />
|
||||
<PackageVersion Include="System.ClientModel" Version="1.9.0" />
|
||||
<PackageVersion Include="System.CodeDom" Version="10.0.0" />
|
||||
<PackageVersion Include="System.Collections.Immutable" Version="10.0.1" />
|
||||
<PackageVersion Include="System.CommandLine" Version="2.0.0-rc.2.25502.107" />
|
||||
<PackageVersion Include="System.Diagnostics.DiagnosticSource" Version="10.0.3" />
|
||||
<PackageVersion Include="System.Linq.AsyncEnumerable" Version="10.0.0" />
|
||||
<PackageVersion Include="System.Diagnostics.DiagnosticSource" Version="10.0.4" />
|
||||
<PackageVersion Include="System.Linq.AsyncEnumerable" Version="10.0.4" />
|
||||
<PackageVersion Include="System.Net.Http.Json" Version="10.0.0" />
|
||||
<PackageVersion Include="System.Net.ServerSentEvents" Version="10.0.3" />
|
||||
<PackageVersion Include="System.Text.Json" Version="10.0.3" />
|
||||
<PackageVersion Include="System.Threading.Channels" Version="10.0.3" />
|
||||
<PackageVersion Include="System.Net.ServerSentEvents" Version="10.0.4" />
|
||||
<PackageVersion Include="System.Text.Json" Version="10.0.4" />
|
||||
<PackageVersion Include="System.Threading.Channels" Version="10.0.4" />
|
||||
<PackageVersion Include="System.Threading.Tasks.Extensions" Version="4.6.3" />
|
||||
<PackageVersion Include="System.Net.Security" Version="4.3.2" />
|
||||
<!-- OpenTelemetry -->
|
||||
@@ -63,24 +63,25 @@
|
||||
<PackageVersion Include="Microsoft.AspNetCore.OpenApi" Version="10.0.0" />
|
||||
<PackageVersion Include="Swashbuckle.AspNetCore.SwaggerUI" Version="10.0.0" />
|
||||
<!-- Microsoft.Extensions.* -->
|
||||
<PackageVersion Include="Microsoft.Extensions.AI" Version="10.3.0" />
|
||||
<PackageVersion Include="Microsoft.Extensions.AI.Abstractions" Version="10.3.0" />
|
||||
<PackageVersion Include="Microsoft.Extensions.AI.Evaluation" Version="10.3.0" />
|
||||
<PackageVersion Include="Microsoft.Extensions.AI.Evaluation.Quality" Version="10.3.0" />
|
||||
<PackageVersion Include="Microsoft.Extensions.AI" Version="10.4.0" />
|
||||
<PackageVersion Include="Microsoft.Extensions.AI.Abstractions" Version="10.4.0" />
|
||||
<PackageVersion Include="Microsoft.Extensions.AI.Evaluation" Version="10.4.0" />
|
||||
<PackageVersion Include="Microsoft.Extensions.AI.Evaluation.Quality" Version="10.4.0" />
|
||||
<PackageVersion Include="Microsoft.Extensions.AI.Evaluation.Safety" Version="10.3.0-preview.1.26109.11" />
|
||||
<PackageVersion Include="Microsoft.Extensions.AI.OpenAI" Version="10.3.0" />
|
||||
<PackageVersion Include="Microsoft.Extensions.AI.OpenAI" Version="10.4.0" />
|
||||
<PackageVersion Include="Microsoft.Extensions.Caching.Memory" Version="10.0.0" />
|
||||
<PackageVersion Include="Microsoft.Extensions.Compliance.Abstractions" Version="10.4.0" />
|
||||
<PackageVersion Include="Microsoft.Extensions.Configuration" Version="10.0.0" />
|
||||
<PackageVersion Include="Microsoft.Extensions.Configuration.Binder" Version="10.0.0" />
|
||||
<PackageVersion Include="Microsoft.Extensions.Configuration.EnvironmentVariables" Version="10.0.0" />
|
||||
<PackageVersion Include="Microsoft.Extensions.Configuration.Json" Version="10.0.0" />
|
||||
<PackageVersion Include="Microsoft.Extensions.Configuration.UserSecrets" Version="10.0.0" />
|
||||
<PackageVersion Include="Microsoft.Extensions.DependencyInjection" Version="10.0.0" />
|
||||
<PackageVersion Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="10.0.3" />
|
||||
<PackageVersion Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="10.0.4" />
|
||||
<PackageVersion Include="Microsoft.Extensions.Hosting" Version="10.0.0" />
|
||||
<PackageVersion Include="Microsoft.Extensions.Http.Resilience" Version="10.0.0" />
|
||||
<PackageVersion Include="Microsoft.Extensions.Logging" Version="10.0.0" />
|
||||
<PackageVersion Include="Microsoft.Extensions.Logging.Abstractions" Version="10.0.3" />
|
||||
<PackageVersion Include="Microsoft.Extensions.Logging.Abstractions" Version="10.0.4" />
|
||||
<PackageVersion Include="Microsoft.Extensions.Logging.Console" Version="10.0.0" />
|
||||
<PackageVersion Include="Microsoft.Extensions.ServiceDiscovery" Version="10.0.0" />
|
||||
<PackageVersion Include="Microsoft.Extensions.VectorData.Abstractions" Version="9.7.0" />
|
||||
@@ -101,18 +102,18 @@
|
||||
<PackageVersion Include="Microsoft.Agents.Authentication.Msal" Version="1.3.171-beta" />
|
||||
<PackageVersion Include="Microsoft.Agents.Hosting.AspNetCore" Version="1.3.171-beta" />
|
||||
<!-- A2A -->
|
||||
<PackageVersion Include="A2A" Version="0.3.3-preview" />
|
||||
<PackageVersion Include="A2A.AspNetCore" Version="0.3.3-preview" />
|
||||
<PackageVersion Include="A2A" Version="0.3.4-preview" />
|
||||
<PackageVersion Include="A2A.AspNetCore" Version="0.3.4-preview" />
|
||||
<!-- MCP -->
|
||||
<PackageVersion Include="ModelContextProtocol" Version="0.8.0-preview.1" />
|
||||
<PackageVersion Include="ModelContextProtocol" Version="1.1.0" />
|
||||
<!-- Inference SDKs -->
|
||||
<PackageVersion Include="AWSSDK.Extensions.Bedrock.MEAI" Version="4.0.5.1" />
|
||||
<PackageVersion Include="Microsoft.ML.OnnxRuntimeGenAI" Version="0.10.0" />
|
||||
<PackageVersion Include="Microsoft.ML.Tokenizers" Version="2.0.0" />
|
||||
<PackageVersion Include="OllamaSharp" Version="5.4.8" />
|
||||
<PackageVersion Include="OpenAI" Version="2.8.0" />
|
||||
<PackageVersion Include="OpenAI" Version="2.9.1" />
|
||||
<!-- Identity -->
|
||||
<PackageVersion Include="Microsoft.Identity.Client.Extensions.Msal" Version="4.78.0" />
|
||||
<PackageVersion Include="Microsoft.Identity.Client.Extensions.Msal" Version="4.83.1" />
|
||||
<!-- Workflows -->
|
||||
<PackageVersion Include="Microsoft.Agents.ObjectModel" Version="2026.2.4.1" />
|
||||
<PackageVersion Include="Microsoft.Agents.ObjectModel.Json" Version="2026.2.4.1" />
|
||||
@@ -126,7 +127,7 @@
|
||||
<!-- Azure Functions -->
|
||||
<PackageVersion Include="Microsoft.Azure.Functions.Worker" Version="2.50.0" />
|
||||
<PackageVersion Include="Microsoft.Azure.Functions.Worker.ApplicationInsights" Version="2.50.0" />
|
||||
<PackageVersion Include="Microsoft.Azure.Functions.Worker.Extensions.DurableTask" Version="1.11.0" />
|
||||
<PackageVersion Include="Microsoft.Azure.Functions.Worker.Extensions.DurableTask" Version="1.12.1" />
|
||||
<PackageVersion Include="Microsoft.Azure.Functions.Worker.Extensions.DurableTask.AzureManaged" Version="1.0.1" />
|
||||
<PackageVersion Include="Microsoft.Azure.Functions.Worker.Extensions.Http" Version="3.3.0" />
|
||||
<PackageVersion Include="Microsoft.Azure.Functions.Worker.Extensions.Http.AspNetCore" Version="2.1.0" />
|
||||
@@ -148,6 +149,7 @@
|
||||
<!-- Symbols -->
|
||||
<PackageVersion Include="Microsoft.SourceLink.GitHub" Version="8.0.0" />
|
||||
<!-- Toolset -->
|
||||
<PackageVersion Include="ReferenceTrimmer" Version="3.4.5" />
|
||||
<PackageVersion Include="Microsoft.CodeAnalysis.Analyzers" Version="3.11.0" />
|
||||
<PackageVersion Include="Microsoft.CodeAnalysis.CSharp" Version="4.14.0" />
|
||||
<PackageVersion Include="Microsoft.CodeAnalysis.NetAnalyzers" Version="10.0.100" />
|
||||
|
||||
@@ -61,6 +61,25 @@
|
||||
<Folder Name="/Samples/02-agents/DeclarativeAgents/">
|
||||
<Project Path="samples/02-agents/DeclarativeAgents/ChatClient/DeclarativeChatClientAgents.csproj" />
|
||||
</Folder>
|
||||
<Folder Name="/Samples/04-hosting/DurableWorkflows/" />
|
||||
<Folder Name="/Samples/04-hosting/DurableWorkflows/ConsoleApps/">
|
||||
<Project Path="samples/04-hosting/DurableWorkflows/ConsoleApps/01_SequentialWorkflow/01_SequentialWorkflow.csproj" />
|
||||
<Project Path="samples/04-hosting/DurableWorkflows/ConsoleApps/02_ConcurrentWorkflow/02_ConcurrentWorkflow.csproj" />
|
||||
<Project Path="samples/04-hosting/DurableWorkflows/ConsoleApps/03_ConditionalEdges/03_ConditionalEdges.csproj" />
|
||||
<Project Path="samples/04-hosting/DurableWorkflows/ConsoleApps/04_WorkflowAndAgents/04_WorkflowAndAgents.csproj" />
|
||||
<Project Path="samples/04-hosting/DurableWorkflows/ConsoleApps/05_WorkflowEvents/05_WorkflowEvents.csproj" />
|
||||
<Project Path="samples/04-hosting/DurableWorkflows/ConsoleApps/06_WorkflowSharedState/06_WorkflowSharedState.csproj" />
|
||||
<Project Path="samples/04-hosting/DurableWorkflows/ConsoleApps/07_SubWorkflows/07_SubWorkflows.csproj" />
|
||||
<Project Path="samples/04-hosting/DurableWorkflows/ConsoleApps/08_WorkflowHITL/08_WorkflowHITL.csproj" />
|
||||
</Folder>
|
||||
<Folder Name="/Samples/04-hosting/DurableWorkflows/AzureFunctions/">
|
||||
<Project Path="samples/04-hosting/DurableWorkflows/AzureFunctions/01_SequentialWorkflow/01_SequentialWorkflow.csproj" />
|
||||
<Project Path="samples/04-hosting/DurableWorkflows/AzureFunctions/02_ConcurrentWorkflow/02_ConcurrentWorkflow.csproj" />
|
||||
<Project Path="samples/04-hosting/DurableWorkflows/AzureFunctions/03_WorkflowHITL/03_WorkflowHITL.csproj" />
|
||||
</Folder>
|
||||
<Folder Name="/Samples/GettingStarted/">
|
||||
<File Path="samples/GettingStarted/README.md" />
|
||||
</Folder>
|
||||
<Folder Name="/Samples/02-agents/AGUI/">
|
||||
<File Path="samples/02-agents/AGUI/README.md" />
|
||||
</Folder>
|
||||
@@ -290,7 +309,6 @@
|
||||
<Project Path="samples/05-end-to-end/HostedAgents/AgentWithHostedMCP/AgentWithHostedMCP.csproj" />
|
||||
<Project Path="samples/05-end-to-end/HostedAgents/AgentWithLocalTools/AgentWithLocalTools.csproj" />
|
||||
<Project Path="samples/05-end-to-end/HostedAgents/AgentWithTextSearchRag/AgentWithTextSearchRag.csproj" />
|
||||
<Project Path="samples/05-end-to-end/HostedAgents/AgentWithTools/AgentWithTools.csproj" />
|
||||
<Project Path="samples/05-end-to-end/HostedAgents/FoundryMultiAgent/FoundryMultiAgent.csproj" />
|
||||
<Project Path="samples/05-end-to-end/HostedAgents/FoundrySingleAgent/FoundrySingleAgent.csproj" />
|
||||
</Folder>
|
||||
@@ -434,6 +452,10 @@
|
||||
<File Path="src/Shared/Samples/TextOutputHelperExtensions.cs" />
|
||||
<File Path="src/Shared/Samples/XunitLogger.cs" />
|
||||
</Folder>
|
||||
<Folder Name="/Solution Items/src/Shared/Redaction/">
|
||||
<File Path="src/Shared/Redaction/README.md" />
|
||||
<File Path="src/Shared/Redaction/ReplacingRedactor.cs" />
|
||||
</Folder>
|
||||
<Folder Name="/Solution Items/src/Shared/Throw/">
|
||||
<File Path="src/Shared/Throw/README.md" />
|
||||
<File Path="src/Shared/Throw/Throw.cs" />
|
||||
@@ -520,4 +542,4 @@
|
||||
<Project Path="tests/Microsoft.Agents.AI.Workflows.Generators.UnitTests/Microsoft.Agents.AI.Workflows.Generators.UnitTests.csproj" />
|
||||
<Project Path="tests/Microsoft.Agents.AI.Workflows.UnitTests/Microsoft.Agents.AI.Workflows.UnitTests.csproj" />
|
||||
</Folder>
|
||||
</Solution>
|
||||
</Solution>
|
||||
|
||||
@@ -29,4 +29,7 @@
|
||||
<ItemGroup Condition="'$(InjectSharedDiagnosticIds)' == 'true'">
|
||||
<Compile Include="$(MSBuildThisFileDirectory)\..\..\src\Shared\DiagnosticIds\*.cs" LinkBase="Shared\DiagnosticIds" />
|
||||
</ItemGroup>
|
||||
<ItemGroup Condition="'$(InjectSharedRedaction)' == 'true'">
|
||||
<Compile Include="$(MSBuildThisFileDirectory)\..\..\src\Shared\Redaction\*.cs" LinkBase="Shared\Redaction" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
|
||||
@@ -59,14 +59,14 @@ while ((input = Console.ReadLine()) != null && !input.Equals("exit", StringCompa
|
||||
{
|
||||
switch (content)
|
||||
{
|
||||
case FunctionApprovalRequestContent approvalRequest:
|
||||
DisplayApprovalRequest(approvalRequest);
|
||||
case ToolApprovalRequestContent approvalRequest when approvalRequest.ToolCall is FunctionCallContent fcc:
|
||||
DisplayApprovalRequest(approvalRequest, fcc);
|
||||
|
||||
Console.Write($"\nApprove '{approvalRequest.FunctionCall.Name}'? (yes/no): ");
|
||||
Console.Write($"\nApprove '{fcc.Name}'? (yes/no): ");
|
||||
string? userInput = Console.ReadLine();
|
||||
bool approved = userInput?.ToUpperInvariant() is "YES" or "Y";
|
||||
|
||||
FunctionApprovalResponseContent approvalResponse = approvalRequest.CreateResponse(approved);
|
||||
ToolApprovalResponseContent approvalResponse = approvalRequest.CreateResponse(approved);
|
||||
|
||||
if (approvalRequest.AdditionalProperties != null)
|
||||
{
|
||||
@@ -128,19 +128,19 @@ while ((input = Console.ReadLine()) != null && !input.Equals("exit", StringCompa
|
||||
}
|
||||
|
||||
#pragma warning disable MEAI001
|
||||
static void DisplayApprovalRequest(FunctionApprovalRequestContent approvalRequest)
|
||||
static void DisplayApprovalRequest(ToolApprovalRequestContent approvalRequest, FunctionCallContent fcc)
|
||||
{
|
||||
Console.ForegroundColor = ConsoleColor.Yellow;
|
||||
Console.WriteLine();
|
||||
Console.WriteLine("============================================================");
|
||||
Console.WriteLine("APPROVAL REQUIRED");
|
||||
Console.WriteLine("============================================================");
|
||||
Console.WriteLine($"Function: {approvalRequest.FunctionCall.Name}");
|
||||
Console.WriteLine($"Function: {fcc.Name}");
|
||||
|
||||
if (approvalRequest.FunctionCall.Arguments != null)
|
||||
if (fcc.Arguments != null)
|
||||
{
|
||||
Console.WriteLine("Arguments:");
|
||||
foreach (var arg in approvalRequest.FunctionCall.Arguments)
|
||||
foreach (var arg in fcc.Arguments)
|
||||
{
|
||||
Console.WriteLine($" {arg.Key} = {arg.Value}");
|
||||
}
|
||||
|
||||
+12
-12
@@ -9,7 +9,7 @@ using ServerFunctionApproval;
|
||||
|
||||
/// <summary>
|
||||
/// A delegating agent that handles server function approval requests and responses.
|
||||
/// Transforms between FunctionApprovalRequestContent/FunctionApprovalResponseContent
|
||||
/// Transforms between ToolApprovalRequestContent/ToolApprovalResponseContent
|
||||
/// and the server's request_approval tool call pattern.
|
||||
/// </summary>
|
||||
internal sealed class ServerFunctionApprovalClientAgent : DelegatingAIAgent
|
||||
@@ -50,14 +50,14 @@ internal sealed class ServerFunctionApprovalClientAgent : DelegatingAIAgent
|
||||
}
|
||||
|
||||
#pragma warning disable MEAI001 // Type is for evaluation purposes only
|
||||
private static FunctionResultContent ConvertApprovalResponseToToolResult(FunctionApprovalResponseContent approvalResponse, JsonSerializerOptions jsonOptions)
|
||||
private static FunctionResultContent ConvertApprovalResponseToToolResult(ToolApprovalResponseContent approvalResponse, JsonSerializerOptions jsonOptions)
|
||||
{
|
||||
return new FunctionResultContent(
|
||||
callId: approvalResponse.Id,
|
||||
callId: approvalResponse.RequestId,
|
||||
result: JsonSerializer.SerializeToElement(
|
||||
new ApprovalResponse
|
||||
{
|
||||
ApprovalId = approvalResponse.Id,
|
||||
ApprovalId = approvalResponse.RequestId,
|
||||
Approved = approvalResponse.Approved
|
||||
},
|
||||
jsonOptions));
|
||||
@@ -89,7 +89,7 @@ internal sealed class ServerFunctionApprovalClientAgent : DelegatingAIAgent
|
||||
{
|
||||
List<ChatMessage>? result = null;
|
||||
|
||||
Dictionary<string, FunctionApprovalRequestContent> approvalRequests = [];
|
||||
Dictionary<string, ToolApprovalRequestContent> approvalRequests = [];
|
||||
for (var messageIndex = 0; messageIndex < messages.Count; messageIndex++)
|
||||
{
|
||||
var message = messages[messageIndex];
|
||||
@@ -102,21 +102,21 @@ internal sealed class ServerFunctionApprovalClientAgent : DelegatingAIAgent
|
||||
var content = message.Contents[contentIndex];
|
||||
|
||||
// Handle pending approval requests (transform to tool call)
|
||||
if (content is FunctionApprovalRequestContent approvalRequest &&
|
||||
if (content is ToolApprovalRequestContent approvalRequest &&
|
||||
approvalRequest.AdditionalProperties?.TryGetValue("original_function", out var originalFunction) == true &&
|
||||
originalFunction is FunctionCallContent original)
|
||||
{
|
||||
approvalRequests[approvalRequest.Id] = approvalRequest;
|
||||
approvalRequests[approvalRequest.RequestId] = approvalRequest;
|
||||
transformedContents ??= CopyContentsUpToIndex(message.Contents, contentIndex);
|
||||
transformedContents.Add(original);
|
||||
}
|
||||
// Handle pending approval responses (transform to tool result)
|
||||
else if (content is FunctionApprovalResponseContent approvalResponse &&
|
||||
approvalRequests.TryGetValue(approvalResponse.Id, out var correspondingRequest))
|
||||
else if (content is ToolApprovalResponseContent approvalResponse &&
|
||||
approvalRequests.TryGetValue(approvalResponse.RequestId, out var correspondingRequest))
|
||||
{
|
||||
transformedContents ??= CopyContentsUpToIndex(message.Contents, contentIndex);
|
||||
transformedContents.Add(ConvertApprovalResponseToToolResult(approvalResponse, jsonSerializerOptions));
|
||||
approvalRequests.Remove(approvalResponse.Id);
|
||||
approvalRequests.Remove(approvalResponse.RequestId);
|
||||
correspondingRequest.AdditionalProperties?.Remove("original_function");
|
||||
}
|
||||
// Skip historical approval content
|
||||
@@ -198,8 +198,8 @@ internal sealed class ServerFunctionApprovalClientAgent : DelegatingAIAgent
|
||||
var functionCallArgs = (Dictionary<string, object?>?)approvalRequest.FunctionArguments?
|
||||
.Deserialize(jsonSerializerOptions.GetTypeInfo(typeof(Dictionary<string, object?>)));
|
||||
|
||||
var approvalRequestContent = new FunctionApprovalRequestContent(
|
||||
id: approvalRequest.ApprovalId,
|
||||
var approvalRequestContent = new ToolApprovalRequestContent(
|
||||
requestId: approvalRequest.ApprovalId,
|
||||
new FunctionCallContent(
|
||||
callId: approvalRequest.ApprovalId,
|
||||
name: approvalRequest.FunctionName,
|
||||
|
||||
+8
-9
@@ -9,7 +9,7 @@ using ServerFunctionApproval;
|
||||
|
||||
/// <summary>
|
||||
/// A delegating agent that handles function approval requests on the server side.
|
||||
/// Transforms between FunctionApprovalRequestContent/FunctionApprovalResponseContent
|
||||
/// Transforms between ToolApprovalRequestContent/ToolApprovalResponseContent
|
||||
/// and the request_approval tool call pattern for client communication.
|
||||
/// </summary>
|
||||
internal sealed class ServerFunctionApprovalAgent : DelegatingAIAgent
|
||||
@@ -50,7 +50,7 @@ internal sealed class ServerFunctionApprovalAgent : DelegatingAIAgent
|
||||
}
|
||||
|
||||
#pragma warning disable MEAI001 // Type is for evaluation purposes only
|
||||
private static FunctionApprovalRequestContent ConvertToolCallToApprovalRequest(FunctionCallContent toolCall, JsonSerializerOptions jsonSerializerOptions)
|
||||
private static ToolApprovalRequestContent ConvertToolCallToApprovalRequest(FunctionCallContent toolCall, JsonSerializerOptions jsonSerializerOptions)
|
||||
{
|
||||
if (toolCall.Name != "request_approval" || toolCall.Arguments == null)
|
||||
{
|
||||
@@ -67,15 +67,15 @@ internal sealed class ServerFunctionApprovalAgent : DelegatingAIAgent
|
||||
throw new InvalidOperationException("Failed to deserialize approval request from tool call");
|
||||
}
|
||||
|
||||
return new FunctionApprovalRequestContent(
|
||||
id: request.ApprovalId,
|
||||
return new ToolApprovalRequestContent(
|
||||
requestId: request.ApprovalId,
|
||||
new FunctionCallContent(
|
||||
callId: request.ApprovalId,
|
||||
name: request.FunctionName,
|
||||
arguments: request.FunctionArguments));
|
||||
}
|
||||
|
||||
private static FunctionApprovalResponseContent ConvertToolResultToApprovalResponse(FunctionResultContent result, FunctionApprovalRequestContent approval, JsonSerializerOptions jsonSerializerOptions)
|
||||
private static ToolApprovalResponseContent ConvertToolResultToApprovalResponse(FunctionResultContent result, ToolApprovalRequestContent approval, JsonSerializerOptions jsonSerializerOptions)
|
||||
{
|
||||
var approvalResponse = result.Result is JsonElement je ?
|
||||
(ApprovalResponse?)je.Deserialize(jsonSerializerOptions.GetTypeInfo(typeof(ApprovalResponse))) :
|
||||
@@ -121,7 +121,7 @@ internal sealed class ServerFunctionApprovalAgent : DelegatingAIAgent
|
||||
// Track approval ID to original call ID mapping
|
||||
_ = new Dictionary<string, string>();
|
||||
#pragma warning disable MEAI001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed.
|
||||
Dictionary<string, FunctionApprovalRequestContent> trackedRequestApprovalToolCalls = new(); // Remote approvals
|
||||
Dictionary<string, ToolApprovalRequestContent> trackedRequestApprovalToolCalls = new(); // Remote approvals
|
||||
for (int messageIndex = 0; messageIndex < messages.Count; messageIndex++)
|
||||
{
|
||||
var message = messages[messageIndex];
|
||||
@@ -181,11 +181,10 @@ internal sealed class ServerFunctionApprovalAgent : DelegatingAIAgent
|
||||
{
|
||||
var content = update.Contents[i];
|
||||
#pragma warning disable MEAI001 // Type is for evaluation purposes only
|
||||
if (content is FunctionApprovalRequestContent request)
|
||||
if (content is ToolApprovalRequestContent request && request.ToolCall is FunctionCallContent functionCall)
|
||||
{
|
||||
updatedContents ??= [.. update.Contents];
|
||||
var functionCall = request.FunctionCall;
|
||||
var approvalId = request.Id;
|
||||
var approvalId = request.RequestId;
|
||||
|
||||
var approvalData = new ApprovalRequest
|
||||
{
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
#pragma warning disable CS0618 // Type or member is obsolete - sample uses deprecated PersistentAgentsClientExtensions
|
||||
|
||||
// This sample shows how to create and use a simple AI agent with Azure Foundry Agents as the backend.
|
||||
|
||||
using Azure.AI.Agents.Persistent;
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
// This sample shows how to create and use a AI agents with Azure Foundry Agents as the backend.
|
||||
|
||||
using Azure.AI.Projects;
|
||||
using Azure.AI.Projects.OpenAI;
|
||||
using Azure.AI.Projects.Agents;
|
||||
using Azure.Identity;
|
||||
using Microsoft.Agents.AI;
|
||||
|
||||
|
||||
@@ -17,8 +17,8 @@ var deploymentName = Environment.GetEnvironmentVariable("AZURE_OPENAI_DEPLOYMENT
|
||||
AIAgent agent = new AzureOpenAIClient(
|
||||
new Uri(endpoint),
|
||||
new DefaultAzureCredential())
|
||||
.GetResponsesClient(deploymentName)
|
||||
.AsAIAgent(instructions: "You are good at telling jokes.", name: "Joker");
|
||||
.GetResponsesClient()
|
||||
.AsAIAgent(model: deploymentName, instructions: "You are good at telling jokes.", name: "Joker");
|
||||
|
||||
// Invoke the agent and output the text result.
|
||||
Console.WriteLine(await agent.RunAsync("Tell me a joke about a pirate."));
|
||||
@@ -29,8 +29,8 @@ Console.WriteLine(await agent.RunAsync("Tell me a joke about a pirate."));
|
||||
AIAgent agentStoreFalse = new AzureOpenAIClient(
|
||||
new Uri(endpoint),
|
||||
new DefaultAzureCredential())
|
||||
.GetResponsesClient(deploymentName)
|
||||
.AsIChatClientWithStoredOutputDisabled()
|
||||
.GetResponsesClient()
|
||||
.AsIChatClientWithStoredOutputDisabled(model: deploymentName)
|
||||
.AsAIAgent(instructions: "You are good at telling jokes.", name: "Joker");
|
||||
|
||||
// Invoke the agent and output the text result.
|
||||
|
||||
+1
-1
@@ -9,7 +9,7 @@
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\..\..\src\Microsoft.Agents.AI\Microsoft.Agents.AI.csproj" />
|
||||
<ProjectReference Include="..\..\..\..\src\Microsoft.Agents.AI.Abstractions\Microsoft.Agents.AI.Abstractions.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
|
||||
@@ -11,8 +11,8 @@ var model = Environment.GetEnvironmentVariable("OPENAI_CHAT_MODEL_NAME") ?? "gpt
|
||||
|
||||
AIAgent agent = new OpenAIClient(
|
||||
apiKey)
|
||||
.GetResponsesClient(model)
|
||||
.AsAIAgent(instructions: "You are good at telling jokes.", name: "Joker");
|
||||
.GetResponsesClient()
|
||||
.AsAIAgent(model: model, instructions: "You are good at telling jokes.", name: "Joker");
|
||||
|
||||
// Invoke the agent and output the text result.
|
||||
Console.WriteLine(await agent.RunAsync("Tell me a joke about a pirate."));
|
||||
|
||||
@@ -23,7 +23,7 @@ var skillsProvider = new FileAgentSkillsProvider(skillPath: Path.Combine(AppCont
|
||||
|
||||
// --- Agent Setup ---
|
||||
AIAgent agent = new AzureOpenAIClient(new Uri(endpoint), new DefaultAzureCredential())
|
||||
.GetResponsesClient(deploymentName)
|
||||
.GetResponsesClient()
|
||||
.AsAIAgent(new ChatClientAgentOptions
|
||||
{
|
||||
Name = "SkillsAgent",
|
||||
@@ -32,7 +32,8 @@ AIAgent agent = new AzureOpenAIClient(new Uri(endpoint), new DefaultAzureCredent
|
||||
Instructions = "You are a helpful assistant.",
|
||||
},
|
||||
AIContextProviders = [skillsProvider],
|
||||
});
|
||||
},
|
||||
model: deploymentName);
|
||||
|
||||
// --- Example 1: Expense policy question (loads FAQ resource) ---
|
||||
Console.WriteLine("Example 1: Checking expense policy FAQ");
|
||||
|
||||
@@ -10,8 +10,8 @@ var apiKey = Environment.GetEnvironmentVariable("OPENAI_API_KEY") ?? throw new I
|
||||
var model = Environment.GetEnvironmentVariable("OPENAI_CHAT_MODEL_NAME") ?? "gpt-5";
|
||||
|
||||
var client = new OpenAIClient(apiKey)
|
||||
.GetResponsesClient(model)
|
||||
.AsIChatClient().AsBuilder()
|
||||
.GetResponsesClient()
|
||||
.AsIChatClient(model).AsBuilder()
|
||||
.ConfigureOptions(o =>
|
||||
{
|
||||
o.Reasoning = new()
|
||||
|
||||
+6
-3
@@ -20,19 +20,21 @@ public class OpenAIResponseClientAgent : DelegatingAIAgent
|
||||
/// <param name="instructions">Optional instructions for the agent.</param>
|
||||
/// <param name="name">Optional name for the agent.</param>
|
||||
/// <param name="description">Optional description for the agent.</param>
|
||||
/// <param name="model">Optional default model ID to use for requests. Required when using a plain <see cref="ResponsesClient"/> (not via Azure OpenAI).</param>
|
||||
/// <param name="loggerFactory">Optional instance of <see cref="ILoggerFactory"/></param>
|
||||
public OpenAIResponseClientAgent(
|
||||
ResponsesClient client,
|
||||
string? instructions = null,
|
||||
string? name = null,
|
||||
string? description = null,
|
||||
string? model = null,
|
||||
ILoggerFactory? loggerFactory = null) :
|
||||
this(client, new()
|
||||
{
|
||||
Name = name,
|
||||
Description = description,
|
||||
ChatOptions = new ChatOptions() { Instructions = instructions },
|
||||
}, loggerFactory)
|
||||
}, model, loggerFactory)
|
||||
{
|
||||
}
|
||||
|
||||
@@ -41,10 +43,11 @@ public class OpenAIResponseClientAgent : DelegatingAIAgent
|
||||
/// </summary>
|
||||
/// <param name="client">Instance of <see cref="ResponsesClient"/></param>
|
||||
/// <param name="options">Options to create the agent.</param>
|
||||
/// <param name="model">Optional default model ID to use for requests. Required when using a plain <see cref="ResponsesClient"/> (not via Azure OpenAI).</param>
|
||||
/// <param name="loggerFactory">Optional instance of <see cref="ILoggerFactory"/></param>
|
||||
public OpenAIResponseClientAgent(
|
||||
ResponsesClient client, ChatClientAgentOptions options, ILoggerFactory? loggerFactory = null) :
|
||||
base(new ChatClientAgent((client ?? throw new ArgumentNullException(nameof(client))).AsIChatClient(), options, loggerFactory))
|
||||
ResponsesClient client, ChatClientAgentOptions options, string? model = null, ILoggerFactory? loggerFactory = null) :
|
||||
base(new ChatClientAgent((client ?? throw new ArgumentNullException(nameof(client))).AsIChatClient(model), options, loggerFactory))
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
+2
-2
@@ -10,10 +10,10 @@ var apiKey = Environment.GetEnvironmentVariable("OPENAI_API_KEY") ?? throw new I
|
||||
var model = Environment.GetEnvironmentVariable("OPENAI_CHAT_MODEL_NAME") ?? "gpt-4o-mini";
|
||||
|
||||
// Create a ResponsesClient directly from OpenAIClient
|
||||
ResponsesClient responseClient = new OpenAIClient(apiKey).GetResponsesClient(model);
|
||||
ResponsesClient responseClient = new OpenAIClient(apiKey).GetResponsesClient();
|
||||
|
||||
// Create an agent directly from the ResponsesClient using OpenAIResponseClientAgent
|
||||
OpenAIResponseClientAgent agent = new(responseClient, instructions: "You are good at telling jokes.", name: "Joker");
|
||||
OpenAIResponseClientAgent agent = new(responseClient, instructions: "You are good at telling jokes.", name: "Joker", model: model);
|
||||
|
||||
ResponseItem userMessage = ResponseItem.CreateUserMessageItem("Tell me a joke about a pirate.");
|
||||
|
||||
|
||||
+1
-1
@@ -22,7 +22,7 @@ OpenAIClient openAIClient = new(apiKey);
|
||||
ConversationClient conversationClient = openAIClient.GetConversationClient();
|
||||
|
||||
// Create an agent directly from the ResponsesClient using OpenAIResponseClientAgent
|
||||
ChatClientAgent agent = new(openAIClient.GetResponsesClient(model).AsIChatClient(), instructions: "You are a helpful assistant.", name: "ConversationAgent");
|
||||
ChatClientAgent agent = new(openAIClient.GetResponsesClient().AsIChatClient(model), instructions: "You are a helpful assistant.", name: "ConversationAgent");
|
||||
|
||||
ClientResult createConversationResult = await conversationClient.CreateConversationAsync(BinaryContent.Create(BinaryData.FromString("{}")));
|
||||
|
||||
|
||||
+5
-5
@@ -36,11 +36,11 @@ AIAgent agent = new AzureOpenAIClient(
|
||||
// For simplicity, we are assuming here that only function approvals are pending.
|
||||
AgentSession session = await agent.CreateSessionAsync();
|
||||
AgentResponse response = await agent.RunAsync("What is the weather like in Amsterdam?", session);
|
||||
List<FunctionApprovalRequestContent> approvalRequests = response.Messages.SelectMany(m => m.Contents).OfType<FunctionApprovalRequestContent>().ToList();
|
||||
List<ToolApprovalRequestContent> approvalRequests = response.Messages.SelectMany(m => m.Contents).OfType<ToolApprovalRequestContent>().ToList();
|
||||
|
||||
// For streaming use:
|
||||
// var updates = await agent.RunStreamingAsync("What is the weather like in Amsterdam?", session).ToListAsync();
|
||||
// approvalRequests = updates.SelectMany(x => x.Contents).OfType<FunctionApprovalRequestContent>().ToList();
|
||||
// approvalRequests = updates.SelectMany(x => x.Contents).OfType<ToolApprovalRequestContent>().ToList();
|
||||
|
||||
while (approvalRequests.Count > 0)
|
||||
{
|
||||
@@ -48,18 +48,18 @@ while (approvalRequests.Count > 0)
|
||||
List<ChatMessage> userInputResponses = approvalRequests
|
||||
.ConvertAll(functionApprovalRequest =>
|
||||
{
|
||||
Console.WriteLine($"The agent would like to invoke the following function, please reply Y to approve: Name {functionApprovalRequest.FunctionCall.Name}");
|
||||
Console.WriteLine($"The agent would like to invoke the following function, please reply Y to approve: Name {((FunctionCallContent)functionApprovalRequest.ToolCall).Name}");
|
||||
return new ChatMessage(ChatRole.User, [functionApprovalRequest.CreateResponse(Console.ReadLine()?.Equals("Y", StringComparison.OrdinalIgnoreCase) ?? false)]);
|
||||
});
|
||||
|
||||
// Pass the user input responses back to the agent for further processing.
|
||||
response = await agent.RunAsync(userInputResponses, session);
|
||||
|
||||
approvalRequests = response.Messages.SelectMany(m => m.Contents).OfType<FunctionApprovalRequestContent>().ToList();
|
||||
approvalRequests = response.Messages.SelectMany(m => m.Contents).OfType<ToolApprovalRequestContent>().ToList();
|
||||
|
||||
// For streaming use:
|
||||
// updates = await agent.RunStreamingAsync(userInputResponses, session).ToListAsync();
|
||||
// approvalRequests = updates.SelectMany(x => x.Contents).OfType<FunctionApprovalRequestContent>().ToList();
|
||||
// approvalRequests = updates.SelectMany(x => x.Contents).OfType<ToolApprovalRequestContent>().ToList();
|
||||
}
|
||||
|
||||
Console.WriteLine($"\nAgent: {response}");
|
||||
|
||||
+2
-2
@@ -10,14 +10,14 @@
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Azure.AI.Agents.Persistent" />
|
||||
<PackageReference Include="Azure.AI.Projects" />
|
||||
<PackageReference Include="Azure.Identity" />
|
||||
<PackageReference Include="Microsoft.Extensions.Hosting" />
|
||||
<PackageReference Include="ModelContextProtocol" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\..\..\src\Microsoft.Agents.AI.AzureAI.Persistent\Microsoft.Agents.AI.AzureAI.Persistent.csproj" />
|
||||
<ProjectReference Include="..\..\..\..\src\Microsoft.Agents.AI.AzureAI\Microsoft.Agents.AI.AzureAI.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
// This sample shows how to expose an AI agent as an MCP tool.
|
||||
|
||||
using Azure.AI.Agents.Persistent;
|
||||
using Azure.AI.Projects;
|
||||
using Azure.Identity;
|
||||
using Microsoft.Agents.AI;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
@@ -15,18 +15,15 @@ var deploymentName = Environment.GetEnvironmentVariable("AZURE_AI_MODEL_DEPLOYME
|
||||
// WARNING: DefaultAzureCredential is convenient for development but requires careful consideration in production.
|
||||
// In production, consider using a specific credential (e.g., ManagedIdentityCredential) to avoid
|
||||
// latency issues, unintended credential probing, and potential security risks from fallback mechanisms.
|
||||
var persistentAgentsClient = new PersistentAgentsClient(endpoint, new DefaultAzureCredential());
|
||||
var aiProjectClient = new AIProjectClient(new Uri(endpoint), new DefaultAzureCredential());
|
||||
|
||||
// Create a server side persistent agent
|
||||
var agentMetadata = await persistentAgentsClient.Administration.CreateAgentAsync(
|
||||
// Create a server side agent and expose it as an AIAgent.
|
||||
AIAgent agent = await aiProjectClient.CreateAIAgentAsync(
|
||||
model: deploymentName,
|
||||
instructions: "You are good at telling jokes, and you always start each joke with 'Aye aye, captain!'.",
|
||||
name: "Joker",
|
||||
description: "An agent that tells jokes.");
|
||||
|
||||
// Retrieve the server side persistent agent as an AIAgent.
|
||||
AIAgent agent = await persistentAgentsClient.GetAIAgentAsync(agentMetadata.Value.Id);
|
||||
|
||||
// Convert the agent to an AIFunction and then to an MCP tool.
|
||||
// The agent name and description will be used as the mcp tool name and description.
|
||||
McpServerTool tool = McpServerTool.Create(agent.AsAIFunction());
|
||||
|
||||
+2
-1
@@ -25,8 +25,9 @@ var stateStore = new Dictionary<string, JsonElement?>();
|
||||
AIAgent agent = new AzureOpenAIClient(
|
||||
new Uri(endpoint),
|
||||
new DefaultAzureCredential())
|
||||
.GetResponsesClient(deploymentName)
|
||||
.GetResponsesClient()
|
||||
.AsAIAgent(
|
||||
model: deploymentName,
|
||||
name: "SpaceNovelWriter",
|
||||
instructions: "You are a space novel writer. Always research relevant facts and generate character profiles for the main characters before writing novels." +
|
||||
"Write complete chapters without asking for approval or feedback. Do not ask the user about tone, style, pace, or format preferences - just write the novel based on the request.",
|
||||
|
||||
@@ -246,7 +246,7 @@ async Task<AgentResponse> ConsolePromptingApprovalMiddleware(IEnumerable<ChatMes
|
||||
AgentResponse response = await innerAgent.RunAsync(messages, session, options, cancellationToken);
|
||||
|
||||
// For simplicity, we are assuming here that only function approvals are pending.
|
||||
List<FunctionApprovalRequestContent> approvalRequests = response.Messages.SelectMany(m => m.Contents).OfType<FunctionApprovalRequestContent>().ToList();
|
||||
List<ToolApprovalRequestContent> approvalRequests = response.Messages.SelectMany(m => m.Contents).OfType<ToolApprovalRequestContent>().ToList();
|
||||
|
||||
while (approvalRequests.Count > 0)
|
||||
{
|
||||
@@ -255,13 +255,13 @@ async Task<AgentResponse> ConsolePromptingApprovalMiddleware(IEnumerable<ChatMes
|
||||
response.Messages = approvalRequests
|
||||
.ConvertAll(functionApprovalRequest =>
|
||||
{
|
||||
Console.WriteLine($"The agent would like to invoke the following function, please reply Y to approve: Name {functionApprovalRequest.FunctionCall.Name}");
|
||||
Console.WriteLine($"The agent would like to invoke the following function, please reply Y to approve: Name {((FunctionCallContent)functionApprovalRequest.ToolCall).Name}");
|
||||
return new ChatMessage(ChatRole.User, [functionApprovalRequest.CreateResponse(Console.ReadLine()?.Equals("Y", StringComparison.OrdinalIgnoreCase) ?? false)]);
|
||||
});
|
||||
|
||||
response = await innerAgent.RunAsync(response.Messages, session, options, cancellationToken);
|
||||
|
||||
approvalRequests = response.Messages.SelectMany(m => m.Contents).OfType<FunctionApprovalRequestContent>().ToList();
|
||||
approvalRequests = response.Messages.SelectMany(m => m.Contents).OfType<ToolApprovalRequestContent>().ToList();
|
||||
}
|
||||
|
||||
return response;
|
||||
|
||||
@@ -16,8 +16,8 @@ var deploymentName = Environment.GetEnvironmentVariable("AZURE_OPENAI_DEPLOYMENT
|
||||
AIAgent agent = new AzureOpenAIClient(
|
||||
new Uri(endpoint),
|
||||
new DefaultAzureCredential())
|
||||
.GetResponsesClient(deploymentName)
|
||||
.AsAIAgent();
|
||||
.GetResponsesClient()
|
||||
.AsAIAgent(model: deploymentName);
|
||||
|
||||
// Enable background responses (only supported by OpenAI Responses at this time).
|
||||
AgentRunOptions options = new() { AllowBackgroundResponses = true };
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
#pragma warning disable CS0618 // Type or member is obsolete - sample uses deprecated PersistentAgentsClientExtensions
|
||||
|
||||
// This sample shows how to create an Azure AI Foundry Agent with the Deep Research Tool.
|
||||
|
||||
using Azure.AI.Agents.Persistent;
|
||||
|
||||
-4
@@ -12,13 +12,9 @@
|
||||
<PackageReference Include="Azure.AI.OpenAI" />
|
||||
<PackageReference Include="Azure.Identity" />
|
||||
<PackageReference Include="Microsoft.Extensions.AI.OpenAI" />
|
||||
<PackageReference Include="Microsoft.Agents.ObjectModel" />
|
||||
<PackageReference Include="Microsoft.Agents.ObjectModel.Json" />
|
||||
<PackageReference Include="Microsoft.Agents.ObjectModel.PowerFx" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\..\..\src\Microsoft.Agents.AI.Declarative\Microsoft.Agents.AI.Declarative.csproj" />
|
||||
<ProjectReference Include="..\..\..\..\src\Microsoft.Agents.AI.OpenAI\Microsoft.Agents.AI.OpenAI.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
// This sample shows how to create and use AI agents with Azure Foundry Agents as the backend.
|
||||
|
||||
using Azure.AI.Projects;
|
||||
using Azure.AI.Projects.OpenAI;
|
||||
using Azure.AI.Projects.Agents;
|
||||
using Azure.Identity;
|
||||
using Microsoft.Agents.AI;
|
||||
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
// This sample shows how to create and use a simple AI agent with Azure Foundry Agents as the backend.
|
||||
|
||||
using Azure.AI.Projects;
|
||||
using Azure.AI.Projects.OpenAI;
|
||||
using Azure.AI.Projects.Agents;
|
||||
using Azure.Identity;
|
||||
using Microsoft.Agents.AI;
|
||||
|
||||
|
||||
+2
-1
@@ -2,8 +2,9 @@
|
||||
|
||||
// This sample shows how to create and use a simple AI agent with a multi-turn conversation.
|
||||
|
||||
using Azure.AI.Extensions.OpenAI;
|
||||
using Azure.AI.Projects;
|
||||
using Azure.AI.Projects.OpenAI;
|
||||
using Azure.AI.Projects.Agents;
|
||||
using Azure.Identity;
|
||||
using Microsoft.Agents.AI;
|
||||
|
||||
|
||||
+3
-3
@@ -40,7 +40,7 @@ AgentResponse response = await agent.RunAsync("What is the weather like in Amste
|
||||
|
||||
// Check if there are any approval requests.
|
||||
// For simplicity, we are assuming here that only function approvals are pending.
|
||||
List<FunctionApprovalRequestContent> approvalRequests = response.Messages.SelectMany(m => m.Contents).OfType<FunctionApprovalRequestContent>().ToList();
|
||||
List<ToolApprovalRequestContent> approvalRequests = response.Messages.SelectMany(m => m.Contents).OfType<ToolApprovalRequestContent>().ToList();
|
||||
|
||||
while (approvalRequests.Count > 0)
|
||||
{
|
||||
@@ -48,7 +48,7 @@ while (approvalRequests.Count > 0)
|
||||
List<ChatMessage> userInputMessages = approvalRequests
|
||||
.ConvertAll(functionApprovalRequest =>
|
||||
{
|
||||
Console.WriteLine($"The agent would like to invoke the following function, please reply Y to approve: Name {functionApprovalRequest.FunctionCall.Name}");
|
||||
Console.WriteLine($"The agent would like to invoke the following function, please reply Y to approve: Name {((FunctionCallContent)functionApprovalRequest.ToolCall).Name}");
|
||||
bool approved = Console.ReadLine()?.Equals("Y", StringComparison.OrdinalIgnoreCase) ?? false;
|
||||
return new ChatMessage(ChatRole.User, [functionApprovalRequest.CreateResponse(approved)]);
|
||||
});
|
||||
@@ -56,7 +56,7 @@ while (approvalRequests.Count > 0)
|
||||
// Pass the user input responses back to the agent for further processing.
|
||||
response = await agent.RunAsync(userInputMessages, session);
|
||||
|
||||
approvalRequests = response.Messages.SelectMany(m => m.Contents).OfType<FunctionApprovalRequestContent>().ToList();
|
||||
approvalRequests = response.Messages.SelectMany(m => m.Contents).OfType<ToolApprovalRequestContent>().ToList();
|
||||
}
|
||||
|
||||
Console.WriteLine($"\nAgent: {response}");
|
||||
|
||||
@@ -197,7 +197,7 @@ async Task<AgentResponse> ConsolePromptingApprovalMiddleware(IEnumerable<ChatMes
|
||||
AgentResponse response = await innerAgent.RunAsync(messages, session, options, cancellationToken);
|
||||
|
||||
// For simplicity, we are assuming here that only function approvals are pending.
|
||||
List<FunctionApprovalRequestContent> approvalRequests = response.Messages.SelectMany(m => m.Contents).OfType<FunctionApprovalRequestContent>().ToList();
|
||||
List<ToolApprovalRequestContent> approvalRequests = response.Messages.SelectMany(m => m.Contents).OfType<ToolApprovalRequestContent>().ToList();
|
||||
|
||||
while (approvalRequests.Count > 0)
|
||||
{
|
||||
@@ -206,14 +206,14 @@ async Task<AgentResponse> ConsolePromptingApprovalMiddleware(IEnumerable<ChatMes
|
||||
response.Messages = approvalRequests
|
||||
.ConvertAll(functionApprovalRequest =>
|
||||
{
|
||||
Console.WriteLine($"The agent would like to invoke the following function, please reply Y to approve: Name {functionApprovalRequest.FunctionCall.Name}");
|
||||
Console.WriteLine($"The agent would like to invoke the following function, please reply Y to approve: Name {((FunctionCallContent)functionApprovalRequest.ToolCall).Name}");
|
||||
bool approved = Console.ReadLine()?.Equals("Y", StringComparison.OrdinalIgnoreCase) ?? false;
|
||||
return new ChatMessage(ChatRole.User, [functionApprovalRequest.CreateResponse(approved)]);
|
||||
});
|
||||
|
||||
response = await innerAgent.RunAsync(response.Messages, session, options, cancellationToken);
|
||||
|
||||
approvalRequests = response.Messages.SelectMany(m => m.Contents).OfType<FunctionApprovalRequestContent>().ToList();
|
||||
approvalRequests = response.Messages.SelectMany(m => m.Contents).OfType<ToolApprovalRequestContent>().ToList();
|
||||
}
|
||||
|
||||
return response;
|
||||
|
||||
+1
-1
@@ -4,7 +4,7 @@
|
||||
|
||||
using System.Text;
|
||||
using Azure.AI.Projects;
|
||||
using Azure.AI.Projects.OpenAI;
|
||||
using Azure.AI.Projects.Agents;
|
||||
using Azure.Identity;
|
||||
using Microsoft.Agents.AI;
|
||||
using Microsoft.Extensions.AI;
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
// This sample shows how to use Computer Use Tool with AI Agents.
|
||||
|
||||
using Azure.AI.Projects;
|
||||
using Azure.AI.Projects.OpenAI;
|
||||
using Azure.AI.Projects.Agents;
|
||||
using Azure.Identity;
|
||||
using Microsoft.Agents.AI;
|
||||
using Microsoft.Extensions.AI;
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
// This sample shows how to use File Search Tool with AI Agents.
|
||||
|
||||
using Azure.AI.Projects;
|
||||
using Azure.AI.Projects.OpenAI;
|
||||
using Azure.AI.Projects.Agents;
|
||||
using Azure.Identity;
|
||||
using Microsoft.Agents.AI;
|
||||
using Microsoft.Extensions.AI;
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
// This sample shows how to use OpenAPI Tools with AI Agents.
|
||||
|
||||
using Azure.AI.Projects;
|
||||
using Azure.AI.Projects.OpenAI;
|
||||
using Azure.AI.Projects.Agents;
|
||||
using Azure.Identity;
|
||||
using Microsoft.Agents.AI;
|
||||
using OpenAI.Responses;
|
||||
@@ -72,7 +72,7 @@ const string CountriesOpenApiSpec = """
|
||||
AIProjectClient aiProjectClient = new(new Uri(endpoint), new DefaultAzureCredential());
|
||||
|
||||
// Create the OpenAPI function definition
|
||||
var openApiFunction = new OpenAPIFunctionDefinition(
|
||||
var openApiFunction = new OpenApiFunctionDefinition(
|
||||
"get_countries",
|
||||
BinaryData.FromString(CountriesOpenApiSpec),
|
||||
new OpenAPIAnonymousAuthenticationDetails())
|
||||
|
||||
+2
-2
@@ -3,7 +3,7 @@
|
||||
// This sample shows how to use Bing Custom Search Tool with AI Agents.
|
||||
|
||||
using Azure.AI.Projects;
|
||||
using Azure.AI.Projects.OpenAI;
|
||||
using Azure.AI.Projects.Agents;
|
||||
using Azure.Identity;
|
||||
using Microsoft.Agents.AI;
|
||||
using OpenAI.Responses;
|
||||
@@ -25,7 +25,7 @@ const string AgentInstructions = """
|
||||
AIProjectClient aiProjectClient = new(new Uri(endpoint), new DefaultAzureCredential());
|
||||
|
||||
// Bing Custom Search tool parameters shared by both options
|
||||
BingCustomSearchToolParameters bingCustomSearchToolParameters = new([
|
||||
BingCustomSearchToolOptions bingCustomSearchToolParameters = new([
|
||||
new BingCustomSearchConfiguration(connectionId, instanceName)
|
||||
]);
|
||||
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
// This sample shows how to use SharePoint Grounding Tool with AI Agents.
|
||||
|
||||
using Azure.AI.Projects;
|
||||
using Azure.AI.Projects.OpenAI;
|
||||
using Azure.AI.Projects.Agents;
|
||||
using Azure.Identity;
|
||||
using Microsoft.Agents.AI;
|
||||
using OpenAI.Responses;
|
||||
|
||||
+1
-1
@@ -3,7 +3,7 @@
|
||||
// This sample shows how to use Microsoft Fabric Tool with AI Agents.
|
||||
|
||||
using Azure.AI.Projects;
|
||||
using Azure.AI.Projects.OpenAI;
|
||||
using Azure.AI.Projects.Agents;
|
||||
using Azure.Identity;
|
||||
using Microsoft.Agents.AI;
|
||||
using OpenAI.Responses;
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
// This sample shows how to use the Responses API Web Search Tool with AI Agents.
|
||||
|
||||
using Azure.AI.Projects;
|
||||
using Azure.AI.Projects.OpenAI;
|
||||
using Azure.AI.Projects.Agents;
|
||||
using Azure.Identity;
|
||||
using Microsoft.Agents.AI;
|
||||
using Microsoft.Extensions.AI;
|
||||
|
||||
-1
@@ -13,7 +13,6 @@
|
||||
<PackageReference Include="Microsoft.Extensions.Logging.Console" />
|
||||
<PackageReference Include="Azure.Identity" />
|
||||
<PackageReference Include="Azure.AI.Projects" />
|
||||
<PackageReference Include="Azure.AI.Projects.OpenAI" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
|
||||
@@ -4,8 +4,9 @@
|
||||
// The Memory Search Tool enables agents to recall information from previous conversations,
|
||||
// supporting user profile persistence and chat summaries across sessions.
|
||||
|
||||
using Azure.AI.Extensions.OpenAI;
|
||||
using Azure.AI.Projects;
|
||||
using Azure.AI.Projects.OpenAI;
|
||||
using Azure.AI.Projects.Agents;
|
||||
using Azure.Identity;
|
||||
using Microsoft.Agents.AI;
|
||||
using OpenAI.Responses;
|
||||
@@ -36,7 +37,7 @@ AIProjectClient aiProjectClient = new(new Uri(endpoint), credential);
|
||||
await EnsureMemoryStoreAsync();
|
||||
|
||||
// Create the Memory Search tool configuration
|
||||
MemorySearchPreviewTool memorySearchTool = new(memoryStoreName, userScope) { UpdateDelay = 0 };
|
||||
MemorySearchPreviewTool memorySearchTool = new(memoryStoreName, userScope) { UpdateDelayInSecs = 0 };
|
||||
|
||||
// Create agent using Option 1 (MEAI) or Option 2 (Native SDK)
|
||||
AIAgent agent = await CreateAgentWithMEAI();
|
||||
@@ -128,8 +129,8 @@ async Task EnsureMemoryStoreAsync()
|
||||
|
||||
MemoryUpdateResult updateResult = await aiProjectClient.MemoryStores.WaitForMemoriesUpdateAsync(
|
||||
memoryStoreName: memoryStoreName,
|
||||
options: memoryOptions,
|
||||
pollingInterval: 500);
|
||||
pollingInterval: 500,
|
||||
options: memoryOptions);
|
||||
|
||||
if (updateResult.Status == MemoryStoreUpdateStatus.Failed)
|
||||
{
|
||||
|
||||
+2
-2
@@ -9,12 +9,12 @@
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Azure.AI.Agents.Persistent" />
|
||||
<PackageReference Include="Azure.AI.Projects" />
|
||||
<PackageReference Include="Azure.Identity" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\..\..\src\Microsoft.Agents.AI.AzureAI.Persistent\Microsoft.Agents.AI.AzureAI.Persistent.csproj" />
|
||||
<ProjectReference Include="..\..\..\..\src\Microsoft.Agents.AI.AzureAI\Microsoft.Agents.AI.AzureAI.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
// In this case the Azure Foundry Agents service will invoke any MCP tools as required. MCP tools are not invoked by the Agent Framework.
|
||||
// The sample first shows how to use MCP tools with auto approval, and then how to set up a tool that requires approval before it can be invoked and how to approve such a tool.
|
||||
|
||||
using Azure.AI.Agents.Persistent;
|
||||
using Azure.AI.Projects;
|
||||
using Azure.Identity;
|
||||
using Microsoft.Agents.AI;
|
||||
using Microsoft.Extensions.AI;
|
||||
@@ -16,7 +16,7 @@ var model = Environment.GetEnvironmentVariable("AZURE_AI_MODEL_DEPLOYMENT_NAME")
|
||||
// WARNING: DefaultAzureCredential is convenient for development but requires careful consideration in production.
|
||||
// In production, consider using a specific credential (e.g., ManagedIdentityCredential) to avoid
|
||||
// latency issues, unintended credential probing, and potential security risks from fallback mechanisms.
|
||||
var persistentAgentsClient = new PersistentAgentsClient(endpoint, new DefaultAzureCredential());
|
||||
var aiProjectClient = new AIProjectClient(new Uri(endpoint), new DefaultAzureCredential());
|
||||
|
||||
// **** MCP Tool with Auto Approval ****
|
||||
// *************************************
|
||||
@@ -31,8 +31,8 @@ var mcpTool = new HostedMcpServerTool(
|
||||
ApprovalMode = HostedMcpServerToolApprovalMode.NeverRequire
|
||||
};
|
||||
|
||||
// Create a server side persistent agent with the mcp tool, and expose it as an AIAgent.
|
||||
AIAgent agent = await persistentAgentsClient.CreateAIAgentAsync(
|
||||
// Create a server side agent with the mcp tool, and expose it as an AIAgent.
|
||||
AIAgent agent = await aiProjectClient.CreateAIAgentAsync(
|
||||
model: model,
|
||||
options: new()
|
||||
{
|
||||
@@ -49,7 +49,7 @@ AgentSession session = await agent.CreateSessionAsync();
|
||||
Console.WriteLine(await agent.RunAsync("Please summarize the Azure AI Agent documentation related to MCP Tool calling?", session));
|
||||
|
||||
// Cleanup for sample purposes.
|
||||
await persistentAgentsClient.Administration.DeleteAgentAsync(agent.Id);
|
||||
aiProjectClient.Agents.DeleteAgent(agent.Name);
|
||||
|
||||
// **** MCP Tool with Approval Required ****
|
||||
// *****************************************
|
||||
@@ -64,8 +64,8 @@ var mcpToolWithApproval = new HostedMcpServerTool(
|
||||
ApprovalMode = HostedMcpServerToolApprovalMode.AlwaysRequire
|
||||
};
|
||||
|
||||
// Create an agent based on Azure OpenAI Responses as the backend.
|
||||
AIAgent agentWithRequiredApproval = await persistentAgentsClient.CreateAIAgentAsync(
|
||||
// Create an agent with the MCP tool that requires approval.
|
||||
AIAgent agentWithRequiredApproval = await aiProjectClient.CreateAIAgentAsync(
|
||||
model: model,
|
||||
options: new()
|
||||
{
|
||||
@@ -81,7 +81,7 @@ AIAgent agentWithRequiredApproval = await persistentAgentsClient.CreateAIAgentAs
|
||||
// For simplicity, we are assuming here that only mcp tool approvals are pending.
|
||||
AgentSession sessionWithRequiredApproval = await agentWithRequiredApproval.CreateSessionAsync();
|
||||
AgentResponse response = await agentWithRequiredApproval.RunAsync("Please summarize the Azure AI Agent documentation related to MCP Tool calling?", sessionWithRequiredApproval);
|
||||
List<McpServerToolApprovalRequestContent> approvalRequests = response.Messages.SelectMany(m => m.Contents).OfType<McpServerToolApprovalRequestContent>().ToList();
|
||||
List<ToolApprovalRequestContent> approvalRequests = response.Messages.SelectMany(m => m.Contents).OfType<ToolApprovalRequestContent>().ToList();
|
||||
|
||||
while (approvalRequests.Count > 0)
|
||||
{
|
||||
@@ -89,11 +89,12 @@ while (approvalRequests.Count > 0)
|
||||
List<ChatMessage> userInputResponses = approvalRequests
|
||||
.ConvertAll(approvalRequest =>
|
||||
{
|
||||
McpServerToolCallContent mcpToolCall = (McpServerToolCallContent)approvalRequest.ToolCall!;
|
||||
Console.WriteLine($"""
|
||||
The agent would like to invoke the following MCP Tool, please reply Y to approve.
|
||||
ServerName: {approvalRequest.ToolCall.ServerName}
|
||||
Name: {approvalRequest.ToolCall.ToolName}
|
||||
Arguments: {string.Join(", ", approvalRequest.ToolCall.Arguments?.Select(x => $"{x.Key}: {x.Value}") ?? [])}
|
||||
ServerName: {mcpToolCall.ServerName}
|
||||
Name: {mcpToolCall.Name}
|
||||
Arguments: {string.Join(", ", mcpToolCall.Arguments?.Select(x => $"{x.Key}: {x.Value}") ?? [])}
|
||||
""");
|
||||
return new ChatMessage(ChatRole.User, [approvalRequest.CreateResponse(Console.ReadLine()?.Equals("Y", StringComparison.OrdinalIgnoreCase) ?? false)]);
|
||||
});
|
||||
@@ -101,7 +102,7 @@ while (approvalRequests.Count > 0)
|
||||
// Pass the user input responses back to the agent for further processing.
|
||||
response = await agentWithRequiredApproval.RunAsync(userInputResponses, sessionWithRequiredApproval);
|
||||
|
||||
approvalRequests = response.Messages.SelectMany(m => m.Contents).OfType<McpServerToolApprovalRequestContent>().ToList();
|
||||
approvalRequests = response.Messages.SelectMany(m => m.Contents).OfType<ToolApprovalRequestContent>().ToList();
|
||||
}
|
||||
|
||||
Console.WriteLine($"\nAgent: {response}");
|
||||
|
||||
@@ -33,8 +33,9 @@ var mcpTool = new HostedMcpServerTool(
|
||||
AIAgent agent = new AzureOpenAIClient(
|
||||
new Uri(endpoint),
|
||||
new DefaultAzureCredential())
|
||||
.GetResponsesClient(deploymentName)
|
||||
.GetResponsesClient()
|
||||
.AsAIAgent(
|
||||
model: deploymentName,
|
||||
instructions: "You answer questions by searching the Microsoft Learn content only.",
|
||||
name: "MicrosoftLearnAgent",
|
||||
tools: [mcpTool]);
|
||||
@@ -60,8 +61,9 @@ var mcpToolWithApproval = new HostedMcpServerTool(
|
||||
AIAgent agentWithRequiredApproval = new AzureOpenAIClient(
|
||||
new Uri(endpoint),
|
||||
new DefaultAzureCredential())
|
||||
.GetResponsesClient(deploymentName)
|
||||
.GetResponsesClient()
|
||||
.AsAIAgent(
|
||||
model: deploymentName,
|
||||
instructions: "You answer questions by searching the Microsoft Learn content only.",
|
||||
name: "MicrosoftLearnAgentWithApproval",
|
||||
tools: [mcpToolWithApproval]);
|
||||
@@ -70,7 +72,7 @@ AIAgent agentWithRequiredApproval = new AzureOpenAIClient(
|
||||
// For simplicity, we are assuming here that only mcp tool approvals are pending.
|
||||
AgentSession sessionWithRequiredApproval = await agentWithRequiredApproval.CreateSessionAsync();
|
||||
AgentResponse response = await agentWithRequiredApproval.RunAsync("Please summarize the Azure AI Agent documentation related to MCP Tool calling?", sessionWithRequiredApproval);
|
||||
List<McpServerToolApprovalRequestContent> approvalRequests = response.Messages.SelectMany(m => m.Contents).OfType<McpServerToolApprovalRequestContent>().ToList();
|
||||
List<ToolApprovalRequestContent> approvalRequests = response.Messages.SelectMany(m => m.Contents).OfType<ToolApprovalRequestContent>().ToList();
|
||||
|
||||
while (approvalRequests.Count > 0)
|
||||
{
|
||||
@@ -78,11 +80,12 @@ while (approvalRequests.Count > 0)
|
||||
List<ChatMessage> userInputResponses = approvalRequests
|
||||
.ConvertAll(approvalRequest =>
|
||||
{
|
||||
McpServerToolCallContent mcpToolCall = (McpServerToolCallContent)approvalRequest.ToolCall!;
|
||||
Console.WriteLine($"""
|
||||
The agent would like to invoke the following MCP Tool, please reply Y to approve.
|
||||
ServerName: {approvalRequest.ToolCall.ServerName}
|
||||
Name: {approvalRequest.ToolCall.ToolName}
|
||||
Arguments: {string.Join(", ", approvalRequest.ToolCall.Arguments?.Select(x => $"{x.Key}: {x.Value}") ?? [])}
|
||||
ServerName: {mcpToolCall.ServerName}
|
||||
Name: {mcpToolCall.Name}
|
||||
Arguments: {string.Join(", ", mcpToolCall.Arguments?.Select(x => $"{x.Key}: {x.Value}") ?? [])}
|
||||
""");
|
||||
return new ChatMessage(ChatRole.User, [approvalRequest.CreateResponse(Console.ReadLine()?.Equals("Y", StringComparison.OrdinalIgnoreCase) ?? false)]);
|
||||
});
|
||||
@@ -90,7 +93,7 @@ while (approvalRequests.Count > 0)
|
||||
// Pass the user input responses back to the agent for further processing.
|
||||
response = await agentWithRequiredApproval.RunAsync(userInputResponses, sessionWithRequiredApproval);
|
||||
|
||||
approvalRequests = response.Messages.SelectMany(m => m.Contents).OfType<McpServerToolApprovalRequestContent>().ToList();
|
||||
approvalRequests = response.Messages.SelectMany(m => m.Contents).OfType<ToolApprovalRequestContent>().ToList();
|
||||
}
|
||||
|
||||
Console.WriteLine($"\nAgent: {response}");
|
||||
|
||||
@@ -61,6 +61,12 @@ public static class Program
|
||||
{
|
||||
Console.WriteLine($"{outputEvent}");
|
||||
}
|
||||
|
||||
if (evt is WorkflowErrorEvent errorEvent)
|
||||
{
|
||||
Console.WriteLine($"Workflow error: {errorEvent.Exception?.Message}");
|
||||
Console.WriteLine($"Details: {errorEvent.Exception}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -175,7 +181,9 @@ internal sealed class FeedbackEvent(FeedbackResult feedbackResult) : WorkflowEve
|
||||
/// <summary>
|
||||
/// A custom executor that uses an AI agent to provide feedback on a slogan.
|
||||
/// </summary>
|
||||
internal sealed class FeedbackExecutor : Executor<SloganResult>
|
||||
[SendsMessage(typeof(FeedbackResult))]
|
||||
[YieldsOutput(typeof(string))]
|
||||
internal sealed partial class FeedbackExecutor : Executor<SloganResult>
|
||||
{
|
||||
private readonly AIAgent _agent;
|
||||
private AgentSession? _session;
|
||||
|
||||
@@ -9,13 +9,13 @@
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Azure.AI.Agents.Persistent" />
|
||||
<PackageReference Include="Azure.AI.Projects" />
|
||||
<PackageReference Include="Azure.Identity" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\..\..\src\Microsoft.Agents.AI.Workflows\Microsoft.Agents.AI.Workflows.csproj" />
|
||||
<ProjectReference Include="..\..\..\..\src\Microsoft.Agents.AI.AzureAI.Persistent\Microsoft.Agents.AI.AzureAI.Persistent.csproj" />
|
||||
<ProjectReference Include="..\..\..\..\src\Microsoft.Agents.AI.AzureAI\Microsoft.Agents.AI.AzureAI.csproj" />
|
||||
<ProjectReference Include="..\..\..\..\src\Microsoft.Agents.AI\Microsoft.Agents.AI.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using Azure.AI.Agents.Persistent;
|
||||
using Azure.AI.Projects;
|
||||
using Azure.Identity;
|
||||
using Microsoft.Agents.AI;
|
||||
using Microsoft.Agents.AI.Workflows;
|
||||
@@ -20,60 +20,63 @@ public static class Program
|
||||
{
|
||||
private static async Task Main()
|
||||
{
|
||||
// Set up the Azure OpenAI client
|
||||
// Set up the Azure AI Project client
|
||||
var endpoint = Environment.GetEnvironmentVariable("AZURE_AI_PROJECT_ENDPOINT")
|
||||
?? throw new InvalidOperationException("AZURE_AI_PROJECT_ENDPOINT is not set.");
|
||||
var deploymentName = Environment.GetEnvironmentVariable("AZURE_AI_MODEL_DEPLOYMENT_NAME") ?? "gpt-4o-mini";
|
||||
var persistentAgentsClient = new PersistentAgentsClient(endpoint, new AzureCliCredential());
|
||||
var aiProjectClient = new AIProjectClient(new Uri(endpoint), new AzureCliCredential());
|
||||
|
||||
// Create agents
|
||||
AIAgent frenchAgent = await GetTranslationAgentAsync("French", persistentAgentsClient, deploymentName);
|
||||
AIAgent spanishAgent = await GetTranslationAgentAsync("Spanish", persistentAgentsClient, deploymentName);
|
||||
AIAgent englishAgent = await GetTranslationAgentAsync("English", persistentAgentsClient, deploymentName);
|
||||
AIAgent frenchAgent = await CreateTranslationAgentAsync("French", aiProjectClient, deploymentName);
|
||||
AIAgent spanishAgent = await CreateTranslationAgentAsync("Spanish", aiProjectClient, deploymentName);
|
||||
AIAgent englishAgent = await CreateTranslationAgentAsync("English", aiProjectClient, deploymentName);
|
||||
|
||||
// Build the workflow by adding executors and connecting them
|
||||
var workflow = new WorkflowBuilder(frenchAgent)
|
||||
.AddEdge(frenchAgent, spanishAgent)
|
||||
.AddEdge(spanishAgent, englishAgent)
|
||||
.Build();
|
||||
|
||||
// Execute the workflow
|
||||
await using StreamingRun run = await InProcessExecution.RunStreamingAsync(workflow, new ChatMessage(ChatRole.User, "Hello World!"));
|
||||
// Must send the turn token to trigger the agents.
|
||||
// The agents are wrapped as executors. When they receive messages,
|
||||
// they will cache the messages and only start processing when they receive a TurnToken.
|
||||
await run.TrySendMessageAsync(new TurnToken(emitEvents: true));
|
||||
await foreach (WorkflowEvent evt in run.WatchStreamAsync())
|
||||
try
|
||||
{
|
||||
if (evt is AgentResponseUpdateEvent executorComplete)
|
||||
// Build the workflow by adding executors and connecting them
|
||||
var workflow = new WorkflowBuilder(frenchAgent)
|
||||
.AddEdge(frenchAgent, spanishAgent)
|
||||
.AddEdge(spanishAgent, englishAgent)
|
||||
.Build();
|
||||
|
||||
// Execute the workflow
|
||||
await using StreamingRun run = await InProcessExecution.RunStreamingAsync(workflow, new ChatMessage(ChatRole.User, "Hello World!"));
|
||||
// Must send the turn token to trigger the agents.
|
||||
// The agents are wrapped as executors. When they receive messages,
|
||||
// they will cache the messages and only start processing when they receive a TurnToken.
|
||||
await run.TrySendMessageAsync(new TurnToken(emitEvents: true));
|
||||
await foreach (WorkflowEvent evt in run.WatchStreamAsync())
|
||||
{
|
||||
Console.WriteLine($"{executorComplete.ExecutorId}: {executorComplete.Data}");
|
||||
if (evt is AgentResponseUpdateEvent executorComplete)
|
||||
{
|
||||
Console.WriteLine($"{executorComplete.ExecutorId}: {executorComplete.Data}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Cleanup the agents created for the sample.
|
||||
await persistentAgentsClient.Administration.DeleteAgentAsync(frenchAgent.Id);
|
||||
await persistentAgentsClient.Administration.DeleteAgentAsync(spanishAgent.Id);
|
||||
await persistentAgentsClient.Administration.DeleteAgentAsync(englishAgent.Id);
|
||||
finally
|
||||
{
|
||||
// Cleanup the agents created for the sample.
|
||||
await aiProjectClient.Agents.DeleteAgentAsync(frenchAgent.Name);
|
||||
await aiProjectClient.Agents.DeleteAgentAsync(spanishAgent.Name);
|
||||
await aiProjectClient.Agents.DeleteAgentAsync(englishAgent.Name);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a translation agent for the specified target language.
|
||||
/// </summary>
|
||||
/// <param name="targetLanguage">The target language for translation</param>
|
||||
/// <param name="persistentAgentsClient">The PersistentAgentsClient to create the agent</param>
|
||||
/// <param name="aiProjectClient">The <see cref="AIProjectClient"/> to create the agent with.</param>
|
||||
/// <param name="model">The model to use for the agent</param>
|
||||
/// <returns>A ChatClientAgent configured for the specified language</returns>
|
||||
private static async Task<ChatClientAgent> GetTranslationAgentAsync(
|
||||
private static async Task<ChatClientAgent> CreateTranslationAgentAsync(
|
||||
string targetLanguage,
|
||||
PersistentAgentsClient persistentAgentsClient,
|
||||
AIProjectClient aiProjectClient,
|
||||
string model)
|
||||
{
|
||||
var agentMetadata = await persistentAgentsClient.Administration.CreateAgentAsync(
|
||||
model: model,
|
||||
return await aiProjectClient.CreateAIAgentAsync(
|
||||
name: $"{targetLanguage} Translator",
|
||||
model: model,
|
||||
instructions: $"You are a translation assistant that translates the provided text to {targetLanguage}.");
|
||||
|
||||
return await persistentAgentsClient.GetAIAgentAsync(agentMetadata.Value.Id);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17,7 +17,7 @@
|
||||
//
|
||||
// Demonstrate:
|
||||
// - Using custom GroupChatManager with agents that have approval-required tools.
|
||||
// - Handling FunctionApprovalRequestContent in group chat scenarios.
|
||||
// - Handling ToolApprovalRequestContent in group chat scenarios.
|
||||
// - Multi-round group chat with tool approval interruption and resumption.
|
||||
|
||||
using System.ComponentModel;
|
||||
@@ -101,16 +101,16 @@ public static class Program
|
||||
{
|
||||
case RequestInfoEvent e:
|
||||
{
|
||||
if (e.Request.TryGetDataAs(out FunctionApprovalRequestContent? approvalRequestContent))
|
||||
if (e.Request.TryGetDataAs(out ToolApprovalRequestContent? approvalRequestContent))
|
||||
{
|
||||
Console.WriteLine();
|
||||
Console.WriteLine($"[APPROVAL REQUIRED] From agent: {e.Request.PortInfo.PortId}");
|
||||
Console.WriteLine($" Tool: {approvalRequestContent.FunctionCall.Name}");
|
||||
Console.WriteLine($" Arguments: {JsonSerializer.Serialize(approvalRequestContent.FunctionCall.Arguments)}");
|
||||
Console.WriteLine($" Tool: {((FunctionCallContent)approvalRequestContent.ToolCall).Name}");
|
||||
Console.WriteLine($" Arguments: {JsonSerializer.Serialize(((FunctionCallContent)approvalRequestContent.ToolCall).Arguments)}");
|
||||
Console.WriteLine();
|
||||
|
||||
// Approve the tool call request
|
||||
Console.WriteLine($"Tool: {approvalRequestContent.FunctionCall.Name} approved");
|
||||
Console.WriteLine($"Tool: {((FunctionCallContent)approvalRequestContent.ToolCall).Name} approved");
|
||||
await run.SendResponseAsync(e.Request.CreateResponse(approvalRequestContent.CreateResponse(approved: true)));
|
||||
}
|
||||
|
||||
|
||||
@@ -9,7 +9,7 @@ using Microsoft.Extensions.AI;
|
||||
namespace WorkflowAsAnAgentSample;
|
||||
|
||||
/// <summary>
|
||||
/// This sample introduces the concepts workflows as agents, where a workflow can be
|
||||
/// This sample introduces the concept of workflows as agents, where a workflow can be
|
||||
/// treated as an <see cref="AIAgent"/>. This allows you to interact with a workflow
|
||||
/// as if it were a single agent.
|
||||
///
|
||||
@@ -18,6 +18,14 @@ namespace WorkflowAsAnAgentSample;
|
||||
///
|
||||
/// You will interact with the workflow in an interactive loop, sending messages and receiving
|
||||
/// streaming responses from the workflow as if it were an agent who responds in both languages.
|
||||
///
|
||||
/// This sample also demonstrates <see cref="IResettableExecutor"/>, which is required
|
||||
/// for stateful executors that are shared across multiple workflow runs. Each iteration
|
||||
/// of the interactive loop triggers a new workflow run against the same workflow instance.
|
||||
/// Between runs, the framework automatically calls <see cref="IResettableExecutor.ResetAsync"/>
|
||||
/// on shared executors so that accumulated state (e.g., collected messages) is cleared
|
||||
/// before the next run begins. See <c>WorkflowFactory.ConcurrentAggregationExecutor</c>
|
||||
/// for the implementation.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Pre-requisites:
|
||||
@@ -39,7 +47,10 @@ public static class Program
|
||||
var agent = workflow.AsAIAgent("workflow-agent", "Workflow Agent");
|
||||
var session = await agent.CreateSessionAsync();
|
||||
|
||||
// Start an interactive loop to interact with the workflow as if it were an agent
|
||||
// Start an interactive loop to interact with the workflow as if it were an agent.
|
||||
// Each iteration runs the workflow again on the same workflow instance. Between runs,
|
||||
// the framework calls IResettableExecutor.ResetAsync() on shared stateful executors
|
||||
// (like ConcurrentAggregationExecutor) to clear accumulated state from the previous run.
|
||||
while (true)
|
||||
{
|
||||
Console.WriteLine();
|
||||
|
||||
@@ -10,6 +10,14 @@ internal static class WorkflowFactory
|
||||
{
|
||||
/// <summary>
|
||||
/// Creates a workflow that uses two language agents to process input concurrently.
|
||||
///
|
||||
/// In this workflow, the <c>Start</c> <see cref="ChatForwardingExecutor"/> and the
|
||||
/// <see cref="ConcurrentAggregationExecutor"/> are provided as shared instances, meaning
|
||||
/// the same executor objects are reused across multiple workflow runs. The language agents
|
||||
/// (French and English) are created via a factory and instantiated per workflow run.
|
||||
/// Stateful shared executors must implement <see cref="IResettableExecutor"/> so the
|
||||
/// framework can clear their state between runs. Framework-provided executors like
|
||||
/// <see cref="ChatForwardingExecutor"/> already implement this interface.
|
||||
/// </summary>
|
||||
/// <param name="chatClient">The chat client to use for the agents</param>
|
||||
/// <returns>A workflow that processes input using two language agents</returns>
|
||||
@@ -40,7 +48,18 @@ internal static class WorkflowFactory
|
||||
|
||||
/// <summary>
|
||||
/// Executor that aggregates the results from the concurrent agents.
|
||||
///
|
||||
/// This executor is stateful — it accumulates messages in <see cref="_messages"/>
|
||||
/// as they arrive from each agent. Because it is provided as a shared instance
|
||||
/// (not via a factory), the same object is reused across workflow runs. Implementing
|
||||
/// <see cref="IResettableExecutor"/> allows the framework to call <see cref="ResetAsync"/>
|
||||
/// between runs, clearing accumulated state so each run starts fresh.
|
||||
///
|
||||
/// Without <see cref="IResettableExecutor"/>, attempting to reuse a workflow containing
|
||||
/// shared executor instances that do not implement this interface would throw an
|
||||
/// <see cref="InvalidOperationException"/>.
|
||||
/// </summary>
|
||||
[YieldsOutput(typeof(string))]
|
||||
private sealed class ConcurrentAggregationExecutor() :
|
||||
Executor<List<ChatMessage>>("ConcurrentAggregationExecutor"), IResettableExecutor
|
||||
{
|
||||
@@ -64,7 +83,11 @@ internal static class WorkflowFactory
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
/// <summary>
|
||||
/// Resets the executor state between workflow runs by clearing accumulated messages.
|
||||
/// The framework calls this automatically when a workflow run completes, before the
|
||||
/// workflow can be used for another run.
|
||||
/// </summary>
|
||||
public ValueTask ResetAsync()
|
||||
{
|
||||
this._messages.Clear();
|
||||
|
||||
@@ -41,6 +41,7 @@ internal enum NumberSignal
|
||||
/// <summary>
|
||||
/// Executor that makes a guess based on the current bounds.
|
||||
/// </summary>
|
||||
[SendsMessage(typeof(int))]
|
||||
internal sealed class GuessNumberExecutor() : Executor<NumberSignal>("Guess")
|
||||
{
|
||||
/// <summary>
|
||||
@@ -104,6 +105,8 @@ internal sealed class GuessNumberExecutor() : Executor<NumberSignal>("Guess")
|
||||
/// <summary>
|
||||
/// Executor that judges the guess and provides feedback.
|
||||
/// </summary>
|
||||
[SendsMessage(typeof(NumberSignal))]
|
||||
[YieldsOutput(typeof(string))]
|
||||
internal sealed class JudgeExecutor() : Executor<int>("Judge")
|
||||
{
|
||||
private readonly int _targetNumber;
|
||||
|
||||
@@ -41,6 +41,7 @@ internal enum NumberSignal
|
||||
/// <summary>
|
||||
/// Executor that makes a guess based on the current bounds.
|
||||
/// </summary>
|
||||
[SendsMessage(typeof(int))]
|
||||
internal sealed class GuessNumberExecutor() : Executor<NumberSignal>("Guess")
|
||||
{
|
||||
/// <summary>
|
||||
@@ -104,6 +105,8 @@ internal sealed class GuessNumberExecutor() : Executor<NumberSignal>("Guess")
|
||||
/// <summary>
|
||||
/// Executor that judges the guess and provides feedback.
|
||||
/// </summary>
|
||||
[SendsMessage(typeof(NumberSignal))]
|
||||
[YieldsOutput(typeof(string))]
|
||||
internal sealed class JudgeExecutor() : Executor<int>("Judge")
|
||||
{
|
||||
private readonly int _targetNumber;
|
||||
|
||||
@@ -53,6 +53,8 @@ internal sealed class SignalWithNumber
|
||||
/// <summary>
|
||||
/// Executor that judges the guess and provides feedback.
|
||||
/// </summary>
|
||||
[SendsMessage(typeof(SignalWithNumber))]
|
||||
[YieldsOutput(typeof(string))]
|
||||
internal sealed class JudgeExecutor() : Executor<int>("Judge")
|
||||
{
|
||||
private readonly int _targetNumber;
|
||||
|
||||
@@ -72,6 +72,8 @@ public static class Program
|
||||
/// <summary>
|
||||
/// Executor that starts the concurrent processing by sending messages to the agents.
|
||||
/// </summary>
|
||||
[SendsMessage(typeof(ChatMessage))]
|
||||
[SendsMessage(typeof(TurnToken))]
|
||||
internal sealed partial class ConcurrentStartExecutor() :
|
||||
Executor("ConcurrentStartExecutor")
|
||||
{
|
||||
@@ -97,7 +99,8 @@ internal sealed partial class ConcurrentStartExecutor() :
|
||||
/// <summary>
|
||||
/// Executor that aggregates the results from the concurrent agents.
|
||||
/// </summary>
|
||||
internal sealed class ConcurrentAggregationExecutor() :
|
||||
[YieldsOutput(typeof(string))]
|
||||
internal sealed partial class ConcurrentAggregationExecutor() :
|
||||
Executor<List<ChatMessage>>("ConcurrentAggregationExecutor")
|
||||
{
|
||||
private readonly List<ChatMessage> _messages = [];
|
||||
|
||||
@@ -128,6 +128,7 @@ public static class Program
|
||||
/// <summary>
|
||||
/// Splits data into roughly equal chunks based on the number of mapper nodes.
|
||||
/// </summary>
|
||||
[SendsMessage(typeof(SplitComplete))]
|
||||
internal sealed class Split(string[] mapperIds, string id) :
|
||||
Executor<string>(id)
|
||||
{
|
||||
@@ -186,6 +187,7 @@ internal sealed class Split(string[] mapperIds, string id) :
|
||||
/// <summary>
|
||||
/// Maps each token to a count of 1 and writes pairs to a per-mapper file.
|
||||
/// </summary>
|
||||
[SendsMessage(typeof(MapComplete))]
|
||||
internal sealed class Mapper(string id) : Executor<SplitComplete>(id)
|
||||
{
|
||||
/// <summary>
|
||||
@@ -212,6 +214,7 @@ internal sealed class Mapper(string id) : Executor<SplitComplete>(id)
|
||||
/// <summary>
|
||||
/// Groups intermediate pairs by key and partitions them across reducers.
|
||||
/// </summary>
|
||||
[SendsMessage(typeof(ShuffleComplete))]
|
||||
internal sealed class Shuffler(string[] reducerIds, string[] mapperIds, string id) :
|
||||
Executor<MapComplete>(id)
|
||||
{
|
||||
@@ -311,6 +314,7 @@ internal sealed class Shuffler(string[] reducerIds, string[] mapperIds, string i
|
||||
/// <summary>
|
||||
/// Sums grouped counts per key for its assigned partition.
|
||||
/// </summary>
|
||||
[SendsMessage(typeof(ReduceComplete))]
|
||||
internal sealed class Reducer(string id) : Executor<ShuffleComplete>(id)
|
||||
{
|
||||
/// <summary>
|
||||
@@ -352,6 +356,7 @@ internal sealed class Reducer(string id) : Executor<ShuffleComplete>(id)
|
||||
/// <summary>
|
||||
/// Joins all reducer outputs and yields the final output.
|
||||
/// </summary>
|
||||
[YieldsOutput(typeof(List<string>))]
|
||||
internal sealed class CompletionExecutor(string id) :
|
||||
Executor<List<ReduceComplete>>(id)
|
||||
{
|
||||
|
||||
@@ -228,6 +228,7 @@ internal sealed class EmailAssistantExecutor : Executor<DetectionResult, EmailRe
|
||||
/// <summary>
|
||||
/// Executor that sends emails.
|
||||
/// </summary>
|
||||
[YieldsOutput(typeof(string))]
|
||||
internal sealed class SendEmailExecutor() : Executor<EmailResponse>("SendEmailExecutor")
|
||||
{
|
||||
/// <summary>
|
||||
@@ -240,6 +241,7 @@ internal sealed class SendEmailExecutor() : Executor<EmailResponse>("SendEmailEx
|
||||
/// <summary>
|
||||
/// Executor that handles spam messages.
|
||||
/// </summary>
|
||||
[YieldsOutput(typeof(string))]
|
||||
internal sealed class HandleSpamExecutor() : Executor<DetectionResult>("HandleSpamExecutor")
|
||||
{
|
||||
/// <summary>
|
||||
|
||||
@@ -252,6 +252,7 @@ internal sealed class EmailAssistantExecutor : Executor<DetectionResult, EmailRe
|
||||
/// <summary>
|
||||
/// Executor that sends emails.
|
||||
/// </summary>
|
||||
[YieldsOutput(typeof(string))]
|
||||
internal sealed class SendEmailExecutor() : Executor<EmailResponse>("SendEmailExecutor")
|
||||
{
|
||||
/// <summary>
|
||||
@@ -264,6 +265,7 @@ internal sealed class SendEmailExecutor() : Executor<EmailResponse>("SendEmailEx
|
||||
/// <summary>
|
||||
/// Executor that handles spam messages.
|
||||
/// </summary>
|
||||
[YieldsOutput(typeof(string))]
|
||||
internal sealed class HandleSpamExecutor() : Executor<DetectionResult>("HandleSpamExecutor")
|
||||
{
|
||||
/// <summary>
|
||||
@@ -285,6 +287,7 @@ internal sealed class HandleSpamExecutor() : Executor<DetectionResult>("HandleSp
|
||||
/// <summary>
|
||||
/// Executor that handles uncertain emails.
|
||||
/// </summary>
|
||||
[YieldsOutput(typeof(string))]
|
||||
internal sealed class HandleUncertainExecutor() : Executor<DetectionResult>("HandleUncertainExecutor")
|
||||
{
|
||||
/// <summary>
|
||||
|
||||
@@ -310,6 +310,7 @@ internal sealed class EmailAssistantExecutor : Executor<AnalysisResult, EmailRes
|
||||
/// <summary>
|
||||
/// Executor that sends emails.
|
||||
/// </summary>
|
||||
[YieldsOutput(typeof(string))]
|
||||
internal sealed class SendEmailExecutor() : Executor<EmailResponse>("SendEmailExecutor")
|
||||
{
|
||||
/// <summary>
|
||||
@@ -322,6 +323,7 @@ internal sealed class SendEmailExecutor() : Executor<EmailResponse>("SendEmailEx
|
||||
/// <summary>
|
||||
/// Executor that handles spam messages.
|
||||
/// </summary>
|
||||
[YieldsOutput(typeof(string))]
|
||||
internal sealed class HandleSpamExecutor() : Executor<AnalysisResult>("HandleSpamExecutor")
|
||||
{
|
||||
/// <summary>
|
||||
@@ -343,6 +345,7 @@ internal sealed class HandleSpamExecutor() : Executor<AnalysisResult>("HandleSpa
|
||||
/// <summary>
|
||||
/// Executor that handles uncertain messages.
|
||||
/// </summary>
|
||||
[YieldsOutput(typeof(string))]
|
||||
internal sealed class HandleUncertainExecutor() : Executor<AnalysisResult>("HandleUncertainExecutor")
|
||||
{
|
||||
/// <summary>
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using Azure.AI.Projects;
|
||||
using Azure.AI.Projects.OpenAI;
|
||||
using Azure.AI.Projects.Agents;
|
||||
using Azure.Identity;
|
||||
using Microsoft.Extensions.AI;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using Azure.AI.Projects;
|
||||
using Azure.AI.Projects.OpenAI;
|
||||
using Azure.AI.Projects.Agents;
|
||||
using Azure.Identity;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using OpenAI.Responses;
|
||||
@@ -275,7 +275,7 @@ internal sealed class Program
|
||||
Tools =
|
||||
{
|
||||
AgentTool.CreateOpenApiTool(
|
||||
new OpenAPIFunctionDefinition(
|
||||
new OpenApiFunctionDefinition(
|
||||
"weather-forecast",
|
||||
BinaryData.FromString(File.ReadAllText(Path.Combine(AppContext.BaseDirectory, "wttr.json"))),
|
||||
new OpenAPIAnonymousAuthenticationDetails()))
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using Azure.AI.Projects;
|
||||
using Azure.AI.Projects.OpenAI;
|
||||
using Azure.AI.Projects.Agents;
|
||||
using Azure.Identity;
|
||||
using Microsoft.Extensions.AI;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
|
||||
@@ -3,8 +3,9 @@
|
||||
// Uncomment this to enable JSON checkpointing to the local file system.
|
||||
//#define CHECKPOINT_JSON
|
||||
|
||||
using Azure.AI.Extensions.OpenAI;
|
||||
using Azure.AI.Projects;
|
||||
using Azure.AI.Projects.OpenAI;
|
||||
using Azure.AI.Projects.Agents;
|
||||
using Azure.Identity;
|
||||
using Microsoft.Agents.AI;
|
||||
using Microsoft.Extensions.AI;
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using Azure.AI.Projects;
|
||||
using Azure.AI.Projects.OpenAI;
|
||||
using Azure.AI.Projects.Agents;
|
||||
using Azure.Identity;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using OpenAI.Responses;
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using Azure.AI.Projects;
|
||||
using Azure.AI.Projects.OpenAI;
|
||||
using Azure.AI.Projects.Agents;
|
||||
using Azure.Identity;
|
||||
using Microsoft.Extensions.AI;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
// invoked to perform specific tasks, like searching documentation or executing operations.
|
||||
|
||||
using Azure.AI.Projects;
|
||||
using Azure.AI.Projects.OpenAI;
|
||||
using Azure.AI.Projects.Agents;
|
||||
using Azure.Core;
|
||||
using Azure.Identity;
|
||||
using Microsoft.Agents.AI.Workflows.Declarative.Mcp;
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using Azure.AI.Projects;
|
||||
using Azure.AI.Projects.OpenAI;
|
||||
using Azure.AI.Projects.Agents;
|
||||
using Azure.Identity;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Shared.Foundry;
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using Azure.AI.Projects;
|
||||
using Azure.AI.Projects.OpenAI;
|
||||
using Azure.AI.Projects.Agents;
|
||||
using Azure.Identity;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Shared.Foundry;
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using Azure.AI.Projects;
|
||||
using Azure.AI.Projects.OpenAI;
|
||||
using Azure.AI.Projects.Agents;
|
||||
using Azure.Identity;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using OpenAI.Responses;
|
||||
|
||||
@@ -38,6 +38,8 @@ internal enum NumberSignal
|
||||
/// <summary>
|
||||
/// Executor that judges the guess and provides feedback.
|
||||
/// </summary>
|
||||
[SendsMessage(typeof(NumberSignal))]
|
||||
[YieldsOutput(typeof(string))]
|
||||
internal sealed class JudgeExecutor() : Executor<int>("Judge")
|
||||
{
|
||||
private readonly int _targetNumber;
|
||||
|
||||
@@ -56,6 +56,7 @@ internal enum NumberSignal
|
||||
/// <summary>
|
||||
/// Executor that makes a guess based on the current bounds.
|
||||
/// </summary>
|
||||
[SendsMessage(typeof(int))]
|
||||
internal sealed class GuessNumberExecutor : Executor<NumberSignal>
|
||||
{
|
||||
/// <summary>
|
||||
@@ -104,6 +105,8 @@ internal sealed class GuessNumberExecutor : Executor<NumberSignal>
|
||||
/// <summary>
|
||||
/// Executor that judges the guess and provides feedback.
|
||||
/// </summary>
|
||||
[SendsMessage(typeof(NumberSignal))]
|
||||
[YieldsOutput(typeof(string))]
|
||||
internal sealed class JudgeExecutor : Executor<int>
|
||||
{
|
||||
private readonly int _targetNumber;
|
||||
@@ -124,8 +127,7 @@ internal sealed class JudgeExecutor : Executor<int>
|
||||
this._tries++;
|
||||
if (message == this._targetNumber)
|
||||
{
|
||||
await context.YieldOutputAsync($"{this._targetNumber} found in {this._tries} tries!", cancellationToken)
|
||||
;
|
||||
await context.YieldOutputAsync($"{this._targetNumber} found in {this._tries} tries!", cancellationToken);
|
||||
}
|
||||
else if (message < this._targetNumber)
|
||||
{
|
||||
|
||||
@@ -99,6 +99,10 @@ internal sealed class ParagraphCountingExecutor() : Executor<string, FileStats>(
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The aggregation executor collects results from both executors and yields the final output.
|
||||
/// </summary>
|
||||
[YieldsOutput(typeof(string))]
|
||||
internal sealed class AggregationExecutor() : Executor<FileStats>("AggregationExecutor")
|
||||
{
|
||||
private readonly List<FileStats> _messages = [];
|
||||
|
||||
@@ -205,6 +205,8 @@ internal sealed class TextInverterExecutor(string id) : Executor<string, string>
|
||||
/// 1. Sending ChatMessage(s)
|
||||
/// 2. Sending a TurnToken to trigger processing
|
||||
/// </summary>
|
||||
[SendsMessage(typeof(ChatMessage))]
|
||||
[SendsMessage(typeof(TurnToken))]
|
||||
internal sealed class StringToChatMessageExecutor(string id) : Executor<string>(id)
|
||||
{
|
||||
public override async ValueTask HandleAsync(string message, IWorkflowContext context, CancellationToken cancellationToken = default)
|
||||
@@ -234,6 +236,8 @@ internal sealed class StringToChatMessageExecutor(string id) : Executor<string>(
|
||||
/// The AIAgentHostExecutor sends response.Messages which has runtime type List<ChatMessage>.
|
||||
/// The message router uses exact type matching via message.GetType().
|
||||
/// </remarks>
|
||||
[SendsMessage(typeof(ChatMessage))]
|
||||
[SendsMessage(typeof(TurnToken))]
|
||||
internal sealed class JailbreakSyncExecutor() : Executor<List<ChatMessage>>("JailbreakSync")
|
||||
{
|
||||
public override async ValueTask HandleAsync(List<ChatMessage> message, IWorkflowContext context, CancellationToken cancellationToken = default)
|
||||
|
||||
-1
@@ -14,7 +14,6 @@
|
||||
<PackageReference Include="Azure.Identity" />
|
||||
<PackageReference Include="Microsoft.Extensions.Hosting" />
|
||||
<PackageReference Include="System.Net.ServerSentEvents" />
|
||||
<PackageReference Include="Microsoft.Bcl.AsyncInterfaces" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
|
||||
+14
-5
@@ -17,7 +17,7 @@ internal sealed class Tools(ILogger<Tools> logger)
|
||||
[Description("Starts a content generation workflow and returns the instance ID for tracking.")]
|
||||
public string StartContentGenerationWorkflow([Description("The topic for content generation")] string topic)
|
||||
{
|
||||
this._logger.LogInformation("Starting content generation workflow for topic: {Topic}", topic);
|
||||
this._logger.LogInformation("Starting content generation workflow for topic: {Topic}", SanitizeLogValue(topic));
|
||||
|
||||
const int MaxReviewAttempts = 3;
|
||||
const float ApprovalTimeoutHours = 72;
|
||||
@@ -34,7 +34,7 @@ internal sealed class Tools(ILogger<Tools> logger)
|
||||
|
||||
this._logger.LogInformation(
|
||||
"Content generation workflow scheduled to be started for topic '{Topic}' with instance ID: {InstanceId}",
|
||||
topic,
|
||||
SanitizeLogValue(topic),
|
||||
instanceId);
|
||||
|
||||
return $"Workflow started with instance ID: {instanceId}";
|
||||
@@ -45,7 +45,7 @@ internal sealed class Tools(ILogger<Tools> logger)
|
||||
[Description("The instance ID of the workflow to check")] string instanceId,
|
||||
[Description("Whether to include detailed information")] bool includeDetails = true)
|
||||
{
|
||||
this._logger.LogInformation("Getting status for workflow instance: {InstanceId}", instanceId);
|
||||
this._logger.LogInformation("Getting status for workflow instance: {InstanceId}", SanitizeLogValue(instanceId));
|
||||
|
||||
// Get the current agent context using the session-static property
|
||||
OrchestrationMetadata? status = await DurableAgentContext.Current.GetOrchestrationStatusAsync(
|
||||
@@ -54,7 +54,7 @@ internal sealed class Tools(ILogger<Tools> logger)
|
||||
|
||||
if (status is null)
|
||||
{
|
||||
this._logger.LogInformation("Workflow instance '{InstanceId}' not found.", instanceId);
|
||||
this._logger.LogInformation("Workflow instance '{InstanceId}' not found.", SanitizeLogValue(instanceId));
|
||||
return new
|
||||
{
|
||||
instanceId,
|
||||
@@ -78,7 +78,16 @@ internal sealed class Tools(ILogger<Tools> logger)
|
||||
[Description("The instance ID of the workflow to submit feedback for")] string instanceId,
|
||||
[Description("Feedback to submit")] HumanApprovalResponse feedback)
|
||||
{
|
||||
this._logger.LogInformation("Submitting human approval for workflow instance: {InstanceId}", instanceId);
|
||||
this._logger.LogInformation("Submitting human approval for workflow instance: {InstanceId}", SanitizeLogValue(instanceId));
|
||||
await DurableAgentContext.Current.RaiseOrchestrationEventAsync(instanceId, "HumanApproval", feedback);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Sanitizes a user-provided value for safe inclusion in log entries
|
||||
/// by removing control characters that could be used for log forging.
|
||||
/// </summary>
|
||||
private static string SanitizeLogValue(string value) =>
|
||||
value
|
||||
.Replace("\r", string.Empty, StringComparison.Ordinal)
|
||||
.Replace("\n", string.Empty, StringComparison.Ordinal);
|
||||
}
|
||||
|
||||
+20
-4
@@ -157,8 +157,8 @@ public sealed class FunctionTriggers
|
||||
|
||||
this._logger.LogInformation(
|
||||
"Resuming stream for conversation {ConversationId} from cursor: {Cursor}",
|
||||
conversationId,
|
||||
cursor ?? "(beginning)");
|
||||
SanitizeLogValue(conversationId),
|
||||
SanitizeLogValue(cursor) ?? "(beginning)");
|
||||
|
||||
// Check Accept header to determine response format
|
||||
// text/plain = raw text output (ideal for terminals)
|
||||
@@ -205,7 +205,7 @@ public sealed class FunctionTriggers
|
||||
{
|
||||
if (chunk.Error != null)
|
||||
{
|
||||
this._logger.LogWarning("Stream error for conversation {ConversationId}: {Error}", conversationId, chunk.Error);
|
||||
this._logger.LogWarning("Stream error for conversation {ConversationId}: {Error}", SanitizeLogValue(conversationId), chunk.Error);
|
||||
await WriteErrorAsync(httpContext.Response, chunk.Error, useSseFormat, cancellationToken);
|
||||
break;
|
||||
}
|
||||
@@ -224,7 +224,7 @@ public sealed class FunctionTriggers
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
this._logger.LogInformation("Client disconnected from stream {ConversationId}", conversationId);
|
||||
this._logger.LogInformation("Client disconnected from stream {ConversationId}", SanitizeLogValue(conversationId));
|
||||
}
|
||||
|
||||
return new EmptyResult();
|
||||
@@ -316,4 +316,20 @@ public sealed class FunctionTriggers
|
||||
|
||||
await response.WriteAsync(sb.ToString());
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Sanitizes a user-provided value for safe inclusion in log entries
|
||||
/// by removing control characters that could be used for log forging.
|
||||
/// </summary>
|
||||
private static string? SanitizeLogValue(string? value)
|
||||
{
|
||||
if (value is null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return value
|
||||
.Replace("\r", string.Empty, StringComparison.Ordinal)
|
||||
.Replace("\n", string.Empty, StringComparison.Ordinal);
|
||||
}
|
||||
}
|
||||
|
||||
+42
@@ -0,0 +1,42 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
<PropertyGroup>
|
||||
<TargetFrameworks>net10.0</TargetFrameworks>
|
||||
<AzureFunctionsVersion>v4</AzureFunctionsVersion>
|
||||
<OutputType>Exe</OutputType>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
<!-- The Functions build tools don't like namespaces that start with a number -->
|
||||
<AssemblyName>SingleAgent</AssemblyName>
|
||||
<RootNamespace>SingleAgent</RootNamespace>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<FrameworkReference Include="Microsoft.AspNetCore.App" />
|
||||
</ItemGroup>
|
||||
|
||||
<!-- Azure Functions packages -->
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.Azure.Functions.Worker" />
|
||||
<PackageReference Include="Microsoft.Azure.Functions.Worker.Extensions.DurableTask" />
|
||||
<PackageReference Include="Microsoft.Azure.Functions.Worker.Extensions.DurableTask.AzureManaged" />
|
||||
<PackageReference Include="Microsoft.Azure.Functions.Worker.Extensions.Http.AspNetCore" />
|
||||
<PackageReference Include="Microsoft.Azure.Functions.Worker.Sdk" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Azure.AI.OpenAI" />
|
||||
<PackageReference Include="Azure.Identity" />
|
||||
</ItemGroup>
|
||||
|
||||
<!-- Local projects that should be switched to package references when using the sample outside of this MAF repo -->
|
||||
<!--
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.Agents.AI.Hosting.AzureFunctions" />
|
||||
<PackageReference Include="Microsoft.Agents.AI.OpenAI" />
|
||||
</ItemGroup>
|
||||
-->
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\..\..\..\src\Microsoft.Agents.AI.Hosting.AzureFunctions\Microsoft.Agents.AI.Hosting.AzureFunctions.csproj" />
|
||||
<ProjectReference Include="..\..\..\..\..\src\Microsoft.Agents.AI.OpenAI\Microsoft.Agents.AI.OpenAI.csproj" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
+215
@@ -0,0 +1,215 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using Microsoft.Agents.AI.Workflows;
|
||||
|
||||
namespace SequentialWorkflow;
|
||||
|
||||
/// <summary>
|
||||
/// Looks up an order by its ID and return an Order object.
|
||||
/// </summary>
|
||||
internal sealed class OrderLookup() : Executor<string, Order>("OrderLookup")
|
||||
{
|
||||
public override async ValueTask<Order> HandleAsync(
|
||||
string message,
|
||||
IWorkflowContext context,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
Console.WriteLine();
|
||||
Console.ForegroundColor = ConsoleColor.Magenta;
|
||||
Console.WriteLine("┌─────────────────────────────────────────────────────────────────┐");
|
||||
Console.WriteLine($"│ [Activity] OrderLookup: Starting lookup for order '{message}'");
|
||||
Console.ResetColor();
|
||||
|
||||
// Simulate database lookup with delay
|
||||
await Task.Delay(TimeSpan.FromMicroseconds(100), cancellationToken);
|
||||
|
||||
Order order = new(
|
||||
Id: message,
|
||||
OrderDate: DateTime.UtcNow.AddDays(-1),
|
||||
IsCancelled: false,
|
||||
Customer: new Customer(Name: "Jerry", Email: "jerry@example.com"));
|
||||
|
||||
Console.ForegroundColor = ConsoleColor.Magenta;
|
||||
Console.WriteLine($"│ [Activity] OrderLookup: Found order '{message}' for customer '{order.Customer.Name}'");
|
||||
Console.WriteLine("└─────────────────────────────────────────────────────────────────┘");
|
||||
Console.ResetColor();
|
||||
|
||||
return order;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Cancels an order.
|
||||
/// </summary>
|
||||
internal sealed class OrderCancel() : Executor<Order, Order>("OrderCancel")
|
||||
{
|
||||
public override async ValueTask<Order> HandleAsync(
|
||||
Order message,
|
||||
IWorkflowContext context,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
Console.WriteLine();
|
||||
Console.ForegroundColor = ConsoleColor.Yellow;
|
||||
Console.WriteLine("┌─────────────────────────────────────────────────────────────────┐");
|
||||
Console.WriteLine($"│ [Activity] OrderCancel: Starting cancellation for order '{message.Id}'");
|
||||
Console.ResetColor();
|
||||
|
||||
// Simulate a slow cancellation process (e.g., calling external payment system)
|
||||
for (int i = 1; i <= 3; i++)
|
||||
{
|
||||
await Task.Delay(TimeSpan.FromMilliseconds(100), cancellationToken);
|
||||
Console.ForegroundColor = ConsoleColor.DarkYellow;
|
||||
Console.WriteLine("│ [Activity] OrderCancel: Processing...");
|
||||
Console.ResetColor();
|
||||
}
|
||||
|
||||
Order cancelledOrder = message with { IsCancelled = true };
|
||||
|
||||
Console.ForegroundColor = ConsoleColor.Yellow;
|
||||
Console.WriteLine($"│ [Activity] OrderCancel: ✓ Order '{cancelledOrder.Id}' has been cancelled");
|
||||
Console.WriteLine("└─────────────────────────────────────────────────────────────────┘");
|
||||
Console.ResetColor();
|
||||
|
||||
return cancelledOrder;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Sends a cancellation confirmation email to the customer.
|
||||
/// </summary>
|
||||
internal sealed class SendEmail() : Executor<Order, string>("SendEmail")
|
||||
{
|
||||
public override ValueTask<string> HandleAsync(
|
||||
Order message,
|
||||
IWorkflowContext context,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
Console.WriteLine();
|
||||
Console.ForegroundColor = ConsoleColor.Cyan;
|
||||
Console.WriteLine("┌─────────────────────────────────────────────────────────────────┐");
|
||||
Console.WriteLine($"│ [Activity] SendEmail: Sending email to '{message.Customer.Email}'...");
|
||||
Console.ResetColor();
|
||||
|
||||
string result = $"Cancellation email sent for order {message.Id} to {message.Customer.Email}.";
|
||||
|
||||
Console.ForegroundColor = ConsoleColor.Cyan;
|
||||
Console.WriteLine("│ [Activity] SendEmail: ✓ Email sent successfully!");
|
||||
Console.WriteLine("└─────────────────────────────────────────────────────────────────┘");
|
||||
Console.ResetColor();
|
||||
|
||||
return ValueTask.FromResult(result);
|
||||
}
|
||||
}
|
||||
|
||||
internal sealed record Order(string Id, DateTime OrderDate, bool IsCancelled, Customer Customer);
|
||||
|
||||
internal sealed record Customer(string Name, string Email);
|
||||
|
||||
/// <summary>
|
||||
/// Represents a batch cancellation request with multiple order IDs and a reason.
|
||||
/// This demonstrates using a complex typed object as workflow input.
|
||||
/// </summary>
|
||||
#pragma warning disable CA1812 // Instantiated via JSON deserialization at runtime
|
||||
internal sealed record BatchCancelRequest(string[] OrderIds, string Reason, bool NotifyCustomers);
|
||||
#pragma warning restore CA1812
|
||||
|
||||
/// <summary>
|
||||
/// Represents the result of processing a batch cancellation.
|
||||
/// </summary>
|
||||
internal sealed record BatchCancelResult(int TotalOrders, int CancelledCount, string Reason);
|
||||
|
||||
/// <summary>
|
||||
/// Generates a status report for an order.
|
||||
/// </summary>
|
||||
internal sealed class StatusReport() : Executor<Order, string>("StatusReport")
|
||||
{
|
||||
public override ValueTask<string> HandleAsync(
|
||||
Order message,
|
||||
IWorkflowContext context,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
Console.WriteLine();
|
||||
Console.ForegroundColor = ConsoleColor.Green;
|
||||
Console.WriteLine("┌─────────────────────────────────────────────────────────────────┐");
|
||||
Console.WriteLine($"│ [Activity] StatusReport: Generating report for order '{message.Id}'");
|
||||
Console.ResetColor();
|
||||
|
||||
string status = message.IsCancelled ? "Cancelled" : "Active";
|
||||
string result = $"Order {message.Id} for {message.Customer.Name}: Status={status}, Date={message.OrderDate:yyyy-MM-dd}";
|
||||
|
||||
Console.ForegroundColor = ConsoleColor.Green;
|
||||
Console.WriteLine($"│ [Activity] StatusReport: ✓ {result}");
|
||||
Console.WriteLine("└─────────────────────────────────────────────────────────────────┘");
|
||||
Console.ResetColor();
|
||||
|
||||
return ValueTask.FromResult(result);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Processes a batch cancellation request. Accepts a complex <see cref="BatchCancelRequest"/> object
|
||||
/// as input, demonstrating how workflows can receive structured JSON input.
|
||||
/// </summary>
|
||||
internal sealed class BatchCancelProcessor() : Executor<BatchCancelRequest, BatchCancelResult>("BatchCancelProcessor")
|
||||
{
|
||||
public override async ValueTask<BatchCancelResult> HandleAsync(
|
||||
BatchCancelRequest message,
|
||||
IWorkflowContext context,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
Console.WriteLine();
|
||||
Console.ForegroundColor = ConsoleColor.Yellow;
|
||||
Console.WriteLine("┌─────────────────────────────────────────────────────────────────┐");
|
||||
Console.WriteLine($"│ [Activity] BatchCancelProcessor: Processing {message.OrderIds.Length} orders");
|
||||
Console.WriteLine($"│ [Activity] BatchCancelProcessor: Reason: {message.Reason}");
|
||||
Console.WriteLine($"│ [Activity] BatchCancelProcessor: Notify customers: {message.NotifyCustomers}");
|
||||
Console.ResetColor();
|
||||
|
||||
// Simulate processing each order
|
||||
int cancelledCount = 0;
|
||||
foreach (string orderId in message.OrderIds)
|
||||
{
|
||||
await Task.Delay(TimeSpan.FromMilliseconds(100), cancellationToken);
|
||||
cancelledCount++;
|
||||
Console.ForegroundColor = ConsoleColor.DarkYellow;
|
||||
Console.WriteLine($"│ [Activity] BatchCancelProcessor: ✓ Cancelled order '{orderId}'");
|
||||
Console.ResetColor();
|
||||
}
|
||||
|
||||
BatchCancelResult result = new(message.OrderIds.Length, cancelledCount, message.Reason);
|
||||
|
||||
Console.ForegroundColor = ConsoleColor.Yellow;
|
||||
Console.WriteLine($"│ [Activity] BatchCancelProcessor: ✓ Batch complete: {cancelledCount}/{message.OrderIds.Length} cancelled");
|
||||
Console.WriteLine("└─────────────────────────────────────────────────────────────────┘");
|
||||
Console.ResetColor();
|
||||
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Generates a summary of the batch cancellation.
|
||||
/// </summary>
|
||||
internal sealed class BatchCancelSummary() : Executor<BatchCancelResult, string>("BatchCancelSummary")
|
||||
{
|
||||
public override ValueTask<string> HandleAsync(
|
||||
BatchCancelResult message,
|
||||
IWorkflowContext context,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
Console.WriteLine();
|
||||
Console.ForegroundColor = ConsoleColor.Cyan;
|
||||
Console.WriteLine("┌─────────────────────────────────────────────────────────────────┐");
|
||||
Console.WriteLine("│ [Activity] BatchCancelSummary: Generating summary");
|
||||
Console.ResetColor();
|
||||
|
||||
string result = $"Batch cancellation complete: {message.CancelledCount}/{message.TotalOrders} orders cancelled. Reason: {message.Reason}";
|
||||
|
||||
Console.ForegroundColor = ConsoleColor.Cyan;
|
||||
Console.WriteLine($"│ [Activity] BatchCancelSummary: ✓ {result}");
|
||||
Console.WriteLine("└─────────────────────────────────────────────────────────────────┘");
|
||||
Console.ResetColor();
|
||||
|
||||
return ValueTask.FromResult(result);
|
||||
}
|
||||
}
|
||||
+52
@@ -0,0 +1,52 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
// This sample demonstrates three workflows that share executors.
|
||||
// The CancelOrder workflow cancels an order and notifies the customer.
|
||||
// The OrderStatus workflow looks up an order and generates a status report.
|
||||
// The BatchCancelOrders workflow accepts a complex JSON input to cancel multiple orders.
|
||||
// Both CancelOrder and OrderStatus reuse the same OrderLookup executor, demonstrating executor sharing.
|
||||
|
||||
using Microsoft.Agents.AI.Hosting.AzureFunctions;
|
||||
using Microsoft.Agents.AI.Workflows;
|
||||
using Microsoft.Azure.Functions.Worker.Builder;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using SequentialWorkflow;
|
||||
|
||||
// Define executors for all workflows
|
||||
OrderLookup orderLookup = new();
|
||||
OrderCancel orderCancel = new();
|
||||
SendEmail sendEmail = new();
|
||||
StatusReport statusReport = new();
|
||||
BatchCancelProcessor batchCancelProcessor = new();
|
||||
BatchCancelSummary batchCancelSummary = new();
|
||||
|
||||
// Build the CancelOrder workflow: OrderLookup -> OrderCancel -> SendEmail
|
||||
Workflow cancelOrder = new WorkflowBuilder(orderLookup)
|
||||
.WithName("CancelOrder")
|
||||
.WithDescription("Cancel an order and notify the customer")
|
||||
.AddEdge(orderLookup, orderCancel)
|
||||
.AddEdge(orderCancel, sendEmail)
|
||||
.Build();
|
||||
|
||||
// Build the OrderStatus workflow: OrderLookup -> StatusReport
|
||||
// This workflow shares the OrderLookup executor with the CancelOrder workflow.
|
||||
Workflow orderStatus = new WorkflowBuilder(orderLookup)
|
||||
.WithName("OrderStatus")
|
||||
.WithDescription("Look up an order and generate a status report")
|
||||
.AddEdge(orderLookup, statusReport)
|
||||
.Build();
|
||||
|
||||
// Build the BatchCancelOrders workflow: BatchCancelProcessor -> BatchCancelSummary
|
||||
// This workflow demonstrates using a complex JSON object as the workflow input.
|
||||
Workflow batchCancelOrders = new WorkflowBuilder(batchCancelProcessor)
|
||||
.WithName("BatchCancelOrders")
|
||||
.WithDescription("Cancel multiple orders in a batch using a complex JSON input")
|
||||
.AddEdge(batchCancelProcessor, batchCancelSummary)
|
||||
.Build();
|
||||
|
||||
using IHost app = FunctionsApplication
|
||||
.CreateBuilder(args)
|
||||
.ConfigureFunctionsWebApplication()
|
||||
.ConfigureDurableWorkflows(workflows => workflows.AddWorkflows(cancelOrder, orderStatus, batchCancelOrders))
|
||||
.Build();
|
||||
app.Run();
|
||||
+100
@@ -0,0 +1,100 @@
|
||||
# Sequential Workflow Sample
|
||||
|
||||
This sample demonstrates how to use the Microsoft Agent Framework to create an Azure Functions app that hosts durable workflows with sequential executor chains. It showcases two workflows that share a common executor, demonstrating executor reuse across workflows.
|
||||
|
||||
## Key Concepts Demonstrated
|
||||
|
||||
- Defining workflows with sequential executor chains using `WorkflowBuilder`
|
||||
- Sharing executors across multiple workflows (the `OrderLookup` executor is used by both workflows)
|
||||
- Registering workflows with the Function app using `ConfigureDurableWorkflows`
|
||||
- Durable orchestration ensuring workflows survive process restarts and failures
|
||||
- Starting workflows via HTTP requests
|
||||
- Viewing workflow execution history and status in the Durable Task Scheduler (DTS) dashboard
|
||||
|
||||
## Workflows
|
||||
|
||||
This sample defines two workflows:
|
||||
|
||||
1. **CancelOrder**: `OrderLookup` → `OrderCancel` → `SendEmail` — Looks up an order, cancels it, and sends a confirmation email.
|
||||
2. **OrderStatus**: `OrderLookup` → `StatusReport` — Looks up an order and generates a status report.
|
||||
|
||||
Both workflows share the `OrderLookup` executor, which is registered only once by the framework.
|
||||
|
||||
## Environment Setup
|
||||
|
||||
See the [README.md](../../README.md) file in the parent directory for more information on how to configure the environment, including how to install and run common sample dependencies.
|
||||
|
||||
## Running the Sample
|
||||
|
||||
With the environment setup and function app running, you can test the sample by sending HTTP requests to the workflow endpoints.
|
||||
|
||||
You can use the `demo.http` file to trigger the workflows, or a command line tool like `curl` as shown below:
|
||||
|
||||
### Cancel an Order
|
||||
|
||||
Bash (Linux/macOS/WSL):
|
||||
|
||||
```bash
|
||||
curl -X POST http://localhost:7071/api/workflows/CancelOrder/run \
|
||||
-H "Content-Type: text/plain" \
|
||||
-d "12345"
|
||||
```
|
||||
|
||||
PowerShell:
|
||||
|
||||
```powershell
|
||||
Invoke-RestMethod -Method Post `
|
||||
-Uri http://localhost:7071/api/workflows/CancelOrder/run `
|
||||
-ContentType text/plain `
|
||||
-Body "12345"
|
||||
```
|
||||
|
||||
The response will confirm the workflow orchestration has started:
|
||||
|
||||
```text
|
||||
Workflow orchestration started for CancelOrder. Orchestration runId: abc123def456
|
||||
```
|
||||
|
||||
> **Tip:** You can provide a custom run ID by appending a `runId` query parameter:
|
||||
>
|
||||
> ```bash
|
||||
> curl -X POST "http://localhost:7071/api/workflows/CancelOrder/run?runId=my-order-123" \
|
||||
> -H "Content-Type: text/plain" \
|
||||
> -d "12345"
|
||||
> ```
|
||||
>
|
||||
> If not provided, a unique run ID is auto-generated.
|
||||
|
||||
In the function app logs, you will see the sequential execution of each executor:
|
||||
|
||||
```text
|
||||
│ [Activity] OrderLookup: Starting lookup for order '12345'
|
||||
│ [Activity] OrderLookup: Found order '12345' for customer 'Jerry'
|
||||
│ [Activity] OrderCancel: Starting cancellation for order '12345'
|
||||
│ [Activity] OrderCancel: ✓ Order '12345' has been cancelled
|
||||
│ [Activity] SendEmail: Sending email to 'jerry@example.com'...
|
||||
│ [Activity] SendEmail: ✓ Email sent successfully!
|
||||
```
|
||||
|
||||
### Get Order Status
|
||||
|
||||
```bash
|
||||
curl -X POST http://localhost:7071/api/workflows/OrderStatus/run \
|
||||
-H "Content-Type: text/plain" \
|
||||
-d "12345"
|
||||
```
|
||||
|
||||
The `OrderStatus` workflow reuses the same `OrderLookup` executor and then generates a status report:
|
||||
|
||||
```text
|
||||
│ [Activity] OrderLookup: Starting lookup for order '12345'
|
||||
│ [Activity] OrderLookup: Found order '12345' for customer 'Jerry'
|
||||
│ [Activity] StatusReport: Generating report for order '12345'
|
||||
│ [Activity] StatusReport: ✓ Order 12345 for Jerry: Status=Active, Date=2025-01-01
|
||||
```
|
||||
|
||||
### Viewing Workflows in the DTS Dashboard
|
||||
|
||||
After running a workflow, you can navigate to the Durable Task Scheduler (DTS) dashboard to visualize the completed orchestration, inspect inputs/outputs for each step, and view execution history.
|
||||
|
||||
If you are using the DTS emulator, the dashboard is available at `http://localhost:8082`.
|
||||
+26
@@ -0,0 +1,26 @@
|
||||
# Default endpoint address for local testing
|
||||
@authority=http://localhost:7071
|
||||
|
||||
### Cancel an order
|
||||
POST {{authority}}/api/workflows/CancelOrder/run
|
||||
Content-Type: text/plain
|
||||
|
||||
12345
|
||||
|
||||
### Cancel an order with a custom run ID
|
||||
POST {{authority}}/api/workflows/CancelOrder/run?runId=my-custom-id-123
|
||||
Content-Type: text/plain
|
||||
|
||||
99999
|
||||
|
||||
### Get order status (shares OrderLookup executor with CancelOrder)
|
||||
POST {{authority}}/api/workflows/OrderStatus/run
|
||||
Content-Type: text/plain
|
||||
|
||||
12345
|
||||
|
||||
### Batch cancel orders with a complex JSON input
|
||||
POST {{authority}}/api/workflows/BatchCancelOrders/run
|
||||
Content-Type: application/json
|
||||
|
||||
{"orderIds": ["1001", "1002", "1003"], "reason": "Customer requested cancellation", "notifyCustomers": true}
|
||||
+20
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"version": "2.0",
|
||||
"logging": {
|
||||
"logLevel": {
|
||||
"Microsoft.Agents.AI.DurableTask": "Information",
|
||||
"Microsoft.Agents.AI.Hosting.AzureFunctions": "Information",
|
||||
"DurableTask": "Information",
|
||||
"Microsoft.DurableTask": "Information"
|
||||
}
|
||||
},
|
||||
"extensions": {
|
||||
"durableTask": {
|
||||
"hubName": "default",
|
||||
"storageProvider": {
|
||||
"type": "AzureManaged",
|
||||
"connectionStringName": "DURABLE_TASK_SCHEDULER_CONNECTION_STRING"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+10
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"IsEncrypted": false,
|
||||
"Values": {
|
||||
"FUNCTIONS_WORKER_RUNTIME": "dotnet-isolated",
|
||||
"AzureWebJobsStorage": "UseDevelopmentStorage=true",
|
||||
"DURABLE_TASK_SCHEDULER_CONNECTION_STRING": "Endpoint=http://localhost:8080;TaskHub=default;Authentication=None",
|
||||
"AZURE_OPENAI_ENDPOINT": "<AZURE_OPENAI_ENDPOINT>",
|
||||
"AZURE_OPENAI_DEPLOYMENT_NAME": "<AZURE_OPENAI_DEPLOYMENT_NAME>"
|
||||
}
|
||||
}
|
||||
+42
@@ -0,0 +1,42 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
<PropertyGroup>
|
||||
<TargetFrameworks>net10.0</TargetFrameworks>
|
||||
<AzureFunctionsVersion>v4</AzureFunctionsVersion>
|
||||
<OutputType>Exe</OutputType>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
<!-- The Functions build tools don't like namespaces that start with a number -->
|
||||
<AssemblyName>SingleAgent</AssemblyName>
|
||||
<RootNamespace>SingleAgent</RootNamespace>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<FrameworkReference Include="Microsoft.AspNetCore.App" />
|
||||
</ItemGroup>
|
||||
|
||||
<!-- Azure Functions packages -->
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.Azure.Functions.Worker" />
|
||||
<PackageReference Include="Microsoft.Azure.Functions.Worker.Extensions.DurableTask" />
|
||||
<PackageReference Include="Microsoft.Azure.Functions.Worker.Extensions.DurableTask.AzureManaged" />
|
||||
<PackageReference Include="Microsoft.Azure.Functions.Worker.Extensions.Http.AspNetCore" />
|
||||
<PackageReference Include="Microsoft.Azure.Functions.Worker.Sdk" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Azure.AI.OpenAI" />
|
||||
<PackageReference Include="Azure.Identity" />
|
||||
</ItemGroup>
|
||||
|
||||
<!-- Local projects that should be switched to package references when using the sample outside of this MAF repo -->
|
||||
<!--
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.Agents.AI.Hosting.AzureFunctions" />
|
||||
<PackageReference Include="Microsoft.Agents.AI.OpenAI" />
|
||||
</ItemGroup>
|
||||
-->
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\..\..\..\src\Microsoft.Agents.AI.Hosting.AzureFunctions\Microsoft.Agents.AI.Hosting.AzureFunctions.csproj" />
|
||||
<ProjectReference Include="..\..\..\..\..\src\Microsoft.Agents.AI.OpenAI\Microsoft.Agents.AI.OpenAI.csproj" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user