Merge branch 'main' into copilot/add-preconfigured-compaction-strategy

This commit is contained in:
Chris
2026-03-24 11:57:30 -07:00
committed by GitHub
Unverified
281 changed files with 6829 additions and 1297 deletions
+1
View File
@@ -3,6 +3,7 @@
"image": "mcr.microsoft.com/devcontainers/dotnet",
"features": {
"ghcr.io/devcontainers/features/azure-cli:1.2.9": {},
"ghcr.io/devcontainers/features/docker-in-docker:2": {},
"ghcr.io/devcontainers/features/github-cli:1": {
"version": "2"
},
+216
View File
@@ -0,0 +1,216 @@
# Copyright (c) Microsoft. All rights reserved.
"""Scan open issues and PRs labeled 'waiting-for-author' for stale follow-ups.
Team members manually add the 'waiting-for-author' label when they need a
response from the external author. If the author hasn't replied within
DAYS_THRESHOLD days of the last team comment, post a reminder and add the
'requested-info' label to prevent duplicate pings.
"""
from __future__ import annotations
import os
import sys
import time
from datetime import datetime, timezone
from github import Auth, Github, GithubException
from github.Issue import Issue
from github.IssueComment import IssueComment
PING_COMMENT = (
"@{author}, friendly reminder — this issue is waiting on your response. "
"Please share any updates when you get a chance. (This is an automated message.)"
)
TRIGGER_LABEL = "waiting-for-author"
PINGED_LABEL = "requested-info"
def get_team_members(g: Github, org: str, team_slug: str) -> set[str]:
"""Fetch active team member usernames."""
try:
org_obj = g.get_organization(org)
team = org_obj.get_team_by_slug(team_slug)
return {m.login for m in team.get_members()}
except GithubException as exc:
if exc.status in (403, 404):
print(
f"ERROR: Failed to fetch team members for {org}/{team_slug} "
f"(HTTP {exc.status}). Check that the token has the 'read:org' "
f"scope and that the team slug '{team_slug}' is correct."
)
else:
print(f"ERROR: Failed to fetch team members for {org}/{team_slug}: {exc}")
sys.exit(1)
except Exception as exc:
print(f"ERROR: Failed to fetch team members for {org}/{team_slug}: {exc}")
sys.exit(1)
def find_last_team_comment(
comments: list[IssueComment], team_members: set[str]
) -> IssueComment | None:
"""Return the most recent comment from a team member, or None."""
for comment in reversed(comments):
if comment.user and comment.user.login in team_members:
return comment
return None
def author_replied_after(
comments: list[IssueComment], author: str, after: datetime
) -> bool:
"""Check if the issue author commented after the given timestamp."""
for comment in comments:
if (
comment.user
and comment.user.login == author
and comment.created_at > after
):
return True
return False
def should_ping(
issue: Issue,
team_members: set[str],
days_threshold: int,
now: datetime,
) -> bool:
"""Determine whether this issue/PR should be pinged.
Only issues/PRs carrying the 'waiting-for-author' label are candidates.
"""
author = issue.user.login
# Skip if the trigger label is not present
if not any(label.name == TRIGGER_LABEL for label in issue.labels):
return False
# Skip if author is a team member
if author in team_members:
return False
# Skip if already pinged
if any(label.name == PINGED_LABEL for label in issue.labels):
return False
# Skip if no comments at all
if issue.comments == 0:
return False
# Fetch comments once for both lookups
comments = list(issue.get_comments())
# Find last team member comment
last_team_comment = find_last_team_comment(comments, team_members)
if last_team_comment is None:
return False
# Skip if author replied after the last team comment
if author_replied_after(comments, author, last_team_comment.created_at):
return False
# Check if enough days have passed
days_since = (now - last_team_comment.created_at.astimezone(timezone.utc)).days
if days_since < days_threshold:
return False
return True
def ping(issue: Issue, dry_run: bool) -> bool:
"""Post a reminder comment and add the 'requested-info' label. Returns True on success."""
author = issue.user.login
kind = "PR" if issue.pull_request else "Issue"
if dry_run:
print(f" [DRY RUN] Would ping {kind} #{issue.number} (@{author})")
return True
max_retries = 3
commented = False
labeled = False
for attempt in range(1, max_retries + 1):
try:
if not commented:
issue.create_comment(PING_COMMENT.format(author=author))
commented = True
if not labeled:
issue.add_to_labels(PINGED_LABEL)
labeled = True
print(f" Pinged {kind} #{issue.number} (@{author})")
return True
except Exception as exc:
if attempt < max_retries:
wait = 2 ** attempt # 2s, 4s
print(f" WARN: Attempt {attempt}/{max_retries} failed for {kind} #{issue.number}: {exc}. Retrying in {wait}s...")
time.sleep(wait)
else:
print(f" ERROR: Failed to ping {kind} #{issue.number} after {max_retries} attempts: {exc}")
return False
def main() -> None:
token = os.environ.get("GITHUB_TOKEN")
if not token:
print("ERROR: GITHUB_TOKEN environment variable is required")
sys.exit(1)
repository = os.environ.get("GITHUB_REPOSITORY")
if not repository:
print("ERROR: GITHUB_REPOSITORY environment variable is required")
sys.exit(1)
team_slug = os.environ.get("TEAM_SLUG")
if not team_slug:
print("ERROR: TEAM_SLUG environment variable is required")
sys.exit(1)
days_threshold_raw = os.environ.get("DAYS_THRESHOLD", "4")
try:
days_threshold = int(days_threshold_raw)
except ValueError:
print(f"ERROR: DAYS_THRESHOLD must be a numeric value, got '{days_threshold_raw}'")
sys.exit(1)
dry_run = os.environ.get("DRY_RUN", "false").lower() == "true"
org = repository.split("/")[0]
if dry_run:
print("Running in DRY RUN mode — no comments or labels will be applied.\n")
g = Github(auth=Auth.Token(token))
repo = g.get_repo(repository)
print(f"Fetching team members for {org}/{team_slug}...")
team_members = get_team_members(g, org, team_slug)
print(f"Found {len(team_members)} team members.\n")
now = datetime.now(timezone.utc)
pinged = []
failed = []
scanned = 0
print(f"Scanning open issues and PRs labeled '{TRIGGER_LABEL}' (threshold: {days_threshold} days)...\n")
for issue in repo.get_issues(state="open", labels=[TRIGGER_LABEL]):
scanned += 1
if should_ping(issue, team_members, days_threshold, now):
if ping(issue, dry_run):
pinged.append(issue.number)
else:
failed.append(issue.number)
print(f"\nDone. Scanned {scanned} items, pinged {len(pinged)}, failed {len(failed)}.")
if pinged:
print(f"Pinged: {', '.join(f'#{n}' for n in pinged)}")
if failed:
print(f"Failed: {', '.join(f'#{n}' for n in failed)}")
sys.exit(1)
if __name__ == "__main__":
main()
+297
View File
@@ -0,0 +1,297 @@
# Copyright (c) Microsoft. All rights reserved.
"""Tests for stale_issue_pr_ping.py."""
from __future__ import annotations
import os
import sys
from datetime import datetime, timezone, timedelta
from unittest.mock import MagicMock, patch
import pytest
# Ensure the script directory is importable
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "scripts"))
from stale_issue_pr_ping import (
PINGED_LABEL,
PING_COMMENT,
TRIGGER_LABEL,
author_replied_after,
find_last_team_comment,
get_team_members,
main,
ping,
should_ping,
)
TEAM = {"alice", "bob"}
NOW = datetime(2026, 3, 15, 12, 0, 0, tzinfo=timezone.utc)
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _make_comment(login: str | None, created_at: datetime) -> MagicMock:
"""Create a mock IssueComment."""
c = MagicMock()
if login is None:
c.user = None
else:
c.user = MagicMock()
c.user.login = login
c.created_at = created_at
return c
def _make_label(name: str) -> MagicMock:
lbl = MagicMock()
lbl.name = name
return lbl
def _make_issue(
author: str = "external",
labels: list[str] | None = None,
comment_count: int = 1,
comments: list[MagicMock] | None = None,
pull_request: bool = False,
number: int = 42,
) -> MagicMock:
issue = MagicMock()
issue.user = MagicMock()
issue.user.login = author
issue.number = number
# Default to having the trigger label, since the API query pre-filters.
if labels is None:
labels = [TRIGGER_LABEL]
issue.labels = [_make_label(n) for n in labels]
issue.comments = comment_count
issue.pull_request = MagicMock() if pull_request else None
if comments is not None:
issue.get_comments.return_value = comments
return issue
# ---------------------------------------------------------------------------
# find_last_team_comment
# ---------------------------------------------------------------------------
class TestFindLastTeamComment:
def test_returns_last_team_comment(self):
c1 = _make_comment("alice", datetime(2026, 3, 1, tzinfo=timezone.utc))
c2 = _make_comment("external", datetime(2026, 3, 2, tzinfo=timezone.utc))
c3 = _make_comment("bob", datetime(2026, 3, 3, tzinfo=timezone.utc))
assert find_last_team_comment([c1, c2, c3], TEAM) is c3
def test_returns_none_when_no_team_comments(self):
c1 = _make_comment("external", datetime(2026, 3, 1, tzinfo=timezone.utc))
assert find_last_team_comment([c1], TEAM) is None
def test_returns_none_for_empty_list(self):
assert find_last_team_comment([], TEAM) is None
def test_skips_deleted_user(self):
c1 = _make_comment(None, datetime(2026, 3, 1, tzinfo=timezone.utc))
c2 = _make_comment("alice", datetime(2026, 3, 2, tzinfo=timezone.utc))
assert find_last_team_comment([c1, c2], TEAM) is c2
def test_only_deleted_users(self):
c1 = _make_comment(None, datetime(2026, 3, 1, tzinfo=timezone.utc))
assert find_last_team_comment([c1], TEAM) is None
# ---------------------------------------------------------------------------
# author_replied_after
# ---------------------------------------------------------------------------
class TestAuthorRepliedAfter:
def test_author_replied(self):
after = datetime(2026, 3, 1, tzinfo=timezone.utc)
c1 = _make_comment("external", datetime(2026, 3, 2, tzinfo=timezone.utc))
assert author_replied_after([c1], "external", after) is True
def test_author_not_replied(self):
after = datetime(2026, 3, 5, tzinfo=timezone.utc)
c1 = _make_comment("external", datetime(2026, 3, 2, tzinfo=timezone.utc))
assert author_replied_after([c1], "external", after) is False
def test_different_user_replied(self):
after = datetime(2026, 3, 1, tzinfo=timezone.utc)
c1 = _make_comment("someone_else", datetime(2026, 3, 2, tzinfo=timezone.utc))
assert author_replied_after([c1], "external", after) is False
def test_deleted_user_comment(self):
after = datetime(2026, 3, 1, tzinfo=timezone.utc)
c1 = _make_comment(None, datetime(2026, 3, 2, tzinfo=timezone.utc))
assert author_replied_after([c1], "external", after) is False
# ---------------------------------------------------------------------------
# should_ping
# ---------------------------------------------------------------------------
class TestShouldPing:
def test_should_ping_stale_issue(self):
team_comment = _make_comment("alice", NOW - timedelta(days=5))
issue = _make_issue(comments=[team_comment], comment_count=1)
assert should_ping(issue, TEAM, 4, NOW) is True
def test_skip_team_member_author(self):
issue = _make_issue(author="alice", labels=[TRIGGER_LABEL], comment_count=1)
assert should_ping(issue, TEAM, 4, NOW) is False
def test_skip_already_pinged(self):
issue = _make_issue(labels=[TRIGGER_LABEL, PINGED_LABEL], comment_count=1)
assert should_ping(issue, TEAM, 4, NOW) is False
def test_skip_no_comments(self):
issue = _make_issue(comment_count=0)
assert should_ping(issue, TEAM, 4, NOW) is False
def test_skip_no_team_comment(self):
c = _make_comment("external", NOW - timedelta(days=5))
issue = _make_issue(comments=[c], comment_count=1)
assert should_ping(issue, TEAM, 4, NOW) is False
def test_skip_author_replied(self):
team_c = _make_comment("alice", NOW - timedelta(days=5))
author_c = _make_comment("external", NOW - timedelta(days=3))
issue = _make_issue(comments=[team_c, author_c], comment_count=2)
assert should_ping(issue, TEAM, 4, NOW) is False
def test_skip_not_enough_days(self):
team_comment = _make_comment("alice", NOW - timedelta(days=2))
issue = _make_issue(comments=[team_comment], comment_count=1)
assert should_ping(issue, TEAM, 4, NOW) is False
def test_aware_datetime_handled(self):
"""Timezone-aware datetimes should not be mangled by astimezone."""
aware_dt = (NOW - timedelta(days=5)).replace(tzinfo=timezone.utc)
team_comment = _make_comment("alice", aware_dt)
issue = _make_issue(comments=[team_comment], comment_count=1)
assert should_ping(issue, TEAM, 4, NOW) is True
def test_naive_datetime_handled(self):
"""Naive datetimes (pre-PyGithub 2.x) should be handled by astimezone."""
naive_dt = (NOW - timedelta(days=5)).replace(tzinfo=None)
team_comment = _make_comment("alice", naive_dt)
issue = _make_issue(comments=[team_comment], comment_count=1)
# astimezone on naive datetime treats it as local time; just verify no crash
should_ping(issue, TEAM, 4, NOW)
# ---------------------------------------------------------------------------
# ping
# ---------------------------------------------------------------------------
class TestPing:
def test_dry_run(self, capsys):
issue = _make_issue()
assert ping(issue, dry_run=True) is True
issue.create_comment.assert_not_called()
assert "DRY RUN" in capsys.readouterr().out
def test_success(self, capsys):
issue = _make_issue()
assert ping(issue, dry_run=False) is True
issue.create_comment.assert_called_once()
issue.add_to_labels.assert_called_once_with(PINGED_LABEL)
@patch("stale_issue_pr_ping.time.sleep")
def test_retry_on_failure(self, mock_sleep):
issue = _make_issue()
issue.create_comment.side_effect = [Exception("net error"), None]
assert ping(issue, dry_run=False) is True
assert issue.create_comment.call_count == 2
mock_sleep.assert_called_once()
@patch("stale_issue_pr_ping.time.sleep")
def test_idempotent_retry_skips_comment_on_label_failure(self, mock_sleep):
"""If create_comment succeeds but add_to_labels fails, retry should not re-comment."""
issue = _make_issue()
issue.add_to_labels.side_effect = [Exception("label error"), None]
assert ping(issue, dry_run=False) is True
# Comment should only be created once even though there were 2 attempts
assert issue.create_comment.call_count == 1
assert issue.add_to_labels.call_count == 2
@patch("stale_issue_pr_ping.time.sleep")
def test_all_retries_fail(self, mock_sleep):
issue = _make_issue()
issue.create_comment.side_effect = Exception("permanent error")
assert ping(issue, dry_run=False) is False
assert issue.create_comment.call_count == 3
# ---------------------------------------------------------------------------
# get_team_members
# ---------------------------------------------------------------------------
class TestGetTeamMembers:
def test_success(self):
g = MagicMock()
member = MagicMock()
member.login = "alice"
g.get_organization.return_value.get_team_by_slug.return_value.get_members.return_value = [member]
assert get_team_members(g, "org", "my-team") == {"alice"}
def test_403_error_message(self, capsys):
from github import GithubException
g = MagicMock()
g.get_organization.return_value.get_team_by_slug.side_effect = GithubException(
403, {"message": "Forbidden"}, None
)
with pytest.raises(SystemExit):
get_team_members(g, "org", "my-team")
out = capsys.readouterr().out
assert "read:org" in out
assert "403" in out
def test_404_error_message(self, capsys):
from github import GithubException
g = MagicMock()
g.get_organization.return_value.get_team_by_slug.side_effect = GithubException(
404, {"message": "Not Found"}, None
)
with pytest.raises(SystemExit):
get_team_members(g, "org", "bad-slug")
out = capsys.readouterr().out
assert "read:org" in out
assert "bad-slug" in out
def test_generic_error(self, capsys):
g = MagicMock()
g.get_organization.side_effect = RuntimeError("boom")
with pytest.raises(SystemExit):
get_team_members(g, "org", "team")
# ---------------------------------------------------------------------------
# main env var validation
# ---------------------------------------------------------------------------
class TestMain:
@patch.dict(os.environ, {
"GITHUB_TOKEN": "tok",
"GITHUB_REPOSITORY": "org/repo",
"TEAM_SLUG": "my-team",
"DAYS_THRESHOLD": "abc",
}, clear=True)
def test_invalid_days_threshold(self, capsys):
with pytest.raises(SystemExit):
main()
assert "numeric" in capsys.readouterr().out
@patch.dict(os.environ, {
"GITHUB_TOKEN": "tok",
"GITHUB_REPOSITORY": "org/repo",
}, clear=True)
def test_missing_team_slug(self, capsys):
with pytest.raises(SystemExit):
main()
assert "TEAM_SLUG" in capsys.readouterr().out
@@ -21,7 +21,7 @@ jobs:
steps:
- uses: actions/checkout@v6
- name: Download coverage report
uses: actions/download-artifact@v7
uses: actions/download-artifact@v8
with:
github-token: ${{ secrets.GH_ACTIONS_PR_WRITE }}
run-id: ${{ github.event.workflow_run.id }}
+49
View File
@@ -0,0 +1,49 @@
name: Stale issue and PR ping
on:
schedule:
- cron: '0 0 * * *' # Midnight UTC daily
workflow_dispatch:
inputs:
days_threshold:
description: 'Days of silence before pinging the author'
required: false
default: '4'
dry_run:
description: 'Log what would be pinged without taking action'
required: false
default: 'false'
type: choice
options:
- 'false'
- 'true'
concurrency:
group: stale-issue-pr-ping
cancel-in-progress: true
jobs:
ping_stale:
name: "Ping stale issues and PRs"
runs-on: ubuntu-latest
permissions:
contents: read
issues: write
pull-requests: write
steps:
- uses: actions/checkout@v6
- uses: actions/setup-python@v5
with:
python-version: '3.13'
- name: Install dependencies
run: pip install PyGithub==2.6.0
- name: Run stale issue/PR ping
run: python .github/scripts/stale_issue_pr_ping.py
env:
GITHUB_TOKEN: ${{ secrets.GH_ACTIONS_PR_WRITE }}
TEAM_SLUG: ${{ secrets.DEVELOPER_TEAM }}
DAYS_THRESHOLD: ${{ github.event.inputs.days_threshold || '4' }}
DRY_RUN: ${{ github.event.inputs.dry_run || 'false' }}
@@ -0,0 +1,815 @@
---
status: accepted
contact: bentho
date: 2026-02-27
deciders: bentho, markwallace-microsoft, westey-m
consulted: Pratyush Mishra, Shivam Shrivastava, Manni Arora (Centrica eval scenario)
informed: Agent Framework team, Foundry Evals team
---
# Agent Evaluation Architecture with Azure AI Foundry Integration
## Context and Problem Statement
Azure AI Foundry provides a rich evaluation service for AI agents — built-in evaluators for agent behavior (task adherence, intent resolution), tool usage (tool call accuracy, tool selection), quality (coherence, fluency, relevance), and safety (violence, self-harm, prohibited actions). Results are viewable in the Foundry portal with dashboards and comparison views.
However, using Foundry Evals with an agent-framework agent today requires significant manual effort. Developers must:
1. Transform agent-framework's `Message`/`Content` types into the OpenAI-style agent message schema that Foundry evaluators expect
2. Map tool definitions from agent-framework's `FunctionTool` format to evaluator-compatible schemas
3. Manually wire up the correct Foundry data source type (`azure_ai_traces`, `jsonl`, `azure_ai_target_completions`, etc.) depending on their scenario
4. Handle App Insights trace ID queries, response ID collection, and eval polling
Additionally, evaluation is a concern that extends beyond any single provider. Developers may want to use local evaluators (LLM-as-judge, regex, keyword matching), third-party evaluation libraries, or multiple providers in combination. The architecture must support this without creating a Foundry-specific lock-in at the API level.
### Functional Requirements for Agent Evaluation
- **Single agents and workflows.** Evaluate both individual agent responses and multi-agent workflow results, with per-agent breakdown to pinpoint underperformance.
- **One-shot and multi-turn conversations.** Capture full conversation trajectories — including tool calls and results — not just final query/response pairs.
- **Conversation factoring.** Support splitting conversations into query/response in multiple ways (last turn, full trajectory, per-turn) because different factorings measure different things.
- **Multiple providers, mix and match.** Run Foundry LLM-as-judge evaluators alongside fast local checks and custom evaluators on the same data, without restructuring code.
- **Third-party extensibility.** Any evaluation library can participate by implementing the `Evaluator` protocol (Python) or `IAgentEvaluator` interface (.NET). No predetermined list of supported libraries — the protocol is intentionally simple (`evaluate(items) → results`) so that wrappers for libraries like DeepEval, RAGAS, or Promptfoo are straightforward to write.
- **Bring your own evaluator.** Creating a custom evaluator should be as simple as writing a function.
- **Evaluate without re-running.** Evaluate existing responses from logs or previous runs without invoking the agent again.
## Decision Drivers
- **Zero-friction evaluation**: Developers should go from "I have an agent" to "I have eval results" with minimal code.
- **Provider-agnostic API**: Core evaluation capabilities must not be tied to any specific provider. Provider configuration should be separate from the evaluation call.
- **Lowest concept count**: Introduce the fewest possible new types, abstractions, and APIs for developers to learn.
- **Leverage existing knowledge**: The framework already knows which agents exist, what tools they have, and what conversations occurred. Evals should use this automatically rather than requiring the developer to re-specify it.
- **Foundry-native results**: When using Foundry, results should be viewable in the Foundry portal with dashboards and comparison views.
- **Progressive disclosure**: Simple scenarios should be near-zero code. Advanced scenarios should build on the same primitives.
- **Cross-language parity**: Design must be implementable in both Python and .NET.
## Considered Options
1. **Provider-specific functions** — Build Foundry-specific helper functions (`evaluate_agent()`, etc.) directly in the Azure package. All eval functions take Foundry connection parameters.
2. **Evaluator protocol with shared orchestration** — Define a provider-agnostic `Evaluator` protocol in the base agent library (`agent_framework` in Python, `Microsoft.Agents.AI` in .NET). Orchestration functions live alongside it. Providers implement the protocol.
3. **Full eval framework** — Build comprehensive eval infrastructure including custom evaluator definitions, scoring profiles, and reporting inside agent-framework.
## Decision Outcome
Proposed option: "Evaluator protocol with shared orchestration", because it delivers the low-friction developer experience, supports multiple providers without API changes, and keeps the concept count low.
### Usage Examples
#### Evaluate an agent
The agent is invoked once per query by default. For statistically meaningful evaluation, provide multiple diverse queries. For measuring **consistency** (does the same query produce reliable results?), use `num_repetitions` to run each query N times independently:
**Python:**
```python
evals = FoundryEvals(
project_client=client,
model_deployment="gpt-4o",
evaluators=[FoundryEvals.RELEVANCE, FoundryEvals.COHERENCE],
)
results = await evaluate_agent(
agent=my_agent,
queries=[
"What's the weather in Seattle?",
"Plan a weekend trip to Portland",
"What restaurants are near Pike Place?",
],
evaluators=evals,
)
for r in results:
r.assert_passed()
```
**C#:**
```csharp
var evals = new FoundryEvals(chatConfiguration, FoundryEvals.Relevance, FoundryEvals.Coherence);
AgentEvaluationResults results = await agent.EvaluateAsync(
new[] {
"What's the weather in Seattle?",
"Plan a weekend trip to Portland",
"What restaurants are near Pike Place?",
},
evals);
results.AssertAllPassed();
```
`evaluate_agent` returns one `EvalResults` per evaluator. Each result contains per-item scores with the evaluated response for auditing:
```
# results[0] (FoundryEvals)
EvalResults(status="completed", passed=3, failed=0, total=3)
items[0]: EvalItemResult(
query="What's the weather in Seattle?",
response="It's currently 72°F and sunny in Seattle.",
scores={"relevance": 5, "coherence": 5})
items[1]: EvalItemResult(
query="Plan a weekend trip to Portland",
response="Here's a 2-day Portland itinerary...",
scores={"relevance": 4, "coherence": 5})
items[2]: EvalItemResult(
query="What restaurants are near Pike Place?",
response="Top restaurants near Pike Place Market: ...",
scores={"relevance": 5, "coherence": 4})
```
#### Measure consistency with repetitions
Run each query multiple times to detect non-deterministic behavior:
**Python:**
```python
results = await evaluate_agent(
agent=my_agent,
queries=["What's the weather in Seattle?"],
evaluators=evals,
num_repetitions=3, # each query runs 3 times independently
)
# results contain 3 items (1 query × 3 repetitions)
```
**C#:**
```csharp
AgentEvaluationResults results = await agent.EvaluateAsync(
new[] { "What's the weather in Seattle?" },
evals,
numRepetitions: 3); // each query runs 3 times independently
// results contain 3 items (1 query × 3 repetitions)
```
#### Evaluate a response you already have
When you already have agent responses, pass them directly to skip re-running the agent. Each query is paired with its corresponding response:
**Python:**
```python
queries = ["What's the weather?", "What's the capital of France?"]
responses = [await agent.run([Message("user", [q])]) for q in queries]
results = await evaluate_agent(
responses=responses,
evaluators=evals,
)
```
**C#:**
```csharp
var queries = new[] { "What's the weather?" };
var responses = new List<AgentResponse>();
foreach (var q in queries)
responses.Add(await agent.RunAsync(new[] { new ChatMessage(ChatRole.User, q) }));
AgentEvaluationResults results = await agent.EvaluateAsync(
responses: responses,
evals);
```
Each `AgentResponse` already contains the conversation (query + response), so the evaluator extracts query/response from the conversation. When you pass `responses` without `queries`, the conversation is the source of truth.
#### Evaluate with conversation split strategies
By default, evaluators see only the last turn (final user message → final assistant response). For multi-turn conversations, you can control how the conversation is factored for evaluation:
**Python:**
```python
results = await evaluate_agent(
agent=agent,
queries=["Plan a 3-day trip to Paris"],
evaluators=evals,
conversation_split=ConversationSplit.FULL, # evaluate entire trajectory
)
# Or per-turn: each user→assistant exchange scored independently
results = await evaluate_agent(
agent=agent,
queries=["Plan a 3-day trip to Paris"],
evaluators=evals,
conversation_split=ConversationSplit.PER_TURN,
)
```
**C#:**
```csharp
// Full conversation as context
AgentEvaluationResults results = await agent.EvaluateAsync(
new[] { "Plan a 3-day trip to Paris" },
evals,
splitter: ConversationSplitters.Full);
// Per-turn splitting
var items = EvalItem.PerTurnItems(conversation); // one EvalItem per user turn
var results = await evals.EvaluateAsync(items);
```
With `PER_TURN`, a 3-turn conversation produces 3 scored items:
```
EvalResults(status="completed", passed=3, failed=0, total=3)
items[0]: query="Plan a 3-day trip to Paris" scores={"relevance": 5}
items[1]: query="What about restaurants?" scores={"relevance": 4}
items[2]: query="Make it budget-friendly" scores={"relevance": 5}
```
#### Evaluate a multi-agent workflow
**Python:**
```python
result = await workflow.run("Plan a trip to Paris")
eval_results = await evaluate_workflow(
workflow=workflow,
workflow_result=result,
evaluators=evals,
)
for r in eval_results:
print(f" overall: {r.passed}/{r.total}")
for name, sub in r.sub_results.items():
print(f" {name}: {sub.passed}/{sub.total}")
```
**C#:**
```csharp
WorkflowRunResult result = await workflow.RunAsync("Plan a trip to Paris");
IReadOnlyList<AgentEvaluationResults> evalResults = await result.EvaluateAsync(evals);
foreach (var r in evalResults)
{
Console.WriteLine($" overall: {r.Passed}/{r.Total}");
foreach (var (name, sub) in r.SubResults)
Console.WriteLine($" {name}: {sub.Passed}/{sub.Total}");
}
```
Workflows return one result per evaluator, with sub-results per agent in the workflow:
```
EvalResults(status="completed", passed=2, failed=0, total=2)
sub_results:
"planner": EvalResults(passed=1, total=1)
"researcher": EvalResults(passed=1, total=1)
```
#### Mix multiple providers
**Python:**
```python
@evaluator
def is_helpful(response: str) -> bool:
return len(response.split()) > 10
foundry = FoundryEvals(
project_client=client,
model_deployment="gpt-4o",
evaluators=[FoundryEvals.RELEVANCE, FoundryEvals.COHERENCE],
)
results = await evaluate_agent(
agent=agent,
queries=queries,
evaluators=[is_helpful, keyword_check("weather"), foundry],
)
```
**C#:**
```csharp
IReadOnlyList<AgentEvaluationResults> results = await agent.EvaluateAsync(
queries,
evaluators: new IAgentEvaluator[]
{
new LocalEvaluator(
EvalChecks.KeywordCheck("weather"),
FunctionEvaluator.Create("is_helpful", (string r) => r.Split(' ').Length > 10)),
new FoundryEvals(chatConfiguration, FoundryEvals.Relevance, FoundryEvals.Coherence),
});
```
Multiple evaluators return one result each — `results[0]` is the local evaluator, `results[1]` is Foundry.
#### Custom function evaluators
**Python:**
```python
@evaluator
def mentions_city(response: str, expected_output: str) -> bool:
return expected_output.lower() in response.lower()
@evaluator
def used_tools(conversation: list, tools: list) -> float:
# ... scoring logic
return score
local = LocalEvaluator(mentions_city, used_tools)
```
`@evaluator` uses **parameter name injection** — the function's parameter names determine what data it receives from the `EvalItem`. Supported names: `query`, `response`, `expected`, `expected_tool_calls`, `conversation`, `tools`, `context`. Any combination is valid.
**C#:**
```csharp
var local = new LocalEvaluator(
FunctionEvaluator.Create("mentions_city",
(EvalItem item) => item.ExpectedOutput != null
&& item.Response.Contains(item.ExpectedOutput, StringComparison.OrdinalIgnoreCase)),
FunctionEvaluator.Create("is_concise",
(string response) => response.Split(' ').Length < 500));
```
## What To Build
### Core: Evaluator Protocol
A runtime-checkable protocol that any evaluation provider implements:
```python
@runtime_checkable
class Evaluator(Protocol):
name: str
async def evaluate(
self, items: Sequence[EvalItem], *, eval_name: str = "Agent Framework Eval"
) -> EvalResults: ...
```
The protocol is minimal — just `name` and `evaluate()`.
### Core: EvalItem
Provider-agnostic data format for items to evaluate:
```python
@dataclass
class ExpectedToolCall:
name: str # Tool/function name
arguments: dict[str, Any] | None = None # None = don't check args
@dataclass
class EvalItem:
conversation: list[Message] # Single source of truth
tools: list[FunctionTool] | None = None # Agent's available tools
context: str | None = None
expected_output: str | None = None # Ground-truth for comparison
expected_tool_calls: list[ExpectedToolCall] | None = None
split_strategy: ConversationSplitter | None = None
query: str # property — derived from conversation split
response: str # property — derived from conversation split
```
`conversation` is the single source of truth. `query` and `response` are derived properties — splitting the conversation at the last user message (default) and extracting text from each side. Changing the `split_strategy` consistently changes all derived values.
`tools` provides typed `FunctionTool` objects — including MCP tools, which are automatically extracted after agent runs.
### Internal: AgentEvalConverter
Internal class that converts agent-framework types to `EvalItem`. Used by `evaluate_agent()` and `evaluate_workflow()` — not part of the public API:
| Agent Framework | Eval Format |
|---|---|
| `Content.function_call` | `tool_call` in OpenAI chat format |
| `Content.function_result` | `tool_result` in OpenAI chat format |
| `FunctionTool` | `{name, description, parameters}` schema |
| `Message` history | `conversation` list + `query`/`response` extraction |
### Core: EvalResults
Rich result type with convenience properties for CI integration:
```python
results.all_passed # bool: no failures or errors (recursive for workflow)
results.passed # int: passing count
results.failed # int: failure count
results.total # int: total = passed + failed + errored
results.items # list[EvalItemResult]: per-item detail with query, response, and scores
results.error # str | None: error details on failure
results.sub_results # dict: per-agent breakdown (workflow evals)
results.report_url # str | None: portal link (Foundry)
results.assert_passed() # raises AssertionError with details
```
### Core: Orchestration Functions
Provider-agnostic functions that extract data and delegate to evaluators:
| Function | What it does |
|---|---|
| `evaluate_agent()` | Runs agent against test queries (or evaluates pre-existing `responses=`), converts to `EvalItem`s, passes to evaluator. Accepts optional `expected_output=` for ground-truth comparison, `expected_tool_calls=` for tool-correctness evaluation, and `num_repetitions=` for consistency measurement |
| `evaluate_workflow()` | Extracts per-agent data from `WorkflowRunResult`, evaluates each agent and overall output. Per-agent breakdown in `sub_results`. Also accepts `num_repetitions=` |
### Core: Conversation Split Strategies
Multi-turn conversations must be split into query (input) and response (output) halves for evaluation. How you split determines *what you're evaluating*:
**Last-turn split** — split at the last user message. Everything up to and including it is the query context; the agent's subsequent actions are the response:
```
conversation: user1 → assistant1 → user2 → assistant2(tool) → tool_result → assistant3
query_messages: [user1, assistant1, user2]
response_messages: [assistant2(tool), tool_result, assistant3]
```
This evaluates: "Given all the context so far, did the agent answer the latest question well?" Best for response quality at a specific point in the conversation.
**Full-conversation split** — the first user message is the query; everything after is the response:
```
query_messages: [user1]
response_messages: [assistant1, user2, assistant2(tool), tool_result, assistant3]
```
This evaluates: "Given the original request, did the entire conversation trajectory serve the user?" Best for task completion and overall conversation quality.
**Per-turn split** — produces N eval items from an N-turn conversation. Each turn is evaluated with its cumulative context:
```
item 1: query = [user1], response = [assistant1]
item 2: query = [user1, assistant1, user2], response = [assistant2(tool), tool_result, assistant3]
```
This evaluates each response independently. Best for fine-grained analysis and pinpointing where a conversation goes wrong.
These factorings produce different scores for the same conversation. The framework ships all three as built-in strategies, defaulting to last-turn. Developers can also provide a custom splitter — a function (Python) or `IConversationSplitter` implementation (.NET) — and override the strategy at the call site or per evaluator.
### Azure AI: FoundryEvals
`Evaluator` implementation backed by Azure AI Foundry:
```python
class FoundryEvals:
def __init__(self, *, project_client=None, openai_client=None,
model_deployment: str, evaluators=None, ...)
async def evaluate(self, items, *, eval_name) -> EvalResults
```
**Smart auto-detection in `evaluate()`:**
- Default evaluators: relevance, coherence, task_adherence
- Auto-adds `tool_call_accuracy` when items have tools/`tool_definitions`
- Filters out tool evaluators for items without tools
### Azure AI: FoundryEvals Constants
```python
from agent_framework_azure_ai import FoundryEvals
evaluators = [FoundryEvals.RELEVANCE, FoundryEvals.TOOL_CALL_ACCURACY]
```
Categories: Agent behavior, Tool usage, Quality, Safety.
### Azure AI: Foundry-Specific Functions
| Function | What it does |
|---|---|
| `evaluate_traces()` | Evaluate from stored response IDs or OTel traces |
| `evaluate_foundry_target()` | Evaluate a Foundry-registered agent or deployment |
### Core: LocalEvaluator and Function Evaluators
`LocalEvaluator` implements the `Evaluator` protocol for fast, API-free evaluation. It runs check functions locally — useful for inner-loop development, CI smoke tests, and combining with cloud-based evaluators.
Built-in checks:
- `keyword_check(*keywords)` — response must contain specified keywords
- `tool_called_check(*tool_names)` — agent must have called specified tools
- `tool_calls_present` — all `expected_tool_calls` names appear in conversation (unordered, extras OK)
- `tool_call_args_match` — expected tool calls match on name + arguments (subset match on args)
Custom function evaluators use `@evaluator` to wrap plain Python functions. The function's **parameter names** determine what data it receives from the `EvalItem`:
```python
from agent_framework import evaluator, LocalEvaluator
# Tier 1: Simple check — just query + response
@evaluator
def is_concise(response: str) -> bool:
return len(response.split()) < 500
# Tier 2: Ground truth — compare against expected output
@evaluator
def mentions_city(response: str, expected_output: str) -> bool:
return expected_output.lower() in response.lower()
# Tier 3: Full context — inspect conversation and tools
@evaluator
def used_tools(conversation: list, tools: list) -> float:
# ... scoring logic
return score
local = LocalEvaluator(is_concise, mentions_city, used_tools)
```
Supported parameters: `query`, `response`, `expected`, `expected_tool_calls`, `conversation`, `tools`, `context`.
Return types: `bool`, `float` (≥0.5 = pass), `dict` with `score` or `passed` key, or `CheckResult`.
Async functions are handled automatically — `@evaluator` detects `async def` and produces the right wrapper.
### Example: GAIA Benchmark
[GAIA](https://huggingface.co/gaia-benchmark) tests real-world multi-step tasks with known expected answers. Each task has a question and a ground-truth answer, with optional file attachments. The framework accommodates GAIA's knobs (difficulty levels, file inputs, multi-step tool use) through the existing `EvalItem` fields:
```python
from datasets import load_dataset
from agent_framework import evaluate_agent, evaluator, LocalEvaluator
gaia = load_dataset("gaia-benchmark/GAIA", "2023_level1", split="test")
@evaluator
def exact_match(response: str, expected_output: str) -> bool:
return expected_output.strip().lower() in response.strip().lower()
# Simple path — evaluate_agent handles running + expected_output stamping
results = await evaluate_agent(
agent=agent,
queries=[task["Question"] for task in gaia],
expected_output=[task["Final answer"] for task in gaia],
evaluators=LocalEvaluator(exact_match),
)
```
### Package Location
- Core types and orchestration: `agent_framework._eval`, `agent_framework._local_eval` (Python), `Microsoft.Agents.AI` (.NET)
- Foundry provider: `agent_framework_azure_ai._foundry_evals` (Python), `Microsoft.Agents.AI.AzureAI` (.NET)
- Azure-AI re-exports core types for convenience (Python)
## Known Limitations
1. **Tool evaluators require query + agent**: Tool evaluators need tool definition schemas. When using these evaluators with `evaluate_agent(responses=...)`, provide `queries=` and pass an agent with tool definitions.
2. **`model_deployment` always required**: Could potentially be inferred from the Foundry project configuration.
## Open Questions
1. **Red teaming non-registered agents**: Requires Foundry API support for callback-based flows.
2. **Datasets with expected outputs**: A dataset abstraction for pre-populating `expected_output` values across eval runs is a natural next step but not yet designed.
3. **Multi-modal evaluation**: The `conversation` field on `EvalItem` already stores full `Message`/`Content` (Python) and `ChatMessage` (.NET) objects, which can represent multi-modal content (images, audio, structured data). Evaluators that accept the full `EvalItem` or `conversation` parameter can access this content today. However, the convenience shortcuts — `query`/`response` string projections and the `FunctionEvaluator` string overloads — are text-only. Multi-modal-aware evaluators should use the full-item path (`Func<EvalItem, CheckResult>` in .NET, `conversation: list` parameter in Python).
## .NET Implementation Design
### Key Difference: MEAI Ecosystem
Unlike Python, the .NET ecosystem already has `Microsoft.Extensions.AI.Evaluation` (v10.3.0) providing:
- `IEvaluator` — per-item evaluation of `(messages, chatResponse) → EvaluationResult`
- `CompositeEvaluator` — combines multiple evaluators
- Quality evaluators — `RelevanceEvaluator`, `CoherenceEvaluator`, `GroundednessEvaluator`
- Safety evaluators — `ContentHarmEvaluator`, `ProtectedMaterialEvaluator`
- Metric types — `NumericMetric`, `BooleanMetric`, `StringMetric`
The .NET integration uses MEAI's `IEvaluator` directly — no new evaluator interface. Our contribution is the **orchestration layer**: extension methods that run agents, extract data, call `IEvaluator` per item, and aggregate results.
### Architecture
```
┌──────────────────────────────────────────────────────────────┐
│ Developer Code │
│ agent.EvaluateAsync(queries, evaluator) │
│ run.EvaluateAsync(evaluator) │
└────────────────┬─────────────────────────────────────────────┘
┌────────────────▼─────────────────────────────────────────────┐
│ Orchestration Layer (Microsoft.Agents.AI) │
│ AgentEvaluationExtensions — runs agents, extracts data, │
│ calls IEvaluator per item, aggregates into │
│ AgentEvaluationResults │
└────────────────┬─────────────────────────────────────────────┘
│ IEvaluator (MEAI)
┌───────────┼────────────┐
│ │ │
┌───▼───-┐ ┌───▼────┐ ┌────▼──────────┐
│ MEAI │ │ Local │ │ Foundry │
│ Quality│ │ Checks │ │ (cloud batch) │
│ Safety │ │ Lambdas│ │ │
└────────┘ └────────┘ └───────────────┘
```
All evaluators implement MEAI's `IEvaluator`. The orchestration layer doesn't need to know which kind — it calls `EvaluateAsync(messages, chatResponse)` per item on all of them. `FoundryEvals` handles batching internally (buffers items, submits once, returns per-item results).
### .NET Core Types
**No new evaluator interface.** Use MEAI's `IEvaluator` directly.
**`AgentEvaluationResults`** — The only new type. Aggregates per-item MEAI `EvaluationResult`s across a batch of queries:
```csharp
public class AgentEvaluationResults
{
public string Provider { get; init; }
public string? ReportUrl { get; init; }
// Per-item — standard MEAI EvaluationResult, unchanged
public IReadOnlyList<EvaluationResult> Items { get; init; }
// Aggregate pass/fail derived from metric interpretations
public int Passed { get; }
public int Failed { get; }
public int Total { get; }
public bool AllPassed { get; }
// Workflow: per-agent breakdown
public IReadOnlyDictionary<string, AgentEvaluationResults>? SubResults { get; init; }
public void AssertAllPassed(string? message = null);
}
```
### .NET Evaluator Implementations
All implement MEAI's `IEvaluator`:
**`LocalEvaluator`** — Runs lambda checks locally, returns `BooleanMetric` per check:
```csharp
var local = new LocalEvaluator(
FunctionEvaluator.Create("is_concise",
(string response) => response.Split().Length < 500),
EvalChecks.KeywordCheck("weather"),
EvalChecks.ToolCalledCheck("get_weather"));
```
**MEAI evaluators** — Used directly, no adapter needed:
```csharp
var quality = new CompositeEvaluator(
new RelevanceEvaluator(),
new CoherenceEvaluator());
```
**`FoundryEvals`** — Implements `IEvaluator` but batches internally. On first call, buffers the item. On the last item (or when explicitly flushed), submits the batch to Foundry and distributes per-item results:
```csharp
var foundry = new FoundryEvals(projectClient, "gpt-4o");
```
### .NET Orchestration: Extension Methods
```csharp
public static class AgentEvaluationExtensions
{
// Evaluate an agent against test queries
public static Task<AgentEvaluationResults> EvaluateAsync(
this AIAgent agent,
IEnumerable<string> queries,
IEvaluator evaluator,
ChatConfiguration? chatConfiguration = null,
IEnumerable<string>? expectedOutput = null,
CancellationToken cancellationToken = default);
// Evaluate pre-existing responses (without re-running the agent)
public static Task<AgentEvaluationResults> EvaluateAsync(
this AIAgent agent,
AgentResponse responses,
IEvaluator evaluator,
IEnumerable<string>? queries = null,
ChatConfiguration? chatConfiguration = null,
IEnumerable<string>? expectedOutput = null,
CancellationToken cancellationToken = default);
// Evaluate with multiple evaluators (one result per evaluator)
public static Task<IReadOnlyList<AgentEvaluationResults>> EvaluateAsync(
this AIAgent agent,
IEnumerable<string> queries,
IEnumerable<IEvaluator> evaluators,
ChatConfiguration? chatConfiguration = null,
IEnumerable<string>? expectedOutput = null,
CancellationToken cancellationToken = default);
// Evaluate a workflow run with per-agent breakdown
public static Task<AgentEvaluationResults> EvaluateAsync(
this Run run,
IEvaluator evaluator,
ChatConfiguration? chatConfiguration = null,
bool includeOverall = true,
bool includePerAgent = true,
CancellationToken cancellationToken = default);
}
```
**Usage:**
```csharp
// MEAI evaluators — just works
var results = await agent.EvaluateAsync(
queries: ["What's the weather?"],
evaluator: new RelevanceEvaluator(),
chatConfiguration: new ChatConfiguration(evalClient));
// Local checks
var results = await agent.EvaluateAsync(
queries: ["What's the weather?"],
evaluator: new LocalEvaluator(
EvalChecks.KeywordCheck("weather")));
// Foundry cloud
var results = await agent.EvaluateAsync(
queries: ["What's the weather?"],
evaluator: new FoundryEvals(projectClient, "gpt-4o"));
// Evaluate existing response (without re-running the agent)
var response = await agent.RunAsync("What's the weather?");
var results = await agent.EvaluateAsync(
responses: response,
queries: ["What's the weather?"],
evaluator: new FoundryEvals(projectClient, "gpt-4o"));
// Mixed — one result per evaluator
var results = await agent.EvaluateAsync(
queries: ["What's the weather?"],
evaluators: [
new LocalEvaluator(EvalChecks.KeywordCheck("weather")),
new RelevanceEvaluator(),
new FoundryEvals(projectClient, "gpt-4o")
],
chatConfiguration: new ChatConfiguration(evalClient));
// Workflow with per-agent breakdown
Run run = await workflowRunner.RunAsync(workflow, "Plan a trip");
var results = await run.EvaluateAsync(
evaluator: new FoundryEvals(projectClient, "gpt-4o"));
```
### .NET Function Evaluators
Typed factory overloads (C# equivalent of Python's `@evaluator`):
```csharp
public static class FunctionEvaluator
{
public static EvalCheck Create(string name, Func<string, bool> check); // response only
public static EvalCheck Create(string name, Func<string, string?, bool> check); // expectedOutput
public static EvalCheck Create(string name, Func<EvalItem, bool> check); // full item
public static EvalCheck Create(string name, Func<EvalItem, CheckResult> check); // full control
public static EvalCheck Create(string name, Func<string, Task<bool>> check); // async
}
```
`EvalItem` is a lightweight record used only by `FunctionEvaluator` and `LocalEvaluator` to pass context to check functions. It is not part of the `IEvaluator` interface:
```csharp
public record ExpectedToolCall(string Name, IReadOnlyDictionary<string, object>? Arguments = null);
public sealed class EvalItem
{
public EvalItem(string query, string response, IReadOnlyList<ChatMessage> conversation);
public string Query { get; }
public string Response { get; }
public IReadOnlyList<ChatMessage> Conversation { get; }
public IReadOnlyList<AITool>? Tools { get; set; }
public string? ExpectedOutput { get; set; }
public IReadOnlyList<ExpectedToolCall>? ExpectedToolCalls { get; set; }
public string? Context { get; set; }
public IConversationSplitter? Splitter { get; set; }
}
```
### Workflow Data Extraction (.NET)
`run.EvaluateAsync()` walks `Run.OutgoingEvents` via LINQ:
1. Pair `ExecutorInvokedEvent` / `ExecutorCompletedEvent` by `ExecutorId`
2. Extract `AgentResponseEvent` for per-agent `ChatResponse`
3. Call `evaluator.EvaluateAsync()` per invocation
4. Group by `ExecutorId` for per-agent `SubResults`
5. Use final workflow output for overall eval
### .NET Package Structure
| Package | Contents |
|---------|----------|
| `Microsoft.Agents.AI` | `IAgentEvaluator`, `AgentEvaluationResults`, `LocalEvaluator`, `FunctionEvaluator`, `EvalChecks`, `EvalItem`, `ExpectedToolCall`, `AgentEvaluationExtensions` |
| `Microsoft.Agents.AI.AzureAI` | `FoundryEvals` (provider + constants) |
### Python ↔ .NET Mapping
| Python | .NET |
|--------|------|
| `Evaluator` protocol | `IAgentEvaluator` (our interface; MEAI provides `IEvaluator` for per-item scoring) |
| `EvalItem` dataclass | `EvalItem` class |
| `EvalResults` | `AgentEvaluationResults` |
| `EvalItemResult` / `EvalScoreResult` | MEAI `EvaluationResult` / `EvaluationMetric` (reused) |
| `LocalEvaluator` | `LocalEvaluator` (implements `IAgentEvaluator`) |
| `@evaluator` | `FunctionEvaluator.Create()` overloads |
| `keyword_check()` / `tool_called_check()` | `EvalChecks.KeywordCheck()` / `EvalChecks.ToolCalledCheck()` |
| `tool_calls_present` / `tool_call_args_match` | (custom `FunctionEvaluator` — same pattern) |
| `ExpectedToolCall` dataclass | `ExpectedToolCall` record |
| `FoundryEvals` | `FoundryEvals` (implements `IAgentEvaluator`, includes evaluator name constants) |
| `evaluate_agent()` | `agent.EvaluateAsync(queries, evaluator)` extension method |
| `evaluate_agent(responses=)` | `agent.EvaluateAsync(responses, evaluator)` extension method |
| `evaluate_workflow()` | `run.EvaluateAsync()` extension method |
## More Information
- [Foundry Evals documentation](https://learn.microsoft.com/azure/ai-foundry/concepts/evaluation-approach-gen-ai) — Azure AI Foundry evaluation overview
+19 -18
View File
@@ -19,11 +19,10 @@
<PackageVersion Include="Aspire.Microsoft.Azure.Cosmos" Version="$(AspireAppHostSdkVersion)" />
<PackageVersion Include="CommunityToolkit.Aspire.OllamaSharp" Version="13.0.0" />
<!-- Azure.* -->
<PackageVersion Include="Azure.AI.Projects" Version="2.0.0-beta.1" />
<PackageVersion Include="Azure.AI.Projects.OpenAI" Version="2.0.0-beta.1" />
<PackageVersion Include="Azure.AI.Agents.Persistent" Version="1.2.0-beta.8" />
<PackageVersion Include="Azure.AI.OpenAI" Version="2.8.0-beta.1" />
<PackageVersion Include="Azure.Identity" Version="1.17.1" />
<PackageVersion Include="Azure.AI.Projects" Version="2.0.0-beta.2" />
<PackageVersion Include="Azure.AI.Agents.Persistent" Version="1.2.0-beta.10" />
<PackageVersion Include="Azure.AI.OpenAI" Version="2.9.0-beta.1" />
<PackageVersion Include="Azure.Identity" Version="1.19.0" />
<PackageVersion Include="Azure.Monitor.OpenTelemetry.Exporter" Version="1.4.0" />
<!-- Google Gemini -->
<PackageVersion Include="Google.GenAI" Version="0.11.0" />
@@ -40,12 +39,12 @@
<PackageVersion Include="System.CodeDom" Version="10.0.0" />
<PackageVersion Include="System.Collections.Immutable" Version="10.0.1" />
<PackageVersion Include="System.CommandLine" Version="2.0.0-rc.2.25502.107" />
<PackageVersion Include="System.Diagnostics.DiagnosticSource" Version="10.0.3" />
<PackageVersion Include="System.Diagnostics.DiagnosticSource" Version="10.0.4" />
<PackageVersion Include="System.Linq.AsyncEnumerable" Version="10.0.4" />
<PackageVersion Include="System.Net.Http.Json" Version="10.0.0" />
<PackageVersion Include="System.Net.ServerSentEvents" Version="10.0.3" />
<PackageVersion Include="System.Text.Json" Version="10.0.3" />
<PackageVersion Include="System.Threading.Channels" Version="10.0.3" />
<PackageVersion Include="System.Net.ServerSentEvents" Version="10.0.4" />
<PackageVersion Include="System.Text.Json" Version="10.0.4" />
<PackageVersion Include="System.Threading.Channels" Version="10.0.4" />
<PackageVersion Include="System.Threading.Tasks.Extensions" Version="4.6.3" />
<PackageVersion Include="System.Net.Security" Version="4.3.2" />
<!-- OpenTelemetry -->
@@ -64,24 +63,25 @@
<PackageVersion Include="Microsoft.AspNetCore.OpenApi" Version="10.0.0" />
<PackageVersion Include="Swashbuckle.AspNetCore.SwaggerUI" Version="10.0.0" />
<!-- Microsoft.Extensions.* -->
<PackageVersion Include="Microsoft.Extensions.AI" Version="10.3.0" />
<PackageVersion Include="Microsoft.Extensions.AI.Abstractions" Version="10.3.0" />
<PackageVersion Include="Microsoft.Extensions.AI.Evaluation" Version="10.3.0" />
<PackageVersion Include="Microsoft.Extensions.AI.Evaluation.Quality" Version="10.3.0" />
<PackageVersion Include="Microsoft.Extensions.AI" Version="10.4.0" />
<PackageVersion Include="Microsoft.Extensions.AI.Abstractions" Version="10.4.0" />
<PackageVersion Include="Microsoft.Extensions.AI.Evaluation" Version="10.4.0" />
<PackageVersion Include="Microsoft.Extensions.AI.Evaluation.Quality" Version="10.4.0" />
<PackageVersion Include="Microsoft.Extensions.AI.Evaluation.Safety" Version="10.3.0-preview.1.26109.11" />
<PackageVersion Include="Microsoft.Extensions.AI.OpenAI" Version="10.3.0" />
<PackageVersion Include="Microsoft.Extensions.AI.OpenAI" Version="10.4.0" />
<PackageVersion Include="Microsoft.Extensions.Caching.Memory" Version="10.0.0" />
<PackageVersion Include="Microsoft.Extensions.Compliance.Abstractions" Version="10.4.0" />
<PackageVersion Include="Microsoft.Extensions.Configuration" Version="10.0.0" />
<PackageVersion Include="Microsoft.Extensions.Configuration.Binder" Version="10.0.0" />
<PackageVersion Include="Microsoft.Extensions.Configuration.EnvironmentVariables" Version="10.0.0" />
<PackageVersion Include="Microsoft.Extensions.Configuration.Json" Version="10.0.0" />
<PackageVersion Include="Microsoft.Extensions.Configuration.UserSecrets" Version="10.0.0" />
<PackageVersion Include="Microsoft.Extensions.DependencyInjection" Version="10.0.0" />
<PackageVersion Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="10.0.3" />
<PackageVersion Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="10.0.4" />
<PackageVersion Include="Microsoft.Extensions.Hosting" Version="10.0.0" />
<PackageVersion Include="Microsoft.Extensions.Http.Resilience" Version="10.0.0" />
<PackageVersion Include="Microsoft.Extensions.Logging" Version="10.0.0" />
<PackageVersion Include="Microsoft.Extensions.Logging.Abstractions" Version="10.0.3" />
<PackageVersion Include="Microsoft.Extensions.Logging.Abstractions" Version="10.0.4" />
<PackageVersion Include="Microsoft.Extensions.Logging.Console" Version="10.0.0" />
<PackageVersion Include="Microsoft.Extensions.ServiceDiscovery" Version="10.0.0" />
<PackageVersion Include="Microsoft.Extensions.VectorData.Abstractions" Version="9.7.0" />
@@ -111,9 +111,9 @@
<PackageVersion Include="Microsoft.ML.OnnxRuntimeGenAI" Version="0.10.0" />
<PackageVersion Include="Microsoft.ML.Tokenizers" Version="2.0.0" />
<PackageVersion Include="OllamaSharp" Version="5.4.8" />
<PackageVersion Include="OpenAI" Version="2.8.0" />
<PackageVersion Include="OpenAI" Version="2.9.1" />
<!-- Identity -->
<PackageVersion Include="Microsoft.Identity.Client.Extensions.Msal" Version="4.78.0" />
<PackageVersion Include="Microsoft.Identity.Client.Extensions.Msal" Version="4.83.1" />
<!-- Workflows -->
<PackageVersion Include="Microsoft.Agents.ObjectModel" Version="2026.2.4.1" />
<PackageVersion Include="Microsoft.Agents.ObjectModel.Json" Version="2026.2.4.1" />
@@ -149,6 +149,7 @@
<!-- Symbols -->
<PackageVersion Include="Microsoft.SourceLink.GitHub" Version="8.0.0" />
<!-- Toolset -->
<PackageVersion Include="ReferenceTrimmer" Version="3.4.5" />
<PackageVersion Include="Microsoft.CodeAnalysis.Analyzers" Version="3.11.0" />
<PackageVersion Include="Microsoft.CodeAnalysis.CSharp" Version="4.14.0" />
<PackageVersion Include="Microsoft.CodeAnalysis.NetAnalyzers" Version="10.0.100" />
+4 -1
View File
@@ -309,7 +309,6 @@
<Project Path="samples/05-end-to-end/HostedAgents/AgentWithHostedMCP/AgentWithHostedMCP.csproj" />
<Project Path="samples/05-end-to-end/HostedAgents/AgentWithLocalTools/AgentWithLocalTools.csproj" />
<Project Path="samples/05-end-to-end/HostedAgents/AgentWithTextSearchRag/AgentWithTextSearchRag.csproj" />
<Project Path="samples/05-end-to-end/HostedAgents/AgentWithTools/AgentWithTools.csproj" />
<Project Path="samples/05-end-to-end/HostedAgents/FoundryMultiAgent/FoundryMultiAgent.csproj" />
<Project Path="samples/05-end-to-end/HostedAgents/FoundrySingleAgent/FoundrySingleAgent.csproj" />
</Folder>
@@ -453,6 +452,10 @@
<File Path="src/Shared/Samples/TextOutputHelperExtensions.cs" />
<File Path="src/Shared/Samples/XunitLogger.cs" />
</Folder>
<Folder Name="/Solution Items/src/Shared/Redaction/">
<File Path="src/Shared/Redaction/README.md" />
<File Path="src/Shared/Redaction/ReplacingRedactor.cs" />
</Folder>
<Folder Name="/Solution Items/src/Shared/Throw/">
<File Path="src/Shared/Throw/README.md" />
<File Path="src/Shared/Throw/Throw.cs" />
+3
View File
@@ -29,4 +29,7 @@
<ItemGroup Condition="'$(InjectSharedDiagnosticIds)' == 'true'">
<Compile Include="$(MSBuildThisFileDirectory)\..\..\src\Shared\DiagnosticIds\*.cs" LinkBase="Shared\DiagnosticIds" />
</ItemGroup>
<ItemGroup Condition="'$(InjectSharedRedaction)' == 'true'">
<Compile Include="$(MSBuildThisFileDirectory)\..\..\src\Shared\Redaction\*.cs" LinkBase="Shared\Redaction" />
</ItemGroup>
</Project>
@@ -59,14 +59,14 @@ while ((input = Console.ReadLine()) != null && !input.Equals("exit", StringCompa
{
switch (content)
{
case FunctionApprovalRequestContent approvalRequest:
DisplayApprovalRequest(approvalRequest);
case ToolApprovalRequestContent approvalRequest when approvalRequest.ToolCall is FunctionCallContent fcc:
DisplayApprovalRequest(approvalRequest, fcc);
Console.Write($"\nApprove '{approvalRequest.FunctionCall.Name}'? (yes/no): ");
Console.Write($"\nApprove '{fcc.Name}'? (yes/no): ");
string? userInput = Console.ReadLine();
bool approved = userInput?.ToUpperInvariant() is "YES" or "Y";
FunctionApprovalResponseContent approvalResponse = approvalRequest.CreateResponse(approved);
ToolApprovalResponseContent approvalResponse = approvalRequest.CreateResponse(approved);
if (approvalRequest.AdditionalProperties != null)
{
@@ -128,19 +128,19 @@ while ((input = Console.ReadLine()) != null && !input.Equals("exit", StringCompa
}
#pragma warning disable MEAI001
static void DisplayApprovalRequest(FunctionApprovalRequestContent approvalRequest)
static void DisplayApprovalRequest(ToolApprovalRequestContent approvalRequest, FunctionCallContent fcc)
{
Console.ForegroundColor = ConsoleColor.Yellow;
Console.WriteLine();
Console.WriteLine("============================================================");
Console.WriteLine("APPROVAL REQUIRED");
Console.WriteLine("============================================================");
Console.WriteLine($"Function: {approvalRequest.FunctionCall.Name}");
Console.WriteLine($"Function: {fcc.Name}");
if (approvalRequest.FunctionCall.Arguments != null)
if (fcc.Arguments != null)
{
Console.WriteLine("Arguments:");
foreach (var arg in approvalRequest.FunctionCall.Arguments)
foreach (var arg in fcc.Arguments)
{
Console.WriteLine($" {arg.Key} = {arg.Value}");
}
@@ -9,7 +9,7 @@ using ServerFunctionApproval;
/// <summary>
/// A delegating agent that handles server function approval requests and responses.
/// Transforms between FunctionApprovalRequestContent/FunctionApprovalResponseContent
/// Transforms between ToolApprovalRequestContent/ToolApprovalResponseContent
/// and the server's request_approval tool call pattern.
/// </summary>
internal sealed class ServerFunctionApprovalClientAgent : DelegatingAIAgent
@@ -50,14 +50,14 @@ internal sealed class ServerFunctionApprovalClientAgent : DelegatingAIAgent
}
#pragma warning disable MEAI001 // Type is for evaluation purposes only
private static FunctionResultContent ConvertApprovalResponseToToolResult(FunctionApprovalResponseContent approvalResponse, JsonSerializerOptions jsonOptions)
private static FunctionResultContent ConvertApprovalResponseToToolResult(ToolApprovalResponseContent approvalResponse, JsonSerializerOptions jsonOptions)
{
return new FunctionResultContent(
callId: approvalResponse.Id,
callId: approvalResponse.RequestId,
result: JsonSerializer.SerializeToElement(
new ApprovalResponse
{
ApprovalId = approvalResponse.Id,
ApprovalId = approvalResponse.RequestId,
Approved = approvalResponse.Approved
},
jsonOptions));
@@ -89,7 +89,7 @@ internal sealed class ServerFunctionApprovalClientAgent : DelegatingAIAgent
{
List<ChatMessage>? result = null;
Dictionary<string, FunctionApprovalRequestContent> approvalRequests = [];
Dictionary<string, ToolApprovalRequestContent> approvalRequests = [];
for (var messageIndex = 0; messageIndex < messages.Count; messageIndex++)
{
var message = messages[messageIndex];
@@ -102,21 +102,21 @@ internal sealed class ServerFunctionApprovalClientAgent : DelegatingAIAgent
var content = message.Contents[contentIndex];
// Handle pending approval requests (transform to tool call)
if (content is FunctionApprovalRequestContent approvalRequest &&
if (content is ToolApprovalRequestContent approvalRequest &&
approvalRequest.AdditionalProperties?.TryGetValue("original_function", out var originalFunction) == true &&
originalFunction is FunctionCallContent original)
{
approvalRequests[approvalRequest.Id] = approvalRequest;
approvalRequests[approvalRequest.RequestId] = approvalRequest;
transformedContents ??= CopyContentsUpToIndex(message.Contents, contentIndex);
transformedContents.Add(original);
}
// Handle pending approval responses (transform to tool result)
else if (content is FunctionApprovalResponseContent approvalResponse &&
approvalRequests.TryGetValue(approvalResponse.Id, out var correspondingRequest))
else if (content is ToolApprovalResponseContent approvalResponse &&
approvalRequests.TryGetValue(approvalResponse.RequestId, out var correspondingRequest))
{
transformedContents ??= CopyContentsUpToIndex(message.Contents, contentIndex);
transformedContents.Add(ConvertApprovalResponseToToolResult(approvalResponse, jsonSerializerOptions));
approvalRequests.Remove(approvalResponse.Id);
approvalRequests.Remove(approvalResponse.RequestId);
correspondingRequest.AdditionalProperties?.Remove("original_function");
}
// Skip historical approval content
@@ -198,8 +198,8 @@ internal sealed class ServerFunctionApprovalClientAgent : DelegatingAIAgent
var functionCallArgs = (Dictionary<string, object?>?)approvalRequest.FunctionArguments?
.Deserialize(jsonSerializerOptions.GetTypeInfo(typeof(Dictionary<string, object?>)));
var approvalRequestContent = new FunctionApprovalRequestContent(
id: approvalRequest.ApprovalId,
var approvalRequestContent = new ToolApprovalRequestContent(
requestId: approvalRequest.ApprovalId,
new FunctionCallContent(
callId: approvalRequest.ApprovalId,
name: approvalRequest.FunctionName,
@@ -9,7 +9,7 @@ using ServerFunctionApproval;
/// <summary>
/// A delegating agent that handles function approval requests on the server side.
/// Transforms between FunctionApprovalRequestContent/FunctionApprovalResponseContent
/// Transforms between ToolApprovalRequestContent/ToolApprovalResponseContent
/// and the request_approval tool call pattern for client communication.
/// </summary>
internal sealed class ServerFunctionApprovalAgent : DelegatingAIAgent
@@ -50,7 +50,7 @@ internal sealed class ServerFunctionApprovalAgent : DelegatingAIAgent
}
#pragma warning disable MEAI001 // Type is for evaluation purposes only
private static FunctionApprovalRequestContent ConvertToolCallToApprovalRequest(FunctionCallContent toolCall, JsonSerializerOptions jsonSerializerOptions)
private static ToolApprovalRequestContent ConvertToolCallToApprovalRequest(FunctionCallContent toolCall, JsonSerializerOptions jsonSerializerOptions)
{
if (toolCall.Name != "request_approval" || toolCall.Arguments == null)
{
@@ -67,15 +67,15 @@ internal sealed class ServerFunctionApprovalAgent : DelegatingAIAgent
throw new InvalidOperationException("Failed to deserialize approval request from tool call");
}
return new FunctionApprovalRequestContent(
id: request.ApprovalId,
return new ToolApprovalRequestContent(
requestId: request.ApprovalId,
new FunctionCallContent(
callId: request.ApprovalId,
name: request.FunctionName,
arguments: request.FunctionArguments));
}
private static FunctionApprovalResponseContent ConvertToolResultToApprovalResponse(FunctionResultContent result, FunctionApprovalRequestContent approval, JsonSerializerOptions jsonSerializerOptions)
private static ToolApprovalResponseContent ConvertToolResultToApprovalResponse(FunctionResultContent result, ToolApprovalRequestContent approval, JsonSerializerOptions jsonSerializerOptions)
{
var approvalResponse = result.Result is JsonElement je ?
(ApprovalResponse?)je.Deserialize(jsonSerializerOptions.GetTypeInfo(typeof(ApprovalResponse))) :
@@ -121,7 +121,7 @@ internal sealed class ServerFunctionApprovalAgent : DelegatingAIAgent
// Track approval ID to original call ID mapping
_ = new Dictionary<string, string>();
#pragma warning disable MEAI001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed.
Dictionary<string, FunctionApprovalRequestContent> trackedRequestApprovalToolCalls = new(); // Remote approvals
Dictionary<string, ToolApprovalRequestContent> trackedRequestApprovalToolCalls = new(); // Remote approvals
for (int messageIndex = 0; messageIndex < messages.Count; messageIndex++)
{
var message = messages[messageIndex];
@@ -181,11 +181,10 @@ internal sealed class ServerFunctionApprovalAgent : DelegatingAIAgent
{
var content = update.Contents[i];
#pragma warning disable MEAI001 // Type is for evaluation purposes only
if (content is FunctionApprovalRequestContent request)
if (content is ToolApprovalRequestContent request && request.ToolCall is FunctionCallContent functionCall)
{
updatedContents ??= [.. update.Contents];
var functionCall = request.FunctionCall;
var approvalId = request.Id;
var approvalId = request.RequestId;
var approvalData = new ApprovalRequest
{
@@ -1,5 +1,7 @@
// Copyright (c) Microsoft. All rights reserved.
#pragma warning disable CS0618 // Type or member is obsolete - sample uses deprecated PersistentAgentsClientExtensions
// This sample shows how to create and use a simple AI agent with Azure Foundry Agents as the backend.
using Azure.AI.Agents.Persistent;
@@ -3,7 +3,7 @@
// This sample shows how to create and use a AI agents with Azure Foundry Agents as the backend.
using Azure.AI.Projects;
using Azure.AI.Projects.OpenAI;
using Azure.AI.Projects.Agents;
using Azure.Identity;
using Microsoft.Agents.AI;
@@ -17,8 +17,8 @@ var deploymentName = Environment.GetEnvironmentVariable("AZURE_OPENAI_DEPLOYMENT
AIAgent agent = new AzureOpenAIClient(
new Uri(endpoint),
new DefaultAzureCredential())
.GetResponsesClient(deploymentName)
.AsAIAgent(instructions: "You are good at telling jokes.", name: "Joker");
.GetResponsesClient()
.AsAIAgent(model: deploymentName, instructions: "You are good at telling jokes.", name: "Joker");
// Invoke the agent and output the text result.
Console.WriteLine(await agent.RunAsync("Tell me a joke about a pirate."));
@@ -29,8 +29,8 @@ Console.WriteLine(await agent.RunAsync("Tell me a joke about a pirate."));
AIAgent agentStoreFalse = new AzureOpenAIClient(
new Uri(endpoint),
new DefaultAzureCredential())
.GetResponsesClient(deploymentName)
.AsIChatClientWithStoredOutputDisabled()
.GetResponsesClient()
.AsIChatClientWithStoredOutputDisabled(model: deploymentName)
.AsAIAgent(instructions: "You are good at telling jokes.", name: "Joker");
// Invoke the agent and output the text result.
@@ -9,7 +9,7 @@
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\..\..\..\src\Microsoft.Agents.AI\Microsoft.Agents.AI.csproj" />
<ProjectReference Include="..\..\..\..\src\Microsoft.Agents.AI.Abstractions\Microsoft.Agents.AI.Abstractions.csproj" />
</ItemGroup>
</Project>
@@ -11,8 +11,8 @@ var model = Environment.GetEnvironmentVariable("OPENAI_CHAT_MODEL_NAME") ?? "gpt
AIAgent agent = new OpenAIClient(
apiKey)
.GetResponsesClient(model)
.AsAIAgent(instructions: "You are good at telling jokes.", name: "Joker");
.GetResponsesClient()
.AsAIAgent(model: model, instructions: "You are good at telling jokes.", name: "Joker");
// Invoke the agent and output the text result.
Console.WriteLine(await agent.RunAsync("Tell me a joke about a pirate."));
@@ -23,7 +23,7 @@ var skillsProvider = new FileAgentSkillsProvider(skillPath: Path.Combine(AppCont
// --- Agent Setup ---
AIAgent agent = new AzureOpenAIClient(new Uri(endpoint), new DefaultAzureCredential())
.GetResponsesClient(deploymentName)
.GetResponsesClient()
.AsAIAgent(new ChatClientAgentOptions
{
Name = "SkillsAgent",
@@ -32,7 +32,8 @@ AIAgent agent = new AzureOpenAIClient(new Uri(endpoint), new DefaultAzureCredent
Instructions = "You are a helpful assistant.",
},
AIContextProviders = [skillsProvider],
});
},
model: deploymentName);
// --- Example 1: Expense policy question (loads FAQ resource) ---
Console.WriteLine("Example 1: Checking expense policy FAQ");
@@ -10,8 +10,8 @@ var apiKey = Environment.GetEnvironmentVariable("OPENAI_API_KEY") ?? throw new I
var model = Environment.GetEnvironmentVariable("OPENAI_CHAT_MODEL_NAME") ?? "gpt-5";
var client = new OpenAIClient(apiKey)
.GetResponsesClient(model)
.AsIChatClient().AsBuilder()
.GetResponsesClient()
.AsIChatClient(model).AsBuilder()
.ConfigureOptions(o =>
{
o.Reasoning = new()
@@ -20,19 +20,21 @@ public class OpenAIResponseClientAgent : DelegatingAIAgent
/// <param name="instructions">Optional instructions for the agent.</param>
/// <param name="name">Optional name for the agent.</param>
/// <param name="description">Optional description for the agent.</param>
/// <param name="model">Optional default model ID to use for requests. Required when using a plain <see cref="ResponsesClient"/> (not via Azure OpenAI).</param>
/// <param name="loggerFactory">Optional instance of <see cref="ILoggerFactory"/></param>
public OpenAIResponseClientAgent(
ResponsesClient client,
string? instructions = null,
string? name = null,
string? description = null,
string? model = null,
ILoggerFactory? loggerFactory = null) :
this(client, new()
{
Name = name,
Description = description,
ChatOptions = new ChatOptions() { Instructions = instructions },
}, loggerFactory)
}, model, loggerFactory)
{
}
@@ -41,10 +43,11 @@ public class OpenAIResponseClientAgent : DelegatingAIAgent
/// </summary>
/// <param name="client">Instance of <see cref="ResponsesClient"/></param>
/// <param name="options">Options to create the agent.</param>
/// <param name="model">Optional default model ID to use for requests. Required when using a plain <see cref="ResponsesClient"/> (not via Azure OpenAI).</param>
/// <param name="loggerFactory">Optional instance of <see cref="ILoggerFactory"/></param>
public OpenAIResponseClientAgent(
ResponsesClient client, ChatClientAgentOptions options, ILoggerFactory? loggerFactory = null) :
base(new ChatClientAgent((client ?? throw new ArgumentNullException(nameof(client))).AsIChatClient(), options, loggerFactory))
ResponsesClient client, ChatClientAgentOptions options, string? model = null, ILoggerFactory? loggerFactory = null) :
base(new ChatClientAgent((client ?? throw new ArgumentNullException(nameof(client))).AsIChatClient(model), options, loggerFactory))
{
}
@@ -10,10 +10,10 @@ var apiKey = Environment.GetEnvironmentVariable("OPENAI_API_KEY") ?? throw new I
var model = Environment.GetEnvironmentVariable("OPENAI_CHAT_MODEL_NAME") ?? "gpt-4o-mini";
// Create a ResponsesClient directly from OpenAIClient
ResponsesClient responseClient = new OpenAIClient(apiKey).GetResponsesClient(model);
ResponsesClient responseClient = new OpenAIClient(apiKey).GetResponsesClient();
// Create an agent directly from the ResponsesClient using OpenAIResponseClientAgent
OpenAIResponseClientAgent agent = new(responseClient, instructions: "You are good at telling jokes.", name: "Joker");
OpenAIResponseClientAgent agent = new(responseClient, instructions: "You are good at telling jokes.", name: "Joker", model: model);
ResponseItem userMessage = ResponseItem.CreateUserMessageItem("Tell me a joke about a pirate.");
@@ -22,7 +22,7 @@ OpenAIClient openAIClient = new(apiKey);
ConversationClient conversationClient = openAIClient.GetConversationClient();
// Create an agent directly from the ResponsesClient using OpenAIResponseClientAgent
ChatClientAgent agent = new(openAIClient.GetResponsesClient(model).AsIChatClient(), instructions: "You are a helpful assistant.", name: "ConversationAgent");
ChatClientAgent agent = new(openAIClient.GetResponsesClient().AsIChatClient(model), instructions: "You are a helpful assistant.", name: "ConversationAgent");
ClientResult createConversationResult = await conversationClient.CreateConversationAsync(BinaryContent.Create(BinaryData.FromString("{}")));
@@ -36,11 +36,11 @@ AIAgent agent = new AzureOpenAIClient(
// For simplicity, we are assuming here that only function approvals are pending.
AgentSession session = await agent.CreateSessionAsync();
AgentResponse response = await agent.RunAsync("What is the weather like in Amsterdam?", session);
List<FunctionApprovalRequestContent> approvalRequests = response.Messages.SelectMany(m => m.Contents).OfType<FunctionApprovalRequestContent>().ToList();
List<ToolApprovalRequestContent> approvalRequests = response.Messages.SelectMany(m => m.Contents).OfType<ToolApprovalRequestContent>().ToList();
// For streaming use:
// var updates = await agent.RunStreamingAsync("What is the weather like in Amsterdam?", session).ToListAsync();
// approvalRequests = updates.SelectMany(x => x.Contents).OfType<FunctionApprovalRequestContent>().ToList();
// approvalRequests = updates.SelectMany(x => x.Contents).OfType<ToolApprovalRequestContent>().ToList();
while (approvalRequests.Count > 0)
{
@@ -48,18 +48,18 @@ while (approvalRequests.Count > 0)
List<ChatMessage> userInputResponses = approvalRequests
.ConvertAll(functionApprovalRequest =>
{
Console.WriteLine($"The agent would like to invoke the following function, please reply Y to approve: Name {functionApprovalRequest.FunctionCall.Name}");
Console.WriteLine($"The agent would like to invoke the following function, please reply Y to approve: Name {((FunctionCallContent)functionApprovalRequest.ToolCall).Name}");
return new ChatMessage(ChatRole.User, [functionApprovalRequest.CreateResponse(Console.ReadLine()?.Equals("Y", StringComparison.OrdinalIgnoreCase) ?? false)]);
});
// Pass the user input responses back to the agent for further processing.
response = await agent.RunAsync(userInputResponses, session);
approvalRequests = response.Messages.SelectMany(m => m.Contents).OfType<FunctionApprovalRequestContent>().ToList();
approvalRequests = response.Messages.SelectMany(m => m.Contents).OfType<ToolApprovalRequestContent>().ToList();
// For streaming use:
// updates = await agent.RunStreamingAsync(userInputResponses, session).ToListAsync();
// approvalRequests = updates.SelectMany(x => x.Contents).OfType<FunctionApprovalRequestContent>().ToList();
// approvalRequests = updates.SelectMany(x => x.Contents).OfType<ToolApprovalRequestContent>().ToList();
}
Console.WriteLine($"\nAgent: {response}");
@@ -10,14 +10,14 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Azure.AI.Agents.Persistent" />
<PackageReference Include="Azure.AI.Projects" />
<PackageReference Include="Azure.Identity" />
<PackageReference Include="Microsoft.Extensions.Hosting" />
<PackageReference Include="ModelContextProtocol" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\..\..\..\src\Microsoft.Agents.AI.AzureAI.Persistent\Microsoft.Agents.AI.AzureAI.Persistent.csproj" />
<ProjectReference Include="..\..\..\..\src\Microsoft.Agents.AI.AzureAI\Microsoft.Agents.AI.AzureAI.csproj" />
</ItemGroup>
</Project>
@@ -2,7 +2,7 @@
// This sample shows how to expose an AI agent as an MCP tool.
using Azure.AI.Agents.Persistent;
using Azure.AI.Projects;
using Azure.Identity;
using Microsoft.Agents.AI;
using Microsoft.Extensions.DependencyInjection;
@@ -15,18 +15,15 @@ var deploymentName = Environment.GetEnvironmentVariable("AZURE_AI_MODEL_DEPLOYME
// WARNING: DefaultAzureCredential is convenient for development but requires careful consideration in production.
// In production, consider using a specific credential (e.g., ManagedIdentityCredential) to avoid
// latency issues, unintended credential probing, and potential security risks from fallback mechanisms.
var persistentAgentsClient = new PersistentAgentsClient(endpoint, new DefaultAzureCredential());
var aiProjectClient = new AIProjectClient(new Uri(endpoint), new DefaultAzureCredential());
// Create a server side persistent agent
var agentMetadata = await persistentAgentsClient.Administration.CreateAgentAsync(
// Create a server side agent and expose it as an AIAgent.
AIAgent agent = await aiProjectClient.CreateAIAgentAsync(
model: deploymentName,
instructions: "You are good at telling jokes, and you always start each joke with 'Aye aye, captain!'.",
name: "Joker",
description: "An agent that tells jokes.");
// Retrieve the server side persistent agent as an AIAgent.
AIAgent agent = await persistentAgentsClient.GetAIAgentAsync(agentMetadata.Value.Id);
// Convert the agent to an AIFunction and then to an MCP tool.
// The agent name and description will be used as the mcp tool name and description.
McpServerTool tool = McpServerTool.Create(agent.AsAIFunction());
@@ -25,8 +25,9 @@ var stateStore = new Dictionary<string, JsonElement?>();
AIAgent agent = new AzureOpenAIClient(
new Uri(endpoint),
new DefaultAzureCredential())
.GetResponsesClient(deploymentName)
.GetResponsesClient()
.AsAIAgent(
model: deploymentName,
name: "SpaceNovelWriter",
instructions: "You are a space novel writer. Always research relevant facts and generate character profiles for the main characters before writing novels." +
"Write complete chapters without asking for approval or feedback. Do not ask the user about tone, style, pace, or format preferences - just write the novel based on the request.",
@@ -246,7 +246,7 @@ async Task<AgentResponse> ConsolePromptingApprovalMiddleware(IEnumerable<ChatMes
AgentResponse response = await innerAgent.RunAsync(messages, session, options, cancellationToken);
// For simplicity, we are assuming here that only function approvals are pending.
List<FunctionApprovalRequestContent> approvalRequests = response.Messages.SelectMany(m => m.Contents).OfType<FunctionApprovalRequestContent>().ToList();
List<ToolApprovalRequestContent> approvalRequests = response.Messages.SelectMany(m => m.Contents).OfType<ToolApprovalRequestContent>().ToList();
while (approvalRequests.Count > 0)
{
@@ -255,13 +255,13 @@ async Task<AgentResponse> ConsolePromptingApprovalMiddleware(IEnumerable<ChatMes
response.Messages = approvalRequests
.ConvertAll(functionApprovalRequest =>
{
Console.WriteLine($"The agent would like to invoke the following function, please reply Y to approve: Name {functionApprovalRequest.FunctionCall.Name}");
Console.WriteLine($"The agent would like to invoke the following function, please reply Y to approve: Name {((FunctionCallContent)functionApprovalRequest.ToolCall).Name}");
return new ChatMessage(ChatRole.User, [functionApprovalRequest.CreateResponse(Console.ReadLine()?.Equals("Y", StringComparison.OrdinalIgnoreCase) ?? false)]);
});
response = await innerAgent.RunAsync(response.Messages, session, options, cancellationToken);
approvalRequests = response.Messages.SelectMany(m => m.Contents).OfType<FunctionApprovalRequestContent>().ToList();
approvalRequests = response.Messages.SelectMany(m => m.Contents).OfType<ToolApprovalRequestContent>().ToList();
}
return response;
@@ -16,8 +16,8 @@ var deploymentName = Environment.GetEnvironmentVariable("AZURE_OPENAI_DEPLOYMENT
AIAgent agent = new AzureOpenAIClient(
new Uri(endpoint),
new DefaultAzureCredential())
.GetResponsesClient(deploymentName)
.AsAIAgent();
.GetResponsesClient()
.AsAIAgent(model: deploymentName);
// Enable background responses (only supported by OpenAI Responses at this time).
AgentRunOptions options = new() { AllowBackgroundResponses = true };
@@ -1,5 +1,7 @@
// Copyright (c) Microsoft. All rights reserved.
#pragma warning disable CS0618 // Type or member is obsolete - sample uses deprecated PersistentAgentsClientExtensions
// This sample shows how to create an Azure AI Foundry Agent with the Deep Research Tool.
using Azure.AI.Agents.Persistent;
@@ -12,13 +12,9 @@
<PackageReference Include="Azure.AI.OpenAI" />
<PackageReference Include="Azure.Identity" />
<PackageReference Include="Microsoft.Extensions.AI.OpenAI" />
<PackageReference Include="Microsoft.Agents.ObjectModel" />
<PackageReference Include="Microsoft.Agents.ObjectModel.Json" />
<PackageReference Include="Microsoft.Agents.ObjectModel.PowerFx" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\..\..\..\src\Microsoft.Agents.AI.Declarative\Microsoft.Agents.AI.Declarative.csproj" />
<ProjectReference Include="..\..\..\..\src\Microsoft.Agents.AI.OpenAI\Microsoft.Agents.AI.OpenAI.csproj" />
</ItemGroup>
@@ -3,7 +3,7 @@
// This sample shows how to create and use AI agents with Azure Foundry Agents as the backend.
using Azure.AI.Projects;
using Azure.AI.Projects.OpenAI;
using Azure.AI.Projects.Agents;
using Azure.Identity;
using Microsoft.Agents.AI;
@@ -3,7 +3,7 @@
// This sample shows how to create and use a simple AI agent with Azure Foundry Agents as the backend.
using Azure.AI.Projects;
using Azure.AI.Projects.OpenAI;
using Azure.AI.Projects.Agents;
using Azure.Identity;
using Microsoft.Agents.AI;
@@ -2,8 +2,9 @@
// This sample shows how to create and use a simple AI agent with a multi-turn conversation.
using Azure.AI.Extensions.OpenAI;
using Azure.AI.Projects;
using Azure.AI.Projects.OpenAI;
using Azure.AI.Projects.Agents;
using Azure.Identity;
using Microsoft.Agents.AI;
@@ -40,7 +40,7 @@ AgentResponse response = await agent.RunAsync("What is the weather like in Amste
// Check if there are any approval requests.
// For simplicity, we are assuming here that only function approvals are pending.
List<FunctionApprovalRequestContent> approvalRequests = response.Messages.SelectMany(m => m.Contents).OfType<FunctionApprovalRequestContent>().ToList();
List<ToolApprovalRequestContent> approvalRequests = response.Messages.SelectMany(m => m.Contents).OfType<ToolApprovalRequestContent>().ToList();
while (approvalRequests.Count > 0)
{
@@ -48,7 +48,7 @@ while (approvalRequests.Count > 0)
List<ChatMessage> userInputMessages = approvalRequests
.ConvertAll(functionApprovalRequest =>
{
Console.WriteLine($"The agent would like to invoke the following function, please reply Y to approve: Name {functionApprovalRequest.FunctionCall.Name}");
Console.WriteLine($"The agent would like to invoke the following function, please reply Y to approve: Name {((FunctionCallContent)functionApprovalRequest.ToolCall).Name}");
bool approved = Console.ReadLine()?.Equals("Y", StringComparison.OrdinalIgnoreCase) ?? false;
return new ChatMessage(ChatRole.User, [functionApprovalRequest.CreateResponse(approved)]);
});
@@ -56,7 +56,7 @@ while (approvalRequests.Count > 0)
// Pass the user input responses back to the agent for further processing.
response = await agent.RunAsync(userInputMessages, session);
approvalRequests = response.Messages.SelectMany(m => m.Contents).OfType<FunctionApprovalRequestContent>().ToList();
approvalRequests = response.Messages.SelectMany(m => m.Contents).OfType<ToolApprovalRequestContent>().ToList();
}
Console.WriteLine($"\nAgent: {response}");
@@ -197,7 +197,7 @@ async Task<AgentResponse> ConsolePromptingApprovalMiddleware(IEnumerable<ChatMes
AgentResponse response = await innerAgent.RunAsync(messages, session, options, cancellationToken);
// For simplicity, we are assuming here that only function approvals are pending.
List<FunctionApprovalRequestContent> approvalRequests = response.Messages.SelectMany(m => m.Contents).OfType<FunctionApprovalRequestContent>().ToList();
List<ToolApprovalRequestContent> approvalRequests = response.Messages.SelectMany(m => m.Contents).OfType<ToolApprovalRequestContent>().ToList();
while (approvalRequests.Count > 0)
{
@@ -206,14 +206,14 @@ async Task<AgentResponse> ConsolePromptingApprovalMiddleware(IEnumerable<ChatMes
response.Messages = approvalRequests
.ConvertAll(functionApprovalRequest =>
{
Console.WriteLine($"The agent would like to invoke the following function, please reply Y to approve: Name {functionApprovalRequest.FunctionCall.Name}");
Console.WriteLine($"The agent would like to invoke the following function, please reply Y to approve: Name {((FunctionCallContent)functionApprovalRequest.ToolCall).Name}");
bool approved = Console.ReadLine()?.Equals("Y", StringComparison.OrdinalIgnoreCase) ?? false;
return new ChatMessage(ChatRole.User, [functionApprovalRequest.CreateResponse(approved)]);
});
response = await innerAgent.RunAsync(response.Messages, session, options, cancellationToken);
approvalRequests = response.Messages.SelectMany(m => m.Contents).OfType<FunctionApprovalRequestContent>().ToList();
approvalRequests = response.Messages.SelectMany(m => m.Contents).OfType<ToolApprovalRequestContent>().ToList();
}
return response;
@@ -4,7 +4,7 @@
using System.Text;
using Azure.AI.Projects;
using Azure.AI.Projects.OpenAI;
using Azure.AI.Projects.Agents;
using Azure.Identity;
using Microsoft.Agents.AI;
using Microsoft.Extensions.AI;
@@ -3,7 +3,7 @@
// This sample shows how to use Computer Use Tool with AI Agents.
using Azure.AI.Projects;
using Azure.AI.Projects.OpenAI;
using Azure.AI.Projects.Agents;
using Azure.Identity;
using Microsoft.Agents.AI;
using Microsoft.Extensions.AI;
@@ -3,7 +3,7 @@
// This sample shows how to use File Search Tool with AI Agents.
using Azure.AI.Projects;
using Azure.AI.Projects.OpenAI;
using Azure.AI.Projects.Agents;
using Azure.Identity;
using Microsoft.Agents.AI;
using Microsoft.Extensions.AI;
@@ -3,7 +3,7 @@
// This sample shows how to use OpenAPI Tools with AI Agents.
using Azure.AI.Projects;
using Azure.AI.Projects.OpenAI;
using Azure.AI.Projects.Agents;
using Azure.Identity;
using Microsoft.Agents.AI;
using OpenAI.Responses;
@@ -72,7 +72,7 @@ const string CountriesOpenApiSpec = """
AIProjectClient aiProjectClient = new(new Uri(endpoint), new DefaultAzureCredential());
// Create the OpenAPI function definition
var openApiFunction = new OpenAPIFunctionDefinition(
var openApiFunction = new OpenApiFunctionDefinition(
"get_countries",
BinaryData.FromString(CountriesOpenApiSpec),
new OpenAPIAnonymousAuthenticationDetails())
@@ -3,7 +3,7 @@
// This sample shows how to use Bing Custom Search Tool with AI Agents.
using Azure.AI.Projects;
using Azure.AI.Projects.OpenAI;
using Azure.AI.Projects.Agents;
using Azure.Identity;
using Microsoft.Agents.AI;
using OpenAI.Responses;
@@ -25,7 +25,7 @@ const string AgentInstructions = """
AIProjectClient aiProjectClient = new(new Uri(endpoint), new DefaultAzureCredential());
// Bing Custom Search tool parameters shared by both options
BingCustomSearchToolParameters bingCustomSearchToolParameters = new([
BingCustomSearchToolOptions bingCustomSearchToolParameters = new([
new BingCustomSearchConfiguration(connectionId, instanceName)
]);
@@ -3,7 +3,7 @@
// This sample shows how to use SharePoint Grounding Tool with AI Agents.
using Azure.AI.Projects;
using Azure.AI.Projects.OpenAI;
using Azure.AI.Projects.Agents;
using Azure.Identity;
using Microsoft.Agents.AI;
using OpenAI.Responses;
@@ -3,7 +3,7 @@
// This sample shows how to use Microsoft Fabric Tool with AI Agents.
using Azure.AI.Projects;
using Azure.AI.Projects.OpenAI;
using Azure.AI.Projects.Agents;
using Azure.Identity;
using Microsoft.Agents.AI;
using OpenAI.Responses;
@@ -3,7 +3,7 @@
// This sample shows how to use the Responses API Web Search Tool with AI Agents.
using Azure.AI.Projects;
using Azure.AI.Projects.OpenAI;
using Azure.AI.Projects.Agents;
using Azure.Identity;
using Microsoft.Agents.AI;
using Microsoft.Extensions.AI;
@@ -13,7 +13,6 @@
<PackageReference Include="Microsoft.Extensions.Logging.Console" />
<PackageReference Include="Azure.Identity" />
<PackageReference Include="Azure.AI.Projects" />
<PackageReference Include="Azure.AI.Projects.OpenAI" />
</ItemGroup>
<ItemGroup>
@@ -4,8 +4,9 @@
// The Memory Search Tool enables agents to recall information from previous conversations,
// supporting user profile persistence and chat summaries across sessions.
using Azure.AI.Extensions.OpenAI;
using Azure.AI.Projects;
using Azure.AI.Projects.OpenAI;
using Azure.AI.Projects.Agents;
using Azure.Identity;
using Microsoft.Agents.AI;
using OpenAI.Responses;
@@ -36,7 +37,7 @@ AIProjectClient aiProjectClient = new(new Uri(endpoint), credential);
await EnsureMemoryStoreAsync();
// Create the Memory Search tool configuration
MemorySearchPreviewTool memorySearchTool = new(memoryStoreName, userScope) { UpdateDelay = 0 };
MemorySearchPreviewTool memorySearchTool = new(memoryStoreName, userScope) { UpdateDelayInSecs = 0 };
// Create agent using Option 1 (MEAI) or Option 2 (Native SDK)
AIAgent agent = await CreateAgentWithMEAI();
@@ -128,8 +129,8 @@ async Task EnsureMemoryStoreAsync()
MemoryUpdateResult updateResult = await aiProjectClient.MemoryStores.WaitForMemoriesUpdateAsync(
memoryStoreName: memoryStoreName,
options: memoryOptions,
pollingInterval: 500);
pollingInterval: 500,
options: memoryOptions);
if (updateResult.Status == MemoryStoreUpdateStatus.Failed)
{
@@ -9,12 +9,12 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Azure.AI.Agents.Persistent" />
<PackageReference Include="Azure.AI.Projects" />
<PackageReference Include="Azure.Identity" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\..\..\..\src\Microsoft.Agents.AI.AzureAI.Persistent\Microsoft.Agents.AI.AzureAI.Persistent.csproj" />
<ProjectReference Include="..\..\..\..\src\Microsoft.Agents.AI.AzureAI\Microsoft.Agents.AI.AzureAI.csproj" />
</ItemGroup>
</Project>
@@ -4,7 +4,7 @@
// In this case the Azure Foundry Agents service will invoke any MCP tools as required. MCP tools are not invoked by the Agent Framework.
// The sample first shows how to use MCP tools with auto approval, and then how to set up a tool that requires approval before it can be invoked and how to approve such a tool.
using Azure.AI.Agents.Persistent;
using Azure.AI.Projects;
using Azure.Identity;
using Microsoft.Agents.AI;
using Microsoft.Extensions.AI;
@@ -16,7 +16,7 @@ var model = Environment.GetEnvironmentVariable("AZURE_AI_MODEL_DEPLOYMENT_NAME")
// WARNING: DefaultAzureCredential is convenient for development but requires careful consideration in production.
// In production, consider using a specific credential (e.g., ManagedIdentityCredential) to avoid
// latency issues, unintended credential probing, and potential security risks from fallback mechanisms.
var persistentAgentsClient = new PersistentAgentsClient(endpoint, new DefaultAzureCredential());
var aiProjectClient = new AIProjectClient(new Uri(endpoint), new DefaultAzureCredential());
// **** MCP Tool with Auto Approval ****
// *************************************
@@ -31,8 +31,8 @@ var mcpTool = new HostedMcpServerTool(
ApprovalMode = HostedMcpServerToolApprovalMode.NeverRequire
};
// Create a server side persistent agent with the mcp tool, and expose it as an AIAgent.
AIAgent agent = await persistentAgentsClient.CreateAIAgentAsync(
// Create a server side agent with the mcp tool, and expose it as an AIAgent.
AIAgent agent = await aiProjectClient.CreateAIAgentAsync(
model: model,
options: new()
{
@@ -49,7 +49,7 @@ AgentSession session = await agent.CreateSessionAsync();
Console.WriteLine(await agent.RunAsync("Please summarize the Azure AI Agent documentation related to MCP Tool calling?", session));
// Cleanup for sample purposes.
await persistentAgentsClient.Administration.DeleteAgentAsync(agent.Id);
aiProjectClient.Agents.DeleteAgent(agent.Name);
// **** MCP Tool with Approval Required ****
// *****************************************
@@ -64,8 +64,8 @@ var mcpToolWithApproval = new HostedMcpServerTool(
ApprovalMode = HostedMcpServerToolApprovalMode.AlwaysRequire
};
// Create an agent based on Azure OpenAI Responses as the backend.
AIAgent agentWithRequiredApproval = await persistentAgentsClient.CreateAIAgentAsync(
// Create an agent with the MCP tool that requires approval.
AIAgent agentWithRequiredApproval = await aiProjectClient.CreateAIAgentAsync(
model: model,
options: new()
{
@@ -81,7 +81,7 @@ AIAgent agentWithRequiredApproval = await persistentAgentsClient.CreateAIAgentAs
// For simplicity, we are assuming here that only mcp tool approvals are pending.
AgentSession sessionWithRequiredApproval = await agentWithRequiredApproval.CreateSessionAsync();
AgentResponse response = await agentWithRequiredApproval.RunAsync("Please summarize the Azure AI Agent documentation related to MCP Tool calling?", sessionWithRequiredApproval);
List<McpServerToolApprovalRequestContent> approvalRequests = response.Messages.SelectMany(m => m.Contents).OfType<McpServerToolApprovalRequestContent>().ToList();
List<ToolApprovalRequestContent> approvalRequests = response.Messages.SelectMany(m => m.Contents).OfType<ToolApprovalRequestContent>().ToList();
while (approvalRequests.Count > 0)
{
@@ -89,11 +89,12 @@ while (approvalRequests.Count > 0)
List<ChatMessage> userInputResponses = approvalRequests
.ConvertAll(approvalRequest =>
{
McpServerToolCallContent mcpToolCall = (McpServerToolCallContent)approvalRequest.ToolCall!;
Console.WriteLine($"""
The agent would like to invoke the following MCP Tool, please reply Y to approve.
ServerName: {approvalRequest.ToolCall.ServerName}
Name: {approvalRequest.ToolCall.ToolName}
Arguments: {string.Join(", ", approvalRequest.ToolCall.Arguments?.Select(x => $"{x.Key}: {x.Value}") ?? [])}
ServerName: {mcpToolCall.ServerName}
Name: {mcpToolCall.Name}
Arguments: {string.Join(", ", mcpToolCall.Arguments?.Select(x => $"{x.Key}: {x.Value}") ?? [])}
""");
return new ChatMessage(ChatRole.User, [approvalRequest.CreateResponse(Console.ReadLine()?.Equals("Y", StringComparison.OrdinalIgnoreCase) ?? false)]);
});
@@ -101,7 +102,7 @@ while (approvalRequests.Count > 0)
// Pass the user input responses back to the agent for further processing.
response = await agentWithRequiredApproval.RunAsync(userInputResponses, sessionWithRequiredApproval);
approvalRequests = response.Messages.SelectMany(m => m.Contents).OfType<McpServerToolApprovalRequestContent>().ToList();
approvalRequests = response.Messages.SelectMany(m => m.Contents).OfType<ToolApprovalRequestContent>().ToList();
}
Console.WriteLine($"\nAgent: {response}");
@@ -33,8 +33,9 @@ var mcpTool = new HostedMcpServerTool(
AIAgent agent = new AzureOpenAIClient(
new Uri(endpoint),
new DefaultAzureCredential())
.GetResponsesClient(deploymentName)
.GetResponsesClient()
.AsAIAgent(
model: deploymentName,
instructions: "You answer questions by searching the Microsoft Learn content only.",
name: "MicrosoftLearnAgent",
tools: [mcpTool]);
@@ -60,8 +61,9 @@ var mcpToolWithApproval = new HostedMcpServerTool(
AIAgent agentWithRequiredApproval = new AzureOpenAIClient(
new Uri(endpoint),
new DefaultAzureCredential())
.GetResponsesClient(deploymentName)
.GetResponsesClient()
.AsAIAgent(
model: deploymentName,
instructions: "You answer questions by searching the Microsoft Learn content only.",
name: "MicrosoftLearnAgentWithApproval",
tools: [mcpToolWithApproval]);
@@ -70,7 +72,7 @@ AIAgent agentWithRequiredApproval = new AzureOpenAIClient(
// For simplicity, we are assuming here that only mcp tool approvals are pending.
AgentSession sessionWithRequiredApproval = await agentWithRequiredApproval.CreateSessionAsync();
AgentResponse response = await agentWithRequiredApproval.RunAsync("Please summarize the Azure AI Agent documentation related to MCP Tool calling?", sessionWithRequiredApproval);
List<McpServerToolApprovalRequestContent> approvalRequests = response.Messages.SelectMany(m => m.Contents).OfType<McpServerToolApprovalRequestContent>().ToList();
List<ToolApprovalRequestContent> approvalRequests = response.Messages.SelectMany(m => m.Contents).OfType<ToolApprovalRequestContent>().ToList();
while (approvalRequests.Count > 0)
{
@@ -78,11 +80,12 @@ while (approvalRequests.Count > 0)
List<ChatMessage> userInputResponses = approvalRequests
.ConvertAll(approvalRequest =>
{
McpServerToolCallContent mcpToolCall = (McpServerToolCallContent)approvalRequest.ToolCall!;
Console.WriteLine($"""
The agent would like to invoke the following MCP Tool, please reply Y to approve.
ServerName: {approvalRequest.ToolCall.ServerName}
Name: {approvalRequest.ToolCall.ToolName}
Arguments: {string.Join(", ", approvalRequest.ToolCall.Arguments?.Select(x => $"{x.Key}: {x.Value}") ?? [])}
ServerName: {mcpToolCall.ServerName}
Name: {mcpToolCall.Name}
Arguments: {string.Join(", ", mcpToolCall.Arguments?.Select(x => $"{x.Key}: {x.Value}") ?? [])}
""");
return new ChatMessage(ChatRole.User, [approvalRequest.CreateResponse(Console.ReadLine()?.Equals("Y", StringComparison.OrdinalIgnoreCase) ?? false)]);
});
@@ -90,7 +93,7 @@ while (approvalRequests.Count > 0)
// Pass the user input responses back to the agent for further processing.
response = await agentWithRequiredApproval.RunAsync(userInputResponses, sessionWithRequiredApproval);
approvalRequests = response.Messages.SelectMany(m => m.Contents).OfType<McpServerToolApprovalRequestContent>().ToList();
approvalRequests = response.Messages.SelectMany(m => m.Contents).OfType<ToolApprovalRequestContent>().ToList();
}
Console.WriteLine($"\nAgent: {response}");
@@ -61,6 +61,12 @@ public static class Program
{
Console.WriteLine($"{outputEvent}");
}
if (evt is WorkflowErrorEvent errorEvent)
{
Console.WriteLine($"Workflow error: {errorEvent.Exception?.Message}");
Console.WriteLine($"Details: {errorEvent.Exception}");
}
}
}
}
@@ -175,7 +181,9 @@ internal sealed class FeedbackEvent(FeedbackResult feedbackResult) : WorkflowEve
/// <summary>
/// A custom executor that uses an AI agent to provide feedback on a slogan.
/// </summary>
internal sealed class FeedbackExecutor : Executor<SloganResult>
[SendsMessage(typeof(FeedbackResult))]
[YieldsOutput(typeof(string))]
internal sealed partial class FeedbackExecutor : Executor<SloganResult>
{
private readonly AIAgent _agent;
private AgentSession? _session;
@@ -9,13 +9,13 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Azure.AI.Agents.Persistent" />
<PackageReference Include="Azure.AI.Projects" />
<PackageReference Include="Azure.Identity" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\..\..\..\src\Microsoft.Agents.AI.Workflows\Microsoft.Agents.AI.Workflows.csproj" />
<ProjectReference Include="..\..\..\..\src\Microsoft.Agents.AI.AzureAI.Persistent\Microsoft.Agents.AI.AzureAI.Persistent.csproj" />
<ProjectReference Include="..\..\..\..\src\Microsoft.Agents.AI.AzureAI\Microsoft.Agents.AI.AzureAI.csproj" />
<ProjectReference Include="..\..\..\..\src\Microsoft.Agents.AI\Microsoft.Agents.AI.csproj" />
</ItemGroup>
@@ -1,6 +1,6 @@
// Copyright (c) Microsoft. All rights reserved.
using Azure.AI.Agents.Persistent;
using Azure.AI.Projects;
using Azure.Identity;
using Microsoft.Agents.AI;
using Microsoft.Agents.AI.Workflows;
@@ -20,60 +20,63 @@ public static class Program
{
private static async Task Main()
{
// Set up the Azure OpenAI client
// Set up the Azure AI Project client
var endpoint = Environment.GetEnvironmentVariable("AZURE_AI_PROJECT_ENDPOINT")
?? throw new InvalidOperationException("AZURE_AI_PROJECT_ENDPOINT is not set.");
var deploymentName = Environment.GetEnvironmentVariable("AZURE_AI_MODEL_DEPLOYMENT_NAME") ?? "gpt-4o-mini";
var persistentAgentsClient = new PersistentAgentsClient(endpoint, new AzureCliCredential());
var aiProjectClient = new AIProjectClient(new Uri(endpoint), new AzureCliCredential());
// Create agents
AIAgent frenchAgent = await GetTranslationAgentAsync("French", persistentAgentsClient, deploymentName);
AIAgent spanishAgent = await GetTranslationAgentAsync("Spanish", persistentAgentsClient, deploymentName);
AIAgent englishAgent = await GetTranslationAgentAsync("English", persistentAgentsClient, deploymentName);
AIAgent frenchAgent = await CreateTranslationAgentAsync("French", aiProjectClient, deploymentName);
AIAgent spanishAgent = await CreateTranslationAgentAsync("Spanish", aiProjectClient, deploymentName);
AIAgent englishAgent = await CreateTranslationAgentAsync("English", aiProjectClient, deploymentName);
// Build the workflow by adding executors and connecting them
var workflow = new WorkflowBuilder(frenchAgent)
.AddEdge(frenchAgent, spanishAgent)
.AddEdge(spanishAgent, englishAgent)
.Build();
// Execute the workflow
await using StreamingRun run = await InProcessExecution.RunStreamingAsync(workflow, new ChatMessage(ChatRole.User, "Hello World!"));
// Must send the turn token to trigger the agents.
// The agents are wrapped as executors. When they receive messages,
// they will cache the messages and only start processing when they receive a TurnToken.
await run.TrySendMessageAsync(new TurnToken(emitEvents: true));
await foreach (WorkflowEvent evt in run.WatchStreamAsync())
try
{
if (evt is AgentResponseUpdateEvent executorComplete)
// Build the workflow by adding executors and connecting them
var workflow = new WorkflowBuilder(frenchAgent)
.AddEdge(frenchAgent, spanishAgent)
.AddEdge(spanishAgent, englishAgent)
.Build();
// Execute the workflow
await using StreamingRun run = await InProcessExecution.RunStreamingAsync(workflow, new ChatMessage(ChatRole.User, "Hello World!"));
// Must send the turn token to trigger the agents.
// The agents are wrapped as executors. When they receive messages,
// they will cache the messages and only start processing when they receive a TurnToken.
await run.TrySendMessageAsync(new TurnToken(emitEvents: true));
await foreach (WorkflowEvent evt in run.WatchStreamAsync())
{
Console.WriteLine($"{executorComplete.ExecutorId}: {executorComplete.Data}");
if (evt is AgentResponseUpdateEvent executorComplete)
{
Console.WriteLine($"{executorComplete.ExecutorId}: {executorComplete.Data}");
}
}
}
// Cleanup the agents created for the sample.
await persistentAgentsClient.Administration.DeleteAgentAsync(frenchAgent.Id);
await persistentAgentsClient.Administration.DeleteAgentAsync(spanishAgent.Id);
await persistentAgentsClient.Administration.DeleteAgentAsync(englishAgent.Id);
finally
{
// Cleanup the agents created for the sample.
await aiProjectClient.Agents.DeleteAgentAsync(frenchAgent.Name);
await aiProjectClient.Agents.DeleteAgentAsync(spanishAgent.Name);
await aiProjectClient.Agents.DeleteAgentAsync(englishAgent.Name);
}
}
/// <summary>
/// Creates a translation agent for the specified target language.
/// </summary>
/// <param name="targetLanguage">The target language for translation</param>
/// <param name="persistentAgentsClient">The PersistentAgentsClient to create the agent</param>
/// <param name="aiProjectClient">The <see cref="AIProjectClient"/> to create the agent with.</param>
/// <param name="model">The model to use for the agent</param>
/// <returns>A ChatClientAgent configured for the specified language</returns>
private static async Task<ChatClientAgent> GetTranslationAgentAsync(
private static async Task<ChatClientAgent> CreateTranslationAgentAsync(
string targetLanguage,
PersistentAgentsClient persistentAgentsClient,
AIProjectClient aiProjectClient,
string model)
{
var agentMetadata = await persistentAgentsClient.Administration.CreateAgentAsync(
model: model,
return await aiProjectClient.CreateAIAgentAsync(
name: $"{targetLanguage} Translator",
model: model,
instructions: $"You are a translation assistant that translates the provided text to {targetLanguage}.");
return await persistentAgentsClient.GetAIAgentAsync(agentMetadata.Value.Id);
}
}
@@ -17,7 +17,7 @@
//
// Demonstrate:
// - Using custom GroupChatManager with agents that have approval-required tools.
// - Handling FunctionApprovalRequestContent in group chat scenarios.
// - Handling ToolApprovalRequestContent in group chat scenarios.
// - Multi-round group chat with tool approval interruption and resumption.
using System.ComponentModel;
@@ -101,16 +101,16 @@ public static class Program
{
case RequestInfoEvent e:
{
if (e.Request.TryGetDataAs(out FunctionApprovalRequestContent? approvalRequestContent))
if (e.Request.TryGetDataAs(out ToolApprovalRequestContent? approvalRequestContent))
{
Console.WriteLine();
Console.WriteLine($"[APPROVAL REQUIRED] From agent: {e.Request.PortInfo.PortId}");
Console.WriteLine($" Tool: {approvalRequestContent.FunctionCall.Name}");
Console.WriteLine($" Arguments: {JsonSerializer.Serialize(approvalRequestContent.FunctionCall.Arguments)}");
Console.WriteLine($" Tool: {((FunctionCallContent)approvalRequestContent.ToolCall).Name}");
Console.WriteLine($" Arguments: {JsonSerializer.Serialize(((FunctionCallContent)approvalRequestContent.ToolCall).Arguments)}");
Console.WriteLine();
// Approve the tool call request
Console.WriteLine($"Tool: {approvalRequestContent.FunctionCall.Name} approved");
Console.WriteLine($"Tool: {((FunctionCallContent)approvalRequestContent.ToolCall).Name} approved");
await run.SendResponseAsync(e.Request.CreateResponse(approvalRequestContent.CreateResponse(approved: true)));
}
@@ -41,6 +41,7 @@ internal static class WorkflowFactory
/// <summary>
/// Executor that aggregates the results from the concurrent agents.
/// </summary>
[YieldsOutput(typeof(string))]
private sealed class ConcurrentAggregationExecutor() :
Executor<List<ChatMessage>>("ConcurrentAggregationExecutor"), IResettableExecutor
{
@@ -41,6 +41,7 @@ internal enum NumberSignal
/// <summary>
/// Executor that makes a guess based on the current bounds.
/// </summary>
[SendsMessage(typeof(int))]
internal sealed class GuessNumberExecutor() : Executor<NumberSignal>("Guess")
{
/// <summary>
@@ -104,6 +105,8 @@ internal sealed class GuessNumberExecutor() : Executor<NumberSignal>("Guess")
/// <summary>
/// Executor that judges the guess and provides feedback.
/// </summary>
[SendsMessage(typeof(NumberSignal))]
[YieldsOutput(typeof(string))]
internal sealed class JudgeExecutor() : Executor<int>("Judge")
{
private readonly int _targetNumber;
@@ -41,6 +41,7 @@ internal enum NumberSignal
/// <summary>
/// Executor that makes a guess based on the current bounds.
/// </summary>
[SendsMessage(typeof(int))]
internal sealed class GuessNumberExecutor() : Executor<NumberSignal>("Guess")
{
/// <summary>
@@ -104,6 +105,8 @@ internal sealed class GuessNumberExecutor() : Executor<NumberSignal>("Guess")
/// <summary>
/// Executor that judges the guess and provides feedback.
/// </summary>
[SendsMessage(typeof(NumberSignal))]
[YieldsOutput(typeof(string))]
internal sealed class JudgeExecutor() : Executor<int>("Judge")
{
private readonly int _targetNumber;
@@ -53,6 +53,8 @@ internal sealed class SignalWithNumber
/// <summary>
/// Executor that judges the guess and provides feedback.
/// </summary>
[SendsMessage(typeof(SignalWithNumber))]
[YieldsOutput(typeof(string))]
internal sealed class JudgeExecutor() : Executor<int>("Judge")
{
private readonly int _targetNumber;
@@ -72,6 +72,8 @@ public static class Program
/// <summary>
/// Executor that starts the concurrent processing by sending messages to the agents.
/// </summary>
[SendsMessage(typeof(ChatMessage))]
[SendsMessage(typeof(TurnToken))]
internal sealed partial class ConcurrentStartExecutor() :
Executor("ConcurrentStartExecutor")
{
@@ -97,7 +99,8 @@ internal sealed partial class ConcurrentStartExecutor() :
/// <summary>
/// Executor that aggregates the results from the concurrent agents.
/// </summary>
internal sealed class ConcurrentAggregationExecutor() :
[YieldsOutput(typeof(string))]
internal sealed partial class ConcurrentAggregationExecutor() :
Executor<List<ChatMessage>>("ConcurrentAggregationExecutor")
{
private readonly List<ChatMessage> _messages = [];
@@ -128,6 +128,7 @@ public static class Program
/// <summary>
/// Splits data into roughly equal chunks based on the number of mapper nodes.
/// </summary>
[SendsMessage(typeof(SplitComplete))]
internal sealed class Split(string[] mapperIds, string id) :
Executor<string>(id)
{
@@ -186,6 +187,7 @@ internal sealed class Split(string[] mapperIds, string id) :
/// <summary>
/// Maps each token to a count of 1 and writes pairs to a per-mapper file.
/// </summary>
[SendsMessage(typeof(MapComplete))]
internal sealed class Mapper(string id) : Executor<SplitComplete>(id)
{
/// <summary>
@@ -212,6 +214,7 @@ internal sealed class Mapper(string id) : Executor<SplitComplete>(id)
/// <summary>
/// Groups intermediate pairs by key and partitions them across reducers.
/// </summary>
[SendsMessage(typeof(ShuffleComplete))]
internal sealed class Shuffler(string[] reducerIds, string[] mapperIds, string id) :
Executor<MapComplete>(id)
{
@@ -311,6 +314,7 @@ internal sealed class Shuffler(string[] reducerIds, string[] mapperIds, string i
/// <summary>
/// Sums grouped counts per key for its assigned partition.
/// </summary>
[SendsMessage(typeof(ReduceComplete))]
internal sealed class Reducer(string id) : Executor<ShuffleComplete>(id)
{
/// <summary>
@@ -352,6 +356,7 @@ internal sealed class Reducer(string id) : Executor<ShuffleComplete>(id)
/// <summary>
/// Joins all reducer outputs and yields the final output.
/// </summary>
[YieldsOutput(typeof(List<string>))]
internal sealed class CompletionExecutor(string id) :
Executor<List<ReduceComplete>>(id)
{
@@ -228,6 +228,7 @@ internal sealed class EmailAssistantExecutor : Executor<DetectionResult, EmailRe
/// <summary>
/// Executor that sends emails.
/// </summary>
[YieldsOutput(typeof(string))]
internal sealed class SendEmailExecutor() : Executor<EmailResponse>("SendEmailExecutor")
{
/// <summary>
@@ -240,6 +241,7 @@ internal sealed class SendEmailExecutor() : Executor<EmailResponse>("SendEmailEx
/// <summary>
/// Executor that handles spam messages.
/// </summary>
[YieldsOutput(typeof(string))]
internal sealed class HandleSpamExecutor() : Executor<DetectionResult>("HandleSpamExecutor")
{
/// <summary>
@@ -252,6 +252,7 @@ internal sealed class EmailAssistantExecutor : Executor<DetectionResult, EmailRe
/// <summary>
/// Executor that sends emails.
/// </summary>
[YieldsOutput(typeof(string))]
internal sealed class SendEmailExecutor() : Executor<EmailResponse>("SendEmailExecutor")
{
/// <summary>
@@ -264,6 +265,7 @@ internal sealed class SendEmailExecutor() : Executor<EmailResponse>("SendEmailEx
/// <summary>
/// Executor that handles spam messages.
/// </summary>
[YieldsOutput(typeof(string))]
internal sealed class HandleSpamExecutor() : Executor<DetectionResult>("HandleSpamExecutor")
{
/// <summary>
@@ -285,6 +287,7 @@ internal sealed class HandleSpamExecutor() : Executor<DetectionResult>("HandleSp
/// <summary>
/// Executor that handles uncertain emails.
/// </summary>
[YieldsOutput(typeof(string))]
internal sealed class HandleUncertainExecutor() : Executor<DetectionResult>("HandleUncertainExecutor")
{
/// <summary>
@@ -310,6 +310,7 @@ internal sealed class EmailAssistantExecutor : Executor<AnalysisResult, EmailRes
/// <summary>
/// Executor that sends emails.
/// </summary>
[YieldsOutput(typeof(string))]
internal sealed class SendEmailExecutor() : Executor<EmailResponse>("SendEmailExecutor")
{
/// <summary>
@@ -322,6 +323,7 @@ internal sealed class SendEmailExecutor() : Executor<EmailResponse>("SendEmailEx
/// <summary>
/// Executor that handles spam messages.
/// </summary>
[YieldsOutput(typeof(string))]
internal sealed class HandleSpamExecutor() : Executor<AnalysisResult>("HandleSpamExecutor")
{
/// <summary>
@@ -343,6 +345,7 @@ internal sealed class HandleSpamExecutor() : Executor<AnalysisResult>("HandleSpa
/// <summary>
/// Executor that handles uncertain messages.
/// </summary>
[YieldsOutput(typeof(string))]
internal sealed class HandleUncertainExecutor() : Executor<AnalysisResult>("HandleUncertainExecutor")
{
/// <summary>
@@ -1,7 +1,7 @@
// Copyright (c) Microsoft. All rights reserved.
using Azure.AI.Projects;
using Azure.AI.Projects.OpenAI;
using Azure.AI.Projects.Agents;
using Azure.Identity;
using Microsoft.Extensions.AI;
using Microsoft.Extensions.Configuration;
@@ -1,7 +1,7 @@
// Copyright (c) Microsoft. All rights reserved.
using Azure.AI.Projects;
using Azure.AI.Projects.OpenAI;
using Azure.AI.Projects.Agents;
using Azure.Identity;
using Microsoft.Extensions.Configuration;
using OpenAI.Responses;
@@ -275,7 +275,7 @@ internal sealed class Program
Tools =
{
AgentTool.CreateOpenApiTool(
new OpenAPIFunctionDefinition(
new OpenApiFunctionDefinition(
"weather-forecast",
BinaryData.FromString(File.ReadAllText(Path.Combine(AppContext.BaseDirectory, "wttr.json"))),
new OpenAPIAnonymousAuthenticationDetails()))
@@ -1,7 +1,7 @@
// Copyright (c) Microsoft. All rights reserved.
using Azure.AI.Projects;
using Azure.AI.Projects.OpenAI;
using Azure.AI.Projects.Agents;
using Azure.Identity;
using Microsoft.Extensions.AI;
using Microsoft.Extensions.Configuration;
@@ -3,8 +3,9 @@
// Uncomment this to enable JSON checkpointing to the local file system.
//#define CHECKPOINT_JSON
using Azure.AI.Extensions.OpenAI;
using Azure.AI.Projects;
using Azure.AI.Projects.OpenAI;
using Azure.AI.Projects.Agents;
using Azure.Identity;
using Microsoft.Agents.AI;
using Microsoft.Extensions.AI;
@@ -1,7 +1,7 @@
// Copyright (c) Microsoft. All rights reserved.
using Azure.AI.Projects;
using Azure.AI.Projects.OpenAI;
using Azure.AI.Projects.Agents;
using Azure.Identity;
using Microsoft.Extensions.Configuration;
using OpenAI.Responses;
@@ -1,7 +1,7 @@
// Copyright (c) Microsoft. All rights reserved.
using Azure.AI.Projects;
using Azure.AI.Projects.OpenAI;
using Azure.AI.Projects.Agents;
using Azure.Identity;
using Microsoft.Extensions.AI;
using Microsoft.Extensions.Configuration;
@@ -5,7 +5,7 @@
// invoked to perform specific tasks, like searching documentation or executing operations.
using Azure.AI.Projects;
using Azure.AI.Projects.OpenAI;
using Azure.AI.Projects.Agents;
using Azure.Core;
using Azure.Identity;
using Microsoft.Agents.AI.Workflows.Declarative.Mcp;
@@ -1,7 +1,7 @@
// Copyright (c) Microsoft. All rights reserved.
using Azure.AI.Projects;
using Azure.AI.Projects.OpenAI;
using Azure.AI.Projects.Agents;
using Azure.Identity;
using Microsoft.Extensions.Configuration;
using Shared.Foundry;
@@ -1,7 +1,7 @@
// Copyright (c) Microsoft. All rights reserved.
using Azure.AI.Projects;
using Azure.AI.Projects.OpenAI;
using Azure.AI.Projects.Agents;
using Azure.Identity;
using Microsoft.Extensions.Configuration;
using Shared.Foundry;
@@ -1,7 +1,7 @@
// Copyright (c) Microsoft. All rights reserved.
using Azure.AI.Projects;
using Azure.AI.Projects.OpenAI;
using Azure.AI.Projects.Agents;
using Azure.Identity;
using Microsoft.Extensions.Configuration;
using OpenAI.Responses;
@@ -38,6 +38,8 @@ internal enum NumberSignal
/// <summary>
/// Executor that judges the guess and provides feedback.
/// </summary>
[SendsMessage(typeof(NumberSignal))]
[YieldsOutput(typeof(string))]
internal sealed class JudgeExecutor() : Executor<int>("Judge")
{
private readonly int _targetNumber;
+4 -2
View File
@@ -56,6 +56,7 @@ internal enum NumberSignal
/// <summary>
/// Executor that makes a guess based on the current bounds.
/// </summary>
[SendsMessage(typeof(int))]
internal sealed class GuessNumberExecutor : Executor<NumberSignal>
{
/// <summary>
@@ -104,6 +105,8 @@ internal sealed class GuessNumberExecutor : Executor<NumberSignal>
/// <summary>
/// Executor that judges the guess and provides feedback.
/// </summary>
[SendsMessage(typeof(NumberSignal))]
[YieldsOutput(typeof(string))]
internal sealed class JudgeExecutor : Executor<int>
{
private readonly int _targetNumber;
@@ -124,8 +127,7 @@ internal sealed class JudgeExecutor : Executor<int>
this._tries++;
if (message == this._targetNumber)
{
await context.YieldOutputAsync($"{this._targetNumber} found in {this._tries} tries!", cancellationToken)
;
await context.YieldOutputAsync($"{this._targetNumber} found in {this._tries} tries!", cancellationToken);
}
else if (message < this._targetNumber)
{
@@ -99,6 +99,10 @@ internal sealed class ParagraphCountingExecutor() : Executor<string, FileStats>(
}
}
/// <summary>
/// The aggregation executor collects results from both executors and yields the final output.
/// </summary>
[YieldsOutput(typeof(string))]
internal sealed class AggregationExecutor() : Executor<FileStats>("AggregationExecutor")
{
private readonly List<FileStats> _messages = [];
@@ -205,6 +205,8 @@ internal sealed class TextInverterExecutor(string id) : Executor<string, string>
/// 1. Sending ChatMessage(s)
/// 2. Sending a TurnToken to trigger processing
/// </summary>
[SendsMessage(typeof(ChatMessage))]
[SendsMessage(typeof(TurnToken))]
internal sealed class StringToChatMessageExecutor(string id) : Executor<string>(id)
{
public override async ValueTask HandleAsync(string message, IWorkflowContext context, CancellationToken cancellationToken = default)
@@ -234,6 +236,8 @@ internal sealed class StringToChatMessageExecutor(string id) : Executor<string>(
/// The AIAgentHostExecutor sends response.Messages which has runtime type List&lt;ChatMessage&gt;.
/// The message router uses exact type matching via message.GetType().
/// </remarks>
[SendsMessage(typeof(ChatMessage))]
[SendsMessage(typeof(TurnToken))]
internal sealed class JailbreakSyncExecutor() : Executor<List<ChatMessage>>("JailbreakSync")
{
public override async ValueTask HandleAsync(List<ChatMessage> message, IWorkflowContext context, CancellationToken cancellationToken = default)
@@ -14,7 +14,6 @@
<PackageReference Include="Azure.Identity" />
<PackageReference Include="Microsoft.Extensions.Hosting" />
<PackageReference Include="System.Net.ServerSentEvents" />
<PackageReference Include="Microsoft.Bcl.AsyncInterfaces" />
</ItemGroup>
<ItemGroup>
@@ -9,7 +9,7 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Azure.AI.Agents.Persistent" />
<PackageReference Include="Azure.AI.Projects" />
<PackageReference Include="Azure.Identity" />
</ItemGroup>
@@ -23,7 +23,7 @@
<ProjectReference Include="..\..\..\..\src\Microsoft.Agents.AI.Hosting.A2A\Microsoft.Agents.AI.Hosting.A2A.csproj" />
<ProjectReference Include="..\..\..\..\src\Microsoft.Agents.AI.A2A\Microsoft.Agents.AI.A2A.csproj" />
<ProjectReference Include="..\..\..\..\src\Microsoft.Agents.AI.AzureAI.Persistent\Microsoft.Agents.AI.AzureAI.Persistent.csproj" />
<ProjectReference Include="..\..\..\..\src\Microsoft.Agents.AI.AzureAI\Microsoft.Agents.AI.AzureAI.csproj" />
<ProjectReference Include="..\..\..\..\src\Microsoft.Agents.AI.OpenAI\Microsoft.Agents.AI.OpenAI.csproj" />
</ItemGroup>
@@ -1,7 +1,7 @@
// Copyright (c) Microsoft. All rights reserved.
using A2A;
using Azure.AI.Agents.Persistent;
using Azure.AI.Projects;
using Azure.Identity;
using Microsoft.Agents.AI;
using Microsoft.Extensions.AI;
@@ -12,16 +12,15 @@ namespace A2AServer;
internal static class HostAgentFactory
{
internal static async Task<(AIAgent, AgentCard)> CreateFoundryHostAgentAsync(string agentType, string model, string endpoint, string assistantId, IList<AITool>? tools = null)
internal static async Task<(AIAgent, AgentCard)> CreateFoundryHostAgentAsync(string agentType, string model, string endpoint, string agentName, IList<AITool>? tools = null)
{
// WARNING: DefaultAzureCredential is convenient for development but requires careful consideration in production.
// In production, consider using a specific credential (e.g., ManagedIdentityCredential) to avoid
// latency issues, unintended credential probing, and potential security risks from fallback mechanisms.
var persistentAgentsClient = new PersistentAgentsClient(endpoint, new DefaultAzureCredential());
PersistentAgent persistentAgent = await persistentAgentsClient.Administration.GetAgentAsync(assistantId);
var aiProjectClient = new AIProjectClient(new Uri(endpoint), new DefaultAzureCredential());
AIAgent agent = await persistentAgentsClient
.GetAIAgentAsync(persistentAgent.Id, chatOptions: new() { Tools = tools });
AIAgent agent = await aiProjectClient
.GetAIAgentAsync(agentName, tools: tools);
AgentCard agentCard = agentType.ToUpperInvariant() switch
{
@@ -8,16 +8,16 @@ using Microsoft.Extensions.AI;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
string agentId = string.Empty;
string agentName = string.Empty;
string agentType = string.Empty;
for (var i = 0; i < args.Length; i++)
{
if (args[i].StartsWith("--agentId", StringComparison.InvariantCultureIgnoreCase) && i + 1 < args.Length)
if (args[i].Equals("--agentName", StringComparison.OrdinalIgnoreCase) && i + 1 < args.Length)
{
agentId = args[++i];
agentName = args[++i];
}
else if (args[i].StartsWith("--agentType", StringComparison.InvariantCultureIgnoreCase) && i + 1 < args.Length)
else if (args[i].Equals("--agentType", StringComparison.OrdinalIgnoreCase) && i + 1 < args.Length)
{
agentType = args[++i];
}
@@ -50,13 +50,13 @@ IList<AITool> tools =
AIAgent hostA2AAgent;
AgentCard hostA2AAgentCard;
if (!string.IsNullOrEmpty(endpoint) && !string.IsNullOrEmpty(agentId))
if (!string.IsNullOrEmpty(endpoint) && !string.IsNullOrEmpty(agentName))
{
(hostA2AAgent, hostA2AAgentCard) = agentType.ToUpperInvariant() switch
{
"INVOICE" => await HostAgentFactory.CreateFoundryHostAgentAsync(agentType, model, endpoint, agentId, tools),
"POLICY" => await HostAgentFactory.CreateFoundryHostAgentAsync(agentType, model, endpoint, agentId),
"LOGISTICS" => await HostAgentFactory.CreateFoundryHostAgentAsync(agentType, model, endpoint, agentId),
"INVOICE" => await HostAgentFactory.CreateFoundryHostAgentAsync(agentType, model, endpoint, agentName, tools),
"POLICY" => await HostAgentFactory.CreateFoundryHostAgentAsync(agentType, model, endpoint, agentName),
"LOGISTICS" => await HostAgentFactory.CreateFoundryHostAgentAsync(agentType, model, endpoint, agentName),
_ => throw new ArgumentException($"Unsupported agent type: {agentType}"),
};
}
@@ -101,7 +101,7 @@ else if (!string.IsNullOrEmpty(apiKey))
}
else
{
throw new ArgumentException("Either A2AServer:ApiKey or A2AServer:ConnectionString & agentId must be provided");
throw new ArgumentException("Either A2AServer:ApiKey or A2AServer:ConnectionString & agentName must be provided");
}
var a2aTaskManager = app.MapA2A(
@@ -90,15 +90,15 @@ $env:AZURE_AI_PROJECT_ENDPOINT="https://ai-foundry-your-project.services.ai.azur
Use the following commands to run each A2A server
```bash
dotnet run --urls "http://localhost:5000;https://localhost:5010" --agentId "<Invoice Agent Id>" --agentType "invoice" --no-build
dotnet run --urls "http://localhost:5000;https://localhost:5010" --agentName "<Invoice Agent Name>" --agentType "invoice" --no-build
```
```bash
dotnet run --urls "http://localhost:5001;https://localhost:5011" --agentId "<Policy Agent Id>" --agentType "policy" --no-build
dotnet run --urls "http://localhost:5001;https://localhost:5011" --agentName "<Policy Agent Name>" --agentType "policy" --no-build
```
```bash
dotnet run --urls "http://localhost:5002;https://localhost:5012" --agentId "<Logistics Agent Id>" --agentType "logistics" --no-build
dotnet run --urls "http://localhost:5002;https://localhost:5012" --agentName "<Logistics Agent Name>" --agentType "logistics" --no-build
```
### Testing the Agents using the Rest Client
@@ -15,7 +15,6 @@
<ItemGroup>
<ProjectReference Include="..\..\..\..\src\Microsoft.Agents.AI.Hosting.AGUI.AspNetCore\Microsoft.Agents.AI.Hosting.AGUI.AspNetCore.csproj" />
<ProjectReference Include="..\..\..\..\src\Microsoft.Agents.AI.AGUI\Microsoft.Agents.AI.AGUI.csproj" />
<ProjectReference Include="..\..\..\..\src\Microsoft.Agents.AI.OpenAI\Microsoft.Agents.AI.OpenAI.csproj" />
</ItemGroup>
@@ -9,10 +9,12 @@
<ItemGroup>
<ProjectReference Include="..\..\..\..\src\Microsoft.Agents.AI.Hosting.A2A\Microsoft.Agents.AI.Hosting.A2A.csproj" />
<ProjectReference Include="..\..\..\..\src\Microsoft.Agents.AI.Hosting.OpenAI\Microsoft.Agents.AI.Hosting.OpenAI.csproj" />
<ProjectReference Include="..\..\..\..\src\Microsoft.Agents.AI.Hosting\Microsoft.Agents.AI.Hosting.csproj" />
<ProjectReference Include="..\AgentWebChat.ServiceDefaults\AgentWebChat.ServiceDefaults.csproj" />
<ProjectReference Include="..\..\..\..\src\Microsoft.Agents.AI.Abstractions\Microsoft.Agents.AI.Abstractions.csproj" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="OpenAI" />
<PackageReference Include="Microsoft.Extensions.AI.OpenAI" />
</ItemGroup>
</Project>
@@ -27,7 +27,7 @@ internal sealed class OpenAIResponsesAgentClient(HttpClient httpClient) : AgentC
Transport = new HttpClientPipelineTransport(httpClient)
};
var openAiClient = new ResponsesClient(model: agentName, credential: new ApiKeyCredential("dummy-key"), options: options).AsIChatClient();
var openAiClient = new ResponsesClient(credential: new ApiKeyCredential("dummy-key"), options: options).AsIChatClient(agentName);
var chatOptions = new ChatOptions()
{
ConversationId = sessionId
@@ -30,8 +30,8 @@ TokenCredential browserCredential = new InteractiveBrowserCredential(
using IChatClient client = new AzureOpenAIClient(
new Uri(endpoint),
new DefaultAzureCredential())
.GetResponsesClient(deploymentName)
.AsIChatClient()
.GetResponsesClient()
.AsIChatClient(deploymentName)
.AsBuilder()
.WithPurview(browserCredential, new PurviewSettings("Agent Framework Test App"))
.Build();
@@ -36,10 +36,10 @@
</ItemGroup>
<ItemGroup>
<PackageReference Include="Azure.AI.AgentServer.AgentFramework" Version="1.0.0-beta.9" />
<PackageReference Include="Azure.AI.OpenAI" Version="2.8.0-beta.1" />
<PackageReference Include="Azure.AI.AgentServer.AgentFramework" Version="1.0.0-beta.11" />
<PackageReference Include="Azure.AI.OpenAI" Version="2.9.0-beta.1" />
<PackageReference Include="Azure.Identity" Version="1.17.1" />
<PackageReference Include="Microsoft.Agents.AI.OpenAI" Version="1.0.0-rc1" />
<PackageReference Include="Microsoft.Agents.AI.OpenAI" Version="1.0.0-rc4" />
</ItemGroup>
<!-- Add analyzers with compatible versions -->
@@ -1,4 +1,4 @@
<Project Sdk="Microsoft.NET.Sdk">
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
@@ -35,10 +35,10 @@
</ItemGroup>
<ItemGroup>
<PackageReference Include="Azure.AI.AgentServer.AgentFramework" Version="1.0.0-beta.9" />
<PackageReference Include="Azure.AI.AgentServer.AgentFramework" Version="1.0.0-beta.11" />
<PackageReference Include="Azure.AI.OpenAI" Version="2.8.0-beta.1" />
<PackageReference Include="Azure.Identity" Version="1.17.1" />
<PackageReference Include="Microsoft.Agents.AI.OpenAI" Version="1.0.0-rc1" />
<PackageReference Include="Microsoft.Agents.AI.OpenAI" Version="1.0.0-rc4" />
</ItemGroup>
<!-- Add analyzers with compatible versions -->
@@ -36,11 +36,11 @@
</ItemGroup>
<ItemGroup>
<PackageReference Include="Azure.AI.AgentServer.AgentFramework" Version="1.0.0-beta.9" />
<PackageReference Include="Azure.AI.Projects" Version="1.2.0-beta.5" />
<PackageReference Include="Azure.AI.OpenAI" Version="2.8.0-beta.1" />
<PackageReference Include="Azure.AI.AgentServer.AgentFramework" Version="1.0.0-beta.11" />
<PackageReference Include="Azure.AI.Projects" Version="2.0.0-beta.1" />
<PackageReference Include="Azure.AI.OpenAI" Version="2.9.0-beta.1" />
<PackageReference Include="Azure.Identity" Version="1.17.1" />
<PackageReference Include="Microsoft.Agents.AI.OpenAI" Version="1.0.0-rc1" />
<PackageReference Include="Microsoft.Agents.AI.OpenAI" Version="1.0.0-rc4" />
</ItemGroup>
<!-- Add analyzers with compatible versions -->
@@ -1,4 +1,4 @@
<Project Sdk="Microsoft.NET.Sdk">
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
@@ -35,10 +35,10 @@
</ItemGroup>
<ItemGroup>
<PackageReference Include="Azure.AI.AgentServer.AgentFramework" Version="1.0.0-beta.9" />
<PackageReference Include="Azure.AI.OpenAI" Version="2.7.0-beta.2" />
<PackageReference Include="Azure.AI.AgentServer.AgentFramework" Version="1.0.0-beta.11" />
<PackageReference Include="Azure.AI.OpenAI" Version="2.9.0-beta.1" />
<PackageReference Include="Azure.Identity" Version="1.17.1" />
<PackageReference Include="Microsoft.Agents.AI.OpenAI" Version="1.0.0-rc1" />
<PackageReference Include="Microsoft.Agents.AI.OpenAI" Version="1.0.0-rc4" />
</ItemGroup>
<!-- Add analyzers with compatible versions -->
@@ -1,68 +0,0 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFrameworks>net10.0</TargetFrameworks>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
<!--
Disable central package management for this project.
This project requires explicit package references with versions specified inline rather than
inheriting them from Directory.Packages.props. This is necessary because a Docker image will
be created from this project, and the Docker build process only has access to this folder
and cannot access parent folders where Directory.Packages.props resides.
-->
<ManagePackageVersionsCentrally>false</ManagePackageVersionsCentrally>
</PropertyGroup>
<!--
Remove analyzer PackageReference items inherited from Directory.Packages.props.
Note: ManagePackageVersionsCentrally only controls PackageVersion items, not PackageReference items.
Directory.Packages.props contains both PackageVersion and PackageReference entries for analyzers,
and the PackageReference items are always inherited through MSBuild imports regardless of the
ManagePackageVersionsCentrally setting. We must explicitly remove them before adding our own versions.
-->
<ItemGroup>
<PackageReference Remove="Microsoft.CodeAnalysis.NetAnalyzers" />
<PackageReference Remove="Microsoft.VisualStudio.Threading.Analyzers" />
<PackageReference Remove="xunit.analyzers" />
<PackageReference Remove="Moq.Analyzers" />
<PackageReference Remove="Roslynator.Analyzers" />
<PackageReference Remove="Roslynator.CodeAnalysis.Analyzers" />
<PackageReference Remove="Roslynator.Formatting.Analyzers" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Azure.AI.AgentServer.AgentFramework" Version="1.0.0-beta.9" />
<PackageReference Include="Azure.AI.OpenAI" Version="2.8.0-beta.1" />
<PackageReference Include="Azure.Identity" Version="1.17.1" />
<PackageReference Include="Microsoft.Agents.AI.OpenAI" Version="1.0.0-rc1" />
</ItemGroup>
<!-- Add analyzers with compatible versions -->
<ItemGroup>
<PackageReference Include="Microsoft.CodeAnalysis.NetAnalyzers" Version="10.0.100">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="Microsoft.VisualStudio.Threading.Analyzers" Version="17.14.15">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="Roslynator.Analyzers" Version="4.14.1">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="Roslynator.CodeAnalysis.Analyzers" Version="4.14.1">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="Roslynator.Formatting.Analyzers" Version="4.14.1">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
</ItemGroup>
</Project>
@@ -1,20 +0,0 @@
# Build the application
FROM mcr.microsoft.com/dotnet/sdk:10.0-alpine AS build
WORKDIR /src
# Copy files from the current directory on the host to the working directory in the container
COPY . .
RUN dotnet restore
RUN dotnet build -c Release --no-restore
RUN dotnet publish -c Release --no-build -o /app -f net10.0
# Run the application
FROM mcr.microsoft.com/dotnet/aspnet:10.0-alpine AS final
WORKDIR /app
# Copy everything needed to run the app from the "build" stage.
COPY --from=build /app .
EXPOSE 8088
ENTRYPOINT ["dotnet", "AgentWithTools.dll"]
@@ -1,46 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
// This sample demonstrates how to use Foundry tools (MCP and code interpreter)
// with an AI agent hosted using the Azure AI AgentServer SDK.
using Azure.AI.AgentServer.AgentFramework.Extensions;
using Azure.AI.OpenAI;
using Azure.Identity;
using Microsoft.Agents.AI;
using Microsoft.Extensions.AI;
string openAiEndpoint = Environment.GetEnvironmentVariable("AZURE_OPENAI_ENDPOINT") ?? throw new InvalidOperationException("AZURE_OPENAI_ENDPOINT is not set.");
string deploymentName = Environment.GetEnvironmentVariable("AZURE_OPENAI_DEPLOYMENT_NAME") ?? "gpt-4o-mini";
string toolConnectionId = Environment.GetEnvironmentVariable("MCP_TOOL_CONNECTION_ID") ?? throw new InvalidOperationException("MCP_TOOL_CONNECTION_ID is not set.");
// WARNING: DefaultAzureCredential is convenient for development but requires careful consideration in production.
// In production, consider using a specific credential (e.g., ManagedIdentityCredential) to avoid
// latency issues, unintended credential probing, and potential security risks from fallback mechanisms.
DefaultAzureCredential credential = new();
IChatClient chatClient = new AzureOpenAIClient(new Uri(openAiEndpoint), credential)
.GetChatClient(deploymentName)
.AsIChatClient()
.AsBuilder()
.UseFoundryTools(new { type = "mcp", project_connection_id = toolConnectionId }, new { type = "code_interpreter" })
.UseOpenTelemetry(sourceName: "Agents", configure: (cfg) => cfg.EnableSensitiveData = true)
.Build();
AIAgent agent = chatClient.AsAIAgent(
name: "AgentWithTools",
instructions: @"You are a helpful assistant with access to tools for fetching Microsoft documentation.
IMPORTANT: When the user asks about Microsoft Learn articles or documentation:
1. You MUST use the microsoft_docs_fetch tool to retrieve the actual content
2. Do NOT rely on your training data
3. Always fetch the latest information from the provided URL
Available tools:
- microsoft_docs_fetch: Fetches and converts Microsoft Learn documentation
- microsoft_docs_search: Searches Microsoft/Azure documentation
- microsoft_code_sample_search: Searches for code examples")
.AsBuilder()
.UseOpenTelemetry(sourceName: "Agents", configure: (cfg) => cfg.EnableSensitiveData = true)
.Build();
await agent.RunAIAgentAsync(telemetrySourceName: "Agents");
@@ -1,45 +0,0 @@
# What this sample demonstrates
This sample demonstrates how to use Foundry tools with an AI agent via the `UseFoundryTools` extension. The agent is configured with two tool types: an MCP (Model Context Protocol) connection for fetching Microsoft Learn documentation and a code interpreter for running code when needed.
Key features:
- Configuring Foundry tools using `UseFoundryTools` with MCP and code interpreter
- Connecting to an external MCP tool via a Foundry project connection
- Using `DefaultAzureCredential` for Azure authentication
- OpenTelemetry instrumentation for both the chat client and agent
> For common prerequisites and setup instructions, see the [Hosted Agent Samples README](../README.md).
## Prerequisites
In addition to the common prerequisites:
1. An **Azure AI Foundry project** with a chat model deployed (e.g., `gpt-5.2`, `gpt-4o-mini`)
2. The **Azure AI Developer** role assigned on the Foundry resource (includes the `agents/write` data action required by `UseFoundryTools`)
3. An **MCP tool connection** configured in your Foundry project pointing to `https://learn.microsoft.com/api/mcp`
## Environment Variables
In addition to the common environment variables in the root README:
```powershell
# Your Azure AI Foundry project endpoint (required by UseFoundryTools)
$env:AZURE_AI_PROJECT_ENDPOINT="https://your-resource.services.ai.azure.com/api/projects/your-project"
# Chat model deployment name (defaults to gpt-4o-mini if not set)
$env:AZURE_OPENAI_DEPLOYMENT_NAME="gpt-4o-mini"
# The MCP tool connection name (just the name, not the full ARM resource ID)
$env:MCP_TOOL_CONNECTION_ID="SampleMCPTool"
```
## How It Works
1. An `AzureOpenAIClient` is created with `DefaultAzureCredential` and used to get a chat client
2. The chat client is wrapped with `UseFoundryTools` which registers two Foundry tool types:
- **MCP connection**: Connects to an external MCP server (Microsoft Learn) via the project connection name, providing documentation fetch and search capabilities
- **Code interpreter**: Allows the agent to execute code snippets when needed
3. `UseFoundryTools` resolves the connection using `AZURE_AI_PROJECT_ENDPOINT` internally
4. A `ChatClientAgent` is created with instructions guiding it to use the MCP tools for documentation queries
5. The agent is hosted using `RunAIAgentAsync` which exposes the OpenAI Responses-compatible API endpoint
@@ -1,31 +0,0 @@
name: AgentWithTools
displayName: "Agent with Tools"
description: >
An AI agent that uses Foundry tools (MCP and code interpreter) with Azure OpenAI.
The agent can fetch Microsoft Learn documentation and run code when needed.
metadata:
authors:
- Microsoft Agent Framework Team
tags:
- Azure AI AgentServer
- Microsoft Agent Framework
- Tools
- MCP
- Code Interpreter
template:
kind: hosted
name: AgentWithTools
protocols:
- protocol: responses
version: v1
environment_variables:
- name: AZURE_OPENAI_ENDPOINT
value: ${AZURE_OPENAI_ENDPOINT}
- name: AZURE_OPENAI_DEPLOYMENT_NAME
value: gpt-4o-mini
- name: MCP_TOOL_CONNECTION_ID
value: ${MCP_TOOL_CONNECTION_ID}
resources:
- name: "gpt-4o-mini"
kind: model
id: gpt-4o-mini
@@ -1,30 +0,0 @@
@host = http://localhost:8088
@endpoint = {{host}}/responses
### Health Check
GET {{host}}/readiness
### Simple string input
POST {{endpoint}}
Content-Type: application/json
{
"input": "Please use the microsoft_docs_fetch tool to fetch and summarize the Microsoft Learn article at https://learn.microsoft.com/azure/ai-services/openai/overview"
}
### Explicit input
POST {{endpoint}}
Content-Type: application/json
{
"input": [
{
"type": "message",
"role": "user",
"content": [
{
"type": "input_text",
"text": "Please use the microsoft_docs_fetch tool to fetch and summarize the Microsoft Learn article at https://learn.microsoft.com/azure/ai-services/openai/overview"
}
]
}
]
}
@@ -35,10 +35,10 @@
</ItemGroup>
<ItemGroup>
<PackageReference Include="Azure.AI.AgentServer.AgentFramework" Version="1.0.0-beta.9" />
<PackageReference Include="Azure.AI.OpenAI" Version="2.7.0-beta.2" />
<PackageReference Include="Azure.AI.AgentServer.AgentFramework" Version="1.0.0-beta.11" />
<PackageReference Include="Azure.AI.OpenAI" Version="2.9.0-beta.1" />
<PackageReference Include="Azure.Identity" Version="1.17.1" />
<PackageReference Include="Microsoft.Agents.AI.OpenAI" Version="1.0.0-rc1" />
<PackageReference Include="Microsoft.Agents.AI.OpenAI" Version="1.0.0-rc4" />
</ItemGroup>
<!-- Add analyzers with compatible versions -->
@@ -1,4 +1,4 @@
<Project Sdk="Microsoft.NET.Sdk">
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFrameworks>net10.0</TargetFrameworks>
@@ -33,12 +33,12 @@
</ItemGroup>
<ItemGroup>
<PackageReference Include="Azure.AI.AgentServer.AgentFramework" Version="1.0.0-beta.8" />
<PackageReference Include="Azure.AI.Projects" Version="1.2.0-beta.5" />
<PackageReference Include="Azure.AI.AgentServer.AgentFramework" Version="1.0.0-beta.11" />
<PackageReference Include="Azure.AI.Projects" Version="2.0.0-beta.1" />
<PackageReference Include="Azure.Identity" Version="1.17.1" />
<PackageReference Include="Microsoft.Agents.AI" Version="1.0.0-preview.251219.1" />
<PackageReference Include="Microsoft.Agents.AI.AzureAI" Version="1.0.0-preview.251219.1" />
<PackageReference Include="Microsoft.Agents.AI.Workflows" Version="1.0.0-preview.251219.1" />
<PackageReference Include="Microsoft.Agents.AI" Version="1.0.0-rc4" />
<PackageReference Include="Microsoft.Agents.AI.AzureAI" Version="1.0.0-rc4" />
<PackageReference Include="Microsoft.Agents.AI.Workflows" Version="1.0.0-rc4" />
<PackageReference Include="OpenTelemetry" Version="1.12.0" />
<PackageReference Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" Version="1.12.0" />
</ItemGroup>
@@ -41,7 +41,7 @@ try
.Build();
Console.WriteLine("Starting Writer-Reviewer Workflow Agent Server on http://localhost:8088");
await workflow.AsAgent().RunAIAgentAsync();
await workflow.AsAIAgent().RunAIAgentAsync();
}
finally
{
@@ -33,11 +33,11 @@
</ItemGroup>
<ItemGroup>
<PackageReference Include="Azure.AI.AgentServer.AgentFramework" Version="1.0.0-beta.8" />
<PackageReference Include="Azure.AI.Projects" Version="1.2.0-beta.5" />
<PackageReference Include="Azure.AI.AgentServer.AgentFramework" Version="1.0.0-beta.11" />
<PackageReference Include="Azure.AI.Projects" Version="2.0.0-beta.1" />
<PackageReference Include="Azure.Identity" Version="1.17.1" />
<PackageReference Include="Microsoft.Agents.AI" Version="1.0.0-preview.251219.1" />
<PackageReference Include="Microsoft.Agents.AI.AzureAI" Version="1.0.0-preview.251219.1" />
<PackageReference Include="Microsoft.Agents.AI" Version="1.0.0-rc4" />
<PackageReference Include="Microsoft.Agents.AI.AzureAI" Version="1.0.0-rc4" />
</ItemGroup>
<!-- Add analyzers with compatible versions -->
@@ -6,7 +6,6 @@ These samples demonstrate how to build and host AI agents using the [Azure AI Ag
| Sample | Description |
|--------|-------------|
| [`AgentWithTools`](./AgentWithTools/) | Foundry tools (MCP + code interpreter) via `UseFoundryTools` |
| [`AgentWithLocalTools`](./AgentWithLocalTools/) | Local C# function tool execution (Seattle hotel search) |
| [`AgentThreadAndHITL`](./AgentThreadAndHITL/) | Human-in-the-loop with `ApprovalRequiredAIFunction` and thread persistence |
| [`AgentWithHostedMCP`](./AgentWithHostedMCP/) | Hosted MCP server tool (Microsoft Learn search) |
@@ -40,19 +39,18 @@ Most samples require one or more of these environment variables:
|----------|---------|-------------|
| `AZURE_OPENAI_ENDPOINT` | Most samples | Your Azure OpenAI resource endpoint URL |
| `AZURE_OPENAI_DEPLOYMENT_NAME` | Most samples | Chat model deployment name (defaults to `gpt-4o-mini`) |
| `AZURE_AI_PROJECT_ENDPOINT` | AgentWithTools, AgentWithLocalTools, FoundryMultiAgent, FoundrySingleAgent | Azure AI Foundry project endpoint |
| `MCP_TOOL_CONNECTION_ID` | AgentWithTools | Foundry MCP tool connection name |
| `AZURE_AI_PROJECT_ENDPOINT` | AgentWithLocalTools, FoundryMultiAgent, FoundrySingleAgent | Azure AI Foundry project endpoint |
| `MODEL_DEPLOYMENT_NAME` | AgentWithLocalTools, FoundryMultiAgent, FoundrySingleAgent | Chat model deployment name (defaults to `gpt-4o-mini`) |
See each sample's README for the specific variables required.
## Azure AI Foundry Setup (for samples that use Foundry)
Some samples (`AgentWithTools`, `AgentWithLocalTools`) connect to an Azure AI Foundry project. If you're using these samples, you'll need additional setup.
Some samples (`AgentWithLocalTools`, `FoundrySingleAgent`, `FoundryMultiAgent`) connect to an Azure AI Foundry project. If you're using these samples, you'll need additional setup.
### Azure AI Developer Role
The `UseFoundryTools` extension requires the **Azure AI Developer** role on the Cognitive Services resource. Even if you created the project, you may not have this role by default.
Some Foundry operations require the **Azure AI Developer** role on the Cognitive Services resource. Even if you created the project, you may not have this role by default.
```powershell
az role assignment create `
@@ -65,23 +63,6 @@ az role assignment create `
For more details on permissions, see [Azure AI Foundry Permissions](https://aka.ms/FoundryPermissions).
### Creating an MCP Tool Connection
The `AgentWithTools` sample requires an MCP tool connection configured in your Foundry project:
1. Go to the [Azure AI Foundry portal](https://ai.azure.com)
2. Navigate to your project
3. Go to **Connected resources****+ New connection** → **Model Context Protocol tool**
4. Fill in:
- **Name**: `SampleMCPTool` (or any name you prefer)
- **Remote MCP Server endpoint**: `https://learn.microsoft.com/api/mcp`
- **Authentication**: `Unauthenticated`
5. Click **Connect**
The connection **name** (e.g., `SampleMCPTool`) is used as the `MCP_TOOL_CONNECTION_ID` environment variable.
> **Important**: Use only the connection **name**, not the full ARM resource ID.
## Running a Sample
Each sample runs as a standalone hosted agent on `http://localhost:8088/`:
@@ -110,14 +91,6 @@ Each sample includes a `Dockerfile` and `agent.yaml` for deployment. To deploy y
Assign the **Azure AI Developer** role to your user. See [Azure AI Developer Role](#azure-ai-developer-role) above.
### `Project connection ... was not found`
Make sure `MCP_TOOL_CONNECTION_ID` contains only the connection **name** (e.g., `SampleMCPTool`), not the full ARM resource ID path.
### `AZURE_AI_PROJECT_ENDPOINT must be set`
The `UseFoundryTools` extension requires `AZURE_AI_PROJECT_ENDPOINT`. Set it to your Foundry project endpoint (e.g., `https://your-resource.services.ai.azure.com/api/projects/your-project`).
### Multi-framework error when running `dotnet run`
If you see "Your project targets multiple frameworks", specify the framework:
@@ -122,7 +122,7 @@ internal sealed class AFAgentApplication : AgentApplication
&& valueElement.GetProperty("requestJson") is JsonElement requestJsonElement
&& requestJsonElement.ValueKind == JsonValueKind.String)
{
var requestContent = JsonSerializer.Deserialize<FunctionApprovalRequestContent>(requestJsonElement.GetString()!, JsonUtilities.DefaultOptions);
var requestContent = JsonSerializer.Deserialize<ToolApprovalRequestContent>(requestJsonElement.GetString()!, JsonUtilities.DefaultOptions);
return new ChatMessage(ChatRole.User, [requestContent!.CreateResponse(approvedJsonElement.ValueKind == JsonValueKind.True)]);
}
@@ -138,7 +138,7 @@ internal sealed class AFAgentApplication : AgentApplication
/// <param name="attachments">The list of <see cref="Attachment"/> to which the adaptive cards will be added.</param>
private static void HandleUserInputRequests(AgentResponse response, ref List<Attachment>? attachments)
{
foreach (FunctionApprovalRequestContent functionApprovalRequest in response.Messages.SelectMany(m => m.Contents).OfType<FunctionApprovalRequestContent>())
foreach (ToolApprovalRequestContent functionApprovalRequest in response.Messages.SelectMany(m => m.Contents).OfType<ToolApprovalRequestContent>())
{
var functionApprovalRequestJson = JsonSerializer.Serialize(functionApprovalRequest, JsonUtilities.DefaultOptions);
@@ -152,7 +152,7 @@ internal sealed class AFAgentApplication : AgentApplication
});
card.Body.Add(new AdaptiveTextBlock
{
Text = $"Function: {functionApprovalRequest.FunctionCall.Name}"
Text = $"Function: {((FunctionCallContent)functionApprovalRequest.ToolCall).Name}"
});
card.Body.Add(new AdaptiveActionSet()
{

Some files were not shown because too many files have changed in this diff Show More