mirror of
https://github.com/microsoft/agent-framework.git
synced 2026-06-16 21:04:09 +08:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
7cc5ff771b | ||
|
|
0269529ccf | ||
|
|
e514fc8837 | ||
|
|
049e823177 | ||
|
|
c4f9d0d4cf | ||
|
|
1929f73959 | ||
|
|
1b0fbb808e | ||
|
|
c799c61ff1 | ||
|
|
158ecb7b40 | ||
|
|
3c8fdb6f49 | ||
|
|
5ec3bcf390 | ||
|
|
9ce21a2a2f | ||
|
|
2a55b35176 | ||
|
|
08dcf74cf4 |
@@ -2,7 +2,7 @@ name: Merge Gatekeeper
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches: ["main", "feature*"]
|
||||
branches: [ "main", "feature*" ]
|
||||
merge_group:
|
||||
branches: ["main"]
|
||||
|
||||
@@ -13,105 +13,23 @@ concurrency:
|
||||
jobs:
|
||||
merge-gatekeeper:
|
||||
runs-on: ubuntu-latest
|
||||
# Restrict permissions of the GITHUB_TOKEN.
|
||||
# Docs: https://docs.github.com/en/actions/using-jobs/assigning-permissions-to-jobs
|
||||
permissions:
|
||||
checks: read
|
||||
statuses: read
|
||||
steps:
|
||||
- name: Wait for required checks
|
||||
- name: Run Merge Gatekeeper
|
||||
# NOTE: v1 is updated to reflect the latest v1.x.y. Please use any tag/branch that suits your needs:
|
||||
# https://github.com/upsidr/merge-gatekeeper/tags
|
||||
# https://github.com/upsidr/merge-gatekeeper/branches
|
||||
uses: upsidr/merge-gatekeeper@v1
|
||||
if: github.event_name == 'pull_request'
|
||||
uses: actions/github-script@v8
|
||||
env:
|
||||
TIMEOUT_SECONDS: "3600"
|
||||
INTERVAL_SECONDS: "30"
|
||||
SELF_JOB_NAME: ${{ github.job }}
|
||||
with:
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
timeout: 3600
|
||||
interval: 30
|
||||
# "Cleanup artifacts", "Agent", "Prepare", and "Upload results" are check runs
|
||||
# created by an org-level GitHub App (MSDO), not by any workflow in this repo.
|
||||
# They are outside our control and their transient failures should not block merges.
|
||||
IGNORED_NAMES: "CodeQL,CodeQL analysis (csharp),Cleanup artifacts,Agent,Prepare,Upload results"
|
||||
with:
|
||||
script: |
|
||||
const timeoutSeconds = Number(process.env.TIMEOUT_SECONDS);
|
||||
const intervalSeconds = Number(process.env.INTERVAL_SECONDS);
|
||||
const selfName = process.env.SELF_JOB_NAME;
|
||||
const ignored = new Set(
|
||||
process.env.IGNORED_NAMES.split(',').map((s) => s.trim()).filter(Boolean),
|
||||
);
|
||||
|
||||
const sha = context.payload.pull_request.head.sha;
|
||||
const { owner, repo } = context.repo;
|
||||
|
||||
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
|
||||
|
||||
// Mirrors upsidr/merge-gatekeeper: merge combined-statuses and check-runs
|
||||
// for the PR head SHA, with combined-statuses winning on name collision.
|
||||
async function collectChecks() {
|
||||
const merged = new Map();
|
||||
|
||||
const combined = await github.rest.repos.getCombinedStatusForRef({
|
||||
owner, repo, ref: sha, per_page: 100,
|
||||
});
|
||||
for (const s of combined.data.statuses ?? []) {
|
||||
if (!merged.has(s.context)) {
|
||||
// Combined-status states: success | pending | error | failure
|
||||
merged.set(s.context, { name: s.context, state: s.state });
|
||||
}
|
||||
}
|
||||
|
||||
const runs = await github.paginate(github.rest.checks.listForRef, {
|
||||
owner, repo, ref: sha, per_page: 100,
|
||||
});
|
||||
for (const r of runs) {
|
||||
if (merged.has(r.name)) continue;
|
||||
let state;
|
||||
if (r.status !== 'completed') {
|
||||
state = 'pending';
|
||||
} else if (r.conclusion === 'skipped') {
|
||||
continue; // Skipped runs are dropped, matching the original action.
|
||||
} else if (r.conclusion === 'success' || r.conclusion === 'neutral') {
|
||||
state = 'success';
|
||||
} else {
|
||||
// cancelled | timed_out | action_required | stale | failure
|
||||
state = 'error';
|
||||
}
|
||||
merged.set(r.name, { name: r.name, state });
|
||||
}
|
||||
|
||||
return [...merged.values()];
|
||||
}
|
||||
|
||||
function evaluate(entries) {
|
||||
const failed = [];
|
||||
const pending = [];
|
||||
const succeeded = [];
|
||||
for (const e of entries) {
|
||||
if (e.name === selfName || ignored.has(e.name)) continue;
|
||||
if (e.state === 'success') succeeded.push(e.name);
|
||||
else if (e.state === 'error' || e.state === 'failure') failed.push(e.name);
|
||||
else pending.push(e.name);
|
||||
}
|
||||
return { failed, pending, succeeded };
|
||||
}
|
||||
|
||||
const deadline = Date.now() + timeoutSeconds * 1000;
|
||||
for (;;) {
|
||||
const entries = await collectChecks();
|
||||
const { failed, pending, succeeded } = evaluate(entries);
|
||||
|
||||
core.info(
|
||||
`succeeded=${succeeded.length} pending=${pending.length} failed=${failed.length}`,
|
||||
);
|
||||
if (failed.length) {
|
||||
core.setFailed(`Failing checks: ${failed.join(', ')}`);
|
||||
return;
|
||||
}
|
||||
if (pending.length === 0) {
|
||||
core.info(`All required checks passed: ${succeeded.join(', ') || '(none)'}`);
|
||||
return;
|
||||
}
|
||||
if (Date.now() > deadline) {
|
||||
core.setFailed(`Timed out waiting for: ${pending.join(', ')}`);
|
||||
return;
|
||||
}
|
||||
core.info(`Waiting on (${pending.length}): ${pending.slice(0, 10).join(', ')}${pending.length > 10 ? ', …' : ''}`);
|
||||
await sleep(intervalSeconds * 1000);
|
||||
}
|
||||
ignored: CodeQL,CodeQL analysis (csharp),Cleanup artifacts,Agent,Prepare,Upload results
|
||||
|
||||
@@ -246,5 +246,3 @@ dotnet/filtered-*.slnx
|
||||
# Local tool state
|
||||
.omc/
|
||||
.omx/
|
||||
|
||||
**/issues/
|
||||
|
||||
@@ -1,191 +0,0 @@
|
||||
---
|
||||
status: accepted
|
||||
contact: lokitoth
|
||||
date: 2026-05-13
|
||||
deciders: lokitoth
|
||||
consulted:
|
||||
informed:
|
||||
---
|
||||
|
||||
# MessageMerger Streaming Merge Invariants
|
||||
|
||||
## Context and Problem Statement
|
||||
|
||||
`Microsoft.Agents.AI.Workflows.MessageMerger` is the internal component that
|
||||
folds a stream of `AgentResponseUpdate` items emitted by an agent (or by a
|
||||
hosting executor wrapping an agent) into a single `AgentResponse` for a turn.
|
||||
Multi-agent workflows (handoff, group chat, orchestration) rely on this
|
||||
merger to produce a coherent transcript even when updates arrive interleaved
|
||||
across responses, messages, and timestamps.
|
||||
|
||||
Prior to this change the contract that hosting executors and the merger
|
||||
should jointly enforce was implicit. The implementation also carried a small
|
||||
amount of dead state (`createdTimes`) that was collected but never consumed,
|
||||
suggesting that an earlier, timestamp-driven ordering scheme had been
|
||||
abandoned without documentation, and the live code still ran an unreliable
|
||||
`CreatedAt`-based sort that could reorder messages across concurrent agents
|
||||
inside a single workflow super-step. There were no tests pinning down the
|
||||
ordering or grouping behavior, so any future refactor risked silently
|
||||
regressing it.
|
||||
|
||||
The problem this ADR addresses is therefore: **what merge guarantees does
|
||||
`MessageMerger` make to its callers, and how do we lock those guarantees in
|
||||
without changing observable behavior in non-pathological cases?**
|
||||
|
||||
## Decision Drivers
|
||||
|
||||
- **A. Predictable ordering.** Developers consuming a merged `AgentResponse`
|
||||
must be able to reason about the order in which messages appear without
|
||||
having to know whether updates carried `CreatedAt`.
|
||||
- **B. Coherent multi-agent transcripts.** When several agents stream into
|
||||
one merger within a single workflow super-step, each agent's contribution
|
||||
must read as a contiguous block; and a step's updates must precede the
|
||||
next step's updates.
|
||||
- **C. Stable hosting-executor contract.** A turn must be addressable by a
|
||||
single `ResponseId`; updates without one are an exceptional, "dangling"
|
||||
case rather than the norm.
|
||||
- **D. Minimal behavioral change for non-pathological inputs.** This work
|
||||
is intended to document and test current behavior, not to alter what
|
||||
users see today in well-formed agent streams.
|
||||
- **E. Discoverability for future contributors.** Known sharp edges (e.g.
|
||||
cross-`ResponseId` ordering, dangling-update placement) should be written
|
||||
down so they are not rediscovered as bugs.
|
||||
|
||||
## Considered Options
|
||||
|
||||
1. **Option 1 — Document invariants, pin them with tests, and use pure
|
||||
emission/insertion order for both responses and the messages inside
|
||||
each response.** Removes the unreliable `CreatedAt`-based sort and the
|
||||
dead `createdTimes` collection. Behavioral change only for inputs that
|
||||
relied on `CreatedAt` to re-order updates after the fact — which the
|
||||
prior comparer could not do correctly anyway (non-transitive).
|
||||
2. **Option 2 — Rewrite `MessageMerger` to group strictly by `AgentId`
|
||||
(rather than `ResponseId`), and to use a stable, transitive comparer
|
||||
that mixes `CreatedAt` with insertion index.** Behavioral change; would
|
||||
also require updating hosting executors that currently assume
|
||||
`ResponseId`-based grouping.
|
||||
3. **Option 3 — Leave the code and tests as-is; capture edge cases only
|
||||
in a working note.** No code or test change.
|
||||
|
||||
## Decision Outcome
|
||||
|
||||
Chosen option: **Option 1 — Document invariants, pin them with tests,
|
||||
and use pure emission/insertion order; remove the dead `createdTimes`
|
||||
collection.**
|
||||
|
||||
This option satisfies driver A (predictable ordering: emission order is
|
||||
the simplest reasoning model), driver B (per-`ResponseId` grouping plus
|
||||
first-seen ordering across responses gives both per-agent blocks within a
|
||||
step and step-ordering across steps), driver C (single ResponseId per
|
||||
turn is unchanged), and driver E (invariants and edge cases are now
|
||||
written down and covered by tests).
|
||||
|
||||
Driver D (minimal behavioral change) is satisfied because well-formed
|
||||
agent streams already emit updates in the order they want to appear; the
|
||||
prior `CreatedAt`-based sort only mattered for pathological inputs (mixed
|
||||
or out-of-order timestamps from concurrent agents), and on those inputs
|
||||
the old comparer was non-transitive and therefore unreliable. Removing
|
||||
the sort makes those inputs deterministic — they now follow emission
|
||||
order — without disturbing the well-formed case.
|
||||
|
||||
Option 2 was rejected for this iteration because it changes what callers
|
||||
observe in well-formed flows and would require a coordinated change
|
||||
across hosting executors. It is a candidate for a follow-up ADR if the
|
||||
known edge cases below are reported as bugs in practice.
|
||||
|
||||
Option 3 was rejected because it leaves the invariants un-tested and the
|
||||
dead code in place, so the next refactor can break the contract without
|
||||
any signal.
|
||||
|
||||
### Invariants
|
||||
|
||||
The following invariants are now part of the contract of
|
||||
`MessageMerger`/hosting executors and are covered by tests in
|
||||
`MessageMergerTests`:
|
||||
|
||||
1. **Single `ResponseId` per turn.** Every `AgentResponseUpdate` produced
|
||||
by a hosting executor in a single agent turn shares one `ResponseId`.
|
||||
If the underlying agent does not supply one, the executor assigns it.
|
||||
Updates with `ResponseId == null` are treated as "dangling" and
|
||||
flattened into loose messages at the end of the merged response.
|
||||
2. **Pure emission-order preservation.** Within a `ResponseId` block,
|
||||
messages appear in the order their updates first arrived at the
|
||||
merger. Across `ResponseId` blocks, blocks appear in first-seen order.
|
||||
`CreatedAt` is **not** consulted when ordering messages or blocks —
|
||||
only when stamping the merged response and its child messages.
|
||||
3. **Per-`ResponseId` grouping (no interleaving).** Messages produced
|
||||
under one `ResponseId` are emitted as a contiguous block in the merged
|
||||
`AgentResponse`. Combined with Invariant 1, this yields:
|
||||
- **Within a workflow super-step with one agent**: all messages
|
||||
appear together, in emission order.
|
||||
- **Within a super-step with multiple agents**: each agent's messages
|
||||
are a contiguous block, ordered by which agent emitted first.
|
||||
- **Across super-steps**: a step's blocks all precede the next step's
|
||||
blocks, because the next step cannot start emitting until the
|
||||
current step's emissions have arrived at the merger.
|
||||
|
||||
### Consequences
|
||||
|
||||
- Good, because the merge contract is now explicit, regression-tested,
|
||||
and trivially reasoned about (it is just emission order with
|
||||
per-`ResponseId` grouping).
|
||||
- Good, because removing the unreliable `CreatedAt`-based sort eliminates
|
||||
a latent bug — the prior comparer was non-transitive on mixed-timestamp
|
||||
inputs, so `List<T>.Sort` could in principle return any of several
|
||||
orderings or throw on some runtimes.
|
||||
- Good, because removing the unused `createdTimes` collection eliminates
|
||||
a misleading code smell.
|
||||
- Good, because hosting-executor authors have a written checklist of
|
||||
invariants to satisfy.
|
||||
- Neutral, because well-formed agent streams (those that emit updates in
|
||||
the order they want to appear) see no change in output.
|
||||
- Bad, because callers who relied on a server-supplied `CreatedAt` to
|
||||
retro-correct out-of-order emissions will no longer see that
|
||||
correction — they must ensure emission order matches desired output
|
||||
order, or attach to `RawRepresentation` for original timestamps.
|
||||
|
||||
## Validation
|
||||
|
||||
- Unit tests in
|
||||
`dotnet/tests/Microsoft.Agents.AI.Workflows.UnitTests/MessageMergerTests.cs`
|
||||
cover each invariant:
|
||||
- Insertion-order preservation with no timestamps.
|
||||
- Insertion-order preservation with mixed timestamps (emission order
|
||||
wins over `CreatedAt`).
|
||||
- Determinism across repeated runs with mixed timestamps.
|
||||
- Per-`ResponseId` grouping for interleaved multi-agent streams within
|
||||
a single step.
|
||||
- Per-`ResponseId` grouping with distinct response ids.
|
||||
- Existing tests for assembly, function-call/result ordering, and
|
||||
`FinishReason` propagation continue to pass.
|
||||
|
||||
## More Information
|
||||
|
||||
### Known edge cases (intentionally not fixed in this ADR)
|
||||
|
||||
These are properties of the *current* `MessageMerger` that callers should
|
||||
be aware of. They are not invariants — they may change in a future ADR —
|
||||
but they are present in the shipped behavior covered by tests above.
|
||||
|
||||
| # | Edge case | Risk | Notes |
|
||||
|---|-----------|------|-------|
|
||||
| 1 | Cross-`ResponseId` ordering follows first-seen-`ResponseId` order, not chronological order across responses. | Medium | Acceptable today because each turn has a single `ResponseId` (Invariant 1); only matters if a caller deliberately interleaves multiple response ids inside one step. |
|
||||
| 2 | Updates with `ResponseId == null` are always emitted **after** all keyed responses, regardless of arrival time. | Medium | Documented as the "dangling" path; agents should always emit a `ResponseId`. |
|
||||
| 3 | Within a response, updates with `MessageId == null` are always emitted **after** keyed messages. | Low | Same rationale as #2, scoped to messages within a response. |
|
||||
| 4 | The merged response's `CreatedAt` is set to `DateTimeOffset.UtcNow`; per-response `CreatedAt` is propagated onto each contained `ChatMessage` instead of being preserved at the response level. | Low | Callers who need original per-update timestamps should read them from `RawRepresentation` or capture them before merging. |
|
||||
| 5 | Metadata on dangling (`ResponseId == null`) updates — `FinishReason`, `Usage`, `AgentId`, `AdditionalProperties` — is **not** merged into the final `AgentResponse`; only their `Messages` are surfaced. | Medium | Hosting executors must attach metadata to a keyed update if they want it reflected in the merged response. |
|
||||
| 6 | Emission order is the **only** ordering signal — `CreatedAt` differences between updates are ignored when ordering. | Low | This is the intended behavior under Invariant 2; producers must emit in the desired output order. |
|
||||
|
||||
If any of these become observable problems in production, the appropriate
|
||||
follow-up is a new ADR that supersedes this one (likely realizing
|
||||
"Option 2") rather than a silent fix.
|
||||
|
||||
### Code references
|
||||
|
||||
- `dotnet/src/Microsoft.Agents.AI.Workflows/MessageMerger.cs` — merger
|
||||
implementation. The previously-unused `createdTimes` collection and
|
||||
the `CreatedAt`-based sort have both been removed in this change.
|
||||
- `dotnet/tests/Microsoft.Agents.AI.Workflows.UnitTests/MessageMergerTests.cs` —
|
||||
invariant tests added in this change.
|
||||
- `AgentInvocationContext.ResponseId` and `ToStreamingResponseAsync` —
|
||||
the hosting-executor side of Invariant 1.
|
||||
@@ -298,7 +298,6 @@
|
||||
</Folder>
|
||||
<Folder Name="/Samples/03-workflows/Evaluation/">
|
||||
<Project Path="samples/03-workflows/Evaluation/Evaluation_WorkflowEval/Evaluation_WorkflowEval.csproj" />
|
||||
<Project Path="samples/03-workflows/Evaluation/Evaluation_WorkflowExpectedOutputs/Evaluation_WorkflowExpectedOutputs.csproj" />
|
||||
</Folder>
|
||||
<Folder Name="/Samples/04-hosting/">
|
||||
</Folder>
|
||||
@@ -583,7 +582,6 @@
|
||||
<Project Path="src/Microsoft.Agents.AI.DurableTask/Microsoft.Agents.AI.DurableTask.csproj" />
|
||||
<Project Path="src/Microsoft.Agents.AI.Foundry.Hosting/Microsoft.Agents.AI.Foundry.Hosting.csproj" />
|
||||
<Project Path="src/Microsoft.Agents.AI.Foundry/Microsoft.Agents.AI.Foundry.csproj" />
|
||||
<Project Path="src/Microsoft.Agents.AI.Harness/Microsoft.Agents.AI.Harness.csproj" />
|
||||
<Project Path="src/Microsoft.Agents.AI.GitHub.Copilot/Microsoft.Agents.AI.GitHub.Copilot.csproj" />
|
||||
<Project Path="src/Microsoft.Agents.AI.Hosting.A2A.AspNetCore/Microsoft.Agents.AI.Hosting.A2A.AspNetCore.csproj" />
|
||||
<Project Path="src/Microsoft.Agents.AI.Hosting.A2A/Microsoft.Agents.AI.Hosting.A2A.csproj" />
|
||||
@@ -638,7 +636,6 @@
|
||||
<Project Path="tests/Microsoft.Agents.AI.Foundry.UnitTests/Microsoft.Agents.AI.Foundry.UnitTests.csproj" />
|
||||
<Project Path="tests/Microsoft.Agents.AI.Foundry.Hosting.UnitTests/Microsoft.Agents.AI.Foundry.Hosting.UnitTests.csproj" />
|
||||
<Project Path="tests/Microsoft.Agents.AI.GitHub.Copilot.UnitTests/Microsoft.Agents.AI.GitHub.Copilot.UnitTests.csproj" />
|
||||
<Project Path="tests/Microsoft.Agents.AI.Harness.UnitTests/Microsoft.Agents.AI.Harness.UnitTests.csproj" />
|
||||
<Project Path="tests/Microsoft.Agents.AI.Hosting.A2A.UnitTests/Microsoft.Agents.AI.Hosting.A2A.UnitTests.csproj" />
|
||||
<Project Path="tests/Microsoft.Agents.AI.Hosting.AGUI.AspNetCore.UnitTests/Microsoft.Agents.AI.Hosting.AGUI.AspNetCore.UnitTests.csproj" />
|
||||
<Project Path="tests/Microsoft.Agents.AI.Hosting.AzureFunctions.UnitTests/Microsoft.Agents.AI.Hosting.AzureFunctions.UnitTests.csproj" />
|
||||
|
||||
@@ -7,7 +7,6 @@
|
||||
"src\\Microsoft.Agents.AI.AGUI\\Microsoft.Agents.AI.AGUI.csproj",
|
||||
"src\\Microsoft.Agents.AI.Anthropic\\Microsoft.Agents.AI.Anthropic.csproj",
|
||||
"src\\Microsoft.Agents.AI.GitHub.Copilot\\Microsoft.Agents.AI.GitHub.Copilot.csproj",
|
||||
"src\\Microsoft.Agents.AI.Harness\\Microsoft.Agents.AI.Harness.csproj",
|
||||
"src\\Microsoft.Agents.AI.AzureAI.Persistent\\Microsoft.Agents.AI.AzureAI.Persistent.csproj",
|
||||
"src\\Microsoft.Agents.AI.Foundry\\Microsoft.Agents.AI.Foundry.csproj",
|
||||
"src\\Microsoft.Agents.AI.Foundry.Hosting\\Microsoft.Agents.AI.Foundry.Hosting.csproj",
|
||||
|
||||
@@ -1,14 +1,14 @@
|
||||
<Project>
|
||||
<PropertyGroup>
|
||||
<!-- Central version prefix - applies to all nuget packages. -->
|
||||
<VersionPrefix>1.6.0</VersionPrefix>
|
||||
<VersionPrefix>1.5.0</VersionPrefix>
|
||||
<RCNumber>1</RCNumber>
|
||||
<DateSuffix>260512</DateSuffix>
|
||||
<DateSuffix>260507</DateSuffix>
|
||||
<PackageVersion Condition="'$(IsReleaseCandidate)' == 'true'">$(VersionPrefix)-rc$(RCNumber)</PackageVersion>
|
||||
<PackageVersion Condition="'$(IsReleaseCandidate)' != 'true' AND '$(VersionSuffix)' != ''">$(VersionPrefix)-$(VersionSuffix).$(DateSuffix).1</PackageVersion>
|
||||
<PackageVersion Condition="'$(IsReleaseCandidate)' != 'true' AND '$(VersionSuffix)' == ''">$(VersionPrefix)-preview.$(DateSuffix).1</PackageVersion>
|
||||
<PackageVersion Condition="'$(IsReleased)' == 'true'">$(VersionPrefix)</PackageVersion>
|
||||
<GitTag>1.6.0</GitTag>
|
||||
<GitTag>1.5.0</GitTag>
|
||||
|
||||
<Configurations>Debug;Release;Publish</Configurations>
|
||||
<IsPackable>true</IsPackable>
|
||||
|
||||
+8
-10
@@ -20,18 +20,22 @@ using OpenAI.Responses;
|
||||
#pragma warning disable OPENAI001 // Experimental API
|
||||
#pragma warning disable AAIP001 // AgentToolboxes is experimental
|
||||
|
||||
// Name of the toolbox to create and connect to.
|
||||
// Must match the `<name>` segment of FOUNDRY_TOOLBOX_ENDPOINT.
|
||||
const string ToolboxName = "research_toolbox";
|
||||
const string Query = "What tools do you have access to?";
|
||||
|
||||
string endpoint = Environment.GetEnvironmentVariable("AZURE_AI_PROJECT_ENDPOINT")
|
||||
?? throw new InvalidOperationException("AZURE_AI_PROJECT_ENDPOINT is not set.");
|
||||
string deploymentName = Environment.GetEnvironmentVariable("AZURE_AI_MODEL_DEPLOYMENT_NAME") ?? "gpt-5.4-mini";
|
||||
string toolboxEndpoint = Environment.GetEnvironmentVariable("FOUNDRY_TOOLBOX_ENDPOINT")
|
||||
?? throw new InvalidOperationException(
|
||||
"FOUNDRY_TOOLBOX_ENDPOINT is not set. Example: " +
|
||||
"https://<account>.services.ai.azure.com/api/projects/<project>/toolsets/<name>/mcp?api-version=2025-05-01-preview");
|
||||
|
||||
TokenCredential credential = new DefaultAzureCredential();
|
||||
|
||||
// Comment out if the toolbox already exists in your Foundry project.
|
||||
var toolboxEndpoint = await CreateSampleToolboxAsync(ToolboxName, endpoint, credential);
|
||||
await CreateSampleToolboxAsync(ToolboxName, endpoint, credential);
|
||||
|
||||
// Inject a fresh Azure AI bearer token on every MCP request.
|
||||
using var httpClient = new HttpClient(new BearerTokenHandler(credential, "https://ai.azure.com/.default")
|
||||
@@ -47,11 +51,6 @@ await using McpClient mcpClient = await McpClient.CreateAsync(
|
||||
{
|
||||
Endpoint = new Uri(toolboxEndpoint),
|
||||
Name = "foundry_toolbox",
|
||||
TransportMode = HttpTransportMode.StreamableHttp,
|
||||
AdditionalHeaders = new Dictionary<string, string>
|
||||
{
|
||||
["Foundry-Features"] = "Toolboxes=V1Preview",
|
||||
},
|
||||
},
|
||||
httpClient));
|
||||
|
||||
@@ -75,7 +74,7 @@ Console.WriteLine($"Assistant: {await agent.RunAsync(Query)}");
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helper: create (or replace) a sample toolbox so the sample runs end-to-end
|
||||
// ---------------------------------------------------------------------------
|
||||
static async Task<string> CreateSampleToolboxAsync(string name, string endpoint, TokenCredential credential)
|
||||
static async Task CreateSampleToolboxAsync(string name, string endpoint, TokenCredential credential)
|
||||
{
|
||||
// Toolboxes are normally configured in the Foundry portal or a deployment
|
||||
// script, not the application itself. This helper exists so the sample can
|
||||
@@ -104,13 +103,12 @@ static async Task<string> CreateSampleToolboxAsync(string name, string endpoint,
|
||||
serverUri: new Uri("https://gitmcp.io/Azure/azure-rest-api-specs"),
|
||||
toolCallApprovalPolicy: new McpToolCallApprovalPolicy(GlobalMcpToolCallApprovalPolicy.NeverRequireApproval)));
|
||||
|
||||
ToolboxVersion created = (await toolboxClient.CreateToolboxVersionAsync(
|
||||
var created = (await toolboxClient.CreateToolboxVersionAsync(
|
||||
name: name,
|
||||
tools: [mcpTool],
|
||||
description: "Sample toolbox with an MCP tool — created by Agent_Step25 sample.")).Value;
|
||||
|
||||
Console.WriteLine($"Created toolbox '{created.Name}' v{created.Version} ({created.Tools.Count} tool(s))");
|
||||
return $"{endpoint}/toolboxes/{created.Name}/mcp?api-version=v{created.Version}";
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
@@ -19,11 +19,10 @@ Set the following environment variables:
|
||||
```powershell
|
||||
$env:AZURE_AI_PROJECT_ENDPOINT="https://your-foundry-service.services.ai.azure.com/api/projects/your-foundry-project"
|
||||
$env:AZURE_AI_MODEL_DEPLOYMENT_NAME="gpt-5.4-mini"
|
||||
$env:FOUNDRY_TOOLBOX_ENDPOINT="https://your-foundry-service.services.ai.azure.com/api/projects/your-foundry-project/toolsets/research_toolbox/mcp?api-version=2025-05-01-preview"
|
||||
```
|
||||
|
||||
The sample creates a toolbox named `research_toolbox` in your Foundry project on
|
||||
startup, then connects to its MCP endpoint at
|
||||
`{AZURE_AI_PROJECT_ENDPOINT}/toolboxes/research_toolbox/mcp?api-version=v{version}`.
|
||||
The `<name>` segment of `FOUNDRY_TOOLBOX_ENDPOINT` must match the `ToolboxName` constant in `Program.cs`.
|
||||
|
||||
## Run the sample
|
||||
|
||||
|
||||
-1
@@ -13,7 +13,6 @@
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\..\..\src\Microsoft.Agents.AI.Harness\Microsoft.Agents.AI.Harness.csproj" />
|
||||
<ProjectReference Include="..\..\..\..\src\Microsoft.Agents.AI.OpenAI\Microsoft.Agents.AI.OpenAI.csproj" />
|
||||
<ProjectReference Include="..\Harness_Shared_Console\Harness_Shared_Console.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
// This sample demonstrates how to use a HarnessAgent with the Harness AIContextProviders
|
||||
// This sample demonstrates how to use a ChatClientAgent with the Harness AIContextProviders
|
||||
// (TodoProvider and AgentModeProvider) for interactive research tasks with web search
|
||||
// capabilities powered by Azure AI Foundry.
|
||||
// The agent plans research tasks, creates a todo list, gets user approval,
|
||||
@@ -17,6 +17,7 @@ using System.ClientModel.Primitives;
|
||||
using Azure.Identity;
|
||||
using Harness.Shared.Console;
|
||||
using Microsoft.Agents.AI;
|
||||
using Microsoft.Agents.AI.Compaction;
|
||||
using Microsoft.Extensions.AI;
|
||||
using OpenAI;
|
||||
using OpenAI.Responses;
|
||||
@@ -28,7 +29,7 @@ var deploymentName = Environment.GetEnvironmentVariable("AZURE_AI_MODEL_DEPLOYME
|
||||
const int MaxContextWindowTokens = 1_050_000;
|
||||
const int MaxOutputTokens = 128_000;
|
||||
|
||||
// Create a HarnessAgent with the Harness providers (TodoProvider and AgentModeProvider)
|
||||
// Create a ChatClientAgent with the Harness providers (TodoProvider and AgentModeProvider)
|
||||
// and research-focused instructions including the mandatory planning workflow.
|
||||
var instructions =
|
||||
"""
|
||||
@@ -109,9 +110,13 @@ var instructions =
|
||||
- Check for relevant previously downloaded data / findings before starting new research.
|
||||
""";
|
||||
|
||||
// Create the agent using AsHarnessAgent, which pre-configures function invocation,
|
||||
// per-service-call chat history persistence, and in-loop compaction.
|
||||
// Then wrap with UseToolApproval to allow auto-approving tools once confirmed.
|
||||
// Create a compaction strategy based on the model's context window.
|
||||
// gpt-5.4: 1,050,000 token context window, 128,000 max output tokens.
|
||||
// Defaults: tool result eviction at 50% of input budget, truncation at 80%.
|
||||
var compactionStrategy = new ContextWindowCompactionStrategy(
|
||||
maxContextWindowTokens: MaxContextWindowTokens,
|
||||
maxOutputTokens: MaxOutputTokens);
|
||||
|
||||
AIAgent agent =
|
||||
// Create an OpenAIClient that communicates with the Foundry responses service.
|
||||
new OpenAIClient(
|
||||
@@ -125,32 +130,49 @@ AIAgent agent =
|
||||
RetryPolicy = new ClientRetryPolicy(3) // Enable retries to improve resiliency.
|
||||
})
|
||||
.GetResponsesClient()
|
||||
.AsIChatClientWithStoredOutputDisabled(deploymentName) // We want to manage chat history locally (not stored in the responses service), so that we can manage compaction ourselves.
|
||||
.AsHarnessAgent(MaxContextWindowTokens, MaxOutputTokens, new HarnessAgentOptions
|
||||
{
|
||||
Name = "ResearchAgent",
|
||||
Description = "A research assistant that plans and executes research tasks.",
|
||||
AIContextProviders =
|
||||
[
|
||||
new TodoProvider(), // Add an AIContextProvider to allow the agent to create a TODO list, which is stored in the session.
|
||||
new AgentModeProvider(), // Add an AIContextProvider that tracks the agent mode and allows switching mode. Current mode is stored in the session.
|
||||
new FileMemoryProvider( // Add an AIContextProvider that can store memories in files under a session specific working folder.
|
||||
new FileSystemAgentFileStore(Path.Combine(AppContext.BaseDirectory, "agent-files")),
|
||||
(_) => new FileMemoryState() { WorkingFolder = DateTime.UtcNow.ToString("yyyyMMdd_HHmmss") + "_" + Guid.NewGuid().ToString() })
|
||||
],
|
||||
ChatOptions = new ChatOptions
|
||||
.AsIChatClientWithStoredOutputDisabled(deploymentName) // We want to manage chat history locally (not stored in the responses service), so that we can manage compaction ourselves.
|
||||
|
||||
// Build a ChatClient Pipeline
|
||||
.AsBuilder()
|
||||
.UseFunctionInvocation() // We are building our own stack from scratch so we need to include Function Invocation ourselves.
|
||||
.UseMessageInjection() // Allow message injection during the function call loop.
|
||||
.UsePerServiceCallChatHistoryPersistence() // Save chat history updates to the session after each service call, rather than only at the end of the run.
|
||||
.UseAIContextProviders(new CompactionProvider(compactionStrategy)) // Add Compaction before each service call to responses so that long function invocation loops don't overflow the context.
|
||||
|
||||
// Build our agent on top of the ChatClient Pipeline
|
||||
.BuildAIAgent(
|
||||
new ChatClientAgentOptions
|
||||
{
|
||||
Instructions = instructions,
|
||||
Tools =
|
||||
Name = "ResearchAgent",
|
||||
Description = "A research assistant that plans and executes research tasks.",
|
||||
UseProvidedChatClientAsIs = true, // Since we built our own stack from scratch we need to tell the agent not to also add defaults like Function Invocation.
|
||||
RequirePerServiceCallChatHistoryPersistence = true, // Since we are added the per service call persistence ChatClient, we need to tell the agent to not also store chat history at the end of the run.
|
||||
ChatHistoryProvider = new InMemoryChatHistoryProvider( // Store chat history in memory in the session object. Will persist if the session is persisted.
|
||||
new InMemoryChatHistoryProviderOptions
|
||||
{
|
||||
ChatReducer = compactionStrategy.AsChatReducer(), // Run compaction on the InMemory chat history when it gets too large.
|
||||
}),
|
||||
AIContextProviders =
|
||||
[
|
||||
ResponseTool.CreateWebSearchTool().AsAITool(), // Add the foundry hosted web search tool that runs in the service.
|
||||
new WebBrowsingTool( // Add a local web browsing tool that converts html to markdown.
|
||||
new WebBrowsingToolOptions { AllowPublicNetworks = true }),
|
||||
new TodoProvider(), // Add an AIContextProvider to allow the agent to create a TODO list, which is stored in the session.
|
||||
new AgentModeProvider(), // Add an AIContextProvider that tracks the agent mode and allows switching mode. Current mode is stored in the session.
|
||||
new FileMemoryProvider( // Add an AIContextProvider that can store memories in files under a session specific working folder.
|
||||
new FileSystemAgentFileStore(Path.Combine(AppContext.BaseDirectory, "agent-files")),
|
||||
(_) => new FileMemoryState() { WorkingFolder = DateTime.UtcNow.ToString("yyyyMMdd_HHmmss") + "_" + Guid.NewGuid().ToString() })
|
||||
],
|
||||
MaxOutputTokens = MaxOutputTokens, // Set a high token limit for long research tasks with many tool calls and long outputs.
|
||||
Reasoning = new() { Effort = ReasoningEffort.Medium },
|
||||
},
|
||||
})
|
||||
ChatOptions = new ChatOptions
|
||||
{
|
||||
Instructions = instructions,
|
||||
Tools =
|
||||
[
|
||||
ResponseTool.CreateWebSearchTool().AsAITool(), // Add the foundry hosted web search tool that runs in the service.
|
||||
new WebBrowsingTool( // Add a local web browsing tool that converts html to markdown.
|
||||
new WebBrowsingToolOptions { AllowPublicNetworks = true }),
|
||||
],
|
||||
MaxOutputTokens = MaxOutputTokens, // Set a high token limit for long research tasks with many tool calls and long outputs.
|
||||
Reasoning = new() { Effort = ReasoningEffort.Medium },
|
||||
},
|
||||
})
|
||||
.AsBuilder()
|
||||
.UseToolApproval() // Add the ability to auto approve tools once a user has said they don't want to be asked again. Approval rules are tied to the session.
|
||||
.Build();
|
||||
|
||||
@@ -1,11 +1,10 @@
|
||||
# What this sample demonstrates
|
||||
|
||||
This sample demonstrates how to use a `HarnessAgent` with the Harness `AIContextProviders` (`TodoProvider` and `AgentModeProvider`) for interactive research tasks with web search capabilities powered by Azure AI Foundry. The `HarnessAgent` pre-configures function invocation, per-service-call chat history persistence, and context-window compaction.
|
||||
This sample demonstrates how to use a `ChatClientAgent` with the Harness `AIContextProviders` (`TodoProvider` and `AgentModeProvider`) for interactive research tasks with web search capabilities powered by Azure AI Foundry.
|
||||
|
||||
Key features showcased:
|
||||
|
||||
- **HarnessAgent** — a pre-configured agent that wraps a `ChatClientAgent` with function invocation, per-service-call persistence, and context-window compaction
|
||||
- **ToolApproval** — the agent is wrapped with `UseToolApproval()` to allow auto-approving tools once confirmed
|
||||
- **ChatClientAgent** — configured directly with Harness providers for planning and task management
|
||||
- **Web Search** — the agent can search the web for current information via `ResponseTool.CreateWebSearchTool()`
|
||||
- **TodoProvider** — the agent creates and manages a todo list to track research questions
|
||||
- **AgentModeProvider** — the agent switches between "plan" mode (breaking down the topic) and "execute" mode (answering each research question)
|
||||
|
||||
-1
@@ -13,7 +13,6 @@
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\..\..\src\Microsoft.Agents.AI.Harness\Microsoft.Agents.AI.Harness.csproj" />
|
||||
<ProjectReference Include="..\..\..\..\src\Microsoft.Agents.AI.OpenAI\Microsoft.Agents.AI.OpenAI.csproj" />
|
||||
<ProjectReference Include="..\Harness_Shared_Console\Harness_Shared_Console.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
@@ -22,9 +22,6 @@ using OpenAI.Responses;
|
||||
var endpoint = Environment.GetEnvironmentVariable("AZURE_FOUNDRY_OPENAI_ENDPOINT") ?? throw new InvalidOperationException("AZURE_FOUNDRY_OPENAI_ENDPOINT is not set.");
|
||||
var deploymentName = Environment.GetEnvironmentVariable("AZURE_AI_MODEL_DEPLOYMENT_NAME") ?? "gpt-5.4";
|
||||
|
||||
const int MaxContextWindowTokens = 1_050_000;
|
||||
const int MaxOutputTokens = 128_000;
|
||||
|
||||
// --- Sub-agent: Web Search Agent ---
|
||||
// This agent can search the web and is used by the parent agent to look up stock prices.
|
||||
AIAgent webSearchAgent =
|
||||
@@ -37,19 +34,20 @@ AIAgent webSearchAgent =
|
||||
})
|
||||
.GetResponsesClient()
|
||||
.AsIChatClientWithStoredOutputDisabled(deploymentName)
|
||||
.AsHarnessAgent(MaxContextWindowTokens, MaxOutputTokens, new HarnessAgentOptions
|
||||
{
|
||||
Name = "WebSearchAgent",
|
||||
Description = "An agent that can search the web to find information.",
|
||||
ChatOptions = new ChatOptions
|
||||
.AsAIAgent(
|
||||
new ChatClientAgentOptions
|
||||
{
|
||||
Instructions = "You are a web search assistant. When asked to find information, use the web search tool to look it up and return a concise, factual answer.",
|
||||
Tools =
|
||||
[
|
||||
ResponseTool.CreateWebSearchTool().AsAITool(),
|
||||
],
|
||||
},
|
||||
});
|
||||
Name = "WebSearchAgent",
|
||||
Description = "An agent that can search the web to find information.",
|
||||
ChatOptions = new ChatOptions
|
||||
{
|
||||
Instructions = "You are a web search assistant. When asked to find information, use the web search tool to look it up and return a concise, factual answer.",
|
||||
Tools =
|
||||
[
|
||||
ResponseTool.CreateWebSearchTool().AsAITool(),
|
||||
],
|
||||
},
|
||||
});
|
||||
|
||||
// --- Parent agent: Stock Price Researcher ---
|
||||
// This agent orchestrates the sub-agent to look up stock prices in parallel.
|
||||
@@ -85,20 +83,21 @@ AIAgent parentAgent =
|
||||
})
|
||||
.GetResponsesClient()
|
||||
.AsIChatClientWithStoredOutputDisabled(deploymentName)
|
||||
.AsHarnessAgent(MaxContextWindowTokens, MaxOutputTokens, new HarnessAgentOptions
|
||||
{
|
||||
Name = "StockPriceResearcher",
|
||||
Description = "An agent that researches stock prices using sub-agents.",
|
||||
AIContextProviders =
|
||||
[
|
||||
new SubAgentsProvider([webSearchAgent]),
|
||||
],
|
||||
ChatOptions = new ChatOptions
|
||||
.AsAIAgent(
|
||||
new ChatClientAgentOptions
|
||||
{
|
||||
Instructions = parentInstructions,
|
||||
MaxOutputTokens = 16_000,
|
||||
},
|
||||
});
|
||||
Name = "StockPriceResearcher",
|
||||
Description = "An agent that researches stock prices using sub-agents.",
|
||||
AIContextProviders =
|
||||
[
|
||||
new SubAgentsProvider([webSearchAgent]),
|
||||
],
|
||||
ChatOptions = new ChatOptions
|
||||
{
|
||||
Instructions = parentInstructions,
|
||||
MaxOutputTokens = 16_000,
|
||||
},
|
||||
});
|
||||
|
||||
// Run the interactive console session.
|
||||
await HarnessConsole.RunAgentAsync(
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# Harness Step 02 — SubAgents (Stock Price Research)
|
||||
|
||||
This sample demonstrates how to use the **SubAgentsProvider** to delegate work from a parent agent to sub-agents. Both agents use `HarnessAgent` for pre-configured function invocation, per-service-call persistence, and context-window compaction.
|
||||
This sample demonstrates how to use the **SubAgentsProvider** to delegate work from a parent agent to sub-agents.
|
||||
|
||||
## What It Does
|
||||
|
||||
|
||||
-1
@@ -13,7 +13,6 @@
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\..\..\src\Microsoft.Agents.AI.Harness\Microsoft.Agents.AI.Harness.csproj" />
|
||||
<ProjectReference Include="..\..\..\..\src\Microsoft.Agents.AI.OpenAI\Microsoft.Agents.AI.OpenAI.csproj" />
|
||||
<ProjectReference Include="..\Harness_Shared_Console\Harness_Shared_Console.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
// This sample demonstrates how to use a HarnessAgent with the FileAccessProvider
|
||||
// This sample demonstrates how to use a ChatClientAgent with the FileAccessProvider
|
||||
// to give an agent access to a folder of CSV data files. The agent can read, analyze,
|
||||
// and extract information from the data, then write results back as new files.
|
||||
//
|
||||
@@ -17,6 +17,7 @@ using System.ClientModel.Primitives;
|
||||
using Azure.Identity;
|
||||
using Harness.Shared.Console;
|
||||
using Microsoft.Agents.AI;
|
||||
using Microsoft.Agents.AI.Compaction;
|
||||
using Microsoft.Extensions.AI;
|
||||
using OpenAI;
|
||||
using OpenAI.Responses;
|
||||
@@ -56,7 +57,11 @@ var instructions =
|
||||
- Always explain what you learned and what you are going to do next between tool calls, so the user can follow along with your thought process.
|
||||
""";
|
||||
|
||||
// Create the chat client from the OpenAI provider.
|
||||
// Create a compaction strategy based on the model's context window.
|
||||
var compactionStrategy = new ContextWindowCompactionStrategy(
|
||||
maxContextWindowTokens: MaxContextWindowTokens,
|
||||
maxOutputTokens: MaxOutputTokens);
|
||||
|
||||
AIAgent agent =
|
||||
new OpenAIClient(
|
||||
new BearerTokenPolicy(new DefaultAzureCredential(), "https://ai.azure.com/.default"),
|
||||
@@ -67,20 +72,36 @@ AIAgent agent =
|
||||
})
|
||||
.GetResponsesClient()
|
||||
.AsIChatClientWithStoredOutputDisabled(deploymentName)
|
||||
.AsHarnessAgent(MaxContextWindowTokens, MaxOutputTokens, new HarnessAgentOptions
|
||||
{
|
||||
Name = "DataAnalyst",
|
||||
Description = "A data analyst assistant that reads, analyzes, and processes data files.",
|
||||
AIContextProviders =
|
||||
[
|
||||
new FileAccessProvider(fileStore),
|
||||
],
|
||||
ChatOptions = new ChatOptions
|
||||
|
||||
.AsBuilder()
|
||||
.UseFunctionInvocation()
|
||||
.UsePerServiceCallChatHistoryPersistence()
|
||||
.UseAIContextProviders(new CompactionProvider(compactionStrategy))
|
||||
|
||||
.BuildAIAgent(
|
||||
new ChatClientAgentOptions
|
||||
{
|
||||
Instructions = instructions,
|
||||
MaxOutputTokens = MaxOutputTokens,
|
||||
},
|
||||
});
|
||||
Name = "DataAnalyst",
|
||||
Description = "A data analyst assistant that reads, analyzes, and processes data files.",
|
||||
UseProvidedChatClientAsIs = true,
|
||||
RequirePerServiceCallChatHistoryPersistence = true,
|
||||
ChatHistoryProvider = new InMemoryChatHistoryProvider(
|
||||
new InMemoryChatHistoryProviderOptions
|
||||
{
|
||||
ChatReducer = compactionStrategy.AsChatReducer(),
|
||||
}),
|
||||
AIContextProviders =
|
||||
[
|
||||
new FileAccessProvider(fileStore),
|
||||
],
|
||||
ChatOptions = new ChatOptions
|
||||
{
|
||||
Instructions = instructions,
|
||||
MaxOutputTokens = MaxOutputTokens,
|
||||
},
|
||||
})
|
||||
.AsBuilder()
|
||||
.Build();
|
||||
|
||||
// Run the interactive console session.
|
||||
await HarnessConsole.RunAgentAsync(
|
||||
|
||||
@@ -1,10 +1,9 @@
|
||||
# What this sample demonstrates
|
||||
|
||||
This sample demonstrates how to use a `HarnessAgent` with the `FileAccessProvider` to give an agent access to a folder of data files for reading, analyzing, and writing results. The `HarnessAgent` pre-configures function invocation, per-service-call chat history persistence, and in-loop compaction — so the sample only needs to supply the chat client, token limits, and application-specific options.
|
||||
This sample demonstrates how to use a `ChatClientAgent` with the `FileAccessProvider` to give an agent access to a folder of data files for reading, analyzing, and writing results.
|
||||
|
||||
Key features showcased:
|
||||
|
||||
- **HarnessAgent** — a pre-configured agent that wraps a `ChatClientAgent` with function invocation, per-service-call persistence, and context-window compaction
|
||||
- **FileAccessProvider** — gives the agent tools to read, write, list, search, and delete files in a shared data folder
|
||||
- **CSV data processing** — the agent reads sales transaction data and performs analysis on demand
|
||||
- **Output file creation** — the agent can write summaries, filtered data, or reports back to the data folder
|
||||
|
||||
-16
@@ -1,16 +0,0 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<OutputType>Exe</OutputType>
|
||||
<TargetFrameworks>net10.0</TargetFrameworks>
|
||||
|
||||
<Nullable>enable</Nullable>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\..\..\src\Microsoft.Agents.AI.Foundry\Microsoft.Agents.AI.Foundry.csproj" />
|
||||
<ProjectReference Include="..\..\..\..\src\Microsoft.Agents.AI.Workflows\Microsoft.Agents.AI.Workflows.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
@@ -1,76 +0,0 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
// This sample demonstrates evaluating a multi-agent workflow against a
|
||||
// golden answer using Foundry's reference-based Similarity evaluator.
|
||||
|
||||
using Azure.AI.Projects;
|
||||
using Azure.Identity;
|
||||
using Microsoft.Agents.AI;
|
||||
using Microsoft.Agents.AI.Workflows;
|
||||
using Microsoft.Extensions.AI;
|
||||
using FoundryEvals = Microsoft.Agents.AI.Foundry.FoundryEvals;
|
||||
|
||||
string endpoint = Environment.GetEnvironmentVariable("AZURE_AI_PROJECT_ENDPOINT")
|
||||
?? throw new InvalidOperationException("AZURE_AI_PROJECT_ENDPOINT is not set.");
|
||||
string deploymentName = Environment.GetEnvironmentVariable("AZURE_AI_MODEL_DEPLOYMENT_NAME") ?? "gpt-4o-mini";
|
||||
|
||||
// WARNING: DefaultAzureCredential is convenient for development but requires careful consideration in production.
|
||||
// In production, consider using a specific credential (e.g., ManagedIdentityCredential) to avoid
|
||||
// latency issues, unintended credential probing, and potential security risks from fallback mechanisms.
|
||||
AIProjectClient projectClient = new(new Uri(endpoint), new DefaultAzureCredential());
|
||||
|
||||
// Build a two-agent workflow: a researcher writes a draft answer, then an
|
||||
// editor polishes it into the final response that we compare to ground truth.
|
||||
// EmitAgentResponseEvents is enabled so the workflow surfaces an AgentResponseEvent
|
||||
// for each agent — this is what EvaluateAsync uses to find the overall final answer.
|
||||
var hostOptions = new AIAgentHostOptions { EmitAgentResponseEvents = true };
|
||||
|
||||
AIAgent researcher = projectClient.AsAIAgent(
|
||||
model: deploymentName,
|
||||
instructions: "You research questions and produce a short factual draft answer.",
|
||||
name: "researcher");
|
||||
|
||||
AIAgent editor = projectClient.AsAIAgent(
|
||||
model: deploymentName,
|
||||
instructions: "You take a draft answer and produce the final concise response.",
|
||||
name: "editor");
|
||||
|
||||
ExecutorBinding researcherExecutor = researcher.BindAsExecutor(hostOptions);
|
||||
ExecutorBinding editorExecutor = editor.BindAsExecutor(hostOptions);
|
||||
|
||||
Workflow workflow = new WorkflowBuilder(researcherExecutor)
|
||||
.AddEdge(researcherExecutor, editorExecutor)
|
||||
.Build();
|
||||
|
||||
// Run the workflow against the user question.
|
||||
const string Query = "What is the capital of France?";
|
||||
const string GroundTruth = "Paris";
|
||||
|
||||
await using Run run = await InProcessExecution.RunAsync(
|
||||
workflow,
|
||||
new ChatMessage(ChatRole.User, Query));
|
||||
|
||||
// Evaluate the overall workflow output against a golden answer using the
|
||||
// reference-based Similarity evaluator. The 'expectedOutput' value is stamped
|
||||
// onto the overall EvalItem.ExpectedOutput and is surfaced to Foundry as
|
||||
// `ground_truth` in the underlying JSONL payload.
|
||||
//
|
||||
// Per-agent breakdown is disabled here: ground truth applies to the workflow's
|
||||
// final answer, not to each sub-agent's intermediate output. Without
|
||||
// includePerAgent: false, the evaluator would be invoked for per-agent items
|
||||
// (which have no ExpectedOutput) and Similarity would fail validation.
|
||||
FoundryEvals similarity = new(projectClient, deploymentName, FoundryEvals.Similarity);
|
||||
|
||||
AgentEvaluationResults results = await run.EvaluateAsync(
|
||||
similarity,
|
||||
includePerAgent: false,
|
||||
expectedOutput: GroundTruth);
|
||||
|
||||
Console.WriteLine($"Query: {Query}");
|
||||
Console.WriteLine($"Expected: {GroundTruth}");
|
||||
Console.WriteLine($"Provider: {results.ProviderName}");
|
||||
Console.WriteLine($"Passed: {results.Passed}/{results.Total}");
|
||||
if (results.ReportUrl is not null)
|
||||
{
|
||||
Console.WriteLine($"Report: {results.ReportUrl}");
|
||||
}
|
||||
@@ -1,37 +0,0 @@
|
||||
# Evaluation - Workflow Expected Outputs
|
||||
|
||||
This sample demonstrates evaluating a multi-agent workflow's final answer
|
||||
against a golden expected output using Foundry's reference-based **Similarity**
|
||||
evaluator.
|
||||
|
||||
## What this sample demonstrates
|
||||
|
||||
- Building a small researcher → editor workflow
|
||||
- Running the workflow and obtaining a `Run`
|
||||
- Calling `run.EvaluateAsync(evaluator, expectedOutput: ...)` to attach a
|
||||
ground-truth answer to the overall workflow item
|
||||
- Using `FoundryEvals.Similarity`, which requires a `ground_truth` value
|
||||
per item
|
||||
|
||||
The `expectedOutput` value is stamped onto the overall `EvalItem.ExpectedOutput`
|
||||
and is surfaced to Foundry as `ground_truth` in the JSONL payload sent to the
|
||||
Evals API.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- .NET 10 SDK or later
|
||||
- Azure CLI installed and authenticated (`az login`)
|
||||
|
||||
Set the following environment variables:
|
||||
|
||||
```powershell
|
||||
$env:AZURE_AI_PROJECT_ENDPOINT="https://your-foundry-service.services.ai.azure.com/api/projects/your-foundry-project"
|
||||
$env:AZURE_AI_MODEL_DEPLOYMENT_NAME="gpt-4o-mini"
|
||||
```
|
||||
|
||||
## Run the sample
|
||||
|
||||
```powershell
|
||||
cd dotnet/samples/03-workflows/Evaluation
|
||||
dotnet run --project .\Evaluation_WorkflowExpectedOutputs
|
||||
```
|
||||
@@ -163,22 +163,10 @@ app.MapA2AHttpJson(knightsKnavesAgentBuilder, path: "/a2a/knights-and-knaves");
|
||||
app.MapDevUI();
|
||||
|
||||
app.MapOpenAIResponses();
|
||||
app.MapOpenAIResponses(pirateAgentBuilder);
|
||||
app.MapOpenAIResponses(knightsKnavesAgentBuilder);
|
||||
app.MapOpenAIResponses(chemistryAgent);
|
||||
app.MapOpenAIResponses(mathsAgent);
|
||||
app.MapOpenAIResponses(literatureAgent);
|
||||
app.MapOpenAIResponses(scienceSequentialWorkflow);
|
||||
app.MapOpenAIResponses(scienceConcurrentWorkflow);
|
||||
app.MapOpenAIConversations();
|
||||
|
||||
app.MapOpenAIChatCompletions(pirateAgentBuilder);
|
||||
app.MapOpenAIChatCompletions(knightsKnavesAgentBuilder);
|
||||
app.MapOpenAIChatCompletions(chemistryAgent);
|
||||
app.MapOpenAIChatCompletions(mathsAgent);
|
||||
app.MapOpenAIChatCompletions(literatureAgent);
|
||||
app.MapOpenAIChatCompletions(scienceSequentialWorkflow);
|
||||
app.MapOpenAIChatCompletions(scienceConcurrentWorkflow);
|
||||
|
||||
// Map the agents HTTP endpoints
|
||||
app.MapAgentDiscovery("/agents");
|
||||
|
||||
+1
-1
@@ -24,7 +24,7 @@ internal sealed class OpenAIChatCompletionsAgentClient(HttpClient httpClient) :
|
||||
{
|
||||
OpenAIClientOptions options = new()
|
||||
{
|
||||
Endpoint = new Uri(httpClient.BaseAddress!, $"/{Uri.EscapeDataString(agentName)}/v1/"),
|
||||
Endpoint = new Uri(httpClient.BaseAddress!, $"/{agentName}/v1/"),
|
||||
Transport = new HttpClientPipelineTransport(httpClient)
|
||||
};
|
||||
|
||||
|
||||
+1
-1
@@ -23,7 +23,7 @@ internal sealed class OpenAIResponsesAgentClient(HttpClient httpClient) : AgentC
|
||||
{
|
||||
OpenAIClientOptions options = new()
|
||||
{
|
||||
Endpoint = new Uri(httpClient.BaseAddress!, $"/{Uri.EscapeDataString(agentName)}/v1/"),
|
||||
Endpoint = new Uri(httpClient.BaseAddress!, "/v1/"),
|
||||
Transport = new HttpClientPipelineTransport(httpClient)
|
||||
};
|
||||
|
||||
|
||||
@@ -130,7 +130,6 @@ internal static class FoundryEvalConverter
|
||||
QueryMessages = ConvertMessages(queryMessages),
|
||||
ResponseMessages = ConvertMessages(responseMessages),
|
||||
Context = item.Context,
|
||||
GroundTruth = item.ExpectedOutput,
|
||||
ToolDefinitions = item.Tools is { Count: > 0 }
|
||||
? item.Tools
|
||||
.OfType<AIFunction>()
|
||||
@@ -186,11 +185,6 @@ internal static class FoundryEvalConverter
|
||||
dataMapping["context"] = "{{item.context}}";
|
||||
}
|
||||
|
||||
if (GroundTruthEvaluators.Contains(qualified))
|
||||
{
|
||||
dataMapping["ground_truth"] = "{{item.ground_truth}}";
|
||||
}
|
||||
|
||||
if (ToolEvaluators.Contains(qualified))
|
||||
{
|
||||
dataMapping["tool_definitions"] = "{{item.tool_definitions}}";
|
||||
@@ -212,7 +206,7 @@ internal static class FoundryEvalConverter
|
||||
/// <summary>
|
||||
/// Builds the <c>item_schema</c> for custom JSONL eval definitions.
|
||||
/// </summary>
|
||||
internal static WireItemSchema BuildItemSchema(bool hasContext = false, bool hasTools = false, bool hasGroundTruth = false)
|
||||
internal static WireItemSchema BuildItemSchema(bool hasContext = false, bool hasTools = false)
|
||||
{
|
||||
var properties = new Dictionary<string, WireSchemaProperty>
|
||||
{
|
||||
@@ -227,11 +221,6 @@ internal static class FoundryEvalConverter
|
||||
properties["context"] = new WireSchemaProperty { Type = "string" };
|
||||
}
|
||||
|
||||
if (hasGroundTruth)
|
||||
{
|
||||
properties["ground_truth"] = new WireSchemaProperty { Type = "string" };
|
||||
}
|
||||
|
||||
if (hasTools)
|
||||
{
|
||||
properties["tool_definitions"] = new WireSchemaProperty { Type = "array" };
|
||||
@@ -244,31 +233,6 @@ internal static class FoundryEvalConverter
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns the subset of <paramref name="evaluators"/> that require a ground-truth
|
||||
/// (reference) value but cannot be evaluated because no item provided one.
|
||||
/// </summary>
|
||||
internal static List<string> FindMissingGroundTruthEvaluators(
|
||||
IEnumerable<string> evaluators,
|
||||
bool hasGroundTruth)
|
||||
{
|
||||
if (hasGroundTruth)
|
||||
{
|
||||
return [];
|
||||
}
|
||||
|
||||
var missing = new List<string>();
|
||||
foreach (var name in evaluators)
|
||||
{
|
||||
if (GroundTruthEvaluators.Contains(ResolveEvaluator(name)))
|
||||
{
|
||||
missing.Add(name);
|
||||
}
|
||||
}
|
||||
|
||||
return missing;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Resolves a short evaluator name to its fully-qualified <c>builtin.*</c> form.
|
||||
/// </summary>
|
||||
@@ -313,12 +277,6 @@ internal static class FoundryEvalConverter
|
||||
"builtin.tool_call_success",
|
||||
};
|
||||
|
||||
// Evaluators that require a ground_truth (reference) value per item.
|
||||
internal static readonly HashSet<string> GroundTruthEvaluators = new(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
"builtin.similarity",
|
||||
};
|
||||
|
||||
// Short name → fully-qualified name mapping.
|
||||
internal static readonly Dictionary<string, string> BuiltinEvaluators = new(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
|
||||
@@ -103,9 +103,6 @@ internal sealed class WireEvalItemPayload
|
||||
[JsonPropertyName("context")]
|
||||
public string? Context { get; init; }
|
||||
|
||||
[JsonPropertyName("ground_truth")]
|
||||
public string? GroundTruth { get; init; }
|
||||
|
||||
[JsonPropertyName("tool_definitions")]
|
||||
public List<WireToolDefinition>? ToolDefinitions { get; init; }
|
||||
}
|
||||
|
||||
@@ -145,8 +145,6 @@ public sealed class FoundryEvals : IAgentEvaluator
|
||||
|
||||
bool hasContext = payloads.Any(p => p.Context is not null);
|
||||
bool hasTools = payloads.Any(p => p.ToolDefinitions is { Count: > 0 });
|
||||
bool hasGroundTruth = payloads.Any(p => p.GroundTruth is not null);
|
||||
bool allHaveGroundTruth = payloads.Count > 0 && payloads.All(p => p.GroundTruth is not null);
|
||||
|
||||
// Filter out tool evaluators if no items have tools; auto-add ToolCallAccuracy if tools present
|
||||
var evaluators = FilterToolEvaluators(this._evaluatorNames, hasTools);
|
||||
@@ -155,27 +153,13 @@ public sealed class FoundryEvals : IAgentEvaluator
|
||||
evaluators = [.. evaluators, ToolCallAccuracy];
|
||||
}
|
||||
|
||||
// Fail fast if a ground-truth evaluator (e.g. similarity) is requested but not
|
||||
// every item carries an ExpectedOutput. Reference-based evaluators score each
|
||||
// item against its own ground truth, so even one missing value will surface as
|
||||
// a provider-side validation error. Catch it here with a clearer message.
|
||||
var missingGroundTruth = FoundryEvalConverter.FindMissingGroundTruthEvaluators(evaluators, allHaveGroundTruth);
|
||||
if (missingGroundTruth.Count > 0)
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
"The following evaluator(s) require a ground-truth/expected output on every item but " +
|
||||
$"at least one item is missing an {nameof(EvalItem.ExpectedOutput)}: {string.Join(", ", missingGroundTruth)}. " +
|
||||
"Provide an expected output per item (for example via the 'expectedOutput' parameter on EvaluateAsync), " +
|
||||
"or set 'includePerAgent: false' so the evaluator only runs on the overall item.");
|
||||
}
|
||||
|
||||
// 2. Create the evaluation definition
|
||||
var createEvalPayload = new WireCreateEvalRequest
|
||||
{
|
||||
Name = evalName,
|
||||
DataSourceConfig = new WireCustomDataSourceConfig
|
||||
{
|
||||
ItemSchema = FoundryEvalConverter.BuildItemSchema(hasContext, hasTools, hasGroundTruth),
|
||||
ItemSchema = FoundryEvalConverter.BuildItemSchema(hasContext, hasTools),
|
||||
},
|
||||
TestingCriteria = FoundryEvalConverter.BuildTestingCriteria(
|
||||
evaluators, this._model, includeDataMapping: true),
|
||||
@@ -838,15 +822,15 @@ public sealed class FoundryEvals : IAgentEvaluator
|
||||
var result = new EvalItemResult(itemId, status, scores);
|
||||
|
||||
// Extract error info from sample
|
||||
if (outputItem.TryGetProperty("sample", out var sample) && sample.ValueKind == JsonValueKind.Object)
|
||||
if (outputItem.TryGetProperty("sample", out var sample))
|
||||
{
|
||||
if (sample.TryGetProperty("error", out var errObj) && errObj.ValueKind == JsonValueKind.Object)
|
||||
if (sample.TryGetProperty("error", out var errObj))
|
||||
{
|
||||
result.ErrorCode = errObj.TryGetProperty("code", out var code) ? code.GetString() : null;
|
||||
result.ErrorMessage = errObj.TryGetProperty("message", out var msg) ? msg.GetString() : null;
|
||||
}
|
||||
|
||||
if (sample.TryGetProperty("usage", out var usage) && usage.ValueKind == JsonValueKind.Object && usage.TryGetProperty("total_tokens", out var tt) && tt.ValueKind == JsonValueKind.Number)
|
||||
if (sample.TryGetProperty("usage", out var usage) && usage.TryGetProperty("total_tokens", out var tt) && tt.ValueKind == JsonValueKind.Number)
|
||||
{
|
||||
var tokenUsage = new Dictionary<string, int>();
|
||||
if (usage.TryGetProperty("prompt_tokens", out var pt) && pt.ValueKind == JsonValueKind.Number)
|
||||
@@ -902,7 +886,7 @@ public sealed class FoundryEvals : IAgentEvaluator
|
||||
}
|
||||
|
||||
// Extract response_id from datasource_item
|
||||
if (outputItem.TryGetProperty("datasource_item", out var dsItem) && dsItem.ValueKind == JsonValueKind.Object)
|
||||
if (outputItem.TryGetProperty("datasource_item", out var dsItem))
|
||||
{
|
||||
if (dsItem.TryGetProperty("resp_id", out var respId))
|
||||
{
|
||||
|
||||
@@ -1,42 +0,0 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using System.Diagnostics.CodeAnalysis;
|
||||
using Microsoft.Agents.AI;
|
||||
using Microsoft.Shared.DiagnosticIds;
|
||||
|
||||
namespace Microsoft.Extensions.AI;
|
||||
|
||||
/// <summary>
|
||||
/// Provides extension methods for creating a <see cref="HarnessAgent"/> from an <see cref="IChatClient"/>.
|
||||
/// </summary>
|
||||
[Experimental(DiagnosticIds.Experiments.AgentsAIExperiments)]
|
||||
public static class ChatClientHarnessExtensions
|
||||
{
|
||||
/// <summary>
|
||||
/// Creates a new <see cref="HarnessAgent"/> that wraps this <see cref="IChatClient"/> with a pre-configured
|
||||
/// pipeline including function invocation, per-service-call chat history persistence, and in-loop compaction.
|
||||
/// </summary>
|
||||
/// <param name="chatClient">
|
||||
/// The <see cref="IChatClient"/> that provides access to the underlying AI model.
|
||||
/// </param>
|
||||
/// <param name="maxContextWindowTokens">
|
||||
/// The maximum number of tokens the model's context window supports (e.g., 1,050,000 for gpt-5.4).
|
||||
/// Used to configure the compaction strategy.
|
||||
/// </param>
|
||||
/// <param name="maxOutputTokens">
|
||||
/// The maximum number of output tokens the model can generate per response (e.g., 128,000 for gpt-5.4).
|
||||
/// Used to configure the compaction strategy.
|
||||
/// </param>
|
||||
/// <param name="options">
|
||||
/// Optional configuration options for the agent, including instructions override, tools,
|
||||
/// additional context providers, and chat history provider.
|
||||
/// When <see langword="null"/>, the agent uses built-in default settings.
|
||||
/// </param>
|
||||
/// <returns>A new <see cref="HarnessAgent"/> instance.</returns>
|
||||
public static HarnessAgent AsHarnessAgent(
|
||||
this IChatClient chatClient,
|
||||
int maxContextWindowTokens,
|
||||
int maxOutputTokens,
|
||||
HarnessAgentOptions? options = null) =>
|
||||
new(chatClient, maxContextWindowTokens, maxOutputTokens, options);
|
||||
}
|
||||
@@ -1,135 +0,0 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using System.Diagnostics.CodeAnalysis;
|
||||
using Microsoft.Agents.AI.Compaction;
|
||||
using Microsoft.Extensions.AI;
|
||||
using Microsoft.Shared.DiagnosticIds;
|
||||
using Microsoft.Shared.Diagnostics;
|
||||
|
||||
namespace Microsoft.Agents.AI;
|
||||
|
||||
/// <summary>
|
||||
/// A pre-configured <see cref="DelegatingAIAgent"/> that wraps a <see cref="ChatClientAgent"/> with
|
||||
/// function invocation, per-service-call chat history persistence, and in-loop compaction.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <see cref="HarnessAgent"/> assembles the following pipeline from a caller-supplied <see cref="IChatClient"/>:
|
||||
/// <list type="number">
|
||||
/// <item><description><see cref="FunctionInvokingChatClient"/> — automatic function/tool invocation.</description></item>
|
||||
/// <item><description><see cref="PerServiceCallChatHistoryPersistingChatClient"/> — persists chat history after every individual service call within a function-invocation loop.</description></item>
|
||||
/// <item><description><see cref="AIContextProviderChatClient"/> with a <see cref="CompactionProvider"/> — applies context-window compaction before each call so long function-invocation loops do not overflow the context window.</description></item>
|
||||
/// </list>
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// The underlying <see cref="ChatClientAgent"/> is configured with
|
||||
/// <see cref="ChatClientAgentOptions.UseProvidedChatClientAsIs"/> and
|
||||
/// <see cref="ChatClientAgentOptions.RequirePerServiceCallChatHistoryPersistence"/> set to <see langword="true"/>
|
||||
/// to match the manually-assembled pipeline.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// When no <see cref="HarnessAgentOptions.ChatHistoryProvider"/> is supplied, the agent defaults to an
|
||||
/// <see cref="InMemoryChatHistoryProvider"/> whose chat reducer applies the same compaction strategy,
|
||||
/// keeping in-memory history from growing unboundedly across sessions.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
[Experimental(DiagnosticIds.Experiments.AgentsAIExperiments)]
|
||||
public sealed class HarnessAgent : DelegatingAIAgent
|
||||
{
|
||||
/// <summary>
|
||||
/// The built-in default system instructions used when <see cref="ChatOptions.Instructions"/> is not set.
|
||||
/// </summary>
|
||||
public const string DefaultInstructions =
|
||||
"""
|
||||
You are a helpful AI assistant that uses tools to complete tasks.
|
||||
|
||||
## General guidelines
|
||||
|
||||
- Think through the task before acting. Break complex work into clear steps.
|
||||
- Use the tools available to you to gather information, perform actions, and verify results.
|
||||
- Explain your reasoning between tool calls so the user can follow your progress.
|
||||
- If a tool call fails or returns unexpected results, adapt your approach rather than repeating the same call.
|
||||
- When you have completed the task, present a clear and concise summary of what you did and what you found.
|
||||
""";
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="HarnessAgent"/> class.
|
||||
/// </summary>
|
||||
/// <param name="chatClient">
|
||||
/// The <see cref="IChatClient"/> that provides access to the underlying AI model.
|
||||
/// The agent wraps this client in a function-invocation, per-service-call persistence,
|
||||
/// and compaction pipeline automatically.
|
||||
/// </param>
|
||||
/// <param name="maxContextWindowTokens">
|
||||
/// The maximum number of tokens the model's context window supports (e.g., 1,050,000 for gpt-5.4).
|
||||
/// Used to configure the compaction strategy.
|
||||
/// </param>
|
||||
/// <param name="maxOutputTokens">
|
||||
/// The maximum number of output tokens the model can generate per response (e.g., 128,000 for gpt-5.4).
|
||||
/// Used to configure the compaction strategy and to limit the model's output.
|
||||
/// </param>
|
||||
/// <param name="options">
|
||||
/// Optional configuration options for the agent, including instructions override, tools,
|
||||
/// additional context providers, and chat history provider.
|
||||
/// When <see langword="null"/>, the agent uses built-in default settings.
|
||||
/// </param>
|
||||
/// <exception cref="System.ArgumentNullException">
|
||||
/// <paramref name="chatClient"/> is <see langword="null"/>.
|
||||
/// </exception>
|
||||
/// <exception cref="System.ArgumentOutOfRangeException">
|
||||
/// <paramref name="maxContextWindowTokens"/> is not positive, or
|
||||
/// <paramref name="maxOutputTokens"/> is negative or greater than or equal to <paramref name="maxContextWindowTokens"/>.
|
||||
/// </exception>
|
||||
public HarnessAgent(IChatClient chatClient, int maxContextWindowTokens, int maxOutputTokens, HarnessAgentOptions? options = null)
|
||||
: base(BuildInnerAgent(
|
||||
Throw.IfNull(chatClient),
|
||||
maxContextWindowTokens,
|
||||
maxOutputTokens,
|
||||
options))
|
||||
{
|
||||
}
|
||||
|
||||
private static ChatClientAgent BuildInnerAgent(IChatClient chatClient, int maxContextWindowTokens, int maxOutputTokens, HarnessAgentOptions? options)
|
||||
{
|
||||
var compactionStrategy = new ContextWindowCompactionStrategy(
|
||||
maxContextWindowTokens: maxContextWindowTokens,
|
||||
maxOutputTokens: maxOutputTokens);
|
||||
|
||||
ChatHistoryProvider chatHistoryProvider = options?.ChatHistoryProvider
|
||||
?? new InMemoryChatHistoryProvider(new InMemoryChatHistoryProviderOptions
|
||||
{
|
||||
ChatReducer = compactionStrategy.AsChatReducer(),
|
||||
});
|
||||
|
||||
string instructions = options?.ChatOptions?.Instructions ?? DefaultInstructions;
|
||||
|
||||
ChatOptions chatOptions = BuildChatOptions(options?.ChatOptions, instructions, maxOutputTokens);
|
||||
|
||||
var compactionProvider = new CompactionProvider(compactionStrategy);
|
||||
|
||||
return chatClient
|
||||
.AsBuilder()
|
||||
.UseFunctionInvocation()
|
||||
.UsePerServiceCallChatHistoryPersistence()
|
||||
.UseAIContextProviders(compactionProvider)
|
||||
.BuildAIAgent(new ChatClientAgentOptions
|
||||
{
|
||||
Id = options?.Id,
|
||||
Name = options?.Name,
|
||||
Description = options?.Description,
|
||||
ChatOptions = chatOptions,
|
||||
ChatHistoryProvider = chatHistoryProvider,
|
||||
AIContextProviders = options?.AIContextProviders,
|
||||
UseProvidedChatClientAsIs = true,
|
||||
RequirePerServiceCallChatHistoryPersistence = true,
|
||||
});
|
||||
}
|
||||
|
||||
private static ChatOptions BuildChatOptions(ChatOptions? source, string instructions, int maxOutputTokens)
|
||||
{
|
||||
ChatOptions result = source?.Clone() ?? new ChatOptions();
|
||||
result.Instructions = instructions;
|
||||
result.MaxOutputTokens ??= maxOutputTokens;
|
||||
return result;
|
||||
}
|
||||
}
|
||||
@@ -1,64 +0,0 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics.CodeAnalysis;
|
||||
using Microsoft.Extensions.AI;
|
||||
using Microsoft.Shared.DiagnosticIds;
|
||||
|
||||
namespace Microsoft.Agents.AI;
|
||||
|
||||
/// <summary>
|
||||
/// Represents configuration options for a <see cref="HarnessAgent"/>.
|
||||
/// </summary>
|
||||
[Experimental(DiagnosticIds.Experiments.AgentsAIExperiments)]
|
||||
public sealed class HarnessAgentOptions
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets or sets the agent id.
|
||||
/// </summary>
|
||||
public string? Id { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the agent name.
|
||||
/// </summary>
|
||||
public string? Name { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the agent description.
|
||||
/// </summary>
|
||||
public string? Description { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets additional chat options such as tools for the agent to use.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// Use <see cref="ChatOptions.Tools"/> to supply additional tools the agent can invoke.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Use <see cref="ChatOptions.Instructions"/> to override the <see cref="HarnessAgent"/>'s built-in
|
||||
/// default instructions. When <see cref="ChatOptions.Instructions"/> is <see langword="null"/> or not set,
|
||||
/// the default instructions are used.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public ChatOptions? ChatOptions { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the <see cref="ChatHistoryProvider"/> to use for storing chat history.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// When <see langword="null"/>, the agent defaults to an <see cref="InMemoryChatHistoryProvider"/>
|
||||
/// configured with a compaction-based chat reducer derived from the <c>maxContextWindowTokens</c>
|
||||
/// and <c>maxOutputTokens</c> constructor parameters of <see cref="HarnessAgent"/>.
|
||||
/// </remarks>
|
||||
public ChatHistoryProvider? ChatHistoryProvider { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets additional <see cref="AIContextProvider"/> instances to include in the agent pipeline.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// These providers are passed to the underlying <see cref="ChatClientAgent"/> via
|
||||
/// <see cref="ChatClientAgentOptions.AIContextProviders"/>.
|
||||
/// </remarks>
|
||||
public IEnumerable<AIContextProvider>? AIContextProviders { get; set; }
|
||||
}
|
||||
@@ -1,28 +0,0 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<IsReleaseCandidate>false</IsReleaseCandidate>
|
||||
<InjectSharedThrow>true</InjectSharedThrow>
|
||||
<InjectSharedDiagnosticIds>true</InjectSharedDiagnosticIds>
|
||||
<InjectExperimentalAttributeOnLegacy>true</InjectExperimentalAttributeOnLegacy>
|
||||
<InjectIsExternalInitOnLegacy>true</InjectIsExternalInitOnLegacy>
|
||||
<InjectTrimAttributesOnLegacy>true</InjectTrimAttributesOnLegacy>
|
||||
</PropertyGroup>
|
||||
|
||||
<Import Project="$(RepoRoot)/dotnet/nuget/nuget-package.props" />
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\Microsoft.Agents.AI\Microsoft.Agents.AI.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
<PropertyGroup>
|
||||
<!-- NuGet Package Settings -->
|
||||
<Title>Microsoft Agent Framework Harness</Title>
|
||||
<Description>Provides the HarnessAgent, a pre-configured AI agent that can be used for long running tasks.</Description>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<InternalsVisibleTo Include="Microsoft.Agents.AI.Harness.UnitTests" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
@@ -32,8 +32,38 @@ internal static class AIAgentsAbstractionsExtensions
|
||||
return message;
|
||||
}
|
||||
|
||||
public static List<ChatMessage> CopyWithAssistantToUserForOtherParticipants(
|
||||
this IEnumerable<ChatMessage> messages,
|
||||
string targetAgentName)
|
||||
=> messages.Select(m => m.ChatAssistantToUserIfNotFromNamed(targetAgentName, out _, false)).ToList();
|
||||
/// <summary>
|
||||
/// Iterates through <paramref name="messages"/> looking for <see cref="ChatRole.Assistant"/> messages and swapping
|
||||
/// any that have a different <see cref="ChatMessage.AuthorName"/> from <paramref name="targetAgentName"/> to
|
||||
/// <see cref="ChatRole.User"/>.
|
||||
/// </summary>
|
||||
public static List<ChatMessage>? ChangeAssistantToUserForOtherParticipants(this IEnumerable<ChatMessage> messages, string targetAgentName)
|
||||
{
|
||||
List<ChatMessage>? roleChanged = null;
|
||||
foreach (var m in messages)
|
||||
{
|
||||
m.ChatAssistantToUserIfNotFromNamed(targetAgentName, out bool changed);
|
||||
if (changed)
|
||||
{
|
||||
(roleChanged ??= []).Add(m);
|
||||
}
|
||||
}
|
||||
|
||||
return roleChanged;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Undoes changes made by <see cref="ChangeAssistantToUserForOtherParticipants"/> when passed the list of changes
|
||||
/// made by that method.
|
||||
/// </summary>
|
||||
public static void ResetUserToAssistantForChangedRoles(this List<ChatMessage>? roleChanged)
|
||||
{
|
||||
if (roleChanged is not null)
|
||||
{
|
||||
foreach (var m in roleChanged)
|
||||
{
|
||||
m.Role = ChatRole.Assistant;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+21
-111
@@ -28,17 +28,6 @@ public static class WorkflowEvaluationExtensions
|
||||
/// Use <see cref="ConversationSplitters.LastTurn"/>, <see cref="ConversationSplitters.Full"/>,
|
||||
/// or a custom <see cref="IConversationSplitter"/> implementation.
|
||||
/// </param>
|
||||
/// <param name="expectedOutput">
|
||||
/// Optional ground-truth/expected output for the workflow's overall final answer.
|
||||
/// When provided, it is stamped onto the overall <see cref="EvalItem.ExpectedOutput"/>
|
||||
/// so reference-based evaluators (for example, similarity) can compare the
|
||||
/// workflow's response against a golden answer. Ground truth is only applied
|
||||
/// to the overall item; per-agent items are intentionally left without an
|
||||
/// expected output, since ground truth is defined against the final response.
|
||||
/// When using a reference-based evaluator that requires ground truth, set
|
||||
/// <paramref name="includePerAgent"/> to <see langword="false"/> to avoid
|
||||
/// invoking the evaluator on per-agent items that have no expected output.
|
||||
/// </param>
|
||||
/// <param name="cancellationToken">Cancellation token.</param>
|
||||
/// <returns>Evaluation results with optional per-agent sub-results.</returns>
|
||||
public static async Task<AgentEvaluationResults> EvaluateAsync(
|
||||
@@ -48,7 +37,6 @@ public static class WorkflowEvaluationExtensions
|
||||
bool includePerAgent = true,
|
||||
string evalName = "Workflow Eval",
|
||||
IConversationSplitter? splitter = null,
|
||||
string? expectedOutput = null,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var events = run.OutgoingEvents.ToList();
|
||||
@@ -60,26 +48,28 @@ public static class WorkflowEvaluationExtensions
|
||||
var overallItems = new List<EvalItem>();
|
||||
if (includeOverall)
|
||||
{
|
||||
var overallItem = BuildOverallItem(events, splitter, expectedOutput);
|
||||
if (overallItem is not null)
|
||||
var finalResponse = events.OfType<AgentResponseEvent>().LastOrDefault();
|
||||
if (finalResponse is not null)
|
||||
{
|
||||
overallItems.Add(overallItem);
|
||||
}
|
||||
else
|
||||
{
|
||||
// The caller asked for an overall evaluation but we couldn't find a final
|
||||
// response to score — almost always because the workflow's agents weren't
|
||||
// built with EmitAgentResponseEvents enabled (so no AgentResponseEvent was
|
||||
// emitted) and no terminal ExecutorCompletedEvent carried an AgentResponse
|
||||
// / ChatMessage / string payload. Fail loudly instead of silently returning
|
||||
// 0/0 (or skipping evaluation against a supplied expectedOutput).
|
||||
throw new InvalidOperationException(
|
||||
"Cannot evaluate the overall workflow output: no AgentResponseEvent or " +
|
||||
"ExecutorCompletedEvent with an AgentResponse/ChatMessage/string payload " +
|
||||
"was found in the run. Bind agents with " +
|
||||
"AIAgentHostOptions { EmitAgentResponseEvents = true } " +
|
||||
"(for example via agent.BindAsExecutor(new AIAgentHostOptions { EmitAgentResponseEvents = true })) " +
|
||||
"so the workflow surfaces the final agent response, or set 'includeOverall: false'.");
|
||||
var firstInvoked = events.OfType<ExecutorInvokedEvent>().FirstOrDefault();
|
||||
var query = firstInvoked?.Data switch
|
||||
{
|
||||
ChatMessage cm => cm.Text ?? string.Empty,
|
||||
IReadOnlyList<ChatMessage> msgs => msgs.LastOrDefault(m => m.Role == ChatRole.User)?.Text ?? string.Empty,
|
||||
string s => s,
|
||||
_ => firstInvoked?.Data?.ToString() ?? string.Empty,
|
||||
};
|
||||
var conversation = new List<ChatMessage>
|
||||
{
|
||||
new(ChatRole.User, query),
|
||||
};
|
||||
|
||||
conversation.AddRange(finalResponse.Response.Messages);
|
||||
|
||||
overallItems.Add(new EvalItem(query, finalResponse.Response.Text, conversation)
|
||||
{
|
||||
Splitter = splitter,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -107,86 +97,6 @@ public static class WorkflowEvaluationExtensions
|
||||
return overallResult;
|
||||
}
|
||||
|
||||
internal static EvalItem? BuildOverallItem(
|
||||
IReadOnlyList<WorkflowEvent> events,
|
||||
IConversationSplitter? splitter,
|
||||
string? expectedOutput)
|
||||
{
|
||||
var firstInvoked = events.OfType<ExecutorInvokedEvent>().FirstOrDefault();
|
||||
var query = firstInvoked?.Data switch
|
||||
{
|
||||
ChatMessage cm => cm.Text ?? string.Empty,
|
||||
IReadOnlyList<ChatMessage> msgs => msgs.LastOrDefault(m => m.Role == ChatRole.User)?.Text ?? string.Empty,
|
||||
string s => s,
|
||||
_ => firstInvoked?.Data?.ToString() ?? string.Empty,
|
||||
};
|
||||
|
||||
var conversation = new List<ChatMessage>
|
||||
{
|
||||
new(ChatRole.User, query),
|
||||
};
|
||||
|
||||
// Prefer AgentResponseEvent (only emitted when AIAgentHostOptions.EmitAgentResponseEvents
|
||||
// is enabled). Otherwise fall back to the last ExecutorCompletedEvent that carries an
|
||||
// AgentResponse / ChatMessage / string payload — these are always emitted by the runtime.
|
||||
var finalResponse = events.OfType<AgentResponseEvent>().LastOrDefault();
|
||||
string responseText;
|
||||
if (finalResponse is not null)
|
||||
{
|
||||
responseText = finalResponse.Response.Text;
|
||||
conversation.AddRange(finalResponse.Response.Messages);
|
||||
}
|
||||
else
|
||||
{
|
||||
ExecutorCompletedEvent? finalCompleted = null;
|
||||
for (int i = events.Count - 1; i >= 0; i--)
|
||||
{
|
||||
if (events[i] is ExecutorCompletedEvent completed
|
||||
&& !IsInternalExecutor(completed.ExecutorId)
|
||||
&& completed.Data is AgentResponse or ChatMessage or string)
|
||||
{
|
||||
finalCompleted = completed;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (finalCompleted is null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
switch (finalCompleted.Data)
|
||||
{
|
||||
case AgentResponse ar:
|
||||
responseText = ar.Text;
|
||||
conversation.AddRange(ar.Messages);
|
||||
break;
|
||||
case ChatMessage cm:
|
||||
responseText = cm.Text ?? string.Empty;
|
||||
conversation.Add(cm);
|
||||
break;
|
||||
case string s:
|
||||
responseText = s;
|
||||
conversation.Add(new ChatMessage(ChatRole.Assistant, s));
|
||||
break;
|
||||
default:
|
||||
// Unreachable — the for-loop above already constrains Data to one of the
|
||||
// three handled types. Throw if the contract drifts so the bug is visible
|
||||
// instead of silently dropping the overall item.
|
||||
throw new InvalidOperationException(
|
||||
"BuildOverallItem: unexpected ExecutorCompletedEvent.Data type " +
|
||||
$"'{finalCompleted.Data?.GetType().FullName ?? "null"}'. Expected " +
|
||||
$"{nameof(AgentResponse)}, {nameof(ChatMessage)}, or string.");
|
||||
}
|
||||
}
|
||||
|
||||
return new EvalItem(query, responseText, conversation)
|
||||
{
|
||||
Splitter = splitter,
|
||||
ExpectedOutput = expectedOutput,
|
||||
};
|
||||
}
|
||||
|
||||
internal static Dictionary<string, List<EvalItem>> ExtractAgentData(
|
||||
List<WorkflowEvent> events,
|
||||
IConversationSplitter? splitter)
|
||||
|
||||
@@ -97,6 +97,28 @@ internal sealed class MessageMerger
|
||||
}
|
||||
}
|
||||
|
||||
private int CompareByDateTimeOffset(AgentResponse left, AgentResponse right)
|
||||
{
|
||||
const int LESS = -1, EQ = 0, GREATER = 1;
|
||||
|
||||
if (left.CreatedAt == right.CreatedAt)
|
||||
{
|
||||
return EQ;
|
||||
}
|
||||
|
||||
if (!left.CreatedAt.HasValue)
|
||||
{
|
||||
return GREATER;
|
||||
}
|
||||
|
||||
if (!right.CreatedAt.HasValue)
|
||||
{
|
||||
return LESS;
|
||||
}
|
||||
|
||||
return left.CreatedAt.Value.CompareTo(right.CreatedAt.Value);
|
||||
}
|
||||
|
||||
public AgentResponse ComputeMerged(string primaryResponseId, string? primaryAgentId = null, string? primaryAgentName = null)
|
||||
{
|
||||
List<ChatMessage> messages = [];
|
||||
@@ -104,15 +126,6 @@ internal sealed class MessageMerger
|
||||
HashSet<string> agentIds = [];
|
||||
HashSet<ChatFinishReason> finishReasons = [];
|
||||
|
||||
// Ordering contract (see docs/decisions/0026-message-merger-invariants.md):
|
||||
// - Outer loop iterates ResponseIds in first-seen order, which preserves step
|
||||
// ordering: each agent invocation owns its own ResponseId, and successive
|
||||
// super-steps emit their first update only after the prior step's updates
|
||||
// have all arrived. Iterating Dictionary<,>.Keys preserves insertion order.
|
||||
// - Inner loop iterates MessageIds in first-seen order, then appends dangling
|
||||
// updates last. This preserves emission order within an agent's block.
|
||||
// We deliberately do NOT sort by CreatedAt: timestamps from concurrent agents
|
||||
// or different clocks would interleave per-agent blocks and break Goal 1.
|
||||
foreach (string responseId in this._mergeStates.Keys)
|
||||
{
|
||||
ResponseMergeState mergeState = this._mergeStates[responseId];
|
||||
@@ -123,12 +136,14 @@ internal sealed class MessageMerger
|
||||
responseList.Add(mergeState.ComputeDangling());
|
||||
}
|
||||
|
||||
responseList.Sort(this.CompareByDateTimeOffset);
|
||||
responses[responseId] = responseList.Aggregate(MergeResponses);
|
||||
messages.AddRange(GetMessagesWithCreatedAt(responses[responseId]));
|
||||
}
|
||||
|
||||
UsageDetails? usage = null;
|
||||
AdditionalPropertiesDictionary? additionalProperties = null;
|
||||
HashSet<DateTimeOffset> createdTimes = [];
|
||||
|
||||
foreach (AgentResponse response in responses.Values)
|
||||
{
|
||||
@@ -137,6 +152,11 @@ internal sealed class MessageMerger
|
||||
agentIds.Add(response.AgentId);
|
||||
}
|
||||
|
||||
if (response.CreatedAt.HasValue)
|
||||
{
|
||||
createdTimes.Add(response.CreatedAt.Value);
|
||||
}
|
||||
|
||||
if (response.FinishReason.HasValue)
|
||||
{
|
||||
finishReasons.Add(response.FinishReason.Value);
|
||||
|
||||
@@ -235,10 +235,11 @@ internal sealed class HandoffAgentExecutor :
|
||||
// This will not filter out tool responses and approval responses that are part of this agent's turn, which is
|
||||
// the expected behavior since those are part of the agent's reasoning process.
|
||||
HandoffMessagesFilter handoffMessagesFilter = new(this._options.ToolCallFilteringBehavior);
|
||||
List<ChatMessage> messagesForAgent = (state.IncomingState.RequestedHandoffTargetAgentId is not null
|
||||
IEnumerable<ChatMessage> messagesForAgent = state.IncomingState.RequestedHandoffTargetAgentId is not null
|
||||
? handoffMessagesFilter.FilterMessages(incomingMessages)
|
||||
: incomingMessages)
|
||||
.CopyWithAssistantToUserForOtherParticipants(this._agent.Name ?? this._agent.Id);
|
||||
: incomingMessages;
|
||||
|
||||
List<ChatMessage>? roleChanges = messagesForAgent.ChangeAssistantToUserForOtherParticipants(this._agent.Name ?? this._agent.Id);
|
||||
|
||||
bool emitUpdateEvents = state.IncomingState!.ShouldEmitStreamingEvents(this._options.EmitAgentResponseUpdateEvents);
|
||||
AgentInvocationResult result = await this.InvokeAgentAsync(messagesForAgent, context, emitUpdateEvents, cancellationToken)
|
||||
@@ -249,6 +250,8 @@ internal sealed class HandoffAgentExecutor :
|
||||
throw new InvalidOperationException("Cannot request a handoff while holding pending requests.");
|
||||
}
|
||||
|
||||
roleChanges.ResetUserToAssistantForChangedRoles();
|
||||
|
||||
int newConversationBookmark = state.ConversationBookmark;
|
||||
await this._sharedStateRef.InvokeWithStateAsync(
|
||||
(sharedState, ctx, ct) =>
|
||||
@@ -433,18 +436,6 @@ internal sealed class HandoffAgentExecutor :
|
||||
FunctionCallContent handoffRequest = candidateRequests[candidateRequests.Count - 1];
|
||||
requestedHandoff = handoffRequest.Name;
|
||||
|
||||
// Stamp the synthetic "Transferred." tool-result update with the same
|
||||
// ResponseId as the agent's preceding updates so it groups with the
|
||||
// rest of this agent's step in MessageMerger (and therefore in
|
||||
// RunAsync's merged AgentResponse and in chat history). Without this,
|
||||
// the synthetic update goes to MessageMerger's null-ResponseId
|
||||
// "dangling" bucket and surfaces after every keyed response, which
|
||||
// re-orders multi-step handoff transcripts versus streaming output
|
||||
// (issue #4544).
|
||||
string? syntheticResponseId = updates
|
||||
.Select(u => u.ResponseId)
|
||||
.LastOrDefault(id => id is not null);
|
||||
|
||||
await AddUpdateAsync(
|
||||
new AgentResponseUpdate
|
||||
{
|
||||
@@ -453,7 +444,6 @@ internal sealed class HandoffAgentExecutor :
|
||||
Contents = [CreateHandoffResult(handoffRequest.CallId)],
|
||||
CreatedAt = DateTimeOffset.UtcNow,
|
||||
MessageId = Guid.NewGuid().ToString("N"),
|
||||
ResponseId = syntheticResponseId,
|
||||
Role = ChatRole.Tool,
|
||||
},
|
||||
cancellationToken
|
||||
|
||||
@@ -3,12 +3,10 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics;
|
||||
using System.Diagnostics.CodeAnalysis;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.AI;
|
||||
using Microsoft.Shared.DiagnosticIds;
|
||||
|
||||
namespace Microsoft.Agents.AI;
|
||||
|
||||
@@ -34,13 +32,6 @@ public sealed class OpenTelemetryAgent : DelegatingAIAgent, IDisposable
|
||||
private readonly OpenTelemetryChatClient _otelClient;
|
||||
/// <summary>The provider name extracted from <see cref="AIAgentMetadata"/>.</summary>
|
||||
private readonly string? _providerName;
|
||||
/// <summary>The resolved source name for telemetry. Always non-empty; defaults to <see cref="OpenTelemetryConsts.DefaultSourceName"/>.</summary>
|
||||
private readonly string _sourceName;
|
||||
/// <summary>
|
||||
/// Indicates whether the underlying <see cref="IChatClient"/> of a <see cref="ChatClientAgent"/> inner agent
|
||||
/// should be automatically wrapped with <see cref="OpenTelemetryChatClient"/> on each invocation.
|
||||
/// </summary>
|
||||
private readonly bool _autoWireChatClient;
|
||||
|
||||
/// <summary>Initializes a new instance of the <see cref="OpenTelemetryAgent"/> class.</summary>
|
||||
/// <param name="innerAgent">The underlying <see cref="AIAgent"/> to be augmented with telemetry capabilities.</param>
|
||||
@@ -53,44 +44,13 @@ public sealed class OpenTelemetryAgent : DelegatingAIAgent, IDisposable
|
||||
/// The constructor automatically extracts provider metadata from the inner agent and configures
|
||||
/// telemetry collection according to OpenTelemetry semantic conventions for AI systems.
|
||||
/// </remarks>
|
||||
public OpenTelemetryAgent(AIAgent innerAgent, string? sourceName = null)
|
||||
#pragma warning disable MAAI001 // Auto-wiring is the new default; the experimental opt-out lives on the 3-arg overload.
|
||||
: this(innerAgent, sourceName, autoWireChatClient: true)
|
||||
#pragma warning restore MAAI001
|
||||
{
|
||||
}
|
||||
|
||||
/// <summary>Initializes a new instance of the <see cref="OpenTelemetryAgent"/> class.</summary>
|
||||
/// <param name="innerAgent">The underlying <see cref="AIAgent"/> to be augmented with telemetry capabilities.</param>
|
||||
/// <param name="sourceName">
|
||||
/// An optional source name that will be used to identify telemetry data from this agent.
|
||||
/// If not provided, a default source name will be used for telemetry identification.
|
||||
/// </param>
|
||||
/// <param name="autoWireChatClient">
|
||||
/// When <see langword="true"/> and the inner agent is a <see cref="ChatClientAgent"/>, the underlying
|
||||
/// <see cref="IChatClient"/> is automatically wrapped with <see cref="OpenTelemetryChatClient"/> for each invocation
|
||||
/// so that chat-level telemetry flows alongside agent-level telemetry. If the underlying chat client is already
|
||||
/// instrumented, no additional wrapping is applied. Set to <see langword="false"/> to opt-out of this behavior.
|
||||
/// </param>
|
||||
/// <exception cref="ArgumentNullException"><paramref name="innerAgent"/> is <see langword="null"/>.</exception>
|
||||
/// <remarks>
|
||||
/// The constructor automatically extracts provider metadata from the inner agent and configures
|
||||
/// telemetry collection according to OpenTelemetry semantic conventions for AI systems.
|
||||
/// </remarks>
|
||||
[Experimental(DiagnosticIds.Experiments.AgentsAIExperiments)]
|
||||
public OpenTelemetryAgent(AIAgent innerAgent, string? sourceName, bool autoWireChatClient) : base(innerAgent)
|
||||
public OpenTelemetryAgent(AIAgent innerAgent, string? sourceName = null) : base(innerAgent)
|
||||
{
|
||||
this._providerName = innerAgent.GetService<AIAgentMetadata>()?.ProviderName;
|
||||
|
||||
// Resolve once so the outer OpenTelemetryChatClient and the auto-wired inner
|
||||
// OpenTelemetryChatClient always emit spans under the same ActivitySource, even when
|
||||
// the caller passes "" or whitespace (which neither client should treat as a real source).
|
||||
this._sourceName = string.IsNullOrWhiteSpace(sourceName) ? OpenTelemetryConsts.DefaultSourceName : sourceName!;
|
||||
this._autoWireChatClient = autoWireChatClient;
|
||||
|
||||
this._otelClient = new OpenTelemetryChatClient(
|
||||
new ForwardingChatClient(this),
|
||||
sourceName: this._sourceName);
|
||||
sourceName: string.IsNullOrEmpty(sourceName) ? OpenTelemetryConsts.DefaultSourceName : sourceName!);
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
@@ -203,85 +163,6 @@ public sealed class OpenTelemetryAgent : DelegatingAIAgent, IDisposable
|
||||
public Activity? CurrentActivity { get; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// If auto-wiring is enabled and the inner agent is a <see cref="ChatClientAgent"/> whose underlying
|
||||
/// <see cref="IChatClient"/> is not already instrumented with <see cref="OpenTelemetryChatClient"/>, returns a
|
||||
/// new <see cref="ChatClientAgentRunOptions"/> with a <see cref="ChatClientAgentRunOptions.ChatClientFactory"/>
|
||||
/// that wraps the chat client with <see cref="OpenTelemetryChatClient"/>. When <paramref name="options"/> is a
|
||||
/// plain <see cref="AgentRunOptions"/> (the base type, not <see cref="ChatClientAgentRunOptions"/>), the base
|
||||
/// properties are copied onto the new <see cref="ChatClientAgentRunOptions"/> so high-level callers that pass
|
||||
/// the abstract <see cref="AgentRunOptions"/> still benefit from auto-wiring and propagate their settings to
|
||||
/// the inner agent. Otherwise, returns <paramref name="options"/> unchanged.
|
||||
/// </summary>
|
||||
private AgentRunOptions? GetRunOptionsWithChatClientWiring(AgentRunOptions? options)
|
||||
{
|
||||
if (!this._autoWireChatClient)
|
||||
{
|
||||
return options;
|
||||
}
|
||||
|
||||
// The auto-wiring only applies when a ChatClientAgent is reachable from the inner agent. Otherwise, no-op.
|
||||
// Use GetService rather than a type check so wrapping agents that expose a nested ChatClientAgent are supported.
|
||||
var chatClientAgent = this.InnerAgent.GetService<ChatClientAgent>();
|
||||
if (chatClientAgent is null)
|
||||
{
|
||||
return options;
|
||||
}
|
||||
|
||||
// Respect ChatClientAgentOptions.UseProvidedChatClientAsIs: don't decorate the chat client when the user opted out.
|
||||
if (chatClientAgent.GetService<ChatClientAgentOptions>()?.UseProvidedChatClientAsIs is true)
|
||||
{
|
||||
return options;
|
||||
}
|
||||
|
||||
// Capture the underlying IChatClient and check whether it is already instrumented.
|
||||
var chatClient = chatClientAgent.GetService<IChatClient>();
|
||||
if (chatClient is null || chatClient.GetService(typeof(OpenTelemetryChatClient)) is not null)
|
||||
{
|
||||
return options;
|
||||
}
|
||||
|
||||
string sourceName = this._sourceName;
|
||||
static IChatClient WrapIfNeeded(IChatClient cc, string sourceName) =>
|
||||
cc.GetService(typeof(OpenTelemetryChatClient)) is not null
|
||||
? cc
|
||||
: cc.AsBuilder().UseOpenTelemetry(sourceName: sourceName).Build();
|
||||
|
||||
if (options is ChatClientAgentRunOptions ccOptions)
|
||||
{
|
||||
// Don't mutate the caller's options; clone and chain any caller-provided factory.
|
||||
// If the user factory already returns an OpenTelemetry-instrumented client, don't double-wrap.
|
||||
var clone = (ChatClientAgentRunOptions)ccOptions.Clone();
|
||||
var userFactory = clone.ChatClientFactory;
|
||||
clone.ChatClientFactory = cc => WrapIfNeeded(userFactory is null ? cc : userFactory(cc), sourceName);
|
||||
return clone;
|
||||
}
|
||||
|
||||
// For a plain AgentRunOptions (or null), create a ChatClientAgentRunOptions and preserve
|
||||
// any base AgentRunOptions properties from the caller so they reach the inner agent.
|
||||
var newOptions = new ChatClientAgentRunOptions
|
||||
{
|
||||
ChatClientFactory = cc => WrapIfNeeded(cc, sourceName),
|
||||
};
|
||||
|
||||
if (options is not null)
|
||||
{
|
||||
CopyBaseAgentRunOptions(options, newOptions);
|
||||
}
|
||||
|
||||
return newOptions;
|
||||
}
|
||||
|
||||
#pragma warning disable MEAI001 // ContinuationToken is experimental; copy it through to preserve caller-provided value.
|
||||
private static void CopyBaseAgentRunOptions(AgentRunOptions source, AgentRunOptions target)
|
||||
{
|
||||
target.ContinuationToken = source.ContinuationToken;
|
||||
target.AllowBackgroundResponses = source.AllowBackgroundResponses;
|
||||
target.AdditionalProperties = source.AdditionalProperties?.Clone();
|
||||
target.ResponseFormat = source.ResponseFormat;
|
||||
}
|
||||
#pragma warning restore MEAI001
|
||||
|
||||
/// <summary>The stub <see cref="IChatClient"/> used to delegate from the <see cref="OpenTelemetryChatClient"/> into the inner <see cref="AIAgent"/>.</summary>
|
||||
/// <param name="parentAgent"></param>
|
||||
private sealed class ForwardingChatClient(OpenTelemetryAgent parentAgent) : IChatClient
|
||||
@@ -294,11 +175,8 @@ public sealed class OpenTelemetryAgent : DelegatingAIAgent, IDisposable
|
||||
// Update the current activity to reflect the agent invocation.
|
||||
parentAgent.UpdateCurrentActivity(fo?.CurrentActivity);
|
||||
|
||||
// If enabled, wire the underlying chat client with OpenTelemetryChatClient via ChatClientFactory.
|
||||
var runOptions = parentAgent.GetRunOptionsWithChatClientWiring(fo?.Options);
|
||||
|
||||
// Invoke the inner agent.
|
||||
var response = await parentAgent.InnerAgent.RunAsync(messages, fo?.Session, runOptions, cancellationToken).ConfigureAwait(false);
|
||||
var response = await parentAgent.InnerAgent.RunAsync(messages, fo?.Session, fo?.Options, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
// Wrap the response in a ChatResponse so we can pass it back through OpenTelemetryChatClient.
|
||||
return response.AsChatResponse();
|
||||
@@ -312,11 +190,8 @@ public sealed class OpenTelemetryAgent : DelegatingAIAgent, IDisposable
|
||||
// Update the current activity to reflect the agent invocation.
|
||||
parentAgent.UpdateCurrentActivity(fo?.CurrentActivity);
|
||||
|
||||
// If enabled, wire the underlying chat client with OpenTelemetryChatClient via ChatClientFactory.
|
||||
var runOptions = parentAgent.GetRunOptionsWithChatClientWiring(fo?.Options);
|
||||
|
||||
// Invoke the inner agent.
|
||||
await foreach (var update in parentAgent.InnerAgent.RunStreamingAsync(messages, fo?.Session, runOptions, cancellationToken).ConfigureAwait(false))
|
||||
await foreach (var update in parentAgent.InnerAgent.RunStreamingAsync(messages, fo?.Session, fo?.Options, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
// Wrap the response updates in ChatResponseUpdates so we can pass them back through OpenTelemetryChatClient.
|
||||
yield return update.AsChatResponseUpdate();
|
||||
|
||||
@@ -179,35 +179,6 @@ public sealed class FoundryEvalConverterTests
|
||||
Assert.Null(payload.Context);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ConvertEvalItem_WithExpectedOutput_PopulatesGroundTruth()
|
||||
{
|
||||
// Arrange
|
||||
var item = new EvalItem(query: "q", response: "r")
|
||||
{
|
||||
ExpectedOutput = "the golden answer",
|
||||
};
|
||||
|
||||
// Act
|
||||
var payload = FoundryEvalConverter.ConvertEvalItem(item);
|
||||
|
||||
// Assert
|
||||
Assert.Equal("the golden answer", payload.GroundTruth);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ConvertEvalItem_WithoutExpectedOutput_OmitsGroundTruth()
|
||||
{
|
||||
// Arrange
|
||||
var item = new EvalItem(query: "q", response: "r");
|
||||
|
||||
// Act
|
||||
var payload = FoundryEvalConverter.ConvertEvalItem(item);
|
||||
|
||||
// Assert
|
||||
Assert.Null(payload.GroundTruth);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// FoundryEvalConverter.BuildTestingCriteria tests
|
||||
// ---------------------------------------------------------------
|
||||
@@ -268,33 +239,6 @@ public sealed class FoundryEvalConverterTests
|
||||
Assert.Equal("{{item.context}}", mapping["context"]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BuildTestingCriteria_SimilarityEvaluator_IncludesGroundTruth()
|
||||
{
|
||||
// Act
|
||||
var criteria = FoundryEvalConverter.BuildTestingCriteria(
|
||||
["similarity"], "gpt-4o-mini", includeDataMapping: true);
|
||||
|
||||
// Assert
|
||||
Assert.Single(criteria);
|
||||
Assert.Equal("builtin.similarity", criteria[0].EvaluatorName);
|
||||
var mapping = criteria[0].DataMapping;
|
||||
Assert.NotNull(mapping);
|
||||
Assert.True(mapping.ContainsKey("ground_truth"));
|
||||
Assert.Equal("{{item.ground_truth}}", mapping["ground_truth"]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BuildTestingCriteria_NonGroundTruthEvaluator_OmitsGroundTruth()
|
||||
{
|
||||
var criteria = FoundryEvalConverter.BuildTestingCriteria(
|
||||
["relevance"], "gpt-4o-mini", includeDataMapping: true);
|
||||
|
||||
var mapping = criteria[0].DataMapping;
|
||||
Assert.NotNull(mapping);
|
||||
Assert.False(mapping.ContainsKey("ground_truth"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BuildTestingCriteria_WithoutDataMapping_OmitsMappingField()
|
||||
{
|
||||
@@ -338,59 +282,6 @@ public sealed class FoundryEvalConverterTests
|
||||
Assert.True(schema.Properties.ContainsKey("tool_definitions"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BuildItemSchema_WithGroundTruth_IncludesGroundTruthProperty()
|
||||
{
|
||||
// Act
|
||||
var schema = FoundryEvalConverter.BuildItemSchema(hasGroundTruth: true);
|
||||
|
||||
// Assert
|
||||
Assert.True(schema.Properties.ContainsKey("ground_truth"));
|
||||
Assert.Equal("string", schema.Properties["ground_truth"].Type);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BuildItemSchema_WithoutGroundTruth_OmitsGroundTruthProperty()
|
||||
{
|
||||
var schema = FoundryEvalConverter.BuildItemSchema();
|
||||
|
||||
Assert.False(schema.Properties.ContainsKey("ground_truth"));
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// FoundryEvalConverter.FindMissingGroundTruthEvaluators tests
|
||||
// ---------------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public void FindMissingGroundTruthEvaluators_NoGroundTruth_ReturnsSimilarity()
|
||||
{
|
||||
// Act
|
||||
var missing = FoundryEvalConverter.FindMissingGroundTruthEvaluators(
|
||||
["similarity", "relevance"], hasGroundTruth: false);
|
||||
|
||||
// Assert
|
||||
Assert.Single(missing);
|
||||
Assert.Equal("similarity", missing[0]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void FindMissingGroundTruthEvaluators_HasGroundTruth_ReturnsEmpty()
|
||||
{
|
||||
var missing = FoundryEvalConverter.FindMissingGroundTruthEvaluators(
|
||||
["similarity"], hasGroundTruth: true);
|
||||
|
||||
Assert.Empty(missing);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void FindMissingGroundTruthEvaluators_NoGroundTruthEvaluators_ReturnsEmpty()
|
||||
{
|
||||
var missing = FoundryEvalConverter.FindMissingGroundTruthEvaluators(
|
||||
["relevance", "coherence"], hasGroundTruth: false);
|
||||
|
||||
Assert.Empty(missing);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// FoundryEvalConverter.ConvertMessage DataContent test
|
||||
// ---------------------------------------------------------------
|
||||
|
||||
@@ -1,56 +0,0 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
namespace Microsoft.Agents.AI.UnitTests;
|
||||
|
||||
public class HarnessAgentOptionsTests
|
||||
{
|
||||
/// <summary>
|
||||
/// Verify that default property values are as expected.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void DefaultPropertyValues()
|
||||
{
|
||||
// Arrange & Act
|
||||
var options = new HarnessAgentOptions();
|
||||
|
||||
// Assert
|
||||
Assert.Null(options.Id);
|
||||
Assert.Null(options.Name);
|
||||
Assert.Null(options.Description);
|
||||
Assert.Null(options.ChatOptions);
|
||||
Assert.Null(options.ChatHistoryProvider);
|
||||
Assert.Null(options.AIContextProviders);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verify that all properties can be set and retrieved.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void PropertiesCanBeSetAndRetrieved()
|
||||
{
|
||||
// Arrange
|
||||
var chatHistoryProvider = new InMemoryChatHistoryProvider();
|
||||
var contextProviders = new AIContextProvider[] { new TodoProvider() };
|
||||
|
||||
// Act
|
||||
var options = new HarnessAgentOptions
|
||||
{
|
||||
Id = "test-id",
|
||||
Name = "test-name",
|
||||
Description = "test-description",
|
||||
ChatOptions = new() { Temperature = 0.5f, Instructions = "custom instructions" },
|
||||
ChatHistoryProvider = chatHistoryProvider,
|
||||
AIContextProviders = contextProviders,
|
||||
};
|
||||
|
||||
// Assert
|
||||
Assert.Equal("test-id", options.Id);
|
||||
Assert.Equal("test-name", options.Name);
|
||||
Assert.Equal("test-description", options.Description);
|
||||
Assert.NotNull(options.ChatOptions);
|
||||
Assert.Equal(0.5f, options.ChatOptions!.Temperature);
|
||||
Assert.Equal("custom instructions", options.ChatOptions.Instructions);
|
||||
Assert.Same(chatHistoryProvider, options.ChatHistoryProvider);
|
||||
Assert.Same(contextProviders, options.AIContextProviders);
|
||||
}
|
||||
}
|
||||
@@ -1,516 +0,0 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.AI;
|
||||
using Moq;
|
||||
|
||||
namespace Microsoft.Agents.AI.UnitTests;
|
||||
|
||||
public class HarnessAgentTests
|
||||
{
|
||||
private const int TestMaxContextWindowTokens = 100_000;
|
||||
private const int TestMaxOutputTokens = 10_000;
|
||||
|
||||
#region Constructor Validation
|
||||
|
||||
/// <summary>
|
||||
/// Verify that the constructor throws when chatClient is null.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void Constructor_ThrowsWhenChatClientIsNull()
|
||||
{
|
||||
// Act & Assert
|
||||
Assert.Throws<ArgumentNullException>(() => new HarnessAgent(null!, TestMaxContextWindowTokens, TestMaxOutputTokens));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verify that the constructor throws when MaxContextWindowTokens is invalid (zero).
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void Constructor_ThrowsWhenMaxContextWindowTokensIsZero()
|
||||
{
|
||||
// Arrange
|
||||
var chatClient = new Mock<IChatClient>().Object;
|
||||
|
||||
// Act & Assert
|
||||
Assert.Throws<ArgumentOutOfRangeException>(() => new HarnessAgent(chatClient, 0, TestMaxOutputTokens));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verify that the constructor throws when MaxOutputTokens equals MaxContextWindowTokens.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void Constructor_ThrowsWhenMaxOutputTokensEqualsContextWindow()
|
||||
{
|
||||
// Arrange
|
||||
var chatClient = new Mock<IChatClient>().Object;
|
||||
|
||||
// Act & Assert
|
||||
Assert.Throws<ArgumentOutOfRangeException>(() => new HarnessAgent(chatClient, 100_000, 100_000));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verify that the constructor succeeds when options is null.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void Constructor_SucceedsWhenOptionsIsNull()
|
||||
{
|
||||
// Arrange
|
||||
var chatClient = new Mock<IChatClient>().Object;
|
||||
|
||||
// Act
|
||||
var agent = new HarnessAgent(chatClient, TestMaxContextWindowTokens, TestMaxOutputTokens);
|
||||
|
||||
// Assert
|
||||
Assert.NotNull(agent);
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Agent Identity
|
||||
|
||||
/// <summary>
|
||||
/// Verify that Name and Description are passed through to the inner agent.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void NameAndDescription_ArePassedThrough()
|
||||
{
|
||||
// Arrange
|
||||
var chatClient = new Mock<IChatClient>().Object;
|
||||
|
||||
// Act
|
||||
var agent = new HarnessAgent(chatClient, TestMaxContextWindowTokens, TestMaxOutputTokens, new HarnessAgentOptions
|
||||
{
|
||||
Name = "TestAgent",
|
||||
Description = "A test agent",
|
||||
});
|
||||
|
||||
// Assert
|
||||
Assert.Equal("TestAgent", agent.Name);
|
||||
Assert.Equal("A test agent", agent.Description);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verify that Id is passed through to the inner agent.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void Id_IsPassedThrough()
|
||||
{
|
||||
// Arrange
|
||||
var chatClient = new Mock<IChatClient>().Object;
|
||||
|
||||
// Act
|
||||
var agent = new HarnessAgent(chatClient, TestMaxContextWindowTokens, TestMaxOutputTokens, new HarnessAgentOptions
|
||||
{
|
||||
Id = "my-agent-id",
|
||||
});
|
||||
|
||||
// Assert
|
||||
Assert.Equal("my-agent-id", agent.Id);
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Instructions
|
||||
|
||||
/// <summary>
|
||||
/// Verify that default instructions are used when none are provided.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void Instructions_DefaultsToBuiltInInstructions()
|
||||
{
|
||||
// Arrange
|
||||
var chatClient = new Mock<IChatClient>().Object;
|
||||
|
||||
// Act
|
||||
var agent = new HarnessAgent(chatClient, TestMaxContextWindowTokens, TestMaxOutputTokens);
|
||||
var innerAgent = agent.GetService<ChatClientAgent>();
|
||||
|
||||
// Assert
|
||||
Assert.NotNull(innerAgent);
|
||||
Assert.Equal(HarnessAgent.DefaultInstructions, innerAgent!.Instructions);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verify that default instructions are used when options is provided but ChatOptions.Instructions is null.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void Instructions_DefaultsWhenChatOptionsInstructionsIsNull()
|
||||
{
|
||||
// Arrange
|
||||
var chatClient = new Mock<IChatClient>().Object;
|
||||
|
||||
// Act
|
||||
var agent = new HarnessAgent(chatClient, TestMaxContextWindowTokens, TestMaxOutputTokens, new HarnessAgentOptions
|
||||
{
|
||||
ChatOptions = new ChatOptions { Temperature = 0.5f },
|
||||
});
|
||||
var innerAgent = agent.GetService<ChatClientAgent>();
|
||||
|
||||
// Assert
|
||||
Assert.NotNull(innerAgent);
|
||||
Assert.Equal(HarnessAgent.DefaultInstructions, innerAgent!.Instructions);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verify that ChatOptions.Instructions overrides the defaults.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void Instructions_CanBeOverriddenViaChatOptions()
|
||||
{
|
||||
// Arrange
|
||||
var chatClient = new Mock<IChatClient>().Object;
|
||||
|
||||
// Act
|
||||
var agent = new HarnessAgent(chatClient, TestMaxContextWindowTokens, TestMaxOutputTokens, new HarnessAgentOptions
|
||||
{
|
||||
ChatOptions = new ChatOptions { Instructions = "You are a custom assistant." },
|
||||
});
|
||||
var innerAgent = agent.GetService<ChatClientAgent>();
|
||||
|
||||
// Assert
|
||||
Assert.NotNull(innerAgent);
|
||||
Assert.Equal("You are a custom assistant.", innerAgent!.Instructions);
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region ChatHistoryProvider
|
||||
|
||||
/// <summary>
|
||||
/// Verify that the default ChatHistoryProvider is InMemoryChatHistoryProvider when none is specified.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void ChatHistoryProvider_DefaultsToInMemory()
|
||||
{
|
||||
// Arrange
|
||||
var chatClient = new Mock<IChatClient>().Object;
|
||||
|
||||
// Act
|
||||
var agent = new HarnessAgent(chatClient, TestMaxContextWindowTokens, TestMaxOutputTokens);
|
||||
var innerAgent = agent.GetService<ChatClientAgent>();
|
||||
|
||||
// Assert
|
||||
Assert.NotNull(innerAgent);
|
||||
Assert.IsType<InMemoryChatHistoryProvider>(innerAgent!.ChatHistoryProvider);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verify that a custom ChatHistoryProvider is used when provided.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void ChatHistoryProvider_UsesCustomProviderWhenSpecified()
|
||||
{
|
||||
// Arrange
|
||||
var chatClient = new Mock<IChatClient>().Object;
|
||||
var customProvider = new InMemoryChatHistoryProvider();
|
||||
|
||||
// Act
|
||||
var agent = new HarnessAgent(chatClient, TestMaxContextWindowTokens, TestMaxOutputTokens, new HarnessAgentOptions
|
||||
{
|
||||
ChatHistoryProvider = customProvider,
|
||||
});
|
||||
var innerAgent = agent.GetService<ChatClientAgent>();
|
||||
|
||||
// Assert
|
||||
Assert.NotNull(innerAgent);
|
||||
Assert.Same(customProvider, innerAgent!.ChatHistoryProvider);
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region ChatClient Pipeline
|
||||
|
||||
/// <summary>
|
||||
/// Verify that the inner agent's ChatClient includes FunctionInvokingChatClient in the pipeline.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void Pipeline_IncludesFunctionInvokingChatClient()
|
||||
{
|
||||
// Arrange
|
||||
var chatClient = new Mock<IChatClient>().Object;
|
||||
|
||||
// Act
|
||||
var agent = new HarnessAgent(chatClient, TestMaxContextWindowTokens, TestMaxOutputTokens);
|
||||
var innerAgent = agent.GetService<ChatClientAgent>();
|
||||
|
||||
// Assert
|
||||
Assert.NotNull(innerAgent);
|
||||
var ficc = innerAgent!.ChatClient.GetService<FunctionInvokingChatClient>();
|
||||
Assert.NotNull(ficc);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verify that the inner agent's ChatClient pipeline includes more than just the raw chat client,
|
||||
/// confirming that per-service-call persistence and other decorators have been applied.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void Pipeline_HasDecoratedChatClient()
|
||||
{
|
||||
// Arrange
|
||||
var mockClient = new Mock<IChatClient>();
|
||||
var rawClient = mockClient.Object;
|
||||
|
||||
// Act
|
||||
var agent = new HarnessAgent(rawClient, TestMaxContextWindowTokens, TestMaxOutputTokens);
|
||||
var innerAgent = agent.GetService<ChatClientAgent>();
|
||||
|
||||
// Assert — the pipeline wraps the raw client, so the outer client is not the same object.
|
||||
Assert.NotNull(innerAgent);
|
||||
Assert.NotSame(rawClient, innerAgent!.ChatClient);
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region AIContextProviders
|
||||
|
||||
/// <summary>
|
||||
/// Verify that additional AIContextProviders from options are passed to the inner ChatClientAgent,
|
||||
/// not merged into the chat client builder pipeline.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void AIContextProviders_ArePassedToInnerAgent()
|
||||
{
|
||||
// Arrange
|
||||
var chatClient = new Mock<IChatClient>().Object;
|
||||
var todoProvider = new TodoProvider();
|
||||
|
||||
// Act
|
||||
var agent = new HarnessAgent(chatClient, TestMaxContextWindowTokens, TestMaxOutputTokens, new HarnessAgentOptions
|
||||
{
|
||||
AIContextProviders = [todoProvider],
|
||||
});
|
||||
var innerAgent = agent.GetService<ChatClientAgent>();
|
||||
|
||||
// Assert — the TodoProvider should appear in the inner agent's AIContextProviders.
|
||||
Assert.NotNull(innerAgent);
|
||||
Assert.NotNull(innerAgent!.AIContextProviders);
|
||||
Assert.Contains(todoProvider, innerAgent.AIContextProviders!);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verify that when no AIContextProviders are specified, the inner agent has no additional providers.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void AIContextProviders_IsNullWhenNoneSpecified()
|
||||
{
|
||||
// Arrange
|
||||
var chatClient = new Mock<IChatClient>().Object;
|
||||
|
||||
// Act
|
||||
var agent = new HarnessAgent(chatClient, TestMaxContextWindowTokens, TestMaxOutputTokens);
|
||||
var innerAgent = agent.GetService<ChatClientAgent>();
|
||||
|
||||
// Assert
|
||||
Assert.NotNull(innerAgent);
|
||||
Assert.Null(innerAgent!.AIContextProviders);
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region ChatOptions and Tools
|
||||
|
||||
/// <summary>
|
||||
/// Verify that tools from ChatOptions are passed to the model during invocation.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task ChatOptions_ToolsArePreservedAsync()
|
||||
{
|
||||
// Arrange
|
||||
var tool = AIFunctionFactory.Create(() => "test", "TestTool");
|
||||
var mockClient = new Mock<IChatClient>();
|
||||
ChatOptions? capturedOptions = null;
|
||||
mockClient
|
||||
.Setup(c => c.GetResponseAsync(
|
||||
It.IsAny<IEnumerable<ChatMessage>>(),
|
||||
It.IsAny<ChatOptions>(),
|
||||
It.IsAny<CancellationToken>()))
|
||||
.Callback<IEnumerable<ChatMessage>, ChatOptions?, CancellationToken>((_, opts, _) => capturedOptions = opts)
|
||||
.ReturnsAsync(new ChatResponse(new ChatMessage(ChatRole.Assistant, "Done")));
|
||||
|
||||
var agent = new HarnessAgent(mockClient.Object, TestMaxContextWindowTokens, TestMaxOutputTokens, new HarnessAgentOptions
|
||||
{
|
||||
ChatOptions = new ChatOptions
|
||||
{
|
||||
Tools = [tool],
|
||||
},
|
||||
});
|
||||
var session = await agent.CreateSessionAsync();
|
||||
|
||||
// Act
|
||||
await agent.RunAsync([new ChatMessage(ChatRole.User, "Hi")], session);
|
||||
|
||||
// Assert — verify the tool was included in the ChatOptions passed to the model.
|
||||
Assert.NotNull(capturedOptions);
|
||||
Assert.NotNull(capturedOptions!.Tools);
|
||||
Assert.Contains(capturedOptions.Tools, t => t == tool);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verify that the source ChatOptions are cloned and not modified.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void ChatOptions_SourceIsNotModified()
|
||||
{
|
||||
// Arrange
|
||||
var chatClient = new Mock<IChatClient>().Object;
|
||||
var sourceChatOptions = new ChatOptions
|
||||
{
|
||||
Instructions = "original instructions",
|
||||
Temperature = 0.7f,
|
||||
};
|
||||
|
||||
// Act
|
||||
_ = new HarnessAgent(chatClient, TestMaxContextWindowTokens, TestMaxOutputTokens, new HarnessAgentOptions
|
||||
{
|
||||
ChatOptions = sourceChatOptions,
|
||||
});
|
||||
|
||||
// Assert — source ChatOptions should not be mutated.
|
||||
Assert.Equal("original instructions", sourceChatOptions.Instructions);
|
||||
Assert.Equal(0.7f, sourceChatOptions.Temperature);
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region GetService
|
||||
|
||||
/// <summary>
|
||||
/// Verify that GetService returns the HarnessAgent for its own type.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void GetService_ReturnsSelfForHarnessAgentType()
|
||||
{
|
||||
// Arrange
|
||||
var chatClient = new Mock<IChatClient>().Object;
|
||||
|
||||
// Act
|
||||
var agent = new HarnessAgent(chatClient, TestMaxContextWindowTokens, TestMaxOutputTokens);
|
||||
|
||||
// Assert
|
||||
Assert.Same(agent, agent.GetService<HarnessAgent>());
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verify that GetService returns the inner ChatClientAgent.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void GetService_ReturnsInnerChatClientAgent()
|
||||
{
|
||||
// Arrange
|
||||
var chatClient = new Mock<IChatClient>().Object;
|
||||
|
||||
// Act
|
||||
var agent = new HarnessAgent(chatClient, TestMaxContextWindowTokens, TestMaxOutputTokens);
|
||||
|
||||
// Assert
|
||||
Assert.NotNull(agent.GetService<ChatClientAgent>());
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region RunAsync Delegation
|
||||
|
||||
/// <summary>
|
||||
/// Verify that RunAsync delegates to the inner ChatClientAgent.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task RunAsync_DelegatesToInnerAgentAsync()
|
||||
{
|
||||
// Arrange
|
||||
var mockClient = new Mock<IChatClient>();
|
||||
mockClient
|
||||
.Setup(c => c.GetResponseAsync(
|
||||
It.IsAny<IEnumerable<ChatMessage>>(),
|
||||
It.IsAny<ChatOptions>(),
|
||||
It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(new ChatResponse(new ChatMessage(ChatRole.Assistant, "Hello!")));
|
||||
|
||||
var agent = new HarnessAgent(mockClient.Object, TestMaxContextWindowTokens, TestMaxOutputTokens);
|
||||
var session = await agent.CreateSessionAsync();
|
||||
|
||||
// Act
|
||||
var response = await agent.RunAsync(
|
||||
[new ChatMessage(ChatRole.User, "Hi")],
|
||||
session);
|
||||
|
||||
// Assert
|
||||
Assert.NotNull(response);
|
||||
Assert.True(response.Messages.Any());
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region DefaultInstructions
|
||||
|
||||
/// <summary>
|
||||
/// Verify that DefaultInstructions is a non-empty public constant.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void DefaultInstructions_IsNonEmpty()
|
||||
{
|
||||
// Assert
|
||||
Assert.False(string.IsNullOrWhiteSpace(HarnessAgent.DefaultInstructions));
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region AsHarnessAgent Extension Method
|
||||
|
||||
/// <summary>
|
||||
/// Verify that AsHarnessAgent creates a HarnessAgent with default options.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void AsHarnessAgent_CreatesAgentWithDefaults()
|
||||
{
|
||||
// Arrange
|
||||
var chatClient = new Mock<IChatClient>().Object;
|
||||
|
||||
// Act
|
||||
var agent = chatClient.AsHarnessAgent(TestMaxContextWindowTokens, TestMaxOutputTokens);
|
||||
|
||||
// Assert
|
||||
Assert.NotNull(agent);
|
||||
Assert.IsType<HarnessAgent>(agent);
|
||||
Assert.Equal(HarnessAgent.DefaultInstructions, agent.GetService<ChatClientAgent>()!.Instructions);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verify that AsHarnessAgent passes options through to the HarnessAgent.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void AsHarnessAgent_PassesOptionsThrough()
|
||||
{
|
||||
// Arrange
|
||||
var chatClient = new Mock<IChatClient>().Object;
|
||||
|
||||
// Act
|
||||
var agent = chatClient.AsHarnessAgent(TestMaxContextWindowTokens, TestMaxOutputTokens, new HarnessAgentOptions
|
||||
{
|
||||
Name = "ExtensionAgent",
|
||||
ChatOptions = new ChatOptions { Instructions = "Custom instructions" },
|
||||
});
|
||||
var innerAgent = agent.GetService<ChatClientAgent>();
|
||||
|
||||
// Assert
|
||||
Assert.Equal("ExtensionAgent", agent.Name);
|
||||
Assert.NotNull(innerAgent);
|
||||
Assert.Equal("Custom instructions", innerAgent!.Instructions);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verify that AsHarnessAgent throws when chatClient is null.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void AsHarnessAgent_ThrowsWhenChatClientIsNull()
|
||||
{
|
||||
// Act & Assert
|
||||
Assert.Throws<ArgumentNullException>(() => ((IChatClient)null!).AsHarnessAgent(TestMaxContextWindowTokens, TestMaxOutputTokens));
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
-11
@@ -1,11 +0,0 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<NoWarn>$(NoWarn);MAAI001</NoWarn>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\src\Microsoft.Agents.AI.Harness\Microsoft.Agents.AI.Harness.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
+1
-37
@@ -267,43 +267,7 @@ public sealed class OpenAIResponsesAgentResolutionIntegrationTests : IAsyncDispo
|
||||
Assert.Equal(System.Net.HttpStatusCode.BadRequest, httpResponse.StatusCode);
|
||||
|
||||
string responseJson = await httpResponse.Content.ReadAsStringAsync();
|
||||
using JsonDocument errorDoc1 = JsonDocument.Parse(responseJson);
|
||||
string? errorCode = errorDoc1.RootElement.GetProperty("error").GetProperty("code").GetString();
|
||||
Assert.Equal("missing_required_parameter", errorCode);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that the model field alone is not used for agent resolution.
|
||||
/// The multi-agent endpoint requires agent.name or metadata.entity_id; setting only model returns 400.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task CreateResponse_WithModelOnly_ReturnsBadRequestAsync()
|
||||
{
|
||||
// Arrange
|
||||
const string AgentName = "test-agent";
|
||||
|
||||
this._httpClient = await this.CreateTestServerWithAgentResolutionAsync(
|
||||
(AgentName, "Instructions", "Response"));
|
||||
|
||||
// Act - Send request with model=agentName but no agent.name or metadata.entity_id
|
||||
using StringContent requestContent = new(JsonSerializer.Serialize(new
|
||||
{
|
||||
model = AgentName,
|
||||
input = new[]
|
||||
{
|
||||
new { type = "message", role = "user", content = "Test message" }
|
||||
}
|
||||
}), Encoding.UTF8, "application/json");
|
||||
|
||||
using HttpResponseMessage httpResponse = await this._httpClient!.PostAsync(new Uri("/v1/responses", UriKind.Relative), requestContent);
|
||||
|
||||
// Assert - model is not used for agent resolution
|
||||
Assert.Equal(System.Net.HttpStatusCode.BadRequest, httpResponse.StatusCode);
|
||||
|
||||
string responseJson = await httpResponse.Content.ReadAsStringAsync();
|
||||
using JsonDocument errorDoc2 = JsonDocument.Parse(responseJson);
|
||||
string? errorCode = errorDoc2.RootElement.GetProperty("error").GetProperty("code").GetString();
|
||||
Assert.Equal("missing_required_parameter", errorCode);
|
||||
Assert.Contains("agent.name", responseJson, StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -627,455 +627,4 @@ public class OpenTelemetryAgentTests
|
||||
}
|
||||
|
||||
private static string ReplaceWhitespace(string? input) => Regex.Replace(input ?? "", @"\s+", "").Trim();
|
||||
|
||||
#region AutoWireChatClient
|
||||
|
||||
[Fact]
|
||||
public async Task AutoWireChatClient_DefaultsToEnabled_EmitsChatSpan_Async()
|
||||
{
|
||||
var sourceName = Guid.NewGuid().ToString();
|
||||
var activities = new List<Activity>();
|
||||
using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder()
|
||||
.AddSource(sourceName)
|
||||
.AddInMemoryExporter(activities)
|
||||
.Build();
|
||||
|
||||
var fakeChatClient = new AutoWireTestChatClient();
|
||||
var inner = new ChatClientAgent(fakeChatClient);
|
||||
using var agent = new OpenTelemetryAgent(inner, sourceName);
|
||||
|
||||
_ = await agent.RunAsync("hi");
|
||||
|
||||
// Expect 2 activities: the inner chat span (from auto-wired OpenTelemetryChatClient) and the invoke_agent span.
|
||||
Assert.Equal(2, activities.Count);
|
||||
Assert.Contains(activities, a => a.DisplayName.StartsWith("invoke_agent", StringComparison.Ordinal));
|
||||
Assert.Contains(activities, a => string.Equals(a.GetTagItem("gen_ai.operation.name") as string, "chat", StringComparison.Ordinal));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task AutoWireChatClient_Streaming_EmitsChatSpan_Async()
|
||||
{
|
||||
var sourceName = Guid.NewGuid().ToString();
|
||||
var activities = new List<Activity>();
|
||||
using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder()
|
||||
.AddSource(sourceName)
|
||||
.AddInMemoryExporter(activities)
|
||||
.Build();
|
||||
|
||||
var fakeChatClient = new AutoWireTestChatClient();
|
||||
var inner = new ChatClientAgent(fakeChatClient);
|
||||
using var agent = new OpenTelemetryAgent(inner, sourceName);
|
||||
|
||||
await foreach (var _ in agent.RunStreamingAsync("hi"))
|
||||
{
|
||||
}
|
||||
|
||||
Assert.Equal(2, activities.Count);
|
||||
Assert.Contains(activities, a => a.DisplayName.StartsWith("invoke_agent", StringComparison.Ordinal));
|
||||
Assert.Contains(activities, a => string.Equals(a.GetTagItem("gen_ai.operation.name") as string, "chat", StringComparison.Ordinal));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task AutoWireChatClient_Disabled_DoesNotEmitChatSpan_Async()
|
||||
{
|
||||
var sourceName = Guid.NewGuid().ToString();
|
||||
var activities = new List<Activity>();
|
||||
using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder()
|
||||
.AddSource(sourceName)
|
||||
.AddInMemoryExporter(activities)
|
||||
.Build();
|
||||
|
||||
var fakeChatClient = new AutoWireTestChatClient();
|
||||
var inner = new ChatClientAgent(fakeChatClient);
|
||||
using var agent = new OpenTelemetryAgent(inner, sourceName, autoWireChatClient: false);
|
||||
|
||||
_ = await agent.RunAsync("hi");
|
||||
|
||||
// Only the invoke_agent activity should be emitted; no chat span.
|
||||
var activity = Assert.Single(activities);
|
||||
Assert.StartsWith("invoke_agent", activity.DisplayName);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task AutoWireChatClient_NonChatClientAgent_NoOp_Async()
|
||||
{
|
||||
// Inner is not a ChatClientAgent — auto-wiring must be a no-op and options must remain null.
|
||||
AgentRunOptions? observedOptions = null;
|
||||
var inner = new TestAIAgent
|
||||
{
|
||||
RunAsyncFunc = (messages, session, options, ct) =>
|
||||
{
|
||||
observedOptions = options;
|
||||
return Task.FromResult(new AgentResponse(new ChatMessage(ChatRole.Assistant, "ok")));
|
||||
},
|
||||
};
|
||||
|
||||
using var agent = new OpenTelemetryAgent(inner);
|
||||
|
||||
_ = await agent.RunAsync("hi");
|
||||
|
||||
Assert.Null(observedOptions);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task AutoWireChatClient_UseProvidedChatClientAsIs_DoesNotEmitChatSpan_Async()
|
||||
{
|
||||
var sourceName = Guid.NewGuid().ToString();
|
||||
var activities = new List<Activity>();
|
||||
using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder()
|
||||
.AddSource(sourceName)
|
||||
.AddInMemoryExporter(activities)
|
||||
.Build();
|
||||
|
||||
var fakeChatClient = new AutoWireTestChatClient();
|
||||
var inner = new ChatClientAgent(fakeChatClient, new ChatClientAgentOptions { UseProvidedChatClientAsIs = true });
|
||||
using var agent = new OpenTelemetryAgent(inner, sourceName);
|
||||
|
||||
_ = await agent.RunAsync("hi");
|
||||
|
||||
// UseProvidedChatClientAsIs opts out of auto-wiring, so only the invoke_agent span should be emitted.
|
||||
var activity = Assert.Single(activities);
|
||||
Assert.StartsWith("invoke_agent", activity.DisplayName);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task AutoWireChatClient_AlreadyInstrumented_DoesNotDoubleWrap_Async()
|
||||
{
|
||||
var sourceName = Guid.NewGuid().ToString();
|
||||
var activities = new List<Activity>();
|
||||
using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder()
|
||||
.AddSource(sourceName)
|
||||
.AddInMemoryExporter(activities)
|
||||
.Build();
|
||||
|
||||
var fakeChatClient = new AutoWireTestChatClient();
|
||||
// Pre-wrap with OpenTelemetryChatClient on the same source so spans flow through the tracer.
|
||||
IChatClient preWrapped = fakeChatClient.AsBuilder().UseOpenTelemetry(sourceName: sourceName).Build();
|
||||
var inner = new ChatClientAgent(preWrapped);
|
||||
using var agent = new OpenTelemetryAgent(inner, sourceName);
|
||||
|
||||
_ = await agent.RunAsync("hi");
|
||||
|
||||
// Expect exactly 2 activities (one invoke_agent + one chat from the pre-existing wrapper). If we had double-wrapped, we would see 3.
|
||||
Assert.Equal(2, activities.Count);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task AutoWireChatClient_PreservesUserChatClientFactory_Async()
|
||||
{
|
||||
var sourceName = Guid.NewGuid().ToString();
|
||||
var activities = new List<Activity>();
|
||||
using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder()
|
||||
.AddSource(sourceName)
|
||||
.AddInMemoryExporter(activities)
|
||||
.Build();
|
||||
|
||||
bool userFactoryCalled = false;
|
||||
var fakeChatClient = new AutoWireTestChatClient();
|
||||
var inner = new ChatClientAgent(fakeChatClient);
|
||||
using var agent = new OpenTelemetryAgent(inner, sourceName);
|
||||
|
||||
var runOptions = new ChatClientAgentRunOptions
|
||||
{
|
||||
ChatClientFactory = cc =>
|
||||
{
|
||||
userFactoryCalled = true;
|
||||
return cc;
|
||||
},
|
||||
};
|
||||
|
||||
_ = await agent.RunAsync("hi", options: runOptions);
|
||||
|
||||
Assert.True(userFactoryCalled);
|
||||
// Auto-wiring should still produce a chat span on top of the user's factory.
|
||||
Assert.Equal(2, activities.Count);
|
||||
Assert.Contains(activities, a => string.Equals(a.GetTagItem("gen_ai.operation.name") as string, "chat", StringComparison.Ordinal));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task AutoWireChatClient_PlainAgentRunOptions_PreservesBaseProperties_Async()
|
||||
{
|
||||
// Auto-wiring converts a plain AgentRunOptions into a ChatClientAgentRunOptions. The base
|
||||
// properties (ContinuationToken, AllowBackgroundResponses, AdditionalProperties, ResponseFormat)
|
||||
// must be preserved so they reach the inner agent.
|
||||
AgentRunOptions? observedOptions = null;
|
||||
var fakeChatClient = new AutoWireTestChatClient();
|
||||
var innerChatClientAgent = new ChatClientAgent(fakeChatClient);
|
||||
|
||||
// Wrapping agent: surfaces the ChatClientAgent via GetService (so auto-wiring activates),
|
||||
// but captures the AgentRunOptions passed to RunAsync by the OpenTelemetryAgent.
|
||||
var wrapper = new TestAIAgent
|
||||
{
|
||||
GetServiceFunc = (type, key) =>
|
||||
type == typeof(ChatClientAgent) ? innerChatClientAgent : null,
|
||||
RunAsyncFunc = (messages, session, options, ct) =>
|
||||
{
|
||||
observedOptions = options;
|
||||
return Task.FromResult(new AgentResponse(new ChatMessage(ChatRole.Assistant, "ok")));
|
||||
},
|
||||
};
|
||||
|
||||
using var agent = new OpenTelemetryAgent(wrapper);
|
||||
|
||||
var additionalProps = new AdditionalPropertiesDictionary { ["customKey"] = "customValue" };
|
||||
var inputOptions = new AgentRunOptions
|
||||
{
|
||||
AllowBackgroundResponses = true,
|
||||
AdditionalProperties = additionalProps,
|
||||
ResponseFormat = ChatResponseFormat.Json,
|
||||
};
|
||||
|
||||
_ = await agent.RunAsync("hi", options: inputOptions);
|
||||
|
||||
Assert.NotNull(observedOptions);
|
||||
Assert.IsType<ChatClientAgentRunOptions>(observedOptions);
|
||||
Assert.Equal(true, observedOptions!.AllowBackgroundResponses);
|
||||
Assert.Same(ChatResponseFormat.Json, observedOptions.ResponseFormat);
|
||||
Assert.NotNull(observedOptions.AdditionalProperties);
|
||||
Assert.Equal("customValue", observedOptions.AdditionalProperties!["customKey"]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task AutoWireChatClient_UserFactoryReturnsInstrumentedClient_DoesNotDoubleWrap_Async()
|
||||
{
|
||||
var sourceName = Guid.NewGuid().ToString();
|
||||
var activities = new List<Activity>();
|
||||
using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder()
|
||||
.AddSource(sourceName)
|
||||
.AddInMemoryExporter(activities)
|
||||
.Build();
|
||||
|
||||
var fakeChatClient = new AutoWireTestChatClient();
|
||||
var inner = new ChatClientAgent(fakeChatClient);
|
||||
using var agent = new OpenTelemetryAgent(inner, sourceName);
|
||||
|
||||
// User factory wraps the chat client with OpenTelemetryChatClient itself.
|
||||
var runOptions = new ChatClientAgentRunOptions
|
||||
{
|
||||
ChatClientFactory = cc => cc.AsBuilder().UseOpenTelemetry(sourceName: sourceName).Build(),
|
||||
};
|
||||
|
||||
_ = await agent.RunAsync("hi", options: runOptions);
|
||||
|
||||
// Expect 2 activities (invoke_agent + a single chat span). If we double-wrapped, we would see 3.
|
||||
Assert.Equal(2, activities.Count);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(null)]
|
||||
[InlineData("")]
|
||||
[InlineData(" ")]
|
||||
[InlineData("\t")]
|
||||
public async Task Ctor_NullOrWhitespaceSourceName_AutoWiredChatClientUsesDefaultSource_Async(string? sourceName)
|
||||
{
|
||||
// Both the agent-level invoke_agent span and the auto-wired chat span must be emitted under
|
||||
// OpenTelemetryConsts.DefaultSourceName when the caller passes null, "", or whitespace, so they reach
|
||||
// the same ActivitySource and are not silently dropped by the exporter.
|
||||
var activities = new List<Activity>();
|
||||
using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder()
|
||||
.AddSource("Experimental.Microsoft.Agents.AI")
|
||||
.AddInMemoryExporter(activities)
|
||||
.Build();
|
||||
|
||||
var fakeChatClient = new AutoWireTestChatClient();
|
||||
var inner = new ChatClientAgent(fakeChatClient);
|
||||
using var agent = new OpenTelemetryAgent(inner, sourceName);
|
||||
|
||||
_ = await agent.RunAsync("hi");
|
||||
|
||||
Assert.Equal(2, activities.Count);
|
||||
Assert.All(activities, a => Assert.Equal("Experimental.Microsoft.Agents.AI", a.Source.Name));
|
||||
Assert.Contains(activities, a => a.DisplayName.StartsWith("invoke_agent", StringComparison.Ordinal));
|
||||
Assert.Contains(activities, a => string.Equals(a.GetTagItem("gen_ai.operation.name") as string, "chat", StringComparison.Ordinal));
|
||||
}
|
||||
|
||||
#pragma warning disable MEAI001 // ResponseContinuationToken is experimental.
|
||||
[Fact]
|
||||
public async Task AutoWireChatClient_PlainAgentRunOptions_PreservesContinuationToken_Async()
|
||||
{
|
||||
// ContinuationToken is the fourth base AgentRunOptions property copied by CopyBaseAgentRunOptions
|
||||
// and is not exercised by AutoWireChatClient_PlainAgentRunOptions_PreservesBaseProperties_Async.
|
||||
AgentRunOptions? observedOptions = null;
|
||||
var fakeChatClient = new AutoWireTestChatClient();
|
||||
var innerChatClientAgent = new ChatClientAgent(fakeChatClient);
|
||||
|
||||
var wrapper = new TestAIAgent
|
||||
{
|
||||
GetServiceFunc = (type, key) =>
|
||||
type == typeof(ChatClientAgent) ? innerChatClientAgent : null,
|
||||
RunAsyncFunc = (messages, session, options, ct) =>
|
||||
{
|
||||
observedOptions = options;
|
||||
return Task.FromResult(new AgentResponse(new ChatMessage(ChatRole.Assistant, "ok")));
|
||||
},
|
||||
};
|
||||
|
||||
using var agent = new OpenTelemetryAgent(wrapper);
|
||||
|
||||
var token = ResponseContinuationToken.FromBytes(new byte[] { 1, 2, 3 });
|
||||
var inputOptions = new AgentRunOptions
|
||||
{
|
||||
ContinuationToken = token,
|
||||
};
|
||||
|
||||
_ = await agent.RunAsync("hi", options: inputOptions);
|
||||
|
||||
Assert.NotNull(observedOptions);
|
||||
Assert.IsType<ChatClientAgentRunOptions>(observedOptions);
|
||||
Assert.Same(token, observedOptions!.ContinuationToken);
|
||||
}
|
||||
#pragma warning restore MEAI001
|
||||
|
||||
[Fact]
|
||||
public async Task AutoWireChatClient_ChatClientAgentRunOptions_NoUserFactory_PreservesChatOptions_Async()
|
||||
{
|
||||
// When the caller passes a ChatClientAgentRunOptions without a ChatClientFactory, the auto-wiring
|
||||
// must clone (not mutate) the caller's options, set the factory, and preserve nested ChatOptions.
|
||||
var sourceName = Guid.NewGuid().ToString();
|
||||
var activities = new List<Activity>();
|
||||
using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder()
|
||||
.AddSource(sourceName)
|
||||
.AddInMemoryExporter(activities)
|
||||
.Build();
|
||||
|
||||
ChatOptions? observedChatOptions = null;
|
||||
var fakeChatClient = new AutoWireTestChatClient
|
||||
{
|
||||
OnGetResponseAsync = (msgs, opts) => observedChatOptions = opts,
|
||||
};
|
||||
var inner = new ChatClientAgent(fakeChatClient);
|
||||
using var agent = new OpenTelemetryAgent(inner, sourceName);
|
||||
|
||||
var inputChatOptions = new ChatOptions { Temperature = 0.42f, ModelId = "test-model" };
|
||||
var inputOptions = new ChatClientAgentRunOptions(inputChatOptions);
|
||||
|
||||
_ = await agent.RunAsync("hi", options: inputOptions);
|
||||
|
||||
// Caller's options must not have been mutated (no factory installed on the caller's instance).
|
||||
Assert.Null(inputOptions.ChatClientFactory);
|
||||
|
||||
// Inner chat client must observe the caller-supplied ChatOptions.
|
||||
Assert.NotNull(observedChatOptions);
|
||||
Assert.Equal(0.42f, observedChatOptions!.Temperature);
|
||||
Assert.Equal("test-model", observedChatOptions.ModelId);
|
||||
|
||||
// Auto-wiring still produces a chat span.
|
||||
Assert.Equal(2, activities.Count);
|
||||
Assert.Contains(activities, a => string.Equals(a.GetTagItem("gen_ai.operation.name") as string, "chat", StringComparison.Ordinal));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task AutoWireChatClient_StreamingDisabled_DoesNotEmitChatSpan_Async()
|
||||
{
|
||||
// Symmetry with AutoWireChatClient_Disabled_DoesNotEmitChatSpan_Async for the streaming path.
|
||||
var sourceName = Guid.NewGuid().ToString();
|
||||
var activities = new List<Activity>();
|
||||
using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder()
|
||||
.AddSource(sourceName)
|
||||
.AddInMemoryExporter(activities)
|
||||
.Build();
|
||||
|
||||
var fakeChatClient = new AutoWireTestChatClient();
|
||||
var inner = new ChatClientAgent(fakeChatClient);
|
||||
using var agent = new OpenTelemetryAgent(inner, sourceName, autoWireChatClient: false);
|
||||
|
||||
await foreach (var _ in agent.RunStreamingAsync("hi"))
|
||||
{
|
||||
}
|
||||
|
||||
var activity = Assert.Single(activities);
|
||||
Assert.StartsWith("invoke_agent", activity.DisplayName);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task AutoWireChatClient_PlainAgentRunOptions_RealChatClientAgent_EmitsChatSpan_Async()
|
||||
{
|
||||
// High-level callers may pass the abstract base AgentRunOptions (not ChatClientAgentRunOptions) when
|
||||
// wiring a ChatClientAgent. Auto-wiring must still kick in: convert to ChatClientAgentRunOptions,
|
||||
// install the OTel-wrapping factory, and produce both the invoke_agent and chat spans end-to-end.
|
||||
var sourceName = Guid.NewGuid().ToString();
|
||||
var activities = new List<Activity>();
|
||||
using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder()
|
||||
.AddSource(sourceName)
|
||||
.AddInMemoryExporter(activities)
|
||||
.Build();
|
||||
|
||||
ChatOptions? observedChatOptions = null;
|
||||
var fakeChatClient = new AutoWireTestChatClient
|
||||
{
|
||||
OnGetResponseAsync = (_, opts) => observedChatOptions = opts,
|
||||
};
|
||||
var inner = new ChatClientAgent(fakeChatClient);
|
||||
using var agent = new OpenTelemetryAgent(inner, sourceName);
|
||||
|
||||
// Pass the base AgentRunOptions, not ChatClientAgentRunOptions.
|
||||
var inputOptions = new AgentRunOptions { AllowBackgroundResponses = false };
|
||||
|
||||
_ = await agent.RunAsync("hi", options: inputOptions);
|
||||
|
||||
// Inner chat client was actually invoked (auto-wired factory ran without breaking the pipeline).
|
||||
Assert.NotNull(observedChatOptions);
|
||||
|
||||
Assert.Equal(2, activities.Count);
|
||||
Assert.Contains(activities, a => a.DisplayName.StartsWith("invoke_agent", StringComparison.Ordinal));
|
||||
Assert.Contains(activities, a => string.Equals(a.GetTagItem("gen_ai.operation.name") as string, "chat", StringComparison.Ordinal));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task AutoWireChatClient_PlainAgentRunOptions_RealChatClientAgent_StreamingEmitsChatSpan_Async()
|
||||
{
|
||||
// Same as the sync test above but for the streaming path so both invocation paths
|
||||
// are covered when callers pass a base AgentRunOptions.
|
||||
var sourceName = Guid.NewGuid().ToString();
|
||||
var activities = new List<Activity>();
|
||||
using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder()
|
||||
.AddSource(sourceName)
|
||||
.AddInMemoryExporter(activities)
|
||||
.Build();
|
||||
|
||||
ChatOptions? observedChatOptions = null;
|
||||
var fakeChatClient = new AutoWireTestChatClient
|
||||
{
|
||||
OnGetResponseAsync = (_, opts) => observedChatOptions = opts,
|
||||
};
|
||||
var inner = new ChatClientAgent(fakeChatClient);
|
||||
using var agent = new OpenTelemetryAgent(inner, sourceName);
|
||||
|
||||
var inputOptions = new AgentRunOptions { AllowBackgroundResponses = false };
|
||||
|
||||
await foreach (var _ in agent.RunStreamingAsync("hi", options: inputOptions))
|
||||
{
|
||||
}
|
||||
|
||||
Assert.NotNull(observedChatOptions);
|
||||
|
||||
Assert.Equal(2, activities.Count);
|
||||
Assert.Contains(activities, a => a.DisplayName.StartsWith("invoke_agent", StringComparison.Ordinal));
|
||||
Assert.Contains(activities, a => string.Equals(a.GetTagItem("gen_ai.operation.name") as string, "chat", StringComparison.Ordinal));
|
||||
}
|
||||
|
||||
private sealed class AutoWireTestChatClient : IChatClient
|
||||
{
|
||||
public Action<IEnumerable<ChatMessage>, ChatOptions?>? OnGetResponseAsync { get; set; }
|
||||
|
||||
public Task<ChatResponse> GetResponseAsync(IEnumerable<ChatMessage> messages, ChatOptions? options = null, CancellationToken cancellationToken = default)
|
||||
{
|
||||
this.OnGetResponseAsync?.Invoke(messages, options);
|
||||
return Task.FromResult(new ChatResponse(new ChatMessage(ChatRole.Assistant, "ok")));
|
||||
}
|
||||
|
||||
public async IAsyncEnumerable<ChatResponseUpdate> GetStreamingResponseAsync(IEnumerable<ChatMessage> messages, ChatOptions? options = null, [EnumeratorCancellation] CancellationToken cancellationToken = default)
|
||||
{
|
||||
this.OnGetResponseAsync?.Invoke(messages, options);
|
||||
await Task.Yield();
|
||||
yield return new ChatResponseUpdate(ChatRole.Assistant, "ok");
|
||||
}
|
||||
|
||||
public object? GetService(Type serviceType, object? serviceKey = null) =>
|
||||
serviceType?.IsInstanceOfType(this) == true ? this : null;
|
||||
|
||||
public void Dispose() { }
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
-26
@@ -1,26 +0,0 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using System.Collections.Generic;
|
||||
using Microsoft.Extensions.AI;
|
||||
|
||||
namespace Microsoft.Agents.AI.Workflows.UnitTests;
|
||||
|
||||
public class AIAgentsAbstractionsExtensionsTests
|
||||
{
|
||||
[Fact]
|
||||
public void CopyWithAssistantToUserForOtherParticipants_DoesNotMutateOriginalMessages()
|
||||
{
|
||||
ChatMessage original = new(ChatRole.Assistant, "from first agent")
|
||||
{
|
||||
AuthorName = "firstAgent"
|
||||
};
|
||||
|
||||
List<ChatMessage> copied = new[] { original }
|
||||
.CopyWithAssistantToUserForOtherParticipants("secondAgent");
|
||||
|
||||
Assert.Single(copied);
|
||||
Assert.Equal(ChatRole.Assistant, original.Role);
|
||||
Assert.Equal(ChatRole.User, copied[0].Role);
|
||||
Assert.NotSame(original, copied[0]);
|
||||
}
|
||||
}
|
||||
@@ -209,36 +209,6 @@ public class HandoffOrchestrationTests
|
||||
Assert.DoesNotContain(capturedNextAgentMessages, m => m.Role == ChatRole.Tool && m.Contents.Any(c => c is FunctionResultContent frc && frc.Result?.ToString() == "Transferred."));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Handoffs_ReassignedMessagesDoNotMutateSharedConversationAsync()
|
||||
{
|
||||
var firstAgent = new ChatClientAgent(new MockChatClient((_, options) =>
|
||||
{
|
||||
string? transferFuncName = options?.Tools?.FirstOrDefault(t => t.Name.StartsWith("handoff_to_", StringComparison.Ordinal))?.Name;
|
||||
Assert.NotNull(transferFuncName);
|
||||
|
||||
return new ChatResponse([
|
||||
new ChatMessage(ChatRole.Assistant, "Context from first agent"),
|
||||
new ChatMessage(ChatRole.Assistant, [new FunctionCallContent("call1", transferFuncName)]),
|
||||
]);
|
||||
}), name: "firstAgent");
|
||||
CapturingAgent secondAgent = new("secondAgent", "The second agent", "Context from first agent");
|
||||
|
||||
var workflow =
|
||||
AgentWorkflowBuilder.CreateHandoffBuilderWith(firstAgent)
|
||||
.WithHandoff(firstAgent, secondAgent)
|
||||
.Build();
|
||||
|
||||
(_, List<ChatMessage>? result, _, _) = await RunWorkflowAsync(workflow, [new ChatMessage(ChatRole.User, "start")]);
|
||||
|
||||
Assert.NotNull(result);
|
||||
Assert.Equal(ChatRole.User, secondAgent.RoleSeenDuringRun);
|
||||
|
||||
ChatMessage sharedMessage = Assert.Single(result, m => m.Text == "Context from first agent");
|
||||
Assert.Equal(ChatRole.Assistant, sharedMessage.Role);
|
||||
Assert.NotSame(sharedMessage, secondAgent.MessageSeenDuringRun);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Handoffs_TwoTransfers_HandoffTargetsDoNotReceiveHandoffFunctionMessagesAsync()
|
||||
{
|
||||
@@ -304,208 +274,6 @@ public class HandoffOrchestrationTests
|
||||
Assert.DoesNotContain(capturedThirdAgentMessages, m => m.Role == ChatRole.Tool && m.Contents.Any(c => c is FunctionResultContent));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Handoffs_MultipleTransfers_MergedMessagesPreserveStepOrderAsync()
|
||||
{
|
||||
// Regression test for https://github.com/microsoft/agent-framework/issues/4544
|
||||
//
|
||||
// Scenario: a multi-step handoff (A -> B -> C) is run through a workflow
|
||||
// exposed as an AIAgent. When invoked via the non-streaming RunAsync
|
||||
// entry-point, the merged AgentResponse.Messages must keep each step's
|
||||
// messages contiguous — in particular, the "Transferred." tool result
|
||||
// synthesized for a handoff function call must appear immediately after
|
||||
// the function call that triggered it, before any messages from later
|
||||
// steps. Streaming output already preserves this order; the bug
|
||||
// manifested only after merging, where the synthesized tool results were
|
||||
// bunched at the end of the response, far from their originating function
|
||||
// calls (and consequently corrupted ChatHistory order).
|
||||
//
|
||||
// Each underlying agent response sets a real ResponseId — that is what
|
||||
// causes MessageMerger to group its updates under that key. If the
|
||||
// synthesized Tool update is emitted without that ResponseId, the merger
|
||||
// routes it to the global "dangling" bucket and flushes it last,
|
||||
// breaking per-step grouping.
|
||||
|
||||
var initialAgent = new ChatClientAgent(new MockChatClient((messages, options) =>
|
||||
{
|
||||
string? transferFuncName = options?.Tools?.FirstOrDefault(t => t.Name.StartsWith("handoff_to_", StringComparison.Ordinal))?.Name;
|
||||
Assert.NotNull(transferFuncName);
|
||||
return new(new ChatMessage(ChatRole.Assistant, [new TextContent("Routing to second agent"), new FunctionCallContent("call1", transferFuncName)]) { MessageId = "msg-initial" }) { ResponseId = "resp-initial" };
|
||||
}), name: "initialAgent");
|
||||
|
||||
var secondAgent = new ChatClientAgent(new MockChatClient((messages, options) =>
|
||||
{
|
||||
string? transferFuncName = options?.Tools?.FirstOrDefault(t => t.Name.StartsWith("handoff_to_", StringComparison.Ordinal))?.Name;
|
||||
Assert.NotNull(transferFuncName);
|
||||
return new(new ChatMessage(ChatRole.Assistant, [new TextContent("Routing to third agent"), new FunctionCallContent("call2", transferFuncName)]) { MessageId = "msg-second" }) { ResponseId = "resp-second" };
|
||||
}), name: "secondAgent", description: "The second agent");
|
||||
|
||||
var thirdAgent = new ChatClientAgent(new MockChatClient((messages, options) =>
|
||||
new(new ChatMessage(ChatRole.Assistant, "Hello from agent3") { MessageId = "msg-third" }) { ResponseId = "resp-third" }),
|
||||
name: "thirdAgent",
|
||||
description: "The third / final agent");
|
||||
|
||||
var workflow =
|
||||
AgentWorkflowBuilder.CreateHandoffBuilderWith(initialAgent)
|
||||
.WithHandoff(initialAgent, secondAgent)
|
||||
.WithHandoff(secondAgent, thirdAgent)
|
||||
.Build();
|
||||
|
||||
AIAgent hostAgent = workflow.AsAIAgent(name: "HandoffWorkflow");
|
||||
|
||||
AgentResponse response = await hostAgent.RunAsync("abc");
|
||||
|
||||
List<ChatMessage> result = response.Messages.ToList();
|
||||
|
||||
// Expected merged sequence keeps each step contiguous:
|
||||
// [0] Assistant (initialAgent): text + FunctionCall(call1)
|
||||
// [1] Tool (initialAgent): FunctionResult(call1, "Transferred.")
|
||||
// [2] Assistant (secondAgent): text + FunctionCall(call2)
|
||||
// [3] Tool (secondAgent): FunctionResult(call2, "Transferred.")
|
||||
// [4] Assistant (thirdAgent): "Hello from agent3"
|
||||
//
|
||||
// The bug surfaced as the two Tool messages being moved to the end of the
|
||||
// list — after thirdAgent's reply — which broke chat-history ordering.
|
||||
Assert.Equal(5, result.Count);
|
||||
|
||||
Assert.Equal(ChatRole.Assistant, result[0].Role);
|
||||
Assert.Contains("initialAgent", result[0].AuthorName);
|
||||
Assert.Contains(result[0].Contents, c => c is FunctionCallContent fcc && fcc.CallId == "call1");
|
||||
|
||||
Assert.Equal(ChatRole.Tool, result[1].Role);
|
||||
Assert.Contains("initialAgent", result[1].AuthorName);
|
||||
Assert.Contains(result[1].Contents, c => c is FunctionResultContent frc && frc.CallId == "call1");
|
||||
|
||||
Assert.Equal(ChatRole.Assistant, result[2].Role);
|
||||
Assert.Contains("secondAgent", result[2].AuthorName);
|
||||
Assert.Contains(result[2].Contents, c => c is FunctionCallContent fcc && fcc.CallId == "call2");
|
||||
|
||||
Assert.Equal(ChatRole.Tool, result[3].Role);
|
||||
Assert.Contains("secondAgent", result[3].AuthorName);
|
||||
Assert.Contains(result[3].Contents, c => c is FunctionResultContent frc && frc.CallId == "call2");
|
||||
|
||||
Assert.Equal(ChatRole.Assistant, result[4].Role);
|
||||
Assert.Contains("thirdAgent", result[4].AuthorName);
|
||||
Assert.Equal("Hello from agent3", result[4].Text);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Handoffs_CoordSpecCoordPingPong_MergedMessagesPreserveStepOrderAsync()
|
||||
{
|
||||
// Regression test for https://github.com/microsoft/agent-framework/issues/4544 /
|
||||
// https://github.com/microsoft/agent-framework/issues/5720.
|
||||
//
|
||||
// Scenario mirrors the user-reported "Coord -> Spec -> Coord" ping-pong where
|
||||
// the same coordinator agent is invoked twice (first to hand off to the
|
||||
// specialist, then a final time to summarize). Additionally exercises the
|
||||
// case where the specialist returns its assistant text and its handoff
|
||||
// FunctionCall in two SEPARATE ChatMessages (i.e. different MessageIds),
|
||||
// because real OpenAI streams sometimes split text and tool calls that way.
|
||||
//
|
||||
// The merged AgentResponse.Messages must preserve per-step grouping:
|
||||
// step 1 (Coord): FunctionCall(call1) then synthesized FunctionResult(call1)
|
||||
// step 2 (Spec) : TextContent "Here are recs" then FunctionCall(call2)
|
||||
// then synthesized FunctionResult(call2)
|
||||
// step 3 (Coord): TextContent "Here are recs"
|
||||
// The bug previously bunched the FunctionResult ("Transferred.") tool messages
|
||||
// at the very end, breaking the contiguity of each step's block.
|
||||
|
||||
int coordCallCount = 0;
|
||||
int specCallCount = 0;
|
||||
var coord = new ChatClientAgent(new MockChatClient((messages, options) =>
|
||||
{
|
||||
coordCallCount++;
|
||||
if (coordCallCount == 1)
|
||||
{
|
||||
string? transferFuncName = options?.Tools?.FirstOrDefault(t => t.Name.StartsWith("handoff_to_", StringComparison.Ordinal))?.Name;
|
||||
Assert.NotNull(transferFuncName);
|
||||
return new(new ChatMessage(ChatRole.Assistant, [new FunctionCallContent("call_coord_1", transferFuncName)]) { MessageId = "coord-msg-1" })
|
||||
{
|
||||
ResponseId = "resp-coord-1",
|
||||
};
|
||||
}
|
||||
|
||||
// Second (final) Coord turn: emit only assistant text - no tool call.
|
||||
// The test asserts this turn does NOT emit a handoff_to_* FunctionCallContent
|
||||
// and that the workflow terminates here (coordCallCount stays at 2).
|
||||
return new(new ChatMessage(ChatRole.Assistant, "Here are two fake Excel course recommendations") { MessageId = "coord-msg-2" })
|
||||
{
|
||||
ResponseId = "resp-coord-2",
|
||||
};
|
||||
}), name: "Coord");
|
||||
|
||||
var spec = new ChatClientAgent(new MockChatClient((messages, options) =>
|
||||
{
|
||||
specCallCount++;
|
||||
string? transferFuncName = options?.Tools?.FirstOrDefault(t => t.Name.StartsWith("handoff_to_", StringComparison.Ordinal))?.Name;
|
||||
Assert.NotNull(transferFuncName);
|
||||
|
||||
// Split text and FunctionCall into two separate ChatMessages (distinct MessageIds)
|
||||
// to mirror the real-world streaming pattern where a specialist's narration
|
||||
// and its handoff tool-call land in different ChatMessages.
|
||||
return new(
|
||||
[
|
||||
new ChatMessage(ChatRole.Assistant, "Here are two fake Excel course recommendations") { MessageId = "spec-msg-text" },
|
||||
new ChatMessage(ChatRole.Assistant, [new FunctionCallContent("call_spec_1", transferFuncName)]) { MessageId = "spec-msg-call" },
|
||||
])
|
||||
{
|
||||
ResponseId = "resp-spec-1",
|
||||
};
|
||||
}), name: "Spec", description: "The specialist agent");
|
||||
|
||||
var workflow =
|
||||
AgentWorkflowBuilder.CreateHandoffBuilderWith(coord)
|
||||
.WithHandoff(coord, spec)
|
||||
.WithHandoff(spec, coord)
|
||||
.Build();
|
||||
|
||||
AIAgent hostAgent = workflow.AsAIAgent(name: "HandoffWorkflow");
|
||||
|
||||
AgentResponse response = await hostAgent.RunAsync("Tell me about Excel courses.");
|
||||
|
||||
List<ChatMessage> result = response.Messages.ToList();
|
||||
|
||||
// Expected merged sequence keeps each step contiguous (6 messages):
|
||||
// [0] Assistant (Coord): FunctionCall(call_coord_1)
|
||||
// [1] Tool (Coord): FunctionResult(call_coord_1, "Transferred.")
|
||||
// [2] Assistant (Spec) : TextContent "Here are recs"
|
||||
// [3] Assistant (Spec) : FunctionCall(call_spec_1)
|
||||
// [4] Tool (Spec) : FunctionResult(call_spec_1, "Transferred.")
|
||||
// [5] Assistant (Coord): TextContent "Here are recs"
|
||||
Assert.Equal(6, result.Count);
|
||||
|
||||
Assert.Equal(ChatRole.Assistant, result[0].Role);
|
||||
Assert.Contains("Coord", result[0].AuthorName);
|
||||
Assert.Contains(result[0].Contents, c => c is FunctionCallContent fcc && fcc.CallId == "call_coord_1");
|
||||
|
||||
Assert.Equal(ChatRole.Tool, result[1].Role);
|
||||
Assert.Contains("Coord", result[1].AuthorName);
|
||||
Assert.Contains(result[1].Contents, c => c is FunctionResultContent frc && frc.CallId == "call_coord_1");
|
||||
|
||||
Assert.Equal(ChatRole.Assistant, result[2].Role);
|
||||
Assert.Contains("Spec", result[2].AuthorName);
|
||||
Assert.Equal("Here are two fake Excel course recommendations", result[2].Text);
|
||||
|
||||
Assert.Equal(ChatRole.Assistant, result[3].Role);
|
||||
Assert.Contains("Spec", result[3].AuthorName);
|
||||
Assert.Contains(result[3].Contents, c => c is FunctionCallContent fcc && fcc.CallId == "call_spec_1");
|
||||
|
||||
Assert.Equal(ChatRole.Tool, result[4].Role);
|
||||
Assert.Contains("Spec", result[4].AuthorName);
|
||||
Assert.Contains(result[4].Contents, c => c is FunctionResultContent frc && frc.CallId == "call_spec_1");
|
||||
|
||||
Assert.Equal(ChatRole.Assistant, result[5].Role);
|
||||
Assert.Contains("Coord", result[5].AuthorName);
|
||||
Assert.Equal("Here are two fake Excel course recommendations", result[5].Text);
|
||||
|
||||
// Final Coord turn must emit ONLY assistant text - no further handoff/tool call.
|
||||
Assert.DoesNotContain(result[5].Contents, c => c is FunctionCallContent);
|
||||
|
||||
// And the workflow must have terminated after that turn - no extra Coord/Spec re-invocations.
|
||||
Assert.Equal(2, coordCallCount);
|
||||
Assert.Equal(1, specCallCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Handoffs_FilteringNone_HandoffTargetReceivesAllMessagesIncludingToolCallsAsync()
|
||||
{
|
||||
@@ -1430,44 +1198,6 @@ public class HandoffOrchestrationTests
|
||||
Workflow workflow, List<ChatMessage> input, ExecutionEnvironment executionEnvironment = ExecutionEnvironment.InProcess_Lockstep)
|
||||
=> RunWorkflowCheckpointedAsync(workflow, input, executionEnvironment.ToWorkflowExecutionEnvironment());
|
||||
|
||||
private sealed class CapturingAgent(string name, string description, string textToCapture) : AIAgent
|
||||
{
|
||||
public override string Name => name;
|
||||
public override string Description => description;
|
||||
public ChatMessage? MessageSeenDuringRun { get; private set; }
|
||||
public ChatRole? RoleSeenDuringRun { get; private set; }
|
||||
|
||||
protected override ValueTask<AgentSession> CreateSessionCoreAsync(CancellationToken cancellationToken = default)
|
||||
=> new(new TestAgentSession());
|
||||
|
||||
protected override ValueTask<AgentSession> DeserializeSessionCoreAsync(JsonElement serializedState, JsonSerializerOptions? jsonSerializerOptions = null, CancellationToken cancellationToken = default)
|
||||
=> new(new TestAgentSession());
|
||||
|
||||
protected override ValueTask<JsonElement> SerializeSessionCoreAsync(AgentSession session, JsonSerializerOptions? jsonSerializerOptions = null, CancellationToken cancellationToken = default)
|
||||
=> default;
|
||||
|
||||
protected override Task<AgentResponse> RunCoreAsync(
|
||||
IEnumerable<ChatMessage> messages, AgentSession? session = null, AgentRunOptions? options = null, CancellationToken cancellationToken = default) =>
|
||||
throw new NotImplementedException();
|
||||
|
||||
protected override async IAsyncEnumerable<AgentResponseUpdate> RunCoreStreamingAsync(
|
||||
IEnumerable<ChatMessage> messages, AgentSession? session = null, AgentRunOptions? options = null, [EnumeratorCancellation] CancellationToken cancellationToken = default)
|
||||
{
|
||||
await Task.Yield();
|
||||
|
||||
this.MessageSeenDuringRun = messages.Single(m => m.Text == textToCapture);
|
||||
this.RoleSeenDuringRun = this.MessageSeenDuringRun.Role;
|
||||
|
||||
yield return new AgentResponseUpdate(ChatRole.Assistant, "Done")
|
||||
{
|
||||
AuthorName = this.Name,
|
||||
MessageId = Guid.NewGuid().ToString("N"),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class TestAgentSession() : AgentSession();
|
||||
|
||||
private sealed class DoubleEchoAgent(string name) : AIAgent
|
||||
{
|
||||
public override string Name => name;
|
||||
|
||||
+4
-9
@@ -36,18 +36,13 @@ public sealed class InputWaiterTests : IDisposable
|
||||
{
|
||||
Task waitTask = this._waiter.WaitForInputAsync(TimeSpan.FromSeconds(5));
|
||||
|
||||
Task completedBeforeSignal = await Task.WhenAny(waitTask, Task.Delay(100));
|
||||
completedBeforeSignal.Should().NotBeSameAs(
|
||||
waitTask,
|
||||
"the waiter should not complete before input is signaled");
|
||||
await Task.Delay(50);
|
||||
waitTask.IsCompleted.Should().BeFalse("the waiter should block until input is signaled");
|
||||
|
||||
this._waiter.SignalInput();
|
||||
|
||||
Task completedAfterSignal = await Task.WhenAny(waitTask, Task.Delay(TimeSpan.FromSeconds(1)));
|
||||
completedAfterSignal.Should().BeSameAs(
|
||||
waitTask,
|
||||
"the wait task should complete after being signaled");
|
||||
|
||||
Task completed = await Task.WhenAny(waitTask, Task.Delay(TimeSpan.FromSeconds(1)));
|
||||
completed.Should().BeSameAs(waitTask, "the wait task should complete after being signaled");
|
||||
await waitTask;
|
||||
}
|
||||
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using System;
|
||||
using System.Linq;
|
||||
using FluentAssertions;
|
||||
using Microsoft.Extensions.AI;
|
||||
|
||||
@@ -43,47 +42,6 @@ public class MessageMergerTests
|
||||
response.FinishReason.Should().BeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Test_MessageMerger_PreservesFunctionCallOrderingWhenToolResultHasCreatedAt()
|
||||
{
|
||||
// Arrange
|
||||
string responseId = Guid.NewGuid().ToString("N");
|
||||
string functionCallMessageId = Guid.NewGuid().ToString("N");
|
||||
string functionResultMessageId = Guid.NewGuid().ToString("N");
|
||||
string callId = Guid.NewGuid().ToString("N");
|
||||
DateTimeOffset toolResultCreatedAt = DateTimeOffset.UtcNow;
|
||||
|
||||
MessageMerger merger = new();
|
||||
|
||||
merger.AddUpdate(new AgentResponseUpdate
|
||||
{
|
||||
ResponseId = responseId,
|
||||
MessageId = functionCallMessageId,
|
||||
AgentId = TestAgentId1,
|
||||
Role = ChatRole.Assistant,
|
||||
Contents = [new FunctionCallContent(callId, "handoff_to_TestAgent2")],
|
||||
});
|
||||
merger.AddUpdate(new AgentResponseUpdate
|
||||
{
|
||||
ResponseId = responseId,
|
||||
MessageId = functionResultMessageId,
|
||||
AgentId = TestAgentId1,
|
||||
CreatedAt = toolResultCreatedAt,
|
||||
Role = ChatRole.Tool,
|
||||
Contents = [new FunctionResultContent(callId, "Transferred.")],
|
||||
});
|
||||
|
||||
// Act
|
||||
AgentResponse response = merger.ComputeMerged(responseId);
|
||||
|
||||
// Assert
|
||||
response.Messages.Should().HaveCount(2);
|
||||
response.Messages[0].Role.Should().Be(ChatRole.Assistant);
|
||||
response.Messages[0].Contents.Should().ContainSingle().Which.Should().BeOfType<FunctionCallContent>();
|
||||
response.Messages[1].Role.Should().Be(ChatRole.Tool);
|
||||
response.Messages[1].Contents.Should().ContainSingle().Which.Should().BeOfType<FunctionResultContent>();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Test_MessageMerger_PropagatesFinishReasonFromUpdates()
|
||||
{
|
||||
@@ -113,494 +71,4 @@ public class MessageMergerTests
|
||||
// Assert - FinishReason from the update should propagate through
|
||||
response.FinishReason.Should().Be(ChatFinishReason.ContentFilter);
|
||||
}
|
||||
|
||||
#region Invariant 2: Output Order Preservation Tests
|
||||
|
||||
[Fact]
|
||||
public void Test_MessageMerger_PreservesInsertionOrder_WhenNoTimestamps()
|
||||
{
|
||||
// Arrange: Multiple updates without CreatedAt, in specific order A, B, C
|
||||
string responseId = Guid.NewGuid().ToString("N");
|
||||
string messageIdA = Guid.NewGuid().ToString("N");
|
||||
string messageIdB = Guid.NewGuid().ToString("N");
|
||||
string messageIdC = Guid.NewGuid().ToString("N");
|
||||
|
||||
MessageMerger merger = new();
|
||||
|
||||
// Add updates without CreatedAt in order A, B, C
|
||||
merger.AddUpdate(new AgentResponseUpdate
|
||||
{
|
||||
ResponseId = responseId,
|
||||
MessageId = messageIdA,
|
||||
Role = ChatRole.Assistant,
|
||||
Contents = [new TextContent("Message A")],
|
||||
// No CreatedAt
|
||||
});
|
||||
merger.AddUpdate(new AgentResponseUpdate
|
||||
{
|
||||
ResponseId = responseId,
|
||||
MessageId = messageIdB,
|
||||
Role = ChatRole.Assistant,
|
||||
Contents = [new TextContent("Message B")],
|
||||
// No CreatedAt
|
||||
});
|
||||
merger.AddUpdate(new AgentResponseUpdate
|
||||
{
|
||||
ResponseId = responseId,
|
||||
MessageId = messageIdC,
|
||||
Role = ChatRole.Assistant,
|
||||
Contents = [new TextContent("Message C")],
|
||||
// No CreatedAt
|
||||
});
|
||||
|
||||
// Act
|
||||
AgentResponse response = merger.ComputeMerged(responseId);
|
||||
|
||||
// Assert: Output order should be A, B, C (insertion order)
|
||||
response.Messages.Should().HaveCount(3);
|
||||
response.Messages[0].Text.Should().Be("Message A");
|
||||
response.Messages[1].Text.Should().Be("Message B");
|
||||
response.Messages[2].Text.Should().Be("Message C");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Test_MessageMerger_PreservesInsertionOrder_WhenMixedTimestamps()
|
||||
{
|
||||
// Arrange: Updates with OUT-OF-ORDER timestamps relative to emission order.
|
||||
// Emission order: A, B, C.
|
||||
// Timestamp order: C (oldest), A, B (newest) - reverse of emission.
|
||||
// Per Invariant 2, emission order wins; timestamps are ignored for ordering.
|
||||
string responseId = Guid.NewGuid().ToString("N");
|
||||
string messageIdA = Guid.NewGuid().ToString("N");
|
||||
string messageIdB = Guid.NewGuid().ToString("N");
|
||||
string messageIdC = Guid.NewGuid().ToString("N");
|
||||
|
||||
DateTimeOffset timeOldest = DateTimeOffset.UtcNow.AddMinutes(-10);
|
||||
DateTimeOffset timeMiddle = DateTimeOffset.UtcNow.AddMinutes(-5);
|
||||
DateTimeOffset timeNewest = DateTimeOffset.UtcNow;
|
||||
|
||||
MessageMerger merger = new();
|
||||
|
||||
// Emit A first but stamp it with timeMiddle.
|
||||
merger.AddUpdate(new AgentResponseUpdate
|
||||
{
|
||||
ResponseId = responseId,
|
||||
MessageId = messageIdA,
|
||||
Role = ChatRole.Assistant,
|
||||
CreatedAt = timeMiddle,
|
||||
Contents = [new TextContent("Message A")],
|
||||
});
|
||||
// Emit B second, without a timestamp.
|
||||
merger.AddUpdate(new AgentResponseUpdate
|
||||
{
|
||||
ResponseId = responseId,
|
||||
MessageId = messageIdB,
|
||||
Role = ChatRole.Assistant,
|
||||
// No CreatedAt
|
||||
Contents = [new TextContent("Message B")],
|
||||
});
|
||||
// Emit C third but stamp it with timeOldest - if we sorted by timestamp,
|
||||
// C would come first; the new merger MUST keep emission order (A, B, C).
|
||||
merger.AddUpdate(new AgentResponseUpdate
|
||||
{
|
||||
ResponseId = responseId,
|
||||
MessageId = messageIdC,
|
||||
Role = ChatRole.Assistant,
|
||||
CreatedAt = timeOldest,
|
||||
Contents = [new TextContent("Message C")],
|
||||
});
|
||||
// Stamp a fourth message with the newest timestamp - it should still come last
|
||||
// because it was emitted last, not because of its timestamp.
|
||||
string messageIdD = Guid.NewGuid().ToString("N");
|
||||
merger.AddUpdate(new AgentResponseUpdate
|
||||
{
|
||||
ResponseId = responseId,
|
||||
MessageId = messageIdD,
|
||||
Role = ChatRole.Assistant,
|
||||
CreatedAt = timeNewest,
|
||||
Contents = [new TextContent("Message D")],
|
||||
});
|
||||
|
||||
// Act
|
||||
AgentResponse response = merger.ComputeMerged(responseId);
|
||||
|
||||
// Assert: Emission order wins; CreatedAt is ignored for ordering.
|
||||
response.Messages.Should().HaveCount(4);
|
||||
response.Messages[0].Text.Should().Be("Message A");
|
||||
response.Messages[1].Text.Should().Be("Message B");
|
||||
response.Messages[2].Text.Should().Be("Message C");
|
||||
response.Messages[3].Text.Should().Be("Message D");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Test_MessageMerger_ReproducibleOrdering_WithMixedTimestamps()
|
||||
{
|
||||
// Arrange: 3+ messages with mixed null/non-null CreatedAt values
|
||||
// This tests that the same input sequence produces the same output
|
||||
// (run-to-run reproducibility for a fixed input)
|
||||
string responseId = Guid.NewGuid().ToString("N");
|
||||
string messageIdA = Guid.NewGuid().ToString("N");
|
||||
string messageIdB = Guid.NewGuid().ToString("N");
|
||||
string messageIdC = Guid.NewGuid().ToString("N");
|
||||
|
||||
DateTimeOffset time10 = DateTimeOffset.UtcNow.AddSeconds(10);
|
||||
DateTimeOffset time5 = DateTimeOffset.UtcNow.AddSeconds(5);
|
||||
|
||||
MessageMerger merger = new();
|
||||
|
||||
// A: CreatedAt = time10, idx=0
|
||||
// B: CreatedAt = null, idx=1
|
||||
// C: CreatedAt = time5, idx=2
|
||||
merger.AddUpdate(new AgentResponseUpdate
|
||||
{
|
||||
ResponseId = responseId,
|
||||
MessageId = messageIdA,
|
||||
Role = ChatRole.Assistant,
|
||||
CreatedAt = time10,
|
||||
Contents = [new TextContent("Message A (T=10)")],
|
||||
});
|
||||
merger.AddUpdate(new AgentResponseUpdate
|
||||
{
|
||||
ResponseId = responseId,
|
||||
MessageId = messageIdB,
|
||||
Role = ChatRole.Assistant,
|
||||
// No CreatedAt
|
||||
Contents = [new TextContent("Message B (no timestamp)")],
|
||||
});
|
||||
merger.AddUpdate(new AgentResponseUpdate
|
||||
{
|
||||
ResponseId = responseId,
|
||||
MessageId = messageIdC,
|
||||
Role = ChatRole.Assistant,
|
||||
CreatedAt = time5,
|
||||
Contents = [new TextContent("Message C (T=5)")],
|
||||
});
|
||||
|
||||
// Act - Run multiple times to verify reproducibility
|
||||
AgentResponse response1 = merger.ComputeMerged(responseId);
|
||||
|
||||
// Create a fresh merger with same data to verify reproducibility
|
||||
MessageMerger merger2 = new();
|
||||
merger2.AddUpdate(new AgentResponseUpdate
|
||||
{
|
||||
ResponseId = responseId,
|
||||
MessageId = messageIdA,
|
||||
Role = ChatRole.Assistant,
|
||||
CreatedAt = time10,
|
||||
Contents = [new TextContent("Message A (T=10)")],
|
||||
});
|
||||
merger2.AddUpdate(new AgentResponseUpdate
|
||||
{
|
||||
ResponseId = responseId,
|
||||
MessageId = messageIdB,
|
||||
Role = ChatRole.Assistant,
|
||||
Contents = [new TextContent("Message B (no timestamp)")],
|
||||
});
|
||||
merger2.AddUpdate(new AgentResponseUpdate
|
||||
{
|
||||
ResponseId = responseId,
|
||||
MessageId = messageIdC,
|
||||
Role = ChatRole.Assistant,
|
||||
CreatedAt = time5,
|
||||
Contents = [new TextContent("Message C (T=5)")],
|
||||
});
|
||||
AgentResponse response2 = merger2.ComputeMerged(responseId);
|
||||
|
||||
// Assert: Result is reproducible and consistent across runs with same input order.
|
||||
// Per Invariant 2, both runs must produce emission order (A, B, C) regardless
|
||||
// of the messages' CreatedAt values.
|
||||
response1.Messages.Should().HaveCount(3);
|
||||
response2.Messages.Should().HaveCount(3);
|
||||
|
||||
response1.Messages.Select(m => m.Text).Should().ContainInOrder(
|
||||
"Message A (T=10)", "Message B (no timestamp)", "Message C (T=5)");
|
||||
|
||||
// Both runs should produce identical ordering
|
||||
for (int i = 0; i < 3; i++)
|
||||
{
|
||||
response1.Messages[i].Text.Should().Be(response2.Messages[i].Text);
|
||||
}
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Invariant 3: Agent Message Grouping Tests
|
||||
|
||||
[Fact]
|
||||
public void Test_MessageMerger_GroupsMessagesByResponseId_InMultiAgentScenario()
|
||||
{
|
||||
// Arrange: Interleaved updates from Agent1 (R1) and Agent2 (R2)
|
||||
string responseIdR1 = Guid.NewGuid().ToString("N");
|
||||
string responseIdR2 = Guid.NewGuid().ToString("N");
|
||||
string messageIdA1M1 = Guid.NewGuid().ToString("N");
|
||||
string messageIdA1M2 = Guid.NewGuid().ToString("N");
|
||||
string messageIdA2M1 = Guid.NewGuid().ToString("N");
|
||||
string messageIdA2M2 = Guid.NewGuid().ToString("N");
|
||||
|
||||
MessageMerger merger = new();
|
||||
|
||||
// Interleaved arrival: A1-msg1, A2-msg1, A1-msg2, A2-msg2
|
||||
merger.AddUpdate(new AgentResponseUpdate
|
||||
{
|
||||
ResponseId = responseIdR1,
|
||||
MessageId = messageIdA1M1,
|
||||
AgentId = TestAgentId1,
|
||||
Role = ChatRole.Assistant,
|
||||
Contents = [new TextContent("Agent1 Message 1")],
|
||||
});
|
||||
merger.AddUpdate(new AgentResponseUpdate
|
||||
{
|
||||
ResponseId = responseIdR2,
|
||||
MessageId = messageIdA2M1,
|
||||
AgentId = TestAgentId2,
|
||||
Role = ChatRole.Assistant,
|
||||
Contents = [new TextContent("Agent2 Message 1")],
|
||||
});
|
||||
merger.AddUpdate(new AgentResponseUpdate
|
||||
{
|
||||
ResponseId = responseIdR1,
|
||||
MessageId = messageIdA1M2,
|
||||
AgentId = TestAgentId1,
|
||||
Role = ChatRole.Assistant,
|
||||
Contents = [new TextContent("Agent1 Message 2")],
|
||||
});
|
||||
merger.AddUpdate(new AgentResponseUpdate
|
||||
{
|
||||
ResponseId = responseIdR2,
|
||||
MessageId = messageIdA2M2,
|
||||
AgentId = TestAgentId2,
|
||||
Role = ChatRole.Assistant,
|
||||
Contents = [new TextContent("Agent2 Message 2")],
|
||||
});
|
||||
|
||||
// Act
|
||||
AgentResponse response = merger.ComputeMerged(responseIdR1);
|
||||
|
||||
// Assert: Messages should be grouped by ResponseId (which groups by agent)
|
||||
// Output should be either [A1-msg1, A1-msg2, A2-msg1, A2-msg2] or [A2-msg1, A2-msg2, A1-msg1, A1-msg2]
|
||||
// The key invariant: Agent1's messages are contiguous, Agent2's messages are contiguous
|
||||
response.Messages.Should().HaveCount(4);
|
||||
|
||||
// Verify grouping - collect message texts and verify they're grouped by agent
|
||||
var messageTexts = response.Messages.Select(m => m.Text).ToList();
|
||||
|
||||
// Find first Agent1 message index and first Agent2 message index
|
||||
int firstA1Index = messageTexts.FindIndex(t => t.StartsWith("Agent1", StringComparison.Ordinal));
|
||||
int firstA2Index = messageTexts.FindIndex(t => t.StartsWith("Agent2", StringComparison.Ordinal));
|
||||
|
||||
// Assert both indices are valid (messages were found)
|
||||
firstA1Index.Should().BeGreaterThanOrEqualTo(0, "Agent1 messages should be present in response");
|
||||
firstA2Index.Should().BeGreaterThanOrEqualTo(0, "Agent2 messages should be present in response");
|
||||
|
||||
// All Agent1 messages should be contiguous (either at start or after all Agent2 messages)
|
||||
var a1Messages = messageTexts.Where(t => t.StartsWith("Agent1", StringComparison.Ordinal)).ToList();
|
||||
var a2Messages = messageTexts.Where(t => t.StartsWith("Agent2", StringComparison.Ordinal)).ToList();
|
||||
|
||||
a1Messages.Should().HaveCount(2);
|
||||
a2Messages.Should().HaveCount(2);
|
||||
|
||||
// Verify no interleaving: if A1 comes first, A2 should come after all A1 messages
|
||||
if (firstA1Index < firstA2Index)
|
||||
{
|
||||
// A1 messages at indices 0, 1 and A2 messages at indices 2, 3
|
||||
messageTexts[0].Should().StartWith("Agent1");
|
||||
messageTexts[1].Should().StartWith("Agent1");
|
||||
messageTexts[2].Should().StartWith("Agent2");
|
||||
messageTexts[3].Should().StartWith("Agent2");
|
||||
}
|
||||
else
|
||||
{
|
||||
// A2 messages at indices 0, 1 and A1 messages at indices 2, 3
|
||||
messageTexts[0].Should().StartWith("Agent2");
|
||||
messageTexts[1].Should().StartWith("Agent2");
|
||||
messageTexts[2].Should().StartWith("Agent1");
|
||||
messageTexts[3].Should().StartWith("Agent1");
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Test_MessageMerger_MaintainsAgentGrouping_WithDifferentResponseIds()
|
||||
{
|
||||
// Arrange: Agent1 uses ResponseId=R1, Agent2 uses ResponseId=R2
|
||||
// Multiple messages per ResponseId to properly test contiguity
|
||||
string responseIdR1 = Guid.NewGuid().ToString("N");
|
||||
string responseIdR2 = Guid.NewGuid().ToString("N");
|
||||
string messageIdA1M1 = Guid.NewGuid().ToString("N");
|
||||
string messageIdA1M2 = Guid.NewGuid().ToString("N");
|
||||
string messageIdA1M3 = Guid.NewGuid().ToString("N");
|
||||
string messageIdA2M1 = Guid.NewGuid().ToString("N");
|
||||
string messageIdA2M2 = Guid.NewGuid().ToString("N");
|
||||
string messageIdA2M3 = Guid.NewGuid().ToString("N");
|
||||
|
||||
MessageMerger merger = new();
|
||||
|
||||
// Interleaved arrival: A1-1, A2-1, A1-2, A2-2, A1-3, A2-3
|
||||
merger.AddUpdate(new AgentResponseUpdate
|
||||
{
|
||||
ResponseId = responseIdR1,
|
||||
MessageId = messageIdA1M1,
|
||||
AgentId = TestAgentId1,
|
||||
Role = ChatRole.Assistant,
|
||||
Contents = [new TextContent("Agent1 Response 1")],
|
||||
});
|
||||
merger.AddUpdate(new AgentResponseUpdate
|
||||
{
|
||||
ResponseId = responseIdR2,
|
||||
MessageId = messageIdA2M1,
|
||||
AgentId = TestAgentId2,
|
||||
Role = ChatRole.Assistant,
|
||||
Contents = [new TextContent("Agent2 Response 1")],
|
||||
});
|
||||
merger.AddUpdate(new AgentResponseUpdate
|
||||
{
|
||||
ResponseId = responseIdR1,
|
||||
MessageId = messageIdA1M2,
|
||||
AgentId = TestAgentId1,
|
||||
Role = ChatRole.Assistant,
|
||||
Contents = [new TextContent("Agent1 Response 2")],
|
||||
});
|
||||
merger.AddUpdate(new AgentResponseUpdate
|
||||
{
|
||||
ResponseId = responseIdR2,
|
||||
MessageId = messageIdA2M2,
|
||||
AgentId = TestAgentId2,
|
||||
Role = ChatRole.Assistant,
|
||||
Contents = [new TextContent("Agent2 Response 2")],
|
||||
});
|
||||
merger.AddUpdate(new AgentResponseUpdate
|
||||
{
|
||||
ResponseId = responseIdR1,
|
||||
MessageId = messageIdA1M3,
|
||||
AgentId = TestAgentId1,
|
||||
Role = ChatRole.Assistant,
|
||||
Contents = [new TextContent("Agent1 Response 3")],
|
||||
});
|
||||
merger.AddUpdate(new AgentResponseUpdate
|
||||
{
|
||||
ResponseId = responseIdR2,
|
||||
MessageId = messageIdA2M3,
|
||||
AgentId = TestAgentId2,
|
||||
Role = ChatRole.Assistant,
|
||||
Contents = [new TextContent("Agent2 Response 3")],
|
||||
});
|
||||
|
||||
// Act
|
||||
AgentResponse response = merger.ComputeMerged(responseIdR1);
|
||||
|
||||
// Assert: Messages from each agent are contiguous (not interleaved)
|
||||
response.Messages.Should().HaveCount(6);
|
||||
|
||||
var messageTexts = response.Messages.Select(m => m.Text).ToList();
|
||||
|
||||
// Verify all messages are present
|
||||
messageTexts.Should().Contain("Agent1 Response 1");
|
||||
messageTexts.Should().Contain("Agent1 Response 2");
|
||||
messageTexts.Should().Contain("Agent1 Response 3");
|
||||
messageTexts.Should().Contain("Agent2 Response 1");
|
||||
messageTexts.Should().Contain("Agent2 Response 2");
|
||||
messageTexts.Should().Contain("Agent2 Response 3");
|
||||
|
||||
// Find indices to verify contiguity
|
||||
int firstA1Index = messageTexts.FindIndex(t => t.StartsWith("Agent1", StringComparison.Ordinal));
|
||||
int lastA1Index = messageTexts.FindLastIndex(t => t.StartsWith("Agent1", StringComparison.Ordinal));
|
||||
int firstA2Index = messageTexts.FindIndex(t => t.StartsWith("Agent2", StringComparison.Ordinal));
|
||||
int lastA2Index = messageTexts.FindLastIndex(t => t.StartsWith("Agent2", StringComparison.Ordinal));
|
||||
|
||||
// Assert indices are valid
|
||||
firstA1Index.Should().BeGreaterThanOrEqualTo(0, "Agent1 messages should be present");
|
||||
firstA2Index.Should().BeGreaterThanOrEqualTo(0, "Agent2 messages should be present");
|
||||
|
||||
// Verify contiguity: all Agent1 messages should span exactly 3 consecutive indices
|
||||
(lastA1Index - firstA1Index).Should().Be(2, "Agent1 messages should be contiguous (3 messages spanning 2 index gaps)");
|
||||
(lastA2Index - firstA2Index).Should().Be(2, "Agent2 messages should be contiguous (3 messages spanning 2 index gaps)");
|
||||
|
||||
// Verify no interleaving: ranges should not overlap
|
||||
bool a1BeforeA2 = lastA1Index < firstA2Index;
|
||||
bool a2BeforeA1 = lastA2Index < firstA1Index;
|
||||
(a1BeforeA2 || a2BeforeA1).Should().BeTrue("Agent message blocks should not interleave");
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Step Ordering Tests (workflow goals)
|
||||
|
||||
[Fact]
|
||||
public void Test_MessageMerger_OrdersStepsBeforeNextStep_AndGroupsAgentsWithinStep()
|
||||
{
|
||||
// Scenario mirroring the workflow ordering goals:
|
||||
// Step 1: Agent1 alone emits 2 updates.
|
||||
// Step 2: Agent1 and Agent2 emit updates interleaved (concurrent in the step).
|
||||
// Step 3: Agent2 alone emits 2 updates.
|
||||
// Each agent invocation has its own ResponseId, so per-ResponseId grouping
|
||||
// yields per-agent grouping. Steps are naturally serialized in emission
|
||||
// order (the next step cannot emit until the prior step's updates arrive),
|
||||
// so the first-seen-ResponseId ordering preserves step boundaries.
|
||||
const string step1Agent1 = "step1-agent1";
|
||||
const string step2Agent1 = "step2-agent1";
|
||||
const string step2Agent2 = "step2-agent2";
|
||||
const string step3Agent2 = "step3-agent2";
|
||||
|
||||
MessageMerger merger = new();
|
||||
|
||||
// Step 1: Agent1 emits 2 messages.
|
||||
AddSimpleUpdate(merger, step1Agent1, TestAgentId1, "S1-A1-m1");
|
||||
AddSimpleUpdate(merger, step1Agent1, TestAgentId1, "S1-A1-m2");
|
||||
|
||||
// Step 2: Agent1 and Agent2 emit interleaved (A1, A2, A1, A2).
|
||||
AddSimpleUpdate(merger, step2Agent1, TestAgentId1, "S2-A1-m1");
|
||||
AddSimpleUpdate(merger, step2Agent2, TestAgentId2, "S2-A2-m1");
|
||||
AddSimpleUpdate(merger, step2Agent1, TestAgentId1, "S2-A1-m2");
|
||||
AddSimpleUpdate(merger, step2Agent2, TestAgentId2, "S2-A2-m2");
|
||||
|
||||
// Step 3: Agent2 emits 2 messages.
|
||||
AddSimpleUpdate(merger, step3Agent2, TestAgentId2, "S3-A2-m1");
|
||||
AddSimpleUpdate(merger, step3Agent2, TestAgentId2, "S3-A2-m2");
|
||||
|
||||
// Act
|
||||
AgentResponse response = merger.ComputeMerged(step1Agent1);
|
||||
|
||||
// Assert: expected output order
|
||||
// Step 1 block (Agent1): S1-A1-m1, S1-A1-m2
|
||||
// Step 2 Agent1 block: S2-A1-m1, S2-A1-m2
|
||||
// Step 2 Agent2 block: S2-A2-m1, S2-A2-m2
|
||||
// Step 3 block (Agent2): S3-A2-m1, S3-A2-m2
|
||||
response.Messages.Should().HaveCount(8);
|
||||
var texts = response.Messages.Select(m => m.Text).ToList();
|
||||
texts.Should().ContainInOrder(
|
||||
"S1-A1-m1", "S1-A1-m2",
|
||||
"S2-A1-m1", "S2-A1-m2",
|
||||
"S2-A2-m1", "S2-A2-m2",
|
||||
"S3-A2-m1", "S3-A2-m2");
|
||||
|
||||
// Step boundaries: no step-2 or step-3 message may appear before the last step-1 message.
|
||||
int lastStep1Index = texts.FindLastIndex(t => t.StartsWith("S1-", StringComparison.Ordinal));
|
||||
int firstStep2Index = texts.FindIndex(t => t.StartsWith("S2-", StringComparison.Ordinal));
|
||||
int lastStep2Index = texts.FindLastIndex(t => t.StartsWith("S2-", StringComparison.Ordinal));
|
||||
int firstStep3Index = texts.FindIndex(t => t.StartsWith("S3-", StringComparison.Ordinal));
|
||||
lastStep1Index.Should().BeLessThan(firstStep2Index, "all step-1 messages precede step-2");
|
||||
lastStep2Index.Should().BeLessThan(firstStep3Index, "all step-2 messages precede step-3");
|
||||
|
||||
// Within step 2 (multi-agent), per-agent blocks must be contiguous.
|
||||
int firstS2A1 = texts.FindIndex(t => t.StartsWith("S2-A1", StringComparison.Ordinal));
|
||||
int lastS2A1 = texts.FindLastIndex(t => t.StartsWith("S2-A1", StringComparison.Ordinal));
|
||||
int firstS2A2 = texts.FindIndex(t => t.StartsWith("S2-A2", StringComparison.Ordinal));
|
||||
int lastS2A2 = texts.FindLastIndex(t => t.StartsWith("S2-A2", StringComparison.Ordinal));
|
||||
(lastS2A1 - firstS2A1).Should().Be(1, "Agent1 step-2 messages should be contiguous");
|
||||
(lastS2A2 - firstS2A2).Should().Be(1, "Agent2 step-2 messages should be contiguous");
|
||||
(lastS2A1 < firstS2A2 || lastS2A2 < firstS2A1).Should().BeTrue("agent blocks within a step must not interleave");
|
||||
|
||||
static void AddSimpleUpdate(MessageMerger m, string responseId, string agentId, string text)
|
||||
{
|
||||
m.AddUpdate(new AgentResponseUpdate
|
||||
{
|
||||
ResponseId = responseId,
|
||||
MessageId = Guid.NewGuid().ToString("N"),
|
||||
AgentId = agentId,
|
||||
Role = ChatRole.Assistant,
|
||||
Contents = [new TextContent(text)],
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.AI;
|
||||
@@ -291,121 +290,6 @@ public sealed class WorkflowEvaluationTests
|
||||
Assert.DoesNotContain("end", result.Keys);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// BuildOverallItem tests (expected output / ground truth)
|
||||
// ---------------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public void BuildOverallItem_NoCompletedExecutorWithResponse_ReturnsNull()
|
||||
{
|
||||
// Arrange — no ExecutorCompletedEvent with usable response data and no AgentResponseEvent
|
||||
var events = new List<WorkflowEvent>
|
||||
{
|
||||
new ExecutorInvokedEvent("agent-1", "query"),
|
||||
};
|
||||
|
||||
// Act
|
||||
var item = WorkflowEvaluationExtensions.BuildOverallItem(events, splitter: null, expectedOutput: null);
|
||||
|
||||
// Assert
|
||||
Assert.Null(item);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BuildOverallItem_NoAgentResponseEvent_FallsBackToLastExecutorCompleted()
|
||||
{
|
||||
// Arrange — only ExecutorCompletedEvent (the default when EmitAgentResponseEvents is false)
|
||||
var finalResponse = new AgentResponse(new ChatMessage(ChatRole.Assistant, "Paris"));
|
||||
var events = new List<WorkflowEvent>
|
||||
{
|
||||
new ExecutorInvokedEvent("researcher", "What is the capital of France?"),
|
||||
new ExecutorCompletedEvent("researcher", new AgentResponse(new ChatMessage(ChatRole.Assistant, "draft"))),
|
||||
new ExecutorInvokedEvent("editor", "draft"),
|
||||
new ExecutorCompletedEvent("editor", finalResponse),
|
||||
};
|
||||
|
||||
// Act
|
||||
var item = WorkflowEvaluationExtensions.BuildOverallItem(
|
||||
events, splitter: null, expectedOutput: "Paris");
|
||||
|
||||
// Assert
|
||||
Assert.NotNull(item);
|
||||
Assert.Equal("What is the capital of France?", item.Query);
|
||||
Assert.Equal("Paris", item.Response);
|
||||
Assert.Equal("Paris", item.ExpectedOutput);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BuildOverallItem_WithFinalResponseAndExpectedOutput_StampsExpectedOutput()
|
||||
{
|
||||
// Arrange
|
||||
var finalResponse = new AgentResponse(new ChatMessage(ChatRole.Assistant, "Ofrece 41 planes"));
|
||||
var events = new List<WorkflowEvent>
|
||||
{
|
||||
new ExecutorInvokedEvent("agent-1", "How many plans does Netlife offer?"),
|
||||
new ExecutorCompletedEvent("agent-1", finalResponse),
|
||||
new AgentResponseEvent("agent-1", finalResponse),
|
||||
};
|
||||
|
||||
// Act
|
||||
var item = WorkflowEvaluationExtensions.BuildOverallItem(
|
||||
events, splitter: null, expectedOutput: "Ofrece 41 planes");
|
||||
|
||||
// Assert
|
||||
Assert.NotNull(item);
|
||||
Assert.Equal("How many plans does Netlife offer?", item.Query);
|
||||
Assert.Equal("Ofrece 41 planes", item.Response);
|
||||
Assert.Equal("Ofrece 41 planes", item.ExpectedOutput);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BuildOverallItem_WithFinalResponseAndNoExpectedOutput_LeavesExpectedOutputNull()
|
||||
{
|
||||
// Arrange
|
||||
var finalResponse = new AgentResponse(new ChatMessage(ChatRole.Assistant, "answer"));
|
||||
var events = new List<WorkflowEvent>
|
||||
{
|
||||
new ExecutorInvokedEvent("agent-1", "query"),
|
||||
new ExecutorCompletedEvent("agent-1", finalResponse),
|
||||
new AgentResponseEvent("agent-1", finalResponse),
|
||||
};
|
||||
|
||||
// Act
|
||||
var item = WorkflowEvaluationExtensions.BuildOverallItem(events, splitter: null, expectedOutput: null);
|
||||
|
||||
// Assert
|
||||
Assert.NotNull(item);
|
||||
Assert.Null(item.ExpectedOutput);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task EvaluateAsync_WithIncludeOverallButNoFinalResponse_ThrowsAsync()
|
||||
{
|
||||
// Arrange — build a workflow whose AIAgentHostExecutor is NOT bound with
|
||||
// EmitAgentResponseEvents=true, so no AgentResponseEvent is emitted, and the
|
||||
// ExecutorCompletedEvent for the host carries null Data. That is the scenario
|
||||
// where BuildOverallItem returns null. When the caller asks for an overall
|
||||
// evaluation (includeOverall: true), we should fail fast rather than silently
|
||||
// returning empty results — regardless of whether expectedOutput was supplied.
|
||||
var agent = new TestEchoAgent(name: "echo");
|
||||
var workflow = AgentWorkflowBuilder.BuildSequential(agent);
|
||||
var input = new List<ChatMessage> { new(ChatRole.User, "Hello") };
|
||||
|
||||
var evaluator = new LocalEvaluator(
|
||||
FunctionEvaluator.Create("noop", (EvalItem _) => true));
|
||||
|
||||
await using var run = await InProcessExecution.RunAsync(workflow, input);
|
||||
|
||||
// Act + Assert — throws even without expectedOutput
|
||||
var ex = await Assert.ThrowsAsync<InvalidOperationException>(() =>
|
||||
run.EvaluateAsync(
|
||||
evaluator,
|
||||
includeOverall: true,
|
||||
includePerAgent: false));
|
||||
|
||||
Assert.Contains("EmitAgentResponseEvents", ex.Message);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// EvaluateAsync integration test
|
||||
// ---------------------------------------------------------------
|
||||
|
||||
@@ -99,30 +99,6 @@ The `AGUIChatClient` supports:
|
||||
- Integration with `Agent` for client-side history management
|
||||
- Interrupt metadata passthrough (`availableInterrupts` and `resume`)
|
||||
|
||||
## Tool Return Helpers
|
||||
|
||||
Use `state_update` when a backend tool needs to send different payloads to the model, the UI, and shared state. The `text` value remains the LLM-bound tool result, `tool_result` becomes the AG-UI `ToolCallResultEvent.content` for frontend rendering, and `state` is merged into durable shared state.
|
||||
|
||||
```python
|
||||
from agent_framework import Content, tool
|
||||
from agent_framework.ag_ui import state_update
|
||||
|
||||
@tool
|
||||
async def get_weather(city: str) -> Content:
|
||||
data = await fetch_weather(city)
|
||||
return state_update(
|
||||
text=f"{city}: {data['temp']}°C and {data['conditions']}",
|
||||
tool_result={
|
||||
"component": "weather-card",
|
||||
"city": city,
|
||||
"temperature": data["temp"],
|
||||
"conditions": data["conditions"],
|
||||
"humidity": data["humidity"],
|
||||
},
|
||||
state={"weather": {"city": city, **data}},
|
||||
)
|
||||
```
|
||||
|
||||
## Documentation
|
||||
|
||||
- **[Getting Started Tutorial](getting_started/)** - Step-by-step guide to building AG-UI servers and clients
|
||||
|
||||
@@ -49,11 +49,8 @@ from ._run_common import (
|
||||
_close_reasoning_block, # type: ignore
|
||||
_emit_content, # type: ignore
|
||||
_extract_resume_payload, # type: ignore
|
||||
_extract_tool_result_display, # type: ignore
|
||||
_has_only_tool_calls, # type: ignore
|
||||
_normalize_resume_interrupts, # type: ignore
|
||||
_resolve_ui_payload, # type: ignore
|
||||
_stringify_tool_result, # type: ignore
|
||||
)
|
||||
from ._utils import (
|
||||
convert_agui_tools_to_agent_framework,
|
||||
@@ -384,23 +381,17 @@ def _handle_step_based_approval(messages: list[Any]) -> list[BaseEvent]:
|
||||
|
||||
|
||||
def _make_approval_tool_result_events(resolved_approval_results: list[Content]) -> list[ToolCallResultEvent]:
|
||||
"""Build TOOL_CALL_RESULT events for tools executed during approval resolution.
|
||||
|
||||
Honors ``TOOL_RESULT_DISPLAY_KEY`` so tools returning
|
||||
``state_update(..., tool_result=...)`` route the display payload to the UI
|
||||
event even when gated by HITL approval.
|
||||
"""
|
||||
"""Build TOOL_CALL_RESULT events for tools executed during approval resolution."""
|
||||
events: list[ToolCallResultEvent] = []
|
||||
for resolved in resolved_approval_results:
|
||||
if resolved.call_id:
|
||||
raw = resolved.result if resolved.result is not None else ""
|
||||
llm_str = _stringify_tool_result(raw)
|
||||
ui_str = _resolve_ui_payload(llm_str, _extract_tool_result_display(resolved))
|
||||
result_str = raw if isinstance(raw, str) else json.dumps(make_json_safe(raw))
|
||||
events.append(
|
||||
ToolCallResultEvent(
|
||||
message_id=generate_event_id(),
|
||||
tool_call_id=resolved.call_id,
|
||||
content=ui_str,
|
||||
content=result_str,
|
||||
role="tool",
|
||||
)
|
||||
)
|
||||
|
||||
@@ -32,14 +32,11 @@ from ag_ui.core import (
|
||||
from agent_framework import Content
|
||||
|
||||
from ._orchestration._predictive_state import PredictiveStateHandler
|
||||
from ._state import TOOL_RESULT_DISPLAY_KEY, TOOL_RESULT_STATE_KEY
|
||||
from ._state import TOOL_RESULT_STATE_KEY
|
||||
from ._utils import generate_event_id, make_json_safe
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Sentinel for an unset display_result; distinguishes "caller didn't pass" from None/{}/"".
|
||||
_UNSET = object()
|
||||
|
||||
|
||||
def _has_only_tool_calls(contents: list[Any]) -> bool:
|
||||
"""Check if contents have only tool calls (no text)."""
|
||||
@@ -238,22 +235,6 @@ def _emit_tool_call(
|
||||
return events
|
||||
|
||||
|
||||
def _extract_tool_result_marker_values(content: Content, key: str) -> list[Any]:
|
||||
"""Extract marker values from outer and inner tool-result content."""
|
||||
values: list[Any] = []
|
||||
|
||||
outer_ap = getattr(content, "additional_properties", None) or {}
|
||||
if key in outer_ap:
|
||||
values.append(outer_ap[key])
|
||||
|
||||
for item in content.items or ():
|
||||
item_ap = getattr(item, "additional_properties", None) or {}
|
||||
if key in item_ap:
|
||||
values.append(item_ap[key])
|
||||
|
||||
return values
|
||||
|
||||
|
||||
def _extract_tool_result_state(content: Content) -> dict[str, Any] | None:
|
||||
"""Extract a deterministic AG-UI state update from a tool-result ``Content``.
|
||||
|
||||
@@ -271,7 +252,14 @@ def _extract_tool_result_state(content: Content) -> dict[str, Any] | None:
|
||||
"""
|
||||
merged: dict[str, Any] | None = None
|
||||
|
||||
for item_state in _extract_tool_result_marker_values(content, TOOL_RESULT_STATE_KEY):
|
||||
outer_ap = getattr(content, "additional_properties", None) or {}
|
||||
outer_state = outer_ap.get(TOOL_RESULT_STATE_KEY)
|
||||
if isinstance(outer_state, dict):
|
||||
merged = dict(outer_state)
|
||||
|
||||
for item in content.items or ():
|
||||
item_ap = getattr(item, "additional_properties", None) or {}
|
||||
item_state = item_ap.get(TOOL_RESULT_STATE_KEY)
|
||||
if isinstance(item_state, dict):
|
||||
if merged is None:
|
||||
merged = dict(item_state)
|
||||
@@ -281,21 +269,6 @@ def _extract_tool_result_state(content: Content) -> dict[str, Any] | None:
|
||||
return merged
|
||||
|
||||
|
||||
def _extract_tool_result_display(content: Content) -> Any: # noqa: ANN401
|
||||
"""Extract a UI-only AG-UI tool result display payload, if present."""
|
||||
display_values = _extract_tool_result_marker_values(content, TOOL_RESULT_DISPLAY_KEY)
|
||||
return display_values[-1] if display_values else _UNSET
|
||||
|
||||
|
||||
def _stringify_tool_result(raw_result: Any) -> str: # noqa: ANN401
|
||||
return raw_result if isinstance(raw_result, str) else json.dumps(make_json_safe(raw_result))
|
||||
|
||||
|
||||
def _resolve_ui_payload(llm_str: str, display_result: Any) -> str: # noqa: ANN401
|
||||
"""Pick the UI-bound string: the serialized display payload when set, else the LLM string."""
|
||||
return llm_str if display_result is _UNSET else _stringify_tool_result(display_result)
|
||||
|
||||
|
||||
def _emit_tool_result_common(
|
||||
call_id: str,
|
||||
raw_result: Any,
|
||||
@@ -303,7 +276,6 @@ def _emit_tool_result_common(
|
||||
predictive_handler: PredictiveStateHandler | None = None,
|
||||
*,
|
||||
state_update: Mapping[str, Any] | None = None,
|
||||
display_result: Any = _UNSET, # noqa: ANN401
|
||||
) -> list[BaseEvent]:
|
||||
"""Shared helper for emitting ToolCallEnd + ToolCallResult events and performing FlowState cleanup.
|
||||
|
||||
@@ -329,14 +301,13 @@ def _emit_tool_result_common(
|
||||
events.append(ToolCallEndEvent(tool_call_id=call_id))
|
||||
flow.tool_calls_ended.add(call_id)
|
||||
|
||||
result_content = _stringify_tool_result(raw_result)
|
||||
ui_result_content = _resolve_ui_payload(result_content, display_result)
|
||||
result_content = raw_result if isinstance(raw_result, str) else json.dumps(make_json_safe(raw_result))
|
||||
message_id = generate_event_id()
|
||||
events.append(
|
||||
ToolCallResultEvent(
|
||||
message_id=message_id,
|
||||
tool_call_id=call_id,
|
||||
content=ui_result_content,
|
||||
content=result_content,
|
||||
role="tool",
|
||||
)
|
||||
)
|
||||
@@ -387,14 +358,12 @@ def _emit_tool_result(
|
||||
return []
|
||||
raw_result = content.result if content.result is not None else ""
|
||||
state_update = _extract_tool_result_state(content)
|
||||
display_result = _extract_tool_result_display(content)
|
||||
return _emit_tool_result_common(
|
||||
content.call_id,
|
||||
raw_result,
|
||||
flow,
|
||||
predictive_handler,
|
||||
state_update=state_update,
|
||||
display_result=display_result,
|
||||
)
|
||||
|
||||
|
||||
@@ -561,14 +530,12 @@ def _emit_mcp_tool_result(
|
||||
return []
|
||||
raw_output = content.output if content.output is not None else ""
|
||||
state_update = _extract_tool_result_state(content)
|
||||
display_result = _extract_tool_result_display(content)
|
||||
return _emit_tool_result_common(
|
||||
content.call_id,
|
||||
raw_output,
|
||||
flow,
|
||||
predictive_handler,
|
||||
state_update=state_update,
|
||||
display_result=display_result,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -1,13 +1,12 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
"""Deterministic tool-driven AG-UI state updates and display payloads.
|
||||
"""Deterministic tool-driven AG-UI state updates.
|
||||
|
||||
Tools wired into the :mod:`agent_framework_ag_ui` endpoint can push a
|
||||
deterministic state update or a per-call tool result display payload by
|
||||
returning :func:`state_update`. Unlike ``predict_state_config`` — which emits
|
||||
``StateDeltaEvent``s optimistically from LLM-predicted tool call arguments —
|
||||
``state_update`` runs *after* the tool executes, so AG-UI state and display
|
||||
content always reflect the tool's actual return value.
|
||||
deterministic state update by returning :func:`state_update`. Unlike
|
||||
``predict_state_config`` — which emits ``StateDeltaEvent``s optimistically from
|
||||
LLM-predicted tool call arguments — ``state_update`` runs *after* the tool
|
||||
executes, so the AG-UI state always reflects the tool's actual return value.
|
||||
|
||||
See issue https://github.com/microsoft/agent-framework/issues/3167 for the
|
||||
motivating discussion.
|
||||
@@ -15,48 +14,33 @@ motivating discussion.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections.abc import Mapping
|
||||
from typing import Any
|
||||
|
||||
from agent_framework import Content
|
||||
|
||||
from ._utils import make_json_safe
|
||||
|
||||
__all__ = ["TOOL_RESULT_DISPLAY_KEY", "TOOL_RESULT_STATE_KEY", "state_update"]
|
||||
__all__ = ["TOOL_RESULT_STATE_KEY", "state_update"]
|
||||
|
||||
|
||||
TOOL_RESULT_STATE_KEY = "__ag_ui_tool_result_state__"
|
||||
"""Reserved ``Content.additional_properties`` key used to carry a tool-driven
|
||||
state snapshot from a tool return value through to the AG-UI emitter."""
|
||||
|
||||
TOOL_RESULT_DISPLAY_KEY = "__ag_ui_tool_result_display__"
|
||||
"""Reserved ``Content.additional_properties`` key used to carry UI-only tool result display content from a tool return value through to the AG-UI emitter."""
|
||||
|
||||
_UNSET = object()
|
||||
|
||||
|
||||
def _serialize_tool_result(value: Any) -> str: # noqa: ANN401
|
||||
return value if isinstance(value, str) else json.dumps(make_json_safe(value))
|
||||
|
||||
|
||||
def state_update(
|
||||
text: str = "",
|
||||
*,
|
||||
state: Mapping[str, Any] | None = None,
|
||||
tool_result: Any = _UNSET, # noqa: ANN401
|
||||
state: Mapping[str, Any],
|
||||
) -> Content:
|
||||
"""Build a tool return value that updates AG-UI shared state or display content.
|
||||
"""Build a tool return value that deterministically updates AG-UI shared state.
|
||||
|
||||
Return the result of this helper from an agent tool to push a state update
|
||||
or UI-only display payload to AG-UI clients using the actual tool output,
|
||||
rather than LLM-predicted tool arguments.
|
||||
to AG-UI clients using the actual tool output, rather than LLM-predicted
|
||||
tool arguments.
|
||||
|
||||
When the AG-UI endpoint emits the tool result, it will:
|
||||
|
||||
* Forward ``text`` to the LLM as the normal ``function_result`` content.
|
||||
* Use ``tool_result`` as the ``ToolCallResultEvent.content`` payload shown
|
||||
to AG-UI clients, falling back to ``text`` when no display payload is set.
|
||||
* Merge ``state`` into ``FlowState.current_state``.
|
||||
* Emit a deterministic ``StateSnapshotEvent`` after the ``ToolCallResult``
|
||||
event so frontends observe the updated state deterministically. If
|
||||
@@ -65,7 +49,7 @@ def state_update(
|
||||
Example:
|
||||
.. code-block:: python
|
||||
|
||||
from agent_framework import Content, tool
|
||||
from agent_framework import tool
|
||||
from agent_framework_ag_ui import state_update
|
||||
|
||||
|
||||
@@ -77,61 +61,24 @@ def state_update(
|
||||
state={"weather": {"city": city, **data}},
|
||||
)
|
||||
|
||||
Example:
|
||||
.. code-block:: python
|
||||
|
||||
from agent_framework import Content, tool
|
||||
from agent_framework_ag_ui import state_update
|
||||
|
||||
|
||||
@tool
|
||||
async def get_weather(city: str) -> Content:
|
||||
data = await _fetch_weather(city)
|
||||
return state_update(
|
||||
text=f"{city}: {data['temp']}°C and {data['conditions']}",
|
||||
tool_result={
|
||||
"component": "weather-card",
|
||||
"city": city,
|
||||
"temperature": data["temp"],
|
||||
"conditions": data["conditions"],
|
||||
"humidity": data["humidity"],
|
||||
},
|
||||
state={"weather": {"city": city, **data}},
|
||||
)
|
||||
|
||||
Args:
|
||||
text: Text passed back to the LLM as the ``function_result`` content.
|
||||
Defaults to an empty string for tools whose only output is a state
|
||||
update.
|
||||
state: A mapping merged into the AG-UI shared state via JSON-compatible
|
||||
``dict.update`` semantics. Nested dicts are replaced, not deep-merged.
|
||||
tool_result: JSON-safe payload emitted to AG-UI clients as
|
||||
``ToolCallResultEvent.content`` for frontend rendering. The LLM
|
||||
still receives ``text``. If ``text`` is empty, the serialized
|
||||
display payload is also used as the LLM-bound text fallback.
|
||||
|
||||
Returns:
|
||||
A ``Content`` object with ``type="text"``. The state payload rides in
|
||||
``additional_properties`` under :data:`TOOL_RESULT_STATE_KEY`
|
||||
(``"__ag_ui_tool_result_state__"``), and the display payload rides
|
||||
under :data:`TOOL_RESULT_DISPLAY_KEY`
|
||||
(``"__ag_ui_tool_result_display__"``). Both reserved keys are extracted
|
||||
by the AG-UI emitter.
|
||||
``additional_properties`` under :data:`TOOL_RESULT_STATE_KEY` and is
|
||||
extracted by the AG-UI emitter.
|
||||
|
||||
Raises:
|
||||
TypeError: If ``state`` is not a ``Mapping``.
|
||||
"""
|
||||
if state is not None and not isinstance(state, Mapping):
|
||||
if not isinstance(state, Mapping):
|
||||
raise TypeError(f"state_update() 'state' must be a Mapping, got {type(state).__name__}")
|
||||
additional_properties: dict[str, Any] = {}
|
||||
if state is not None:
|
||||
additional_properties[TOOL_RESULT_STATE_KEY] = dict(state)
|
||||
if tool_result is not _UNSET:
|
||||
display_content = _serialize_tool_result(tool_result)
|
||||
additional_properties[TOOL_RESULT_DISPLAY_KEY] = display_content
|
||||
if not text:
|
||||
text = display_content
|
||||
return Content.from_text(
|
||||
text,
|
||||
additional_properties=additional_properties,
|
||||
additional_properties={TOOL_RESULT_STATE_KEY: dict(state)},
|
||||
)
|
||||
|
||||
@@ -68,19 +68,6 @@ def _tool_result_with_state(call_id: str, text: str, state: dict[str, Any]) -> A
|
||||
)
|
||||
|
||||
|
||||
def _tool_result_with_display(call_id: str, text: str, tool_result: Any, **kwargs: Any) -> AgentResponseUpdate:
|
||||
"""Build a function_result update carrying an optional UI display marker."""
|
||||
return AgentResponseUpdate(
|
||||
contents=[
|
||||
Content.from_function_result(
|
||||
call_id=call_id,
|
||||
result=[state_update(text=text, tool_result=tool_result, **kwargs)],
|
||||
)
|
||||
],
|
||||
role="assistant",
|
||||
)
|
||||
|
||||
|
||||
# ── Golden stream tests ──
|
||||
|
||||
|
||||
@@ -278,87 +265,3 @@ async def test_deterministic_state_coexists_with_predict_state_config() -> None:
|
||||
# The final observed state must contain both the deterministic and predictive contributions.
|
||||
final = stream.snapshot()
|
||||
assert final["weather"] == {"city": "SF", "temp": 14}, f"Deterministic state missing from final snapshot: {final}"
|
||||
|
||||
|
||||
async def test_tool_result_display_payload_reaches_ui_event_only() -> None:
|
||||
"""Rich display payload overrides TOOL_CALL_RESULT without leaking marker keys."""
|
||||
updates = [
|
||||
_tool_call("call-1", "get_weather", '{"city": "SF"}'),
|
||||
_tool_result_with_display(
|
||||
"call-1",
|
||||
text="Weather in SF: 14°C foggy",
|
||||
tool_result={"city": "SF", "temp": 14, "conditions": "foggy"},
|
||||
),
|
||||
]
|
||||
agent = _build_agent(updates)
|
||||
stream = await _run(agent, PAYLOAD)
|
||||
|
||||
stream.assert_bookends()
|
||||
stream.assert_no_run_error()
|
||||
stream.assert_tool_calls_balanced()
|
||||
|
||||
result = stream.first("TOOL_CALL_RESULT")
|
||||
assert result.content == '{"city": "SF", "temp": 14, "conditions": "foggy"}'
|
||||
assert "__ag_ui_tool_result_display__" not in result.content
|
||||
assert "__ag_ui_tool_result_state__" not in result.content
|
||||
|
||||
|
||||
async def test_tool_result_display_falls_back_to_text_when_unset() -> None:
|
||||
"""Without a display marker, the UI event keeps the existing text content."""
|
||||
updates = [
|
||||
_tool_call("call-1", "get_weather", '{"city": "SF"}'),
|
||||
_tool_result_with_state(
|
||||
"call-1",
|
||||
text="Weather in SF: 14°C foggy",
|
||||
state={"weather": {"city": "SF", "temp": 14}},
|
||||
),
|
||||
]
|
||||
agent = _build_agent(updates)
|
||||
stream = await _run(agent, PAYLOAD)
|
||||
|
||||
stream.assert_bookends()
|
||||
stream.assert_no_run_error()
|
||||
stream.assert_tool_calls_balanced()
|
||||
|
||||
result = stream.first("TOOL_CALL_RESULT")
|
||||
assert result.content == "Weather in SF: 14°C foggy"
|
||||
assert "__ag_ui_tool_result_display__" not in result.content
|
||||
assert "__ag_ui_tool_result_state__" not in result.content
|
||||
|
||||
|
||||
async def test_tool_result_display_coexists_with_state_snapshot() -> None:
|
||||
"""Display and durable state markers produce one deterministic state snapshot."""
|
||||
updates = [
|
||||
_tool_call("call-1", "get_weather", '{"city": "SF"}'),
|
||||
_tool_result_with_display(
|
||||
"call-1",
|
||||
text="Weather in SF: 14°C foggy",
|
||||
tool_result={"city": "SF", "temp": 14, "conditions": "foggy"},
|
||||
state={"weather": {"city": "SF", "temp": 14, "conditions": "foggy"}},
|
||||
),
|
||||
]
|
||||
agent = _build_agent(updates)
|
||||
stream = await _run(agent, PAYLOAD)
|
||||
|
||||
stream.assert_bookends()
|
||||
stream.assert_no_run_error()
|
||||
stream.assert_tool_calls_balanced()
|
||||
stream.assert_ordered_types(["TOOL_CALL_RESULT", "STATE_SNAPSHOT", "RUN_FINISHED"])
|
||||
|
||||
result = stream.first("TOOL_CALL_RESULT")
|
||||
assert result.content == '{"city": "SF", "temp": 14, "conditions": "foggy"}'
|
||||
|
||||
result_idx = stream.events.index(result)
|
||||
deterministic_snapshots = [
|
||||
event
|
||||
for event in stream.events[result_idx + 1 :]
|
||||
if getattr(getattr(event, "type", None), "value", getattr(event, "type", None)) == "STATE_SNAPSHOT"
|
||||
]
|
||||
assert len(deterministic_snapshots) == 1
|
||||
assert deterministic_snapshots[0].snapshot["weather"] == {
|
||||
"city": "SF",
|
||||
"temp": 14,
|
||||
"conditions": "foggy",
|
||||
}
|
||||
assert "__ag_ui_tool_result_display__" not in str(deterministic_snapshots[0].snapshot)
|
||||
assert "__ag_ui_tool_result_state__" not in str(deterministic_snapshots[0].snapshot)
|
||||
|
||||
@@ -448,37 +448,3 @@ async def test_resolve_approval_responses_returns_only_approved() -> None:
|
||||
rejection_results = [c for c in all_contents if c.type == "function_result" and c.call_id == rejected_call_id]
|
||||
assert len(rejection_results) == 1
|
||||
assert "rejected" in str(rejection_results[0].result).lower()
|
||||
|
||||
|
||||
class TestApprovalToolResultDisplayChannel:
|
||||
"""Approved tools using ``state_update(..., tool_result=...)`` must route the
|
||||
display payload to the UI event while ``flow.tool_results`` still receives
|
||||
the LLM-bound text. The HITL approval emitter is separate from the standard
|
||||
streaming emitter, so it gets its own coverage.
|
||||
"""
|
||||
|
||||
def test_approval_emits_display_payload_when_marker_present(self) -> None:
|
||||
from agent_framework_ag_ui import state_update
|
||||
from agent_framework_ag_ui._agent_run import _make_approval_tool_result_events
|
||||
|
||||
display_payload = {"city": "Seattle", "temp": 14, "conditions": "foggy"}
|
||||
inner = state_update(text="14°C, foggy", tool_result=display_payload)
|
||||
resolved = Content.from_function_result(call_id="call_disp", result=[inner])
|
||||
|
||||
events = _make_approval_tool_result_events([resolved])
|
||||
|
||||
assert len(events) == 1
|
||||
# UI event must carry the serialized display payload, NOT the LLM text.
|
||||
assert json.loads(events[0].content) == display_payload
|
||||
assert events[0].content != "14°C, foggy"
|
||||
|
||||
def test_approval_falls_back_to_text_when_no_marker(self) -> None:
|
||||
"""Backward compat: without a display marker, behaviour is unchanged."""
|
||||
from agent_framework_ag_ui._agent_run import _make_approval_tool_result_events
|
||||
|
||||
resolved = Content.from_function_result(call_id="call_plain", result="Sunny in Seattle")
|
||||
|
||||
events = _make_approval_tool_result_events([resolved])
|
||||
|
||||
assert len(events) == 1
|
||||
assert events[0].content == "Sunny in Seattle"
|
||||
|
||||
@@ -15,7 +15,7 @@ from agent_framework_ag_ui._run_common import (
|
||||
_extract_tool_result_state,
|
||||
_normalize_resume_interrupts,
|
||||
)
|
||||
from agent_framework_ag_ui._state import TOOL_RESULT_DISPLAY_KEY, TOOL_RESULT_STATE_KEY
|
||||
from agent_framework_ag_ui._state import TOOL_RESULT_STATE_KEY
|
||||
|
||||
|
||||
class TestNormalizeResumeInterrupts:
|
||||
@@ -140,15 +140,6 @@ class TestStateUpdateHelper:
|
||||
TOOL_RESULT_STATE_KEY: {"weather": {"temp": 14}},
|
||||
}
|
||||
|
||||
def test_builds_text_content_with_display_marker(self):
|
||||
"""state_update can carry a UI display payload without requiring state."""
|
||||
c = state_update(text="14°C, foggy", tool_result={"temp": 14, "conditions": "foggy"})
|
||||
assert c.type == "text"
|
||||
assert c.text == "14°C, foggy"
|
||||
assert c.additional_properties == {
|
||||
TOOL_RESULT_DISPLAY_KEY: '{"temp": 14, "conditions": "foggy"}',
|
||||
}
|
||||
|
||||
def test_empty_text_is_allowed(self):
|
||||
"""State-only tools can omit the text argument."""
|
||||
c = state_update(state={"steps": ["a", "b"]})
|
||||
@@ -174,18 +165,6 @@ class TestStateUpdateHelper:
|
||||
inner = c.additional_properties[TOOL_RESULT_STATE_KEY]
|
||||
assert inner is not caller_state
|
||||
|
||||
def test_tool_result_without_text_falls_back_to_display_payload(self):
|
||||
"""Display-only tools use the serialized display payload as LLM text."""
|
||||
c = state_update(tool_result={"temp": 14, "conditions": "foggy"})
|
||||
assert c.text == '{"temp": 14, "conditions": "foggy"}'
|
||||
assert c.additional_properties[TOOL_RESULT_DISPLAY_KEY] == '{"temp": 14, "conditions": "foggy"}'
|
||||
|
||||
def test_string_tool_result_is_not_json_encoded_again(self):
|
||||
"""A pre-serialized display string passes through verbatim."""
|
||||
c = state_update(text="Weather summary", tool_result='{"temp":14}')
|
||||
assert c.text == "Weather summary"
|
||||
assert c.additional_properties[TOOL_RESULT_DISPLAY_KEY] == '{"temp":14}'
|
||||
|
||||
|
||||
class TestExtractToolResultState:
|
||||
"""Tests for ``_extract_tool_result_state``."""
|
||||
@@ -286,60 +265,6 @@ class TestEmitToolResultWithState:
|
||||
assert result_events[0].content == "Weather: 14°C"
|
||||
assert TOOL_RESULT_STATE_KEY not in result_events[0].content
|
||||
|
||||
def test_display_payload_routes_to_ui_only(self):
|
||||
"""A display marker overrides only the UI event, not the LLM-bound tool result."""
|
||||
tool_return = state_update(
|
||||
text="Weather: 14°C",
|
||||
tool_result={"temp": 14, "conditions": "foggy"},
|
||||
)
|
||||
content = Content.from_function_result(call_id="c1", result=[tool_return])
|
||||
flow = FlowState()
|
||||
|
||||
events = _emit_tool_result(content, flow)
|
||||
result_events = [e for e in events if e.type == EventType.TOOL_CALL_RESULT]
|
||||
|
||||
assert len(result_events) == 1
|
||||
assert result_events[0].content == '{"temp": 14, "conditions": "foggy"}'
|
||||
assert flow.tool_results[-1]["content"] == "Weather: 14°C"
|
||||
assert TOOL_RESULT_DISPLAY_KEY not in result_events[0].content
|
||||
assert TOOL_RESULT_DISPLAY_KEY not in flow.tool_results[-1]["content"]
|
||||
|
||||
def test_plain_tool_result_uses_existing_content_for_both_channels(self):
|
||||
"""Without a display marker, UI and LLM channels keep the existing derivation."""
|
||||
content = Content.from_function_result(call_id="c1", result="plain result")
|
||||
flow = FlowState()
|
||||
|
||||
events = _emit_tool_result(content, flow)
|
||||
result_events = [e for e in events if e.type == EventType.TOOL_CALL_RESULT]
|
||||
|
||||
assert len(result_events) == 1
|
||||
assert result_events[0].content == "plain result"
|
||||
assert flow.tool_results[-1]["content"] == "plain result"
|
||||
|
||||
def test_display_only_payload_falls_back_to_llm_content(self):
|
||||
"""When text is empty, both channels receive the serialized display payload."""
|
||||
tool_return = state_update(tool_result={"temp": 14})
|
||||
content = Content.from_function_result(call_id="c1", result=[tool_return])
|
||||
flow = FlowState()
|
||||
|
||||
events = _emit_tool_result(content, flow)
|
||||
result_events = [e for e in events if e.type == EventType.TOOL_CALL_RESULT]
|
||||
|
||||
assert result_events[0].content == '{"temp": 14}'
|
||||
assert flow.tool_results[-1]["content"] == '{"temp": 14}'
|
||||
|
||||
def test_pre_serialized_display_string_routes_verbatim(self):
|
||||
"""String display payloads pass through without JSON double-encoding."""
|
||||
tool_return = state_update(text="Weather summary", tool_result='{"temp":14}')
|
||||
content = Content.from_function_result(call_id="c1", result=[tool_return])
|
||||
flow = FlowState()
|
||||
|
||||
events = _emit_tool_result(content, flow)
|
||||
result_events = [e for e in events if e.type == EventType.TOOL_CALL_RESULT]
|
||||
|
||||
assert result_events[0].content == '{"temp":14}'
|
||||
assert flow.tool_results[-1]["content"] == "Weather summary"
|
||||
|
||||
def test_coexists_with_active_predictive_state_handler(self):
|
||||
"""Both predictive and deterministic state produce a single coalesced snapshot.
|
||||
|
||||
@@ -421,31 +346,3 @@ class TestEmitMcpToolResultWithState:
|
||||
|
||||
events = _emit_mcp_tool_result(content, flow)
|
||||
assert all(e.type != EventType.STATE_SNAPSHOT for e in events)
|
||||
|
||||
|
||||
class TestEmitMcpToolResultWithDisplay:
|
||||
"""MCP tool results must honour the display marker so UI consumers can
|
||||
render structured payloads while ``flow.tool_results`` keeps the LLM
|
||||
string. MCP outputs do not pass through ``parse_result``; the marker
|
||||
rides on the outer content's ``additional_properties``.
|
||||
"""
|
||||
|
||||
def test_mcp_tool_result_routes_display_payload_to_ui_only(self):
|
||||
import json as _json
|
||||
|
||||
display_payload = {"rows": [{"id": 1, "name": "alpha"}, {"id": 2, "name": "beta"}]}
|
||||
content = Content.from_mcp_server_tool_result(
|
||||
call_id="mcp_disp",
|
||||
output="2 rows returned",
|
||||
additional_properties={TOOL_RESULT_DISPLAY_KEY: display_payload},
|
||||
)
|
||||
flow = FlowState()
|
||||
|
||||
events = _emit_mcp_tool_result(content, flow)
|
||||
result_events = [e for e in events if e.type == EventType.TOOL_CALL_RESULT]
|
||||
|
||||
assert len(result_events) == 1
|
||||
# UI event carries the structured display payload.
|
||||
assert _json.loads(result_events[0].content) == display_payload
|
||||
# LLM-side accumulator keeps the short text.
|
||||
assert flow.tool_results[-1]["content"] == "2 rows returned"
|
||||
|
||||
@@ -69,8 +69,7 @@ agent_framework/
|
||||
|
||||
### Skills (`_skills.py`)
|
||||
|
||||
- **`Skill`** - Abstract base for a skill definition bundling instructions (`content`) with frontmatter metadata, resources, and scripts. Concrete subclasses (`InlineSkill`, `FileSkill`, `ClassSkill`) accept a `frontmatter=SkillFrontmatter(...)` argument carrying the spec fields. Adding new spec fields is done in one place — on `SkillFrontmatter` — keeping the subclass constructors stable.
|
||||
- **`SkillFrontmatter`** - L1 discovery metadata for a skill (`name`, `description`, `license`, `compatibility`, `allowed_tools`, `metadata`). All fields are mutable plain attributes; the constructor validates `name`, `description`, and `compatibility` against the spec but post-construction assignments are not re-validated. Spec fields are reachable on every skill via `skill.frontmatter`.
|
||||
- **`Skill`** - A skill definition bundling instructions (`content`) with metadata, resources, and scripts. Supports `@skill.resource` and `@skill.script` decorators for adding components.
|
||||
- **`SkillResource`** - Named supplementary content attached to a skill; holds either static `content` or a dynamic `function` (sync or async). Exactly one must be provided.
|
||||
- **`SkillScript`** - An executable script attached to a skill; holds either an inline `function` (code-defined, runs in-process) or a `path` to a file on disk (file-based, delegated to a runner). Exactly one must be provided.
|
||||
- **`SkillScriptRunner`** - Protocol for file-based script execution. Any callable matching `(skill, script, args) -> Any` satisfies it. Code-defined scripts do not use a runner.
|
||||
|
||||
@@ -147,7 +147,6 @@ from ._skills import (
|
||||
InlineSkillScript,
|
||||
InMemorySkillsSource,
|
||||
Skill,
|
||||
SkillFrontmatter,
|
||||
SkillResource,
|
||||
SkillScript,
|
||||
SkillScriptRunner,
|
||||
@@ -433,7 +432,6 @@ __all__ = [
|
||||
"SessionContext",
|
||||
"SingleEdgeGroup",
|
||||
"Skill",
|
||||
"SkillFrontmatter",
|
||||
"SkillResource",
|
||||
"SkillScript",
|
||||
"SkillScriptRunner",
|
||||
|
||||
@@ -10,7 +10,7 @@ import logging
|
||||
import re
|
||||
import sys
|
||||
from abc import abstractmethod
|
||||
from collections.abc import Callable, Collection, Coroutine, Sequence
|
||||
from collections.abc import Callable, Collection, Sequence
|
||||
from contextlib import AsyncExitStack, _AsyncGeneratorContextManager # type: ignore
|
||||
from datetime import timedelta
|
||||
from functools import partial
|
||||
@@ -264,7 +264,6 @@ class MCPTool:
|
||||
self.is_connected: bool = False
|
||||
self._tools_loaded: bool = False
|
||||
self._prompts_loaded: bool = False
|
||||
self._pending_reload_tasks: set[asyncio.Task[None]] = set()
|
||||
|
||||
def __str__(self) -> str:
|
||||
return f"MCPTool(name={self.name}, description={self.description})"
|
||||
@@ -906,47 +905,12 @@ class MCPTool:
|
||||
if isinstance(message, types.ServerNotification):
|
||||
match message.root.method:
|
||||
case "notifications/tools/list_changed":
|
||||
self._schedule_reload(self.load_tools())
|
||||
await self.load_tools()
|
||||
case "notifications/prompts/list_changed":
|
||||
self._schedule_reload(self.load_prompts())
|
||||
await self.load_prompts()
|
||||
case _:
|
||||
logger.debug("Unhandled notification: %s", message.root.method)
|
||||
|
||||
def _schedule_reload(self, coro: Coroutine[Any, Any, None]) -> None:
|
||||
"""Schedule a reload coroutine as a background task.
|
||||
|
||||
Reloads (load_tools / load_prompts) triggered by MCP server
|
||||
notifications must NOT be awaited inside the message handler because
|
||||
the handler runs on the MCP SDK's single-threaded receive loop.
|
||||
Awaiting a session request (e.g. ``list_tools``) from within that loop
|
||||
deadlocks: the receive loop cannot read the response while it is
|
||||
blocked waiting for the handler to return.
|
||||
|
||||
Instead we fire the reload as an independent ``asyncio.Task`` and keep
|
||||
a strong reference in ``_pending_reload_tasks`` so it is not garbage-
|
||||
collected before completion. Only one reload per kind (tools / prompts)
|
||||
is kept in flight; a new notification cancels the previous pending task
|
||||
for the same coroutine name to avoid unbounded growth.
|
||||
"""
|
||||
# Cancel-and-replace: only one reload per kind should be in flight.
|
||||
reload_name = f"mcp-reload:{self.name}:{coro.__qualname__}"
|
||||
for existing in list(self._pending_reload_tasks):
|
||||
if existing.get_name() == reload_name and not existing.done():
|
||||
logger.debug("Cancelling in-flight reload %s; superseded by new notification", reload_name)
|
||||
existing.cancel()
|
||||
|
||||
async def _safe_reload() -> None:
|
||||
try:
|
||||
await coro
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception:
|
||||
logger.warning("Background MCP reload failed", exc_info=True)
|
||||
|
||||
task = asyncio.create_task(_safe_reload(), name=reload_name)
|
||||
self._pending_reload_tasks.add(task)
|
||||
task.add_done_callback(self._pending_reload_tasks.discard)
|
||||
|
||||
def _determine_approval_mode(
|
||||
self,
|
||||
*candidate_names: str,
|
||||
@@ -1083,14 +1047,6 @@ class MCPTool:
|
||||
params = types.PaginatedRequestParams(cursor=tool_list.nextCursor)
|
||||
|
||||
async def _close_on_owner(self) -> None:
|
||||
# Cancel any pending reload tasks before tearing down the session.
|
||||
tasks = list(self._pending_reload_tasks)
|
||||
for task in tasks:
|
||||
task.cancel()
|
||||
self._pending_reload_tasks.clear()
|
||||
if tasks:
|
||||
await asyncio.gather(*tasks, return_exceptions=True)
|
||||
|
||||
await self._safe_close_exit_stack()
|
||||
self._exit_stack = AsyncExitStack()
|
||||
self.session = None
|
||||
|
||||
@@ -465,23 +465,41 @@ class Skill(ABC):
|
||||
|
||||
A skill represents a domain-specific capability with instructions,
|
||||
resources, and scripts. Concrete implementations include
|
||||
:class:`FileSkill` (filesystem-backed), :class:`InlineSkill`
|
||||
(code-defined), and :class:`ClassSkill` (class-based).
|
||||
:class:`FileSkill` (filesystem-backed) and :class:`InlineSkill`
|
||||
(code-defined).
|
||||
|
||||
Skill spec metadata (name, description, license, compatibility,
|
||||
allowed_tools, metadata) is exposed via the :attr:`frontmatter`
|
||||
property, which returns a :class:`SkillFrontmatter` instance.
|
||||
Skill metadata follows the
|
||||
`Agent Skills specification <https://agentskills.io/>`_.
|
||||
|
||||
Attributes:
|
||||
name: Skill name (lowercase letters, numbers, hyphens only).
|
||||
description: Human-readable description of the skill.
|
||||
"""
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def frontmatter(self) -> SkillFrontmatter:
|
||||
"""The L1 discovery metadata for this skill.
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
name: str,
|
||||
description: str,
|
||||
) -> None:
|
||||
"""Initialize a Skill.
|
||||
|
||||
Contains the name, description, and other spec fields as defined by
|
||||
the `Agent Skills specification <https://agentskills.io/specification>`_.
|
||||
Validates the skill name and description against specification rules.
|
||||
|
||||
Args:
|
||||
name: Skill name (lowercase letters, numbers, hyphens only;
|
||||
max 64 characters; no leading/trailing/consecutive hyphens).
|
||||
description: Human-readable description of the skill
|
||||
(≤1024 characters).
|
||||
|
||||
Raises:
|
||||
ValueError: If the name or description is invalid.
|
||||
"""
|
||||
...
|
||||
_validate_skill_name(name)
|
||||
_validate_skill_description(name, description)
|
||||
|
||||
self.name = name
|
||||
self.description = description
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
@@ -517,68 +535,6 @@ class Skill(ABC):
|
||||
return []
|
||||
|
||||
|
||||
@experimental(feature_id=ExperimentalFeature.SKILLS)
|
||||
class SkillFrontmatter:
|
||||
"""L1 discovery metadata for a :class:`Skill`.
|
||||
|
||||
Encapsulates all `Agent Skills specification <https://agentskills.io/specification>`_
|
||||
frontmatter fields in a single object. All fields are mutable plain
|
||||
attributes; callers may freely reassign them after construction.
|
||||
|
||||
The constructor validates ``name``, ``description``, and ``compatibility``
|
||||
against specification rules and raises :class:`ValueError` on invalid
|
||||
input. Assignments made after construction are **not** re-validated;
|
||||
callers are expected to honor the spec.
|
||||
|
||||
Attributes:
|
||||
name: Skill name (lowercase letters, numbers, hyphens only).
|
||||
description: Human-readable description of the skill.
|
||||
license: Optional license name or reference.
|
||||
compatibility: Optional compatibility information (≤500 characters).
|
||||
allowed_tools: Optional space-delimited pre-approved tool names.
|
||||
metadata: Optional arbitrary key-value pairs (shallow-copied on
|
||||
construction to avoid caller-owned dict aliasing).
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
name: str,
|
||||
description: str,
|
||||
license: str | None = None,
|
||||
compatibility: str | None = None,
|
||||
allowed_tools: str | None = None,
|
||||
metadata: dict[str, str] | None = None,
|
||||
) -> None:
|
||||
"""Initialize a SkillFrontmatter.
|
||||
|
||||
Args:
|
||||
name: Skill name (lowercase letters, numbers, hyphens only;
|
||||
max 64 characters; no leading/trailing/consecutive hyphens).
|
||||
description: Human-readable description of the skill
|
||||
(≤1024 characters).
|
||||
license: Optional license name or reference.
|
||||
compatibility: Optional compatibility information
|
||||
(≤500 characters).
|
||||
allowed_tools: Optional space-delimited pre-approved tool names.
|
||||
metadata: Optional arbitrary key-value pairs.
|
||||
|
||||
Raises:
|
||||
ValueError: If the name, description, or compatibility is invalid.
|
||||
"""
|
||||
_validate_skill_name(name)
|
||||
_validate_skill_description(name, description)
|
||||
_validate_compatibility(compatibility)
|
||||
|
||||
self.name = name
|
||||
self.description = description
|
||||
self.compatibility = compatibility
|
||||
self.license = license
|
||||
self.allowed_tools = allowed_tools
|
||||
# Shallow-copy to avoid aliasing with caller-owned dict.
|
||||
self.metadata: dict[str, str] | None = dict(metadata) if metadata is not None else None
|
||||
|
||||
|
||||
def _validate_skill_name(name: str) -> None:
|
||||
"""Validate a skill name against specification rules.
|
||||
|
||||
@@ -617,21 +573,6 @@ def _validate_skill_description(name: str, description: str) -> None:
|
||||
)
|
||||
|
||||
|
||||
def _validate_compatibility(compatibility: str | None) -> None:
|
||||
"""Validate an optional compatibility value against specification rules.
|
||||
|
||||
Args:
|
||||
compatibility: The optional compatibility value to validate.
|
||||
|
||||
Raises:
|
||||
ValueError: If the value exceeds the maximum allowed length.
|
||||
"""
|
||||
if compatibility is not None and len(compatibility) > MAX_COMPATIBILITY_LENGTH:
|
||||
raise ValueError(
|
||||
f"Skill compatibility must be {MAX_COMPATIBILITY_LENGTH} characters or fewer."
|
||||
)
|
||||
|
||||
|
||||
def _build_skill_content(
|
||||
name: str,
|
||||
description: str,
|
||||
@@ -698,17 +639,23 @@ class InlineSkill(Skill):
|
||||
All resources and scripts should be configured before the skill is
|
||||
registered with a :class:`SkillsProvider`.
|
||||
|
||||
Attributes:
|
||||
name: Skill name (lowercase letters, numbers, hyphens only).
|
||||
description: Human-readable description of the skill.
|
||||
instructions: The skill instructions text.
|
||||
|
||||
Examples:
|
||||
With the decorator:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
skill = InlineSkill(
|
||||
frontmatter=SkillFrontmatter(
|
||||
name="db-skill",
|
||||
description="Database operations",
|
||||
),
|
||||
name="db-skill",
|
||||
description="Database operations",
|
||||
instructions="Use this skill for DB tasks.",
|
||||
)
|
||||
|
||||
|
||||
@skill.resource
|
||||
def get_schema() -> str:
|
||||
return "CREATE TABLE ..."
|
||||
@@ -717,7 +664,8 @@ class InlineSkill(Skill):
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
frontmatter: SkillFrontmatter,
|
||||
name: str,
|
||||
description: str,
|
||||
instructions: str,
|
||||
resources: Sequence[SkillResource] | None = None,
|
||||
scripts: Sequence[SkillScript] | None = None,
|
||||
@@ -725,25 +673,19 @@ class InlineSkill(Skill):
|
||||
"""Initialize an InlineSkill.
|
||||
|
||||
Args:
|
||||
frontmatter: Skill specification metadata (name, description,
|
||||
and optional spec fields). Construct a :class:`SkillFrontmatter`
|
||||
with the desired fields.
|
||||
name: Skill name (lowercase letters, numbers, hyphens only).
|
||||
description: Human-readable description of the skill (≤1024 chars).
|
||||
instructions: The skill instructions text.
|
||||
resources: Pre-built resources to attach to this skill.
|
||||
scripts: Pre-built scripts to attach to this skill.
|
||||
"""
|
||||
self._frontmatter = frontmatter
|
||||
super().__init__(name=name, description=description)
|
||||
|
||||
self.instructions = instructions
|
||||
self._resources: list[SkillResource] = list(resources) if resources is not None else []
|
||||
self._scripts: list[SkillScript] = list(scripts) if scripts is not None else []
|
||||
self._cached_content: str | None = None
|
||||
|
||||
@property
|
||||
def frontmatter(self) -> SkillFrontmatter:
|
||||
"""The L1 discovery metadata for this skill."""
|
||||
return self._frontmatter
|
||||
|
||||
@property
|
||||
def content(self) -> str:
|
||||
"""Synthesized XML content with name, description, instructions, resources, and scripts.
|
||||
@@ -755,11 +697,7 @@ class InlineSkill(Skill):
|
||||
return self._cached_content
|
||||
|
||||
self._cached_content = _build_skill_content(
|
||||
self._frontmatter.name,
|
||||
self._frontmatter.description,
|
||||
self.instructions,
|
||||
self._resources,
|
||||
self._scripts,
|
||||
self.name, self.description, self.instructions, self._resources, self._scripts
|
||||
)
|
||||
return self._cached_content
|
||||
|
||||
@@ -994,6 +932,10 @@ class ClassSkill(Skill, ABC):
|
||||
Class-based skills can be distributed via shared libraries or PyPI
|
||||
packages, making them easy to reuse across projects.
|
||||
|
||||
Attributes:
|
||||
name: Skill name (lowercase letters, numbers, hyphens only).
|
||||
description: Human-readable description of the skill.
|
||||
|
||||
Examples:
|
||||
Decorator-based (recommended):
|
||||
|
||||
@@ -1002,10 +944,8 @@ class ClassSkill(Skill, ABC):
|
||||
class UnitConverterSkill(ClassSkill):
|
||||
def __init__(self) -> None:
|
||||
super().__init__(
|
||||
frontmatter=SkillFrontmatter(
|
||||
name="unit-converter",
|
||||
description="Convert between common units.",
|
||||
),
|
||||
name="unit-converter",
|
||||
description="Convert between common units.",
|
||||
)
|
||||
|
||||
@property
|
||||
@@ -1027,10 +967,8 @@ class ClassSkill(Skill, ABC):
|
||||
class UnitConverterSkill(ClassSkill):
|
||||
def __init__(self) -> None:
|
||||
super().__init__(
|
||||
frontmatter=SkillFrontmatter(
|
||||
name="unit-converter",
|
||||
description="Convert between common units.",
|
||||
),
|
||||
name="unit-converter",
|
||||
description="Convert between common units.",
|
||||
)
|
||||
|
||||
@property
|
||||
@@ -1051,25 +989,22 @@ class ClassSkill(Skill, ABC):
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
frontmatter: SkillFrontmatter,
|
||||
name: str,
|
||||
description: str,
|
||||
) -> None:
|
||||
"""Initialize a ClassSkill.
|
||||
|
||||
Args:
|
||||
frontmatter: Skill specification metadata (name, description,
|
||||
and optional spec fields). Construct a :class:`SkillFrontmatter`
|
||||
with the desired fields.
|
||||
name: Skill name (lowercase letters, numbers, hyphens only;
|
||||
max 64 characters).
|
||||
description: Human-readable description of the skill
|
||||
(≤1024 characters).
|
||||
"""
|
||||
self._frontmatter = frontmatter
|
||||
super().__init__(name=name, description=description)
|
||||
self._cached_content: str | None = None
|
||||
self._cached_resources: list[SkillResource] | None = None
|
||||
self._cached_scripts: list[SkillScript] | None = None
|
||||
|
||||
@property
|
||||
def frontmatter(self) -> SkillFrontmatter:
|
||||
"""The L1 discovery metadata for this skill."""
|
||||
return self._frontmatter
|
||||
|
||||
@staticmethod
|
||||
def resource(
|
||||
func: Callable[..., Any] | None = None,
|
||||
@@ -1217,7 +1152,7 @@ class ClassSkill(Skill, ABC):
|
||||
resource_name = marker.get("name") or _make_method_name(attr_name)
|
||||
if resource_name in seen_names:
|
||||
raise ValueError(
|
||||
f"Skill '{self._frontmatter.name}' already has a resource named '{resource_name}'. "
|
||||
f"Skill '{self.name}' already has a resource named '{resource_name}'. "
|
||||
"Ensure each @ClassSkill.resource has a unique name."
|
||||
)
|
||||
seen_names.add(resource_name)
|
||||
@@ -1277,7 +1212,7 @@ class ClassSkill(Skill, ABC):
|
||||
script_name = marker.get("name") or _make_method_name(attr_name)
|
||||
if script_name in seen_names:
|
||||
raise ValueError(
|
||||
f"Skill '{self._frontmatter.name}' already has a script named '{script_name}'. "
|
||||
f"Skill '{self.name}' already has a script named '{script_name}'. "
|
||||
"Ensure each @ClassSkill.script has a unique name."
|
||||
)
|
||||
seen_names.add(script_name)
|
||||
@@ -1305,11 +1240,7 @@ class ClassSkill(Skill, ABC):
|
||||
return self._cached_content
|
||||
|
||||
self._cached_content = _build_skill_content(
|
||||
self._frontmatter.name,
|
||||
self._frontmatter.description,
|
||||
self.instructions,
|
||||
self.resources,
|
||||
self.scripts,
|
||||
self.name, self.description, self.instructions, self.resources, self.scripts
|
||||
)
|
||||
return self._cached_content
|
||||
|
||||
@@ -1319,13 +1250,16 @@ class FileSkill(Skill):
|
||||
"""A :class:`Skill` discovered from a filesystem directory backed by a SKILL.md file.
|
||||
|
||||
Attributes:
|
||||
name: Skill name (lowercase letters, numbers, hyphens only).
|
||||
description: Human-readable description of the skill.
|
||||
path: Absolute path to the directory containing this skill.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
frontmatter: SkillFrontmatter,
|
||||
name: str,
|
||||
description: str,
|
||||
content: str,
|
||||
path: str,
|
||||
resources: Sequence[SkillResource] | None = None,
|
||||
@@ -1334,26 +1268,20 @@ class FileSkill(Skill):
|
||||
"""Initialize a FileSkill.
|
||||
|
||||
Args:
|
||||
frontmatter: Skill specification metadata parsed from the
|
||||
SKILL.md file's YAML frontmatter (name, description,
|
||||
and optional spec fields).
|
||||
name: Skill name (lowercase letters, numbers, hyphens only).
|
||||
description: Human-readable description of the skill (≤1024 chars).
|
||||
content: The full raw SKILL.md file content including YAML frontmatter.
|
||||
path: Absolute path to the skill directory on disk.
|
||||
resources: Resources discovered for this skill.
|
||||
scripts: Scripts discovered for this skill.
|
||||
"""
|
||||
self._frontmatter = frontmatter
|
||||
super().__init__(name=name, description=description)
|
||||
|
||||
self._content = content
|
||||
self.path = path
|
||||
self._resources: list[SkillResource] = list(resources) if resources is not None else []
|
||||
self._scripts: list[SkillScript] = list(scripts) if scripts is not None else []
|
||||
|
||||
@property
|
||||
def frontmatter(self) -> SkillFrontmatter:
|
||||
"""The L1 discovery metadata for this skill."""
|
||||
return self._frontmatter
|
||||
|
||||
@property
|
||||
def content(self) -> str:
|
||||
"""The skill content provided at construction time."""
|
||||
@@ -1418,7 +1346,6 @@ SKILL_FILE_NAME: Final[str] = "SKILL.md"
|
||||
MAX_SEARCH_DEPTH: Final[int] = 2
|
||||
MAX_NAME_LENGTH: Final[int] = 64
|
||||
MAX_DESCRIPTION_LENGTH: Final[int] = 1024
|
||||
MAX_COMPATIBILITY_LENGTH: Final[int] = 500
|
||||
DEFAULT_RESOURCE_EXTENSIONS: Final[tuple[str, ...]] = (
|
||||
".md",
|
||||
".json",
|
||||
@@ -1439,24 +1366,10 @@ FRONTMATTER_RE = re.compile(
|
||||
re.MULTILINE | re.DOTALL,
|
||||
)
|
||||
|
||||
# Matches top-level YAML "key: value" lines (unindented). Group 1 = key,
|
||||
# Group 2 = quoted value, Group 3 = unquoted value. Only matches keys at
|
||||
# column 0 so that indented children (e.g. under "metadata:") are not
|
||||
# mistakenly captured as top-level fields.
|
||||
# Matches YAML "key: value" lines. Group 1 = key, Group 2 = quoted value,
|
||||
# Group 3 = unquoted value.
|
||||
YAML_KV_RE = re.compile(
|
||||
r"^([\w-]+)\s*:\s*(?:[\"'](.+?)[\"']|(.+?))\s*$",
|
||||
re.MULTILINE,
|
||||
)
|
||||
|
||||
# Matches a YAML "metadata:" block followed by indented key-value pairs.
|
||||
YAML_METADATA_BLOCK_RE = re.compile(
|
||||
r"^metadata\s*:\s*$\n((?:[ \t]+\S.*\n?)+)",
|
||||
re.MULTILINE,
|
||||
)
|
||||
|
||||
# Matches indented "key: value" lines within a metadata block.
|
||||
YAML_INDENTED_KV_RE = re.compile(
|
||||
r"^\s+([\w-]+)\s*:\s*(?:[\"'](.+?)[\"']|(.+?))\s*$",
|
||||
r"^\s*(\w+)\s*:\s*(?:[\"'](.+?)[\"']|(.+?))\s*$",
|
||||
re.MULTILINE,
|
||||
)
|
||||
|
||||
@@ -1464,7 +1377,6 @@ YAML_INDENTED_KV_RE = re.compile(
|
||||
# must not start or end with a hyphen, and must not contain consecutive hyphens.
|
||||
VALID_NAME_RE = re.compile(r"^[a-z0-9]([a-z0-9]*-[a-z0-9])*[a-z0-9]*$")
|
||||
|
||||
|
||||
# Default system prompt template for advertising available skills to the model.
|
||||
# Use {skills} as the placeholder for the generated skills XML list.
|
||||
DEFAULT_SKILLS_INSTRUCTION_PROMPT = """\
|
||||
@@ -1551,7 +1463,7 @@ class SkillsProvider(ContextProvider):
|
||||
FileSkillsSource("./skills", script_runner=my_runner),
|
||||
InMemorySkillsSource([my_code_skill]),
|
||||
]),
|
||||
predicate=lambda s: s.frontmatter.name != "internal",
|
||||
predicate=lambda s: s.name != "internal",
|
||||
)
|
||||
)
|
||||
provider = SkillsProvider(source)
|
||||
@@ -1786,10 +1698,10 @@ class SkillsProvider(ContextProvider):
|
||||
|
||||
lines: list[str] = []
|
||||
# Sort by name for deterministic output
|
||||
for skill in sorted(skills, key=lambda s: s.frontmatter.name):
|
||||
for skill in sorted(skills, key=lambda s: s.name):
|
||||
lines.append(" <skill>")
|
||||
lines.append(f" <name>{xml_escape(skill.frontmatter.name)}</name>")
|
||||
lines.append(f" <description>{xml_escape(skill.frontmatter.description)}</description>")
|
||||
lines.append(f" <name>{xml_escape(skill.name)}</name>")
|
||||
lines.append(f" <description>{xml_escape(skill.description)}</description>")
|
||||
lines.append(" </skill>")
|
||||
|
||||
return template.format(
|
||||
@@ -2008,7 +1920,7 @@ class SkillsProvider(ContextProvider):
|
||||
def _find_skill(skills: Sequence[Skill], name: str) -> Skill | None:
|
||||
"""Find a skill by name (case-insensitive linear scan)."""
|
||||
name_lower = name.lower()
|
||||
return next((s for s in skills if s.frontmatter.name.lower() == name_lower), None)
|
||||
return next((s for s in skills if s.name.lower() == name_lower), None)
|
||||
|
||||
def _load_skill(self, skills: Sequence[Skill], skill_name: str) -> str:
|
||||
"""Return the full content for the named skill.
|
||||
@@ -2267,18 +2179,19 @@ class FileSkillsSource(SkillsSource):
|
||||
if parsed is None:
|
||||
continue
|
||||
|
||||
frontmatter, content = parsed
|
||||
name, description, content = parsed
|
||||
|
||||
if frontmatter.name in skills:
|
||||
if name in skills:
|
||||
logger.warning(
|
||||
"Duplicate skill name '%s': skill from '%s' skipped in favor of existing skill",
|
||||
frontmatter.name,
|
||||
name,
|
||||
skill_path,
|
||||
)
|
||||
continue
|
||||
|
||||
file_skill = FileSkill(
|
||||
frontmatter=frontmatter,
|
||||
name=name,
|
||||
description=description,
|
||||
content=content,
|
||||
path=skill_path,
|
||||
)
|
||||
@@ -2295,8 +2208,8 @@ class FileSkillsSource(SkillsSource):
|
||||
FileSkillScript(name=sn, full_path=script_full_path, runner=self._script_runner)
|
||||
)
|
||||
|
||||
skills[file_skill.frontmatter.name] = file_skill
|
||||
logger.info("Loaded skill: %s", file_skill.frontmatter.name)
|
||||
skills[file_skill.name] = file_skill
|
||||
logger.info("Loaded skill: %s", file_skill.name)
|
||||
|
||||
logger.info("Successfully loaded %d skills", len(skills))
|
||||
return list(skills.values())
|
||||
@@ -2525,9 +2438,8 @@ class FileSkillsSource(SkillsSource):
|
||||
name: str | None,
|
||||
description: str | None,
|
||||
source: str,
|
||||
compatibility: str | None = None,
|
||||
) -> str | None:
|
||||
"""Validate a skill's name, description, and compatibility against naming rules.
|
||||
"""Validate a skill's name and description against naming rules.
|
||||
|
||||
Enforces length limits, character-set restrictions, and non-emptiness
|
||||
for both file-based and code-defined skills.
|
||||
@@ -2537,7 +2449,6 @@ class FileSkillsSource(SkillsSource):
|
||||
description: Skill description to validate.
|
||||
source: Human-readable label for diagnostics (e.g. a file path
|
||||
or ``"code skill"``).
|
||||
compatibility: Optional compatibility value to validate.
|
||||
|
||||
Returns:
|
||||
A diagnostic error string if validation fails, or ``None`` if valid.
|
||||
@@ -2561,32 +2472,24 @@ class FileSkillsSource(SkillsSource):
|
||||
f"Must be {MAX_DESCRIPTION_LENGTH} characters or fewer."
|
||||
)
|
||||
|
||||
if compatibility is not None and len(compatibility) > MAX_COMPATIBILITY_LENGTH:
|
||||
return (
|
||||
f"Skill '{name}' from '{source}' has an invalid compatibility: "
|
||||
f"Must be {MAX_COMPATIBILITY_LENGTH} characters or fewer."
|
||||
)
|
||||
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _extract_frontmatter(
|
||||
content: str,
|
||||
skill_file_path: str,
|
||||
) -> SkillFrontmatter | None:
|
||||
) -> tuple[str, str] | None:
|
||||
"""Extract and validate YAML frontmatter from a SKILL.md file.
|
||||
|
||||
Parses the ``---``-delimited frontmatter block for all
|
||||
`agentskills.io specification <https://agentskills.io/specification>`_
|
||||
fields: ``name``, ``description``, ``license``, ``compatibility``,
|
||||
``allowed-tools``, and ``metadata``.
|
||||
Parses the ``---``-delimited frontmatter block for ``name`` and
|
||||
``description`` fields.
|
||||
|
||||
Args:
|
||||
content: Raw text content of the SKILL.md file.
|
||||
skill_file_path: Path to the file (used in diagnostic messages only).
|
||||
|
||||
Returns:
|
||||
A :class:`SkillFrontmatter` on success, or ``None`` if the
|
||||
A ``(name, description)`` tuple on success, or ``None`` if the
|
||||
frontmatter is missing, malformed, or fails validation.
|
||||
"""
|
||||
match = FRONTMATTER_RE.search(content)
|
||||
@@ -2597,63 +2500,35 @@ class FileSkillsSource(SkillsSource):
|
||||
yaml_content = match.group(1).strip()
|
||||
name: str | None = None
|
||||
description: str | None = None
|
||||
license_value: str | None = None
|
||||
compatibility: str | None = None
|
||||
allowed_tools: str | None = None
|
||||
|
||||
for kv_match in YAML_KV_RE.finditer(yaml_content):
|
||||
key = kv_match.group(1)
|
||||
value = kv_match.group(2) if kv_match.group(2) is not None else kv_match.group(3)
|
||||
|
||||
key_lower = key.lower()
|
||||
if key_lower == "name":
|
||||
if key.lower() == "name":
|
||||
name = value
|
||||
elif key_lower == "description":
|
||||
elif key.lower() == "description":
|
||||
description = value
|
||||
elif key_lower == "license":
|
||||
license_value = value
|
||||
elif key_lower == "compatibility":
|
||||
compatibility = value
|
||||
elif key_lower == "allowed-tools":
|
||||
allowed_tools = value
|
||||
|
||||
# Parse metadata block (indented key-value pairs under "metadata:").
|
||||
metadata: dict[str, str] | None = None
|
||||
metadata_match = YAML_METADATA_BLOCK_RE.search(yaml_content)
|
||||
if metadata_match:
|
||||
metadata = {}
|
||||
for kv_match in YAML_INDENTED_KV_RE.finditer(metadata_match.group(1)):
|
||||
mk = kv_match.group(1)
|
||||
mv = kv_match.group(2) if kv_match.group(2) is not None else kv_match.group(3)
|
||||
metadata[mk] = mv
|
||||
|
||||
error = FileSkillsSource._validate_skill_metadata(name, description, skill_file_path, compatibility)
|
||||
error = FileSkillsSource._validate_skill_metadata(name, description, skill_file_path)
|
||||
if error:
|
||||
logger.error(error)
|
||||
return None
|
||||
|
||||
# name and description are guaranteed non-None after validation;
|
||||
# SkillFrontmatter re-validates as a defense-in-depth invariant.
|
||||
return SkillFrontmatter(
|
||||
name=cast(str, name),
|
||||
description=cast(str, description),
|
||||
license=license_value,
|
||||
compatibility=compatibility,
|
||||
allowed_tools=allowed_tools,
|
||||
metadata=metadata,
|
||||
)
|
||||
# name and description are guaranteed non-None after validation
|
||||
return name, description # type: ignore[return-value]
|
||||
|
||||
@staticmethod
|
||||
def _read_and_parse_skill_file(
|
||||
skill_dir_path: str,
|
||||
) -> tuple[SkillFrontmatter, str] | None:
|
||||
) -> tuple[str, str, str] | None:
|
||||
"""Read and parse the SKILL.md file in *skill_dir_path*.
|
||||
|
||||
Args:
|
||||
skill_dir_path: Absolute path to the directory containing ``SKILL.md``.
|
||||
|
||||
Returns:
|
||||
A ``(frontmatter, content)`` tuple where *content* is the
|
||||
A ``(name, description, content)`` tuple where *content* is the
|
||||
full raw file text, or ``None`` if the file cannot be read or
|
||||
its frontmatter is invalid.
|
||||
"""
|
||||
@@ -2665,21 +2540,23 @@ class FileSkillsSource(SkillsSource):
|
||||
logger.error("Failed to read SKILL.md at '%s'", skill_file)
|
||||
return None
|
||||
|
||||
frontmatter = FileSkillsSource._extract_frontmatter(content, str(skill_file))
|
||||
if frontmatter is None:
|
||||
result = FileSkillsSource._extract_frontmatter(content, str(skill_file))
|
||||
if result is None:
|
||||
return None
|
||||
|
||||
name, description = result
|
||||
|
||||
dir_name = Path(skill_dir_path).name
|
||||
if frontmatter.name != dir_name:
|
||||
if name != dir_name:
|
||||
logger.error(
|
||||
"SKILL.md at '%s' has frontmatter name '%s' that does not match the directory name '%s'; skipping.",
|
||||
skill_file,
|
||||
frontmatter.name,
|
||||
name,
|
||||
dir_name,
|
||||
)
|
||||
return None
|
||||
|
||||
return frontmatter, content
|
||||
return name, description, content
|
||||
|
||||
@staticmethod
|
||||
def _discover_skill_directories(skill_paths: Sequence[str]) -> list[str]:
|
||||
@@ -2827,12 +2704,12 @@ class DeduplicatingSkillsSource(DelegatingSkillsSource):
|
||||
result: list[Skill] = []
|
||||
|
||||
for skill in skills:
|
||||
key = skill.frontmatter.name.lower()
|
||||
key = skill.name.lower()
|
||||
if key in seen:
|
||||
logger.warning(
|
||||
"Duplicate skill name '%s': skill skipped in favor of existing skill '%s'",
|
||||
skill.frontmatter.name,
|
||||
seen[key].frontmatter.name,
|
||||
skill.name,
|
||||
seen[key].name,
|
||||
)
|
||||
continue
|
||||
seen[key] = skill
|
||||
@@ -2853,7 +2730,7 @@ class FilteringSkillsSource(DelegatingSkillsSource):
|
||||
|
||||
filtered = FilteringSkillsSource(
|
||||
inner_source=my_source,
|
||||
predicate=lambda s: s.frontmatter.name != "internal",
|
||||
predicate=lambda s: s.name != "internal",
|
||||
)
|
||||
skills = await filtered.get_skills()
|
||||
"""
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
# type: ignore[reportPrivateUsage]
|
||||
import asyncio
|
||||
import contextlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
@@ -1616,7 +1615,7 @@ async def test_mcp_connection_reset_integration():
|
||||
|
||||
async def test_mcp_tool_message_handler_notification():
|
||||
"""Test that message_handler correctly processes tools/list_changed and prompts/list_changed
|
||||
notifications by scheduling reloads as background tasks."""
|
||||
notifications."""
|
||||
tool = MCPStdioTool(name="test_tool", command="python")
|
||||
|
||||
# Mock the load_tools and load_prompts methods
|
||||
@@ -1630,8 +1629,6 @@ async def test_mcp_tool_message_handler_notification():
|
||||
|
||||
result = await tool.message_handler(tools_notification)
|
||||
assert result is None
|
||||
# The reload is scheduled as a background task; let it run.
|
||||
await asyncio.sleep(0)
|
||||
tool.load_tools.assert_called_once()
|
||||
|
||||
# Reset mock
|
||||
@@ -1644,7 +1641,6 @@ async def test_mcp_tool_message_handler_notification():
|
||||
|
||||
result = await tool.message_handler(prompts_notification)
|
||||
assert result is None
|
||||
await asyncio.sleep(0)
|
||||
tool.load_prompts.assert_called_once()
|
||||
|
||||
# Test unhandled notification
|
||||
@@ -1668,112 +1664,6 @@ async def test_mcp_tool_message_handler_error():
|
||||
assert result is None
|
||||
|
||||
|
||||
async def test_mcp_tool_message_handler_does_not_block_receive_loop():
|
||||
"""Test that message_handler does not deadlock the MCP receive loop.
|
||||
|
||||
Regression test for https://github.com/microsoft/agent-framework/issues/4828.
|
||||
When the MCP server sends a ``notifications/tools/list_changed``
|
||||
notification, the handler must NOT await ``load_tools()`` synchronously
|
||||
because that would block the single-threaded MCP receive loop, preventing
|
||||
it from delivering the ``list_tools`` response — a classic deadlock.
|
||||
"""
|
||||
tool = MCPStdioTool(name="test_tool", command="python")
|
||||
|
||||
# Use an event to make load_tools block until we release it.
|
||||
# This simulates load_tools waiting for a session response that the
|
||||
# receive loop would need to deliver.
|
||||
release = asyncio.Event()
|
||||
|
||||
async def slow_load_tools():
|
||||
await release.wait()
|
||||
|
||||
tool.load_tools = slow_load_tools # type: ignore[assignment]
|
||||
|
||||
tools_notification = Mock(spec=types.ServerNotification)
|
||||
tools_notification.root = Mock()
|
||||
tools_notification.root.method = "notifications/tools/list_changed"
|
||||
|
||||
# message_handler must return immediately even though load_tools blocks.
|
||||
await tool.message_handler(tools_notification)
|
||||
|
||||
# If the handler had awaited load_tools synchronously, we would never
|
||||
# reach this line (deadlock). Verify the reload task is pending.
|
||||
assert len(tool._pending_reload_tasks) == 1
|
||||
|
||||
# Unblock the reload so the background task finishes cleanly.
|
||||
release.set()
|
||||
# Wait for the pending reload task(s) to complete so their done-callbacks
|
||||
# have a chance to remove them from _pending_reload_tasks.
|
||||
await asyncio.wait_for(asyncio.gather(*tool._pending_reload_tasks), timeout=1)
|
||||
assert len(tool._pending_reload_tasks) == 0
|
||||
|
||||
|
||||
async def test_mcp_tool_message_handler_reload_failure_is_logged(caplog: pytest.LogCaptureFixture):
|
||||
"""Background reload errors are logged, not raised into the receive loop."""
|
||||
tool = MCPStdioTool(name="test_tool", command="python")
|
||||
tool.load_tools = AsyncMock(side_effect=RuntimeError("connection lost"))
|
||||
|
||||
tools_notification = Mock(spec=types.ServerNotification)
|
||||
tools_notification.root = Mock()
|
||||
tools_notification.root.method = "notifications/tools/list_changed"
|
||||
|
||||
await tool.message_handler(tools_notification)
|
||||
# Let the background task run — it should not propagate the exception.
|
||||
# Snapshot tasks and await them to ensure done-callbacks fire.
|
||||
pending = list(tool._pending_reload_tasks)
|
||||
if pending:
|
||||
await asyncio.wait_for(asyncio.gather(*pending, return_exceptions=True), timeout=1)
|
||||
tool.load_tools.assert_called_once()
|
||||
assert len(tool._pending_reload_tasks) == 0
|
||||
|
||||
# Verify the warning was actually logged with exception info.
|
||||
reload_warnings = [r for r in caplog.records if "Background MCP reload failed" in r.message]
|
||||
assert len(reload_warnings) == 1
|
||||
assert reload_warnings[0].levelname == "WARNING"
|
||||
assert reload_warnings[0].exc_info is not None
|
||||
|
||||
|
||||
async def test_mcp_tool_message_handler_cancel_and_replace():
|
||||
"""Sending two notifications in quick succession cancels the first reload task."""
|
||||
tool = MCPStdioTool(name="test_tool", command="python")
|
||||
|
||||
release = asyncio.Event()
|
||||
call_count = 0
|
||||
|
||||
async def blocking_load_tools():
|
||||
nonlocal call_count
|
||||
call_count += 1
|
||||
await release.wait()
|
||||
|
||||
tool.load_tools = blocking_load_tools # type: ignore[assignment]
|
||||
|
||||
notification = Mock(spec=types.ServerNotification)
|
||||
notification.root = Mock()
|
||||
notification.root.method = "notifications/tools/list_changed"
|
||||
|
||||
# First notification — starts a blocking reload task.
|
||||
await tool.message_handler(notification)
|
||||
assert len(tool._pending_reload_tasks) == 1
|
||||
first_task = next(iter(tool._pending_reload_tasks))
|
||||
|
||||
# Second notification — should cancel the first and replace it.
|
||||
await tool.message_handler(notification)
|
||||
# Yield to the event loop so the cancellation is processed.
|
||||
with contextlib.suppress(asyncio.CancelledError):
|
||||
await first_task
|
||||
|
||||
assert first_task.cancelled()
|
||||
|
||||
assert len(tool._pending_reload_tasks) == 1
|
||||
second_task = next(iter(tool._pending_reload_tasks))
|
||||
assert second_task is not first_task
|
||||
|
||||
# Unblock and let the second task finish.
|
||||
release.set()
|
||||
await asyncio.wait_for(asyncio.gather(*tool._pending_reload_tasks), timeout=1)
|
||||
assert len(tool._pending_reload_tasks) == 0
|
||||
|
||||
|
||||
async def test_mcp_tool_sampling_callback_no_client():
|
||||
"""Test sampling callback error path when no chat client is available."""
|
||||
tool = MCPStdioTool(name="test_tool", command="python")
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -11,7 +11,7 @@ import os
|
||||
from textwrap import dedent
|
||||
from typing import Any
|
||||
|
||||
from agent_framework import Agent, InlineSkill, InlineSkillResource, SkillFrontmatter, SkillsProvider
|
||||
from agent_framework import Agent, InlineSkill, InlineSkillResource, SkillsProvider
|
||||
from agent_framework.foundry import FoundryChatClient
|
||||
from azure.identity import AzureCliCredential
|
||||
from dotenv import load_dotenv
|
||||
@@ -47,9 +47,8 @@ load_dotenv()
|
||||
# 1. Static Resources — inline content passed at construction time
|
||||
# ---------------------------------------------------------------------------
|
||||
unit_converter_skill = InlineSkill(
|
||||
frontmatter=SkillFrontmatter(
|
||||
name="unit-converter", description="Convert between common units using a conversion factor"
|
||||
),
|
||||
name="unit-converter",
|
||||
description="Convert between common units using a conversion factor",
|
||||
instructions=dedent("""\
|
||||
Use this skill when the user asks to convert between units.
|
||||
|
||||
|
||||
@@ -1,12 +1,6 @@
|
||||
---
|
||||
name: unit-converter
|
||||
description: Convert between common units using a multiplication factor. Use when asked to convert miles, kilometers, pounds, or kilograms.
|
||||
license: MIT
|
||||
compatibility: Works with any model that supports tool use.
|
||||
allowed-tools: convert
|
||||
metadata:
|
||||
author: agent-framework-samples
|
||||
version: "1.0"
|
||||
---
|
||||
|
||||
## Usage
|
||||
|
||||
@@ -21,7 +21,6 @@ from agent_framework import (
|
||||
FileSkillsSource,
|
||||
InlineSkill,
|
||||
InMemorySkillsSource,
|
||||
SkillFrontmatter,
|
||||
SkillsProvider,
|
||||
)
|
||||
from agent_framework.foundry import FoundryChatClient
|
||||
@@ -74,9 +73,8 @@ load_dotenv()
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
volume_converter_skill = InlineSkill(
|
||||
frontmatter=SkillFrontmatter(
|
||||
name="volume-converter", description="Convert between gallons and liters using a conversion factor"
|
||||
),
|
||||
name="volume-converter",
|
||||
description="Convert between gallons and liters using a conversion factor",
|
||||
instructions=dedent("""\
|
||||
Use this skill when the user asks to convert between gallons and liters.
|
||||
|
||||
@@ -120,7 +118,6 @@ def convert_volume(value: float, factor: float) -> str:
|
||||
# 2. Define a class-based skill for temperature conversion
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TemperatureConverterSkill(ClassSkill):
|
||||
"""A temperature-converter skill defined as a Python class.
|
||||
|
||||
@@ -130,10 +127,8 @@ class TemperatureConverterSkill(ClassSkill):
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__(
|
||||
frontmatter=SkillFrontmatter(
|
||||
name="temperature-converter",
|
||||
description="Convert between temperature scales (Fahrenheit, Celsius, Kelvin).",
|
||||
)
|
||||
name="temperature-converter",
|
||||
description="Convert between temperature scales (Fahrenheit, Celsius, Kelvin).",
|
||||
)
|
||||
|
||||
@property
|
||||
@@ -183,7 +178,6 @@ class TemperatureConverterSkill(ClassSkill):
|
||||
# 3. Wire everything together and run the agent
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
"""Run the combined skills demo."""
|
||||
endpoint = os.environ["FOUNDRY_PROJECT_ENDPOINT"]
|
||||
|
||||
@@ -1,12 +1,6 @@
|
||||
---
|
||||
name: unit-converter
|
||||
description: Convert between common units using a multiplication factor. Use when asked to convert miles, kilometers, pounds, or kilograms.
|
||||
license: MIT
|
||||
compatibility: Works with any model that supports tool use.
|
||||
allowed-tools: convert
|
||||
metadata:
|
||||
author: agent-framework-samples
|
||||
version: "1.0"
|
||||
---
|
||||
|
||||
## Usage
|
||||
|
||||
@@ -9,7 +9,7 @@ import os
|
||||
# warnings.filterwarnings("ignore", message=r"\[SKILLS\].*", category=FutureWarning)
|
||||
from textwrap import dedent
|
||||
|
||||
from agent_framework import Agent, InlineSkill, SkillFrontmatter, SkillsProvider
|
||||
from agent_framework import Agent, InlineSkill, SkillsProvider
|
||||
from agent_framework.foundry import FoundryChatClient
|
||||
from azure.identity import AzureCliCredential
|
||||
from dotenv import load_dotenv
|
||||
@@ -43,9 +43,8 @@ load_dotenv()
|
||||
|
||||
# Define a code skill with a script that performs a sensitive operation
|
||||
deployment_skill = InlineSkill(
|
||||
frontmatter=SkillFrontmatter(
|
||||
name="deployment", description="Tools for deploying application versions to production"
|
||||
),
|
||||
name="deployment",
|
||||
description="Tools for deploying application versions to production",
|
||||
instructions=dedent("""\
|
||||
Use this skill when the user asks to deploy an application.
|
||||
|
||||
|
||||
@@ -75,7 +75,7 @@ async def main() -> None:
|
||||
FilteringSkillsSource(
|
||||
FileSkillsSource(str(skills_dir), script_runner=subprocess_script_runner),
|
||||
# Only keep the volume-converter skill
|
||||
predicate=lambda s: s.frontmatter.name != "length-converter",
|
||||
predicate=lambda s: s.name != "length-converter",
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@@ -1,12 +1,6 @@
|
||||
---
|
||||
name: length-converter
|
||||
description: Convert between common length units (miles, km, feet, meters) using a multiplication factor.
|
||||
license: MIT
|
||||
compatibility: Works with any model that supports tool use.
|
||||
allowed-tools: convert
|
||||
metadata:
|
||||
author: agent-framework-samples
|
||||
version: "1.0"
|
||||
---
|
||||
|
||||
## Usage
|
||||
|
||||
@@ -1,12 +1,6 @@
|
||||
---
|
||||
name: volume-converter
|
||||
description: Convert between gallons and liters using a conversion factor.
|
||||
license: MIT
|
||||
compatibility: Works with any model that supports tool use.
|
||||
allowed-tools: convert
|
||||
metadata:
|
||||
author: agent-framework-samples
|
||||
version: "1.0"
|
||||
---
|
||||
|
||||
## Usage
|
||||
|
||||
Generated
+1
-1
@@ -602,7 +602,7 @@ dependencies = [
|
||||
[package.metadata]
|
||||
requires-dist = [
|
||||
{ name = "agent-framework-core", editable = "packages/core" },
|
||||
{ name = "github-copilot-sdk", marker = "python_full_version >= '3.11'", specifier = ">=1.0.0b2,<=1.0.0b2" },
|
||||
{ name = "github-copilot-sdk", marker = "python_full_version >= '3.11'", specifier = "<=1.0.0b2,>=1.0.0b2" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
||||
Reference in New Issue
Block a user