mirror of
https://github.com/microsoft/agent-framework.git
synced 2026-06-16 21:04:09 +08:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
cb5cbbc17b | ||
|
|
f38c17a406 |
@@ -8,7 +8,6 @@ function getPullRequest(context) {
|
||||
|
||||
return {
|
||||
author: pullRequest.user.login,
|
||||
authorType: pullRequest.user.type,
|
||||
labels: pullRequest.labels?.map((label) => label.name).filter(Boolean) ?? [],
|
||||
number: pullRequest.number,
|
||||
};
|
||||
@@ -50,10 +49,6 @@ function hasLabel(labels, labelName) {
|
||||
return labels.some((label) => label.toLowerCase() === labelName.toLowerCase());
|
||||
}
|
||||
|
||||
function isDependabotAuthor({ author, authorType }) {
|
||||
return authorType === 'Bot' && author.toLowerCase() === 'dependabot[bot]';
|
||||
}
|
||||
|
||||
function buildLimitMessage({ author, exemptLabelName, maxOpenPrs, openPrCount }) {
|
||||
return [
|
||||
`Thank you for your contribution, @${author}.`,
|
||||
@@ -68,37 +63,24 @@ function buildLimitMessage({ author, exemptLabelName, maxOpenPrs, openPrCount })
|
||||
}
|
||||
|
||||
async function getOpenPrCount({ github, owner, repo, author, pullRequestNumber }) {
|
||||
const openPullRequests = await github.paginate(github.rest.pulls.list, {
|
||||
owner,
|
||||
repo,
|
||||
state: 'open',
|
||||
const query = `repo:${owner}/${repo} is:pr is:open author:${author}`;
|
||||
const response = await github.rest.search.issuesAndPullRequests({
|
||||
q: query,
|
||||
per_page: 100,
|
||||
});
|
||||
|
||||
const authorOpenPullRequestNumbers = openPullRequests
|
||||
.filter((pullRequest) => pullRequest.user?.login === author)
|
||||
.map((pullRequest) => pullRequest.number);
|
||||
const currentPrIsOpen = authorOpenPullRequestNumbers.includes(pullRequestNumber);
|
||||
const existingOpenPrCount = currentPrIsOpen
|
||||
? authorOpenPullRequestNumbers.length - 1
|
||||
: authorOpenPullRequestNumbers.length;
|
||||
const indexedPrNumbers = response.data.items.map((item) => item.number);
|
||||
const currentPrIsIndexed = indexedPrNumbers.includes(pullRequestNumber);
|
||||
if (currentPrIsIndexed || response.data.total_count >= 100) {
|
||||
return response.data.total_count;
|
||||
}
|
||||
|
||||
return existingOpenPrCount + 1;
|
||||
return response.data.total_count + 1;
|
||||
}
|
||||
|
||||
async function enforcePrLimit({ github, context, core, exemptLabelName, maxOpenPrs, labelName }) {
|
||||
const { owner, repo } = context.repo;
|
||||
const { author, authorType, labels, number } = getPullRequest(context);
|
||||
|
||||
if (isDependabotAuthor({ author, authorType })) {
|
||||
core.info(`Author ${author} is Dependabot; skipping open PR limit enforcement.`);
|
||||
return {
|
||||
author,
|
||||
closed: false,
|
||||
dependabotExempt: true,
|
||||
openPrCount: null,
|
||||
};
|
||||
}
|
||||
const { author, labels, number } = getPullRequest(context);
|
||||
|
||||
if (hasLabel(labels, exemptLabelName)) {
|
||||
core.info(`PR #${number} has the ${exemptLabelName} label; skipping open PR limit enforcement.`);
|
||||
|
||||
@@ -16,7 +16,7 @@ const { enforcePrLimit } = require('../scripts/pr_limit_moderation.js');
|
||||
// Helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function createContext({ author = 'community-user', authorType = 'User', labels = [], number = 123 } = {}) {
|
||||
function createContext({ author = 'community-user', labels = [], number = 123 } = {}) {
|
||||
return {
|
||||
repo: {
|
||||
owner: 'microsoft',
|
||||
@@ -28,7 +28,6 @@ function createContext({ author = 'community-user', authorType = 'User', labels
|
||||
labels: labels.map((name) => ({ name })),
|
||||
user: {
|
||||
login: author,
|
||||
type: authorType,
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -45,20 +44,23 @@ function createCore() {
|
||||
};
|
||||
}
|
||||
|
||||
function createGithub({
|
||||
itemNumbers,
|
||||
labelExists = true,
|
||||
pullRequests = createPullRequestPage({ numbers: itemNumbers }),
|
||||
}) {
|
||||
function createGithub({ totalCount, itemNumbers, labelExists = true }) {
|
||||
const calls = [];
|
||||
|
||||
return {
|
||||
calls,
|
||||
async paginate(method, params) {
|
||||
calls.push({ api: 'paginate', method, params });
|
||||
return pullRequests;
|
||||
},
|
||||
rest: {
|
||||
search: {
|
||||
async issuesAndPullRequests(params) {
|
||||
calls.push({ api: 'search.issuesAndPullRequests', params });
|
||||
return {
|
||||
data: {
|
||||
total_count: totalCount,
|
||||
items: itemNumbers.map((number) => ({ number })),
|
||||
},
|
||||
};
|
||||
},
|
||||
},
|
||||
issues: {
|
||||
async getLabel(params) {
|
||||
calls.push({ api: 'issues.getLabel', params });
|
||||
@@ -83,10 +85,6 @@ function createGithub({
|
||||
},
|
||||
},
|
||||
pulls: {
|
||||
async list(params) {
|
||||
calls.push({ api: 'pulls.list', params });
|
||||
return { data: pullRequests };
|
||||
},
|
||||
async update(params) {
|
||||
calls.push({ api: 'pulls.update', params });
|
||||
return { data: { state: params.state } };
|
||||
@@ -96,15 +94,6 @@ function createGithub({
|
||||
};
|
||||
}
|
||||
|
||||
function createPullRequestPage({ author = 'community-user', numbers }) {
|
||||
return numbers.map((number) => ({
|
||||
number,
|
||||
user: {
|
||||
login: author,
|
||||
},
|
||||
}));
|
||||
}
|
||||
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// PR limit enforcement
|
||||
@@ -113,6 +102,7 @@ function createPullRequestPage({ author = 'community-user', numbers }) {
|
||||
describe('PR limit enforcement', () => {
|
||||
it('does not close the PR when the author is at the open PR limit', async () => {
|
||||
const github = createGithub({
|
||||
totalCount: 10,
|
||||
itemNumbers: [1, 2, 3, 4, 5, 6, 7, 8, 9, 123],
|
||||
});
|
||||
|
||||
@@ -129,13 +119,14 @@ describe('PR limit enforcement', () => {
|
||||
assert.equal(result.openPrCount, 10);
|
||||
assert.deepEqual(
|
||||
github.calls.map((call) => call.api),
|
||||
['paginate'],
|
||||
['search.issuesAndPullRequests'],
|
||||
);
|
||||
});
|
||||
|
||||
it('counts the new PR when the pull list includes it', async () => {
|
||||
it('counts the new PR when search has not indexed it yet', async () => {
|
||||
const github = createGithub({
|
||||
itemNumbers: [123, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10],
|
||||
totalCount: 10,
|
||||
itemNumbers: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10],
|
||||
});
|
||||
|
||||
const result = await enforcePrLimit({
|
||||
@@ -152,7 +143,7 @@ describe('PR limit enforcement', () => {
|
||||
assert.deepEqual(
|
||||
github.calls.map((call) => call.api),
|
||||
[
|
||||
'paginate',
|
||||
'search.issuesAndPullRequests',
|
||||
'issues.getLabel',
|
||||
'issues.addLabels',
|
||||
'issues.createComment',
|
||||
@@ -161,31 +152,9 @@ describe('PR limit enforcement', () => {
|
||||
);
|
||||
});
|
||||
|
||||
it('counts the current PR on top of existing open PRs', async () => {
|
||||
const github = createGithub({
|
||||
itemNumbers: [123, ...Array.from({ length: 24 }, (_, index) => index + 1)],
|
||||
pullRequests: createPullRequestPage({
|
||||
numbers: [123, ...Array.from({ length: 25 }, (_, index) => index + 1)],
|
||||
}),
|
||||
});
|
||||
|
||||
const result = await enforcePrLimit({
|
||||
github,
|
||||
context: createContext(),
|
||||
core: createCore(),
|
||||
exemptLabelName: 'pr-limit-exempt',
|
||||
maxOpenPrs: 10,
|
||||
labelName: 'too-many-prs',
|
||||
});
|
||||
|
||||
assert.equal(result.closed, true);
|
||||
assert.equal(result.openPrCount, 26);
|
||||
const comment = github.calls.find((call) => call.api === 'issues.createComment').params.body;
|
||||
assert.match(comment, /This PR would put you at 26 open pull requests/);
|
||||
});
|
||||
|
||||
it('creates the label when it does not already exist', async () => {
|
||||
const github = createGithub({
|
||||
totalCount: 11,
|
||||
itemNumbers: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 123],
|
||||
labelExists: false,
|
||||
});
|
||||
@@ -203,7 +172,7 @@ describe('PR limit enforcement', () => {
|
||||
assert.deepEqual(
|
||||
github.calls.map((call) => call.api),
|
||||
[
|
||||
'paginate',
|
||||
'search.issuesAndPullRequests',
|
||||
'issues.getLabel',
|
||||
'issues.createLabel',
|
||||
'issues.addLabels',
|
||||
@@ -219,6 +188,7 @@ describe('PR limit enforcement', () => {
|
||||
|
||||
it('tolerates a 422 race when creating the label', async () => {
|
||||
const github = createGithub({
|
||||
totalCount: 11,
|
||||
itemNumbers: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 123],
|
||||
labelExists: false,
|
||||
});
|
||||
@@ -242,7 +212,7 @@ describe('PR limit enforcement', () => {
|
||||
assert.deepEqual(
|
||||
github.calls.map((call) => call.api),
|
||||
[
|
||||
'paginate',
|
||||
'search.issuesAndPullRequests',
|
||||
'issues.getLabel',
|
||||
'issues.createLabel',
|
||||
'issues.addLabels',
|
||||
@@ -254,11 +224,8 @@ describe('PR limit enforcement', () => {
|
||||
|
||||
it('uses a diplomatic close message with the configured limit', async () => {
|
||||
const github = createGithub({
|
||||
totalCount: 11,
|
||||
itemNumbers: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 123],
|
||||
pullRequests: createPullRequestPage({
|
||||
author: 'octo-contributor',
|
||||
numbers: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 123],
|
||||
}),
|
||||
});
|
||||
|
||||
await enforcePrLimit({
|
||||
@@ -279,6 +246,7 @@ describe('PR limit enforcement', () => {
|
||||
|
||||
it('does not close an exempt PR when it is reopened', async () => {
|
||||
const github = createGithub({
|
||||
totalCount: 11,
|
||||
itemNumbers: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 123],
|
||||
});
|
||||
|
||||
@@ -297,33 +265,10 @@ describe('PR limit enforcement', () => {
|
||||
assert.deepEqual(github.calls, []);
|
||||
});
|
||||
|
||||
it('does not close Dependabot PRs', async () => {
|
||||
it('does not over-count when the current PR is not on the first search page', async () => {
|
||||
const github = createGithub({
|
||||
itemNumbers: [123, ...Array.from({ length: 25 }, (_, index) => index + 1)],
|
||||
pullRequests: createPullRequestPage({
|
||||
author: 'dependabot[bot]',
|
||||
numbers: [123, ...Array.from({ length: 25 }, (_, index) => index + 1)],
|
||||
}),
|
||||
});
|
||||
|
||||
const result = await enforcePrLimit({
|
||||
github,
|
||||
context: createContext({ author: 'dependabot[bot]', authorType: 'Bot' }),
|
||||
core: createCore(),
|
||||
exemptLabelName: 'pr-limit-exempt',
|
||||
maxOpenPrs: 10,
|
||||
labelName: 'too-many-prs',
|
||||
});
|
||||
|
||||
assert.equal(result.closed, false);
|
||||
assert.equal(result.dependabotExempt, true);
|
||||
assert.equal(result.openPrCount, null);
|
||||
assert.deepEqual(github.calls, []);
|
||||
});
|
||||
|
||||
it('counts the current PR when the author has more than one page of open PRs', async () => {
|
||||
const github = createGithub({
|
||||
itemNumbers: [123, ...Array.from({ length: 100 }, (_, index) => index + 1)],
|
||||
totalCount: 101,
|
||||
itemNumbers: Array.from({ length: 100 }, (_, index) => index + 1),
|
||||
});
|
||||
|
||||
const result = await enforcePrLimit({
|
||||
|
||||
@@ -8,7 +8,6 @@ on:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
pull-requests: write
|
||||
|
||||
jobs:
|
||||
@@ -24,7 +23,7 @@ jobs:
|
||||
- name: Download coverage report
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
|
||||
with:
|
||||
github-token: ${{ github.token }}
|
||||
github-token: ${{ secrets.GH_ACTIONS_PR_WRITE }}
|
||||
run-id: ${{ github.event.workflow_run.id }}
|
||||
path: ./python
|
||||
merge-multiple: true
|
||||
@@ -39,9 +38,9 @@ jobs:
|
||||
echo "PR number file 'pr_number' is missing or empty"
|
||||
exit 1
|
||||
fi
|
||||
PR_NUMBER=$(cat pr_number)
|
||||
if ! [[ "$PR_NUMBER" =~ ^[0-9]+$ ]]; then
|
||||
echo "::error::PR number file contains invalid content"
|
||||
PR_NUMBER=$(head -1 pr_number | tr -dc '0-9')
|
||||
if [ -z "$PR_NUMBER" ]; then
|
||||
echo "PR number file 'pr_number' does not contain a valid PR number"
|
||||
exit 1
|
||||
fi
|
||||
echo "PR_NUMBER=$PR_NUMBER" >> "$GITHUB_ENV"
|
||||
@@ -49,7 +48,7 @@ jobs:
|
||||
id: coverageComment
|
||||
uses: MishaKav/pytest-coverage-comment@26f986d2599c288bb62f623d29c2da98609e9cd4 # v1.6.0
|
||||
with:
|
||||
github-token: ${{ github.token }}
|
||||
github-token: ${{ secrets.GH_ACTIONS_PR_WRITE }}
|
||||
issue-number: ${{ env.PR_NUMBER }}
|
||||
pytest-xml-coverage-path: python/python-coverage.xml
|
||||
title: "Python Test Coverage Report"
|
||||
|
||||
@@ -248,4 +248,3 @@ dotnet/filtered-*.slnx
|
||||
.omx/
|
||||
|
||||
**/issues/
|
||||
.test_*
|
||||
|
||||
+17
-17
@@ -1,17 +1,17 @@
|
||||
# Support
|
||||
|
||||
## How to file issues and get help
|
||||
|
||||
This project uses GitHub Issues to track bugs and feature requests. Please search the existing
|
||||
issues before filing new issues to avoid duplicates. For new issues, file your bug or
|
||||
feature request as a new Issue.
|
||||
|
||||
For help and questions about using this project, please create a GitHub issue.
|
||||
|
||||
AI Support team will support Microsoft Agent Framework issues for customers under a **Unified support agreement when the issue arises from usage of Azure AI services** (Foundry Models, Foundry Agents etc.) in conjunction with the SDK. Conversely, if customer has any other / non unified support agreement and/or Agent Framework SDK is used in a way **not involving an Azure service**, it is treated as a purely open-source tool – Microsoft’s support organization will not handle it, and users should use GitHub or forums for assistance
|
||||
|
||||
For Copilot Studio SDK implementation issues, customers should use GitHub Issues for assistance, as outlined above. Conversely, for prerequisites managed within the Copilot Studio portal, customers can rely on the standard Microsoft Copilot Studio support channels.
|
||||
|
||||
## Microsoft Support Policy
|
||||
|
||||
Support for this **PROJECT or PRODUCT** is limited to the resources listed above.
|
||||
# Support
|
||||
|
||||
## How to file issues and get help
|
||||
|
||||
This project uses GitHub Issues to track bugs and feature requests. Please search the existing
|
||||
issues before filing new issues to avoid duplicates. For new issues, file your bug or
|
||||
feature request as a new Issue.
|
||||
|
||||
For help and questions about using this project, please create a GitHub issue.
|
||||
|
||||
AI Support team will support Microsoft Agent Framework issues for customers under a **Unified support agreement when the issue arises from usage of Azure AI services** (Foundry Models, Foundry Agents etc.) in conjunction with the SDK. Conversely, if customer has any other / non unified support agreement and/or Agent Framework SDK is used in a way **not involving an Azure service**, it is treated as a purely open-source tool – Microsoft’s support organization will not handle it, and users should use GitHub or forums for assistance
|
||||
|
||||
For Copilot Studio SDK implementation issues, customers should use GitHub Issues for assistance, as outlined above. Conversely, for prerequisites managed within the Copilot Studio portal, customers can rely on the standard Microsoft Copilot Studio support channels.
|
||||
|
||||
## Microsoft Support Policy
|
||||
|
||||
Support for this **PROJECT or PRODUCT** is limited to the resources listed above.
|
||||
|
||||
@@ -344,9 +344,6 @@
|
||||
<Folder Name="/Samples/04-hosting/FoundryHostedAgents/responses/Hosted-Toolbox/">
|
||||
<Project Path="samples/04-hosting/FoundryHostedAgents/responses/Hosted-Toolbox/HostedToolbox.csproj" />
|
||||
</Folder>
|
||||
<Folder Name="/Samples/04-hosting/FoundryHostedAgents/responses/Hosted-ToolboxMcpSkills/">
|
||||
<Project Path="samples/04-hosting/FoundryHostedAgents/responses/Hosted-ToolboxMcpSkills/HostedToolboxMcpSkills.csproj" />
|
||||
</Folder>
|
||||
<Folder Name="/Samples/04-hosting/FoundryHostedAgents/responses/Hosted-AzureSearchRag/">
|
||||
<Project Path="samples/04-hosting/FoundryHostedAgents/responses/Hosted-AzureSearchRag/HostedAzureSearchRag.csproj" />
|
||||
</Folder>
|
||||
|
||||
@@ -1,14 +1,14 @@
|
||||
<Project>
|
||||
<PropertyGroup>
|
||||
<!-- Central version prefix - applies to all nuget packages. -->
|
||||
<VersionPrefix>1.9.0</VersionPrefix>
|
||||
<VersionPrefix>1.8.0</VersionPrefix>
|
||||
<RCNumber>1</RCNumber>
|
||||
<DateSuffix>260603</DateSuffix>
|
||||
<DateSuffix>260528</DateSuffix>
|
||||
<PackageVersion Condition="'$(IsReleaseCandidate)' == 'true'">$(VersionPrefix)-rc$(RCNumber)</PackageVersion>
|
||||
<PackageVersion Condition="'$(IsReleaseCandidate)' != 'true' AND '$(VersionSuffix)' != ''">$(VersionPrefix)-$(VersionSuffix).$(DateSuffix).1</PackageVersion>
|
||||
<PackageVersion Condition="'$(IsReleaseCandidate)' != 'true' AND '$(VersionSuffix)' == ''">$(VersionPrefix)-preview.$(DateSuffix).1</PackageVersion>
|
||||
<PackageVersion Condition="'$(IsReleased)' == 'true'">$(VersionPrefix)</PackageVersion>
|
||||
<GitTag>1.9.0</GitTag>
|
||||
<GitTag>1.8.0</GitTag>
|
||||
|
||||
<Configurations>Debug;Release;Publish</Configurations>
|
||||
<IsPackable>true</IsPackable>
|
||||
|
||||
@@ -10,11 +10,6 @@ WebApplicationBuilder builder = WebApplication.CreateBuilder(args);
|
||||
builder.Services.AddHttpClient().AddLogging();
|
||||
builder.Services.AddAGUI();
|
||||
|
||||
// WARNING: When adding session persistence (e.g., WithInMemorySessionStore), or running in production,
|
||||
// make sure to also register a SessionIsolationKeyProvider to scope sessions by principal in multi-user
|
||||
// deployments, e.g.:
|
||||
// builder.Services.UseClaimsBasedSessionIsolation(new() { ClaimType = ClaimTypes.NameIdentifier });
|
||||
|
||||
WebApplication app = builder.Build();
|
||||
|
||||
string endpoint = builder.Configuration["AZURE_OPENAI_ENDPOINT"]
|
||||
|
||||
@@ -14,7 +14,6 @@
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\..\..\..\src\Microsoft.Agents.AI.Hosting.AGUI.AspNetCore\Microsoft.Agents.AI.Hosting.AGUI.AspNetCore.csproj" />
|
||||
<ProjectReference Include="..\..\..\..\..\src\Microsoft.Agents.AI.Hosting.AspNetCore\Microsoft.Agents.AI.Hosting.AspNetCore.csproj" />
|
||||
<ProjectReference Include="..\..\..\..\..\src\Microsoft.Agents.AI.OpenAI\Microsoft.Agents.AI.OpenAI.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
|
||||
@@ -16,11 +16,6 @@ builder.Services.ConfigureHttpJsonOptions(options =>
|
||||
options.SerializerOptions.TypeInfoResolverChain.Add(SampleJsonSerializerContext.Default));
|
||||
builder.Services.AddAGUI();
|
||||
|
||||
// WARNING: When adding session persistence (e.g., WithInMemorySessionStore), or running in production,
|
||||
// make sure to also register a SessionIsolationKeyProvider to scope sessions by principal in multi-user
|
||||
// deployments, e.g.:
|
||||
// builder.Services.UseClaimsBasedSessionIsolation(new() { ClaimType = ClaimTypes.NameIdentifier });
|
||||
|
||||
WebApplication app = builder.Build();
|
||||
|
||||
string endpoint = builder.Configuration["AZURE_OPENAI_ENDPOINT"]
|
||||
|
||||
@@ -14,7 +14,6 @@
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\..\..\..\src\Microsoft.Agents.AI.Hosting.AGUI.AspNetCore\Microsoft.Agents.AI.Hosting.AGUI.AspNetCore.csproj" />
|
||||
<ProjectReference Include="..\..\..\..\..\src\Microsoft.Agents.AI.Hosting.AspNetCore\Microsoft.Agents.AI.Hosting.AspNetCore.csproj" />
|
||||
<ProjectReference Include="..\..\..\..\..\src\Microsoft.Agents.AI.OpenAI\Microsoft.Agents.AI.OpenAI.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
|
||||
@@ -10,11 +10,6 @@ WebApplicationBuilder builder = WebApplication.CreateBuilder(args);
|
||||
builder.Services.AddHttpClient().AddLogging();
|
||||
builder.Services.AddAGUI();
|
||||
|
||||
// WARNING: When adding session persistence (e.g., WithInMemorySessionStore), or running in production,
|
||||
// make sure to also register a SessionIsolationKeyProvider to scope sessions by principal in multi-user
|
||||
// deployments, e.g.:
|
||||
// builder.Services.UseClaimsBasedSessionIsolation(new() { ClaimType = ClaimTypes.NameIdentifier });
|
||||
|
||||
WebApplication app = builder.Build();
|
||||
|
||||
string endpoint = builder.Configuration["AZURE_OPENAI_ENDPOINT"]
|
||||
|
||||
@@ -14,7 +14,6 @@
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\..\..\..\src\Microsoft.Agents.AI.Hosting.AGUI.AspNetCore\Microsoft.Agents.AI.Hosting.AGUI.AspNetCore.csproj" />
|
||||
<ProjectReference Include="..\..\..\..\..\src\Microsoft.Agents.AI.Hosting.AspNetCore\Microsoft.Agents.AI.Hosting.AspNetCore.csproj" />
|
||||
<ProjectReference Include="..\..\..\..\..\src\Microsoft.Agents.AI.OpenAI\Microsoft.Agents.AI.OpenAI.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
|
||||
@@ -27,11 +27,6 @@ builder.Services.ConfigureHttpJsonOptions(options =>
|
||||
options.SerializerOptions.TypeInfoResolverChain.Add(ApprovalJsonContext.Default));
|
||||
builder.Services.AddAGUI();
|
||||
|
||||
// WARNING: When adding session persistence (e.g., WithInMemorySessionStore), or running in production,
|
||||
// make sure to also register a SessionIsolationKeyProvider to scope sessions by principal in multi-user
|
||||
// deployments, e.g.:
|
||||
// builder.Services.UseClaimsBasedSessionIsolation(new() { ClaimType = ClaimTypes.NameIdentifier });
|
||||
|
||||
WebApplication app = builder.Build();
|
||||
|
||||
app.UseHttpLogging();
|
||||
|
||||
@@ -14,7 +14,6 @@
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\..\..\..\src\Microsoft.Agents.AI.Hosting.AGUI.AspNetCore\Microsoft.Agents.AI.Hosting.AGUI.AspNetCore.csproj" />
|
||||
<ProjectReference Include="..\..\..\..\..\src\Microsoft.Agents.AI.Hosting.AspNetCore\Microsoft.Agents.AI.Hosting.AspNetCore.csproj" />
|
||||
<ProjectReference Include="..\..\..\..\..\src\Microsoft.Agents.AI.OpenAI\Microsoft.Agents.AI.OpenAI.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
|
||||
@@ -17,11 +17,6 @@ builder.Services.AddAGUI();
|
||||
// Configure to listen on port 8888
|
||||
builder.WebHost.UseUrls("http://localhost:8888");
|
||||
|
||||
// WARNING: When adding session persistence (e.g., WithInMemorySessionStore), or running in production,
|
||||
// make sure to also register a SessionIsolationKeyProvider to scope sessions by principal in multi-user
|
||||
// deployments, e.g.:
|
||||
// builder.Services.UseClaimsBasedSessionIsolation(new() { ClaimType = ClaimTypes.NameIdentifier });
|
||||
|
||||
WebApplication app = builder.Build();
|
||||
|
||||
string endpoint = builder.Configuration["AZURE_OPENAI_ENDPOINT"]
|
||||
|
||||
@@ -14,7 +14,6 @@
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\..\..\..\src\Microsoft.Agents.AI.Hosting.AGUI.AspNetCore\Microsoft.Agents.AI.Hosting.AGUI.AspNetCore.csproj" />
|
||||
<ProjectReference Include="..\..\..\..\..\src\Microsoft.Agents.AI.Hosting.AspNetCore\Microsoft.Agents.AI.Hosting.AspNetCore.csproj" />
|
||||
<ProjectReference Include="..\..\..\..\..\src\Microsoft.Agents.AI.OpenAI\Microsoft.Agents.AI.OpenAI.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
|
||||
@@ -50,16 +50,12 @@ internal static partial class WorkflowHelper
|
||||
/// <summary>
|
||||
/// Executor that starts the concurrent processing by sending messages to the agents.
|
||||
/// </summary>
|
||||
[SendsMessage(typeof(List<ChatMessage>))]
|
||||
[SendsMessage(typeof(TurnToken))]
|
||||
private sealed partial class ConcurrentStartExecutor()
|
||||
: Executor("ConcurrentStartExecutor", declareCrossRunShareable: true), IResettableExecutor
|
||||
private sealed partial class ConcurrentStartExecutor() : Executor("ConcurrentStartExecutor")
|
||||
{
|
||||
[MessageHandler]
|
||||
internal ValueTask RouteMessages(IEnumerable<ChatMessage> messages, IWorkflowContext context, CancellationToken cancellationToken)
|
||||
internal ValueTask RouteMessages(List<ChatMessage> messages, IWorkflowContext context, CancellationToken cancellationToken)
|
||||
{
|
||||
List<ChatMessage> payload = messages as List<ChatMessage> ?? messages.ToList();
|
||||
return context.SendMessageAsync(payload, cancellationToken: cancellationToken);
|
||||
return context.SendMessageAsync(messages, cancellationToken: cancellationToken);
|
||||
}
|
||||
|
||||
[MessageHandler]
|
||||
@@ -67,16 +63,13 @@ internal static partial class WorkflowHelper
|
||||
{
|
||||
return context.SendMessageAsync(token, cancellationToken: cancellationToken);
|
||||
}
|
||||
|
||||
public ValueTask ResetAsync() => default;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Executor that aggregates the results from the concurrent agents.
|
||||
/// </summary>
|
||||
[YieldsOutput(typeof(string))]
|
||||
private sealed partial class ConcurrentAggregationExecutor() :
|
||||
Executor<List<ChatMessage>>("ConcurrentAggregationExecutor"), IResettableExecutor
|
||||
[YieldsOutput(typeof(List<ChatMessage>))]
|
||||
private sealed partial class ConcurrentAggregationExecutor() : Executor<List<ChatMessage>>("ConcurrentAggregationExecutor")
|
||||
{
|
||||
private readonly List<ChatMessage> _messages = [];
|
||||
|
||||
@@ -97,11 +90,5 @@ internal static partial class WorkflowHelper
|
||||
await context.YieldOutputAsync(formattedMessages, cancellationToken);
|
||||
}
|
||||
}
|
||||
|
||||
public ValueTask ResetAsync()
|
||||
{
|
||||
this._messages.Clear();
|
||||
return default;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+10
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"IsEncrypted": false,
|
||||
"Values": {
|
||||
"FUNCTIONS_WORKER_RUNTIME": "dotnet-isolated",
|
||||
"AzureWebJobsStorage": "UseDevelopmentStorage=true",
|
||||
"DURABLE_TASK_SCHEDULER_CONNECTION_STRING": "Endpoint=http://localhost:8080;TaskHub=default;Authentication=None",
|
||||
"AZURE_OPENAI_ENDPOINT": "<AZURE_OPENAI_ENDPOINT>",
|
||||
"AZURE_OPENAI_DEPLOYMENT_NAME": "<AZURE_OPENAI_DEPLOYMENT_NAME>"
|
||||
}
|
||||
}
|
||||
+10
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"IsEncrypted": false,
|
||||
"Values": {
|
||||
"FUNCTIONS_WORKER_RUNTIME": "dotnet-isolated",
|
||||
"AzureWebJobsStorage": "UseDevelopmentStorage=true",
|
||||
"DURABLE_TASK_SCHEDULER_CONNECTION_STRING": "Endpoint=http://localhost:8080;TaskHub=default;Authentication=None",
|
||||
"AZURE_OPENAI_ENDPOINT": "<AZURE_OPENAI_ENDPOINT>",
|
||||
"AZURE_OPENAI_DEPLOYMENT_NAME": "<AZURE_OPENAI_DEPLOYMENT_NAME>"
|
||||
}
|
||||
}
|
||||
+10
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"IsEncrypted": false,
|
||||
"Values": {
|
||||
"FUNCTIONS_WORKER_RUNTIME": "dotnet-isolated",
|
||||
"AzureWebJobsStorage": "UseDevelopmentStorage=true",
|
||||
"DURABLE_TASK_SCHEDULER_CONNECTION_STRING": "Endpoint=http://localhost:8080;TaskHub=default;Authentication=None",
|
||||
"AZURE_OPENAI_ENDPOINT": "<AZURE_OPENAI_ENDPOINT>",
|
||||
"AZURE_OPENAI_DEPLOYMENT_NAME": "<AZURE_OPENAI_DEPLOYMENT_NAME>"
|
||||
}
|
||||
}
|
||||
+10
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"IsEncrypted": false,
|
||||
"Values": {
|
||||
"FUNCTIONS_WORKER_RUNTIME": "dotnet-isolated",
|
||||
"AzureWebJobsStorage": "UseDevelopmentStorage=true",
|
||||
"DURABLE_TASK_SCHEDULER_CONNECTION_STRING": "Endpoint=http://localhost:8080;TaskHub=default;Authentication=None",
|
||||
"AZURE_OPENAI_ENDPOINT": "<AZURE_OPENAI_ENDPOINT>",
|
||||
"AZURE_OPENAI_DEPLOYMENT_NAME": "<AZURE_OPENAI_DEPLOYMENT_NAME>"
|
||||
}
|
||||
}
|
||||
+10
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"IsEncrypted": false,
|
||||
"Values": {
|
||||
"FUNCTIONS_WORKER_RUNTIME": "dotnet-isolated",
|
||||
"AzureWebJobsStorage": "UseDevelopmentStorage=true",
|
||||
"DURABLE_TASK_SCHEDULER_CONNECTION_STRING": "Endpoint=http://localhost:8080;TaskHub=default;Authentication=None",
|
||||
"AZURE_OPENAI_ENDPOINT": "<AZURE_OPENAI_ENDPOINT>",
|
||||
"AZURE_OPENAI_DEPLOYMENT_NAME": "<AZURE_OPENAI_DEPLOYMENT_NAME>"
|
||||
}
|
||||
}
|
||||
+10
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"IsEncrypted": false,
|
||||
"Values": {
|
||||
"FUNCTIONS_WORKER_RUNTIME": "dotnet-isolated",
|
||||
"AzureWebJobsStorage": "UseDevelopmentStorage=true",
|
||||
"DURABLE_TASK_SCHEDULER_CONNECTION_STRING": "Endpoint=http://localhost:8080;TaskHub=default;Authentication=None",
|
||||
"AZURE_OPENAI_ENDPOINT": "<AZURE_OPENAI_ENDPOINT>",
|
||||
"AZURE_OPENAI_DEPLOYMENT_NAME": "<AZURE_OPENAI_DEPLOYMENT_NAME>"
|
||||
}
|
||||
}
|
||||
+10
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"IsEncrypted": false,
|
||||
"Values": {
|
||||
"FUNCTIONS_WORKER_RUNTIME": "dotnet-isolated",
|
||||
"AzureWebJobsStorage": "UseDevelopmentStorage=true",
|
||||
"DURABLE_TASK_SCHEDULER_CONNECTION_STRING": "Endpoint=http://localhost:8080;TaskHub=default;Authentication=None",
|
||||
"AZURE_OPENAI_ENDPOINT": "<AZURE_OPENAI_ENDPOINT>",
|
||||
"AZURE_OPENAI_DEPLOYMENT_NAME": "<AZURE_OPENAI_DEPLOYMENT_NAME>"
|
||||
}
|
||||
}
|
||||
+12
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"IsEncrypted": false,
|
||||
"Values": {
|
||||
"FUNCTIONS_WORKER_RUNTIME": "dotnet-isolated",
|
||||
"AzureWebJobsStorage": "UseDevelopmentStorage=true",
|
||||
"DURABLE_TASK_SCHEDULER_CONNECTION_STRING": "Endpoint=http://localhost:8080;TaskHub=default;Authentication=None",
|
||||
"AZURE_OPENAI_ENDPOINT": "<AZURE_OPENAI_ENDPOINT>",
|
||||
"AZURE_OPENAI_DEPLOYMENT_NAME": "<AZURE_OPENAI_DEPLOYMENT_NAME>",
|
||||
"REDIS_CONNECTION_STRING": "localhost:6379",
|
||||
"REDIS_STREAM_TTL_MINUTES": "10"
|
||||
}
|
||||
}
|
||||
+10
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"IsEncrypted": false,
|
||||
"Values": {
|
||||
"FUNCTIONS_WORKER_RUNTIME": "dotnet-isolated",
|
||||
"AzureWebJobsStorage": "UseDevelopmentStorage=true",
|
||||
"DURABLE_TASK_SCHEDULER_CONNECTION_STRING": "Endpoint=http://localhost:8080;TaskHub=default;Authentication=None",
|
||||
"AZURE_OPENAI_ENDPOINT": "<AZURE_OPENAI_ENDPOINT>",
|
||||
"AZURE_OPENAI_DEPLOYMENT_NAME": "<AZURE_OPENAI_DEPLOYMENT_NAME>"
|
||||
}
|
||||
}
|
||||
+10
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"IsEncrypted": false,
|
||||
"Values": {
|
||||
"FUNCTIONS_WORKER_RUNTIME": "dotnet-isolated",
|
||||
"AzureWebJobsStorage": "UseDevelopmentStorage=true",
|
||||
"DURABLE_TASK_SCHEDULER_CONNECTION_STRING": "Endpoint=http://localhost:8080;TaskHub=default;Authentication=None",
|
||||
"AZURE_OPENAI_ENDPOINT": "<AZURE_OPENAI_ENDPOINT>",
|
||||
"AZURE_OPENAI_DEPLOYMENT_NAME": "<AZURE_OPENAI_DEPLOYMENT_NAME>"
|
||||
}
|
||||
}
|
||||
+10
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"IsEncrypted": false,
|
||||
"Values": {
|
||||
"FUNCTIONS_WORKER_RUNTIME": "dotnet-isolated",
|
||||
"AzureWebJobsStorage": "UseDevelopmentStorage=true",
|
||||
"DURABLE_TASK_SCHEDULER_CONNECTION_STRING": "Endpoint=http://localhost:8080;TaskHub=default;Authentication=None",
|
||||
"AZURE_OPENAI_ENDPOINT": "<AZURE_OPENAI_ENDPOINT>",
|
||||
"AZURE_OPENAI_DEPLOYMENT_NAME": "<AZURE_OPENAI_DEPLOYMENT_NAME>"
|
||||
}
|
||||
}
|
||||
+8
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"IsEncrypted": false,
|
||||
"Values": {
|
||||
"FUNCTIONS_WORKER_RUNTIME": "dotnet-isolated",
|
||||
"AzureWebJobsStorage": "UseDevelopmentStorage=true",
|
||||
"DURABLE_TASK_SCHEDULER_CONNECTION_STRING": "Endpoint=http://localhost:8080;TaskHub=default;Authentication=None"
|
||||
}
|
||||
}
|
||||
+10
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"IsEncrypted": false,
|
||||
"Values": {
|
||||
"FUNCTIONS_WORKER_RUNTIME": "dotnet-isolated",
|
||||
"AzureWebJobsStorage": "UseDevelopmentStorage=true",
|
||||
"DURABLE_TASK_SCHEDULER_CONNECTION_STRING": "Endpoint=http://localhost:8080;TaskHub=default;Authentication=None",
|
||||
"AZURE_OPENAI_ENDPOINT": "<AZURE_OPENAI_ENDPOINT>",
|
||||
"AZURE_OPENAI_DEPLOYMENT_NAME": "<AZURE_OPENAI_DEPLOYMENT_NAME>"
|
||||
}
|
||||
}
|
||||
-6
@@ -1,6 +0,0 @@
|
||||
AZURE_AI_PROJECT_ENDPOINT=<your-azure-ai-project-endpoint>
|
||||
ASPNETCORE_URLS=http://+:8088
|
||||
ASPNETCORE_ENVIRONMENT=Development
|
||||
AZURE_AI_MODEL_DEPLOYMENT_NAME=gpt-5
|
||||
FOUNDRY_TOOLBOX_NAME=<your-toolbox-name>
|
||||
AZURE_BEARER_TOKEN=DefaultAzureCredential
|
||||
-26
@@ -1,26 +0,0 @@
|
||||
# Dockerfile for end-users consuming the Agent Framework via NuGet packages.
|
||||
#
|
||||
# This Dockerfile performs a full `dotnet restore` and `dotnet publish` inside the container,
|
||||
# which only succeeds when the project references its dependencies via PackageReference (see the
|
||||
# commented-out section in HostedToolboxMcpSkills.csproj). Contributors building from the
|
||||
# agent-framework repository source must use Dockerfile.contributor instead because
|
||||
# ProjectReference dependencies live outside this folder and cannot be restored from inside
|
||||
# this build context.
|
||||
#
|
||||
# Use the official .NET 10.0 ASP.NET runtime as a parent image
|
||||
FROM mcr.microsoft.com/dotnet/aspnet:10.0 AS base
|
||||
WORKDIR /app
|
||||
|
||||
FROM mcr.microsoft.com/dotnet/sdk:10.0 AS build
|
||||
WORKDIR /src
|
||||
COPY . .
|
||||
RUN dotnet restore
|
||||
RUN dotnet publish -c Release -o /app/publish
|
||||
|
||||
# Final stage
|
||||
FROM base AS final
|
||||
WORKDIR /app
|
||||
COPY --from=build /app/publish .
|
||||
EXPOSE 8088
|
||||
ENV ASPNETCORE_URLS=http://+:8088
|
||||
ENTRYPOINT ["dotnet", "HostedToolboxMcpSkills.dll"]
|
||||
-18
@@ -1,18 +0,0 @@
|
||||
# Dockerfile for contributors building from the agent-framework repository source.
|
||||
#
|
||||
# This project uses ProjectReference to the local source, which means a standard
|
||||
# multi-stage Docker build cannot resolve dependencies outside this folder.
|
||||
# Pre-publish the app targeting the container runtime and copy the output:
|
||||
#
|
||||
# dotnet publish -c Debug -f net10.0 -r linux-musl-x64 --self-contained false -o out
|
||||
# docker build -f Dockerfile.contributor -t hosted-toolbox-mcp-skills .
|
||||
# docker run --rm -p 8088:8088 -e AGENT_NAME=hosted-toolbox-mcp-skills -e AZURE_BEARER_TOKEN=$AZURE_BEARER_TOKEN --env-file .env hosted-toolbox-mcp-skills
|
||||
#
|
||||
# For end-users consuming the NuGet package (not ProjectReference), use the standard
|
||||
# Dockerfile which performs a full dotnet restore + publish inside the container.
|
||||
FROM mcr.microsoft.com/dotnet/aspnet:10.0-alpine AS final
|
||||
WORKDIR /app
|
||||
COPY out/ .
|
||||
EXPOSE 8088
|
||||
ENV ASPNETCORE_URLS=http://+:8088
|
||||
ENTRYPOINT ["dotnet", "HostedToolboxMcpSkills.dll"]
|
||||
-36
@@ -1,36 +0,0 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk.Web">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFrameworks>net10.0</TargetFrameworks>
|
||||
<Nullable>enable</Nullable>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<CentralPackageTransitivePinningEnabled>false</CentralPackageTransitivePinningEnabled>
|
||||
<RootNamespace>HostedToolboxMcpSkills</RootNamespace>
|
||||
<AssemblyName>HostedToolboxMcpSkills</AssemblyName>
|
||||
<NoWarn>$(NoWarn);</NoWarn>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Azure.AI.Projects" />
|
||||
<PackageReference Include="Azure.Identity" />
|
||||
<PackageReference Include="ModelContextProtocol" VersionOverride="1.2.0" />
|
||||
<PackageReference Include="DotNetEnv" />
|
||||
</ItemGroup>
|
||||
|
||||
<!-- For contributors: uses ProjectReference to build against local source -->
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\..\..\..\src\Microsoft.Agents.AI.Foundry\Microsoft.Agents.AI.Foundry.csproj" />
|
||||
<ProjectReference Include="..\..\..\..\..\src\Microsoft.Agents.AI.Foundry.Hosting\Microsoft.Agents.AI.Foundry.Hosting.csproj" />
|
||||
<ProjectReference Include="..\..\..\..\..\src\Microsoft.Agents.AI.Mcp\Microsoft.Agents.AI.Mcp.csproj" />
|
||||
<ProjectReference Include="..\Hosted_Shared_Contributor_Setup\Hosted_Shared_Contributor_Setup.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
<!-- For end-users: uncomment the PackageReference below and remove the ProjectReference above
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.Agents.AI.Foundry" Version="1.6.1-preview.260514.1" />
|
||||
<PackageReference Include="Microsoft.Agents.AI.Foundry.Hosting" Version="1.6.1-preview.260514.1" />
|
||||
<PackageReference Include="Microsoft.Agents.AI.Mcp" Version="1.6.1-preview.260514.1" />
|
||||
</ItemGroup>
|
||||
-->
|
||||
|
||||
</Project>
|
||||
-109
@@ -1,109 +0,0 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
// Hosted Toolbox MCP Skills Agent
|
||||
//
|
||||
// Demonstrates how to host an agent that discovers MCP-based skills from a
|
||||
// Foundry Toolbox MCP endpoint and injects them as AIContextProviders using
|
||||
// AgentSkillsProviderBuilder.UseMcpSkills().
|
||||
//
|
||||
// Required environment variables:
|
||||
// AZURE_AI_PROJECT_ENDPOINT - Azure AI Foundry project endpoint
|
||||
// FOUNDRY_TOOLBOX_NAME - Name of the Foundry Toolbox to connect to
|
||||
// AZURE_AI_MODEL_DEPLOYMENT_NAME - Model deployment name (default: gpt-5)
|
||||
|
||||
using System.Net.Http.Headers;
|
||||
using Azure.AI.Projects;
|
||||
using Azure.Core;
|
||||
using Azure.Identity;
|
||||
using DotNetEnv;
|
||||
using Hosted_Shared_Contributor_Setup;
|
||||
using Microsoft.Agents.AI;
|
||||
using Microsoft.Agents.AI.Foundry.Hosting;
|
||||
using ModelContextProtocol.Client;
|
||||
|
||||
// Load .env file if present (for local development)
|
||||
Env.TraversePath().Load();
|
||||
|
||||
var projectEndpoint = Environment.GetEnvironmentVariable("AZURE_AI_PROJECT_ENDPOINT")
|
||||
?? throw new InvalidOperationException("AZURE_AI_PROJECT_ENDPOINT is not set.");
|
||||
var deployment = Environment.GetEnvironmentVariable("AZURE_AI_MODEL_DEPLOYMENT_NAME") ?? "gpt-5";
|
||||
var toolboxName = Environment.GetEnvironmentVariable("FOUNDRY_TOOLBOX_NAME")
|
||||
?? throw new InvalidOperationException("FOUNDRY_TOOLBOX_NAME is not set.");
|
||||
|
||||
// Build the Toolbox MCP URL from the project endpoint and toolbox name.
|
||||
var toolboxMcpServerUrl = $"{projectEndpoint.TrimEnd('/')}/toolboxes/{toolboxName}/mcp?api-version=v1";
|
||||
|
||||
// Use a chained credential: try a temporary dev token first (for local Docker debugging),
|
||||
// then fall back to DefaultAzureCredential (for local dev via dotnet run / managed identity in production).
|
||||
TokenCredential credential = new ChainedTokenCredential(
|
||||
new DevTemporaryTokenCredential(),
|
||||
new DefaultAzureCredential());
|
||||
|
||||
// ── Connect to the Foundry Toolbox MCP endpoint ─────────────────────────────
|
||||
// Create an HttpClient that attaches a fresh Foundry bearer token to every request.
|
||||
using var httpClient = new HttpClient(new BearerTokenHandler(credential, "https://ai.azure.com/.default") { CheckCertificateRevocationList = true });
|
||||
|
||||
Console.WriteLine($"Connecting to Foundry Toolbox '{toolboxName}' MCP server...");
|
||||
|
||||
await using var mcpClient = await McpClient.CreateAsync(
|
||||
new HttpClientTransport(
|
||||
new HttpClientTransportOptions
|
||||
{
|
||||
Endpoint = new Uri(toolboxMcpServerUrl),
|
||||
Name = toolboxName,
|
||||
TransportMode = HttpTransportMode.StreamableHttp,
|
||||
AdditionalHeaders = new Dictionary<string, string>
|
||||
{
|
||||
["Foundry-Features"] = "Toolboxes=V1Preview",
|
||||
},
|
||||
},
|
||||
httpClient));
|
||||
|
||||
// ── Configure MCP-based skills provider ──────────────────────────────────────
|
||||
var skillsProvider = new AgentSkillsProviderBuilder()
|
||||
.UseMcpSkills(mcpClient)
|
||||
.Build();
|
||||
|
||||
// ── Create the agent ─────────────────────────────────────────────────────────
|
||||
AIAgent agent = new AIProjectClient(new Uri(projectEndpoint), credential)
|
||||
.AsAIAgent(new ChatClientAgentOptions
|
||||
{
|
||||
Name = Environment.GetEnvironmentVariable("AGENT_NAME") ?? "hosted-toolbox-mcp-skills",
|
||||
Description = "Hosted agent with MCP skills discovered from a Foundry Toolbox",
|
||||
ChatOptions = new()
|
||||
{
|
||||
ModelId = deployment,
|
||||
Instructions = "You are a helpful assistant.",
|
||||
},
|
||||
AIContextProviders = [skillsProvider],
|
||||
});
|
||||
|
||||
// ── Build the host ───────────────────────────────────────────────────────────
|
||||
var builder = WebApplication.CreateBuilder(args);
|
||||
builder.Services.AddFoundryResponses(agent);
|
||||
builder.Services.AddDevTemporaryLocalContributorSetup(); // Local Docker debugging only - must not be used in production.
|
||||
|
||||
var app = builder.Build();
|
||||
app.MapFoundryResponses();
|
||||
|
||||
// Contributor-only: in Development, also map the per-agent OpenAI route shape that live Foundry uses
|
||||
// so a local REPL client can target this server via AIProjectClient.AsAIAgent(Uri agentEndpoint).
|
||||
// Do not use this in production. Hosted Foundry agents only support the agent-endpoint path.
|
||||
app.MapDevTemporaryLocalAgentEndpoint();
|
||||
|
||||
app.Run();
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// HttpClientHandler: attaches a fresh Foundry bearer token to every request
|
||||
// ---------------------------------------------------------------------------
|
||||
internal sealed class BearerTokenHandler(TokenCredential credential, string scope) : HttpClientHandler
|
||||
{
|
||||
private readonly TokenRequestContext _tokenContext = new([scope]);
|
||||
|
||||
protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
|
||||
{
|
||||
AccessToken token = await credential.GetTokenAsync(this._tokenContext, cancellationToken).ConfigureAwait(false);
|
||||
request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", token.Token);
|
||||
return await base.SendAsync(request, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
-103
@@ -1,103 +0,0 @@
|
||||
# Hosted-ToolboxMcpSkills
|
||||
|
||||
A hosted agent that discovers **MCP-based skills from a Foundry Toolbox** and makes them available to the agent using `AgentSkillsProviderBuilder.UseMcpSkills(mcpClient)`.
|
||||
|
||||
The `AgentSkillsProvider` is attached to the agent as a context provider and implements the [Agent Skills](https://agentskills.io/) progressive-disclosure pattern. When the agent is prompted, it discovers available skills in the Foundry Toolbox via the provider:
|
||||
|
||||
1. **Advertise** - skill names and descriptions are injected into the system prompt so the agent knows what is available.
|
||||
2. **Load** - when the agent decides a skill is relevant, it retrieves the full skill body with detailed instructions via the provider.
|
||||
3. **Read resources** - if a skill includes supplementary content (reference documents, assets), the agent reads them on demand via the provider.
|
||||
|
||||
This way the full skill body and resources are only loaded when the agent actually needs them, reducing token usage.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- [.NET 10 SDK](https://dotnet.microsoft.com/download/dotnet/10.0)
|
||||
- An Azure AI Foundry project with a deployed model (e.g., `gpt-5`)
|
||||
- A Foundry Toolbox already configured with skills provisioned
|
||||
- Azure CLI logged in (`az login`)
|
||||
|
||||
## Configuration
|
||||
|
||||
Copy the template and fill in your values:
|
||||
|
||||
```bash
|
||||
cp .env.example .env
|
||||
```
|
||||
|
||||
Edit `.env` and set your Azure AI Foundry project endpoint and toolbox name:
|
||||
|
||||
```env
|
||||
AZURE_AI_PROJECT_ENDPOINT=https://<your-account>.services.ai.azure.com/api/projects/<your-project>
|
||||
ASPNETCORE_URLS=http://+:8088
|
||||
ASPNETCORE_ENVIRONMENT=Development
|
||||
AZURE_AI_MODEL_DEPLOYMENT_NAME=gpt-5
|
||||
FOUNDRY_TOOLBOX_NAME=my-toolbox
|
||||
```
|
||||
|
||||
> **Note:** `.env` is gitignored. The `.env.example` template is checked in as a reference.
|
||||
|
||||
## Running directly (contributors)
|
||||
|
||||
This project uses `ProjectReference` to build against the local Agent Framework source.
|
||||
|
||||
```bash
|
||||
cd dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-ToolboxMcpSkills
|
||||
dotnet run
|
||||
```
|
||||
|
||||
The agent will start on `http://localhost:8088`.
|
||||
|
||||
### Test it
|
||||
|
||||
Using the Azure Developer CLI:
|
||||
|
||||
```bash
|
||||
azd ai agent invoke --local "What skills do you have available?"
|
||||
```
|
||||
|
||||
## Running with Docker
|
||||
|
||||
Since this project uses `ProjectReference`, use `Dockerfile.contributor` which takes a pre-published output.
|
||||
|
||||
### 1. Publish for the container runtime (Linux Alpine)
|
||||
|
||||
```bash
|
||||
dotnet publish -c Debug -f net10.0 -r linux-musl-x64 --self-contained false -o out
|
||||
```
|
||||
|
||||
### 2. Build the Docker image
|
||||
|
||||
```bash
|
||||
docker build -f Dockerfile.contributor -t hosted-toolbox-mcp-skills .
|
||||
```
|
||||
|
||||
### 3. Run the container
|
||||
|
||||
Generate a bearer token on your host and pass it to the container:
|
||||
|
||||
```bash
|
||||
# Generate token (expires in ~1 hour)
|
||||
export AZURE_BEARER_TOKEN=$(az account get-access-token --resource https://ai.azure.com --query accessToken -o tsv)
|
||||
|
||||
# Run with token
|
||||
docker run --rm -p 8088:8088 \
|
||||
-e AGENT_NAME=hosted-toolbox-mcp-skills \
|
||||
-e AZURE_BEARER_TOKEN=$AZURE_BEARER_TOKEN \
|
||||
--env-file .env \
|
||||
hosted-toolbox-mcp-skills
|
||||
```
|
||||
|
||||
> **Note:** `AGENT_NAME` is passed via `-e` to simulate the platform injection. `AZURE_BEARER_TOKEN` provides Azure credentials to the container (tokens expire after ~1 hour). The `.env` file provides the remaining configuration.
|
||||
|
||||
### 4. Test it
|
||||
|
||||
Using the Azure Developer CLI:
|
||||
|
||||
```bash
|
||||
azd ai agent invoke --local "What skills do you have available?"
|
||||
```
|
||||
|
||||
## NuGet package users
|
||||
|
||||
If you are consuming the Agent Framework as a NuGet package (not building from source), use the standard `Dockerfile` instead of `Dockerfile.contributor`. See the commented section in `HostedToolboxMcpSkills.csproj` for the `PackageReference` alternative.
|
||||
-43
@@ -1,43 +0,0 @@
|
||||
# yaml-language-server: $schema=https://raw.githubusercontent.com/microsoft/AgentSchema/refs/heads/main/schemas/v1.0/AgentManifest.yaml
|
||||
name: hosted-toolbox-mcp-skills
|
||||
displayName: "Hosted Toolbox MCP Skills Agent"
|
||||
|
||||
description: >
|
||||
A hosted agent that discovers MCP-based skills from a Foundry Toolbox
|
||||
and makes them available to the agent via the agent skills provider.
|
||||
|
||||
metadata:
|
||||
tags:
|
||||
- AI Agent Hosting
|
||||
- Azure AI AgentServer
|
||||
- Responses Protocol
|
||||
- Agent Framework
|
||||
- MCP
|
||||
- Model Context Protocol
|
||||
- Agent Skills
|
||||
- Foundry Toolbox
|
||||
- Foundry Toolbox Skills
|
||||
|
||||
template:
|
||||
name: hosted-toolbox-mcp-skills
|
||||
kind: hosted
|
||||
protocols:
|
||||
- protocol: responses
|
||||
version: 1.0.0
|
||||
resources:
|
||||
cpu: "0.25"
|
||||
memory: 0.5Gi
|
||||
environment_variables:
|
||||
- name: AZURE_AI_MODEL_DEPLOYMENT_NAME
|
||||
value: "{{AZURE_AI_MODEL_DEPLOYMENT_NAME}}"
|
||||
- name: FOUNDRY_TOOLBOX_NAME
|
||||
value: "{{FOUNDRY_TOOLBOX_NAME}}"
|
||||
parameters:
|
||||
properties:
|
||||
- name: FOUNDRY_TOOLBOX_NAME
|
||||
secret: false
|
||||
description: Name of the Foundry Toolbox to connect to for MCP skill discovery
|
||||
resources:
|
||||
- kind: model
|
||||
id: gpt-5
|
||||
name: AZURE_AI_MODEL_DEPLOYMENT_NAME
|
||||
-14
@@ -1,14 +0,0 @@
|
||||
# yaml-language-server: $schema=https://raw.githubusercontent.com/microsoft/AgentSchema/refs/heads/main/schemas/v1.0/ContainerAgent.yaml
|
||||
kind: hosted
|
||||
name: hosted-toolbox-mcp-skills
|
||||
protocols:
|
||||
- protocol: responses
|
||||
version: 1.0.0
|
||||
resources:
|
||||
cpu: "0.25"
|
||||
memory: 0.5Gi
|
||||
environment_variables:
|
||||
- name: AZURE_AI_MODEL_DEPLOYMENT_NAME
|
||||
value: ${AZURE_AI_MODEL_DEPLOYMENT_NAME}
|
||||
- name: FOUNDRY_TOOLBOX_NAME
|
||||
value: ${FOUNDRY_TOOLBOX_NAME}
|
||||
@@ -15,7 +15,6 @@
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\..\..\src\Microsoft.Agents.AI.Hosting.AGUI.AspNetCore\Microsoft.Agents.AI.Hosting.AGUI.AspNetCore.csproj" />
|
||||
<ProjectReference Include="..\..\..\..\src\Microsoft.Agents.AI.Hosting.AspNetCore\Microsoft.Agents.AI.Hosting.AspNetCore.csproj" />
|
||||
<ProjectReference Include="..\..\..\..\src\Microsoft.Agents.AI.OpenAI\Microsoft.Agents.AI.OpenAI.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
|
||||
@@ -19,11 +19,6 @@ builder.Services.AddHttpClient().AddLogging();
|
||||
builder.Services.ConfigureHttpJsonOptions(options => options.SerializerOptions.TypeInfoResolverChain.Add(AGUIDojoServerSerializerContext.Default));
|
||||
builder.Services.AddAGUI();
|
||||
|
||||
// WARNING: When adding session persistence (e.g., WithInMemorySessionStore), or running in production,
|
||||
// make sure to also register a SessionIsolationKeyProvider to scope sessions by principal in multi-user
|
||||
// deployments, e.g.:
|
||||
// builder.Services.UseClaimsBasedSessionIsolation(new() { ClaimType = ClaimTypes.NameIdentifier });
|
||||
|
||||
WebApplication app = builder.Build();
|
||||
|
||||
app.UseHttpLogging();
|
||||
|
||||
@@ -49,9 +49,8 @@ var agent = new AzureOpenAIClient(
|
||||
AGUIServerSerializerContext.Default.Options)
|
||||
]);
|
||||
|
||||
// WARNING: When adding session persistence (e.g., WithInMemorySessionStore), or running in production,
|
||||
// make sure to also register a SessionIsolationKeyProvider to scope sessions by principal in multi-user
|
||||
// deployments, e.g.:
|
||||
// When running in production, make sure to use an SessionIsolationKeyProvider, e.g. ClaimsIdentity-based
|
||||
// if using Claims-based Identity for Authentication/Authorization
|
||||
// builder.Services.UseClaimsBasedSessionIsolation(new() { ClaimType = ClaimTypes.NameIdentifier });
|
||||
|
||||
// Register the agent with the host and configure it to use an in-memory session store
|
||||
|
||||
@@ -14,7 +14,6 @@
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\..\..\src\Microsoft.Agents.AI.Hosting.AGUI.AspNetCore\Microsoft.Agents.AI.Hosting.AGUI.AspNetCore.csproj" />
|
||||
<ProjectReference Include="..\..\..\..\src\Microsoft.Agents.AI.Hosting.AspNetCore\Microsoft.Agents.AI.Hosting.AspNetCore.csproj" />
|
||||
<ProjectReference Include="..\..\..\..\src\Microsoft.Agents.AI.AGUI\Microsoft.Agents.AI.AGUI.csproj" />
|
||||
<ProjectReference Include="..\..\..\..\src\Microsoft.Agents.AI.OpenAI\Microsoft.Agents.AI.OpenAI.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
@@ -12,11 +12,6 @@ WebApplicationBuilder builder = WebApplication.CreateBuilder(args);
|
||||
builder.Services.AddHttpClient().AddLogging();
|
||||
builder.Services.AddAGUI();
|
||||
|
||||
// WARNING: When adding session persistence (e.g., WithInMemorySessionStore), or running in production,
|
||||
// make sure to also register a SessionIsolationKeyProvider to scope sessions by principal in multi-user
|
||||
// deployments, e.g.:
|
||||
// builder.Services.UseClaimsBasedSessionIsolation(new() { ClaimType = ClaimTypes.NameIdentifier });
|
||||
|
||||
WebApplication app = builder.Build();
|
||||
|
||||
string endpoint = builder.Configuration["AZURE_OPENAI_ENDPOINT"] ?? throw new InvalidOperationException("AZURE_OPENAI_ENDPOINT is not set.");
|
||||
|
||||
Regular → Executable
@@ -24,13 +24,11 @@
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Azure.AI.Projects" />
|
||||
<PackageReference Include="Azure.Core" />
|
||||
<PackageReference Include="Microsoft.Extensions.AI" />
|
||||
<PackageReference Include="Microsoft.Extensions.AI.Abstractions" />
|
||||
<PackageReference Include="Microsoft.Extensions.AI.OpenAI" />
|
||||
<PackageReference Include="Microsoft.Extensions.Compliance.Abstractions" />
|
||||
<PackageReference Include="OpenAI" />
|
||||
<PackageReference Include="System.ClientModel" />
|
||||
</ItemGroup>
|
||||
|
||||
<!-- Evaluation support requires net8.0+ (MEAI.Evaluation does not support legacy TFMs) -->
|
||||
|
||||
@@ -1,9 +1,7 @@
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
using System;
|
||||
using System.Diagnostics.CodeAnalysis;
|
||||
using Microsoft.Agents.AI;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Shared.DiagnosticIds;
|
||||
|
||||
namespace Microsoft.Extensions.AI;
|
||||
@@ -34,19 +32,11 @@ public static class ChatClientHarnessExtensions
|
||||
/// additional context providers, and chat history provider.
|
||||
/// When <see langword="null"/>, the agent uses built-in default settings.
|
||||
/// </param>
|
||||
/// <param name="loggerFactory">
|
||||
/// Optional logger factory for creating loggers used by the agent and its components.
|
||||
/// </param>
|
||||
/// <param name="services">
|
||||
/// Optional service provider for resolving dependencies required by AI functions and other agent components.
|
||||
/// </param>
|
||||
/// <returns>A new <see cref="HarnessAgent"/> instance.</returns>
|
||||
public static HarnessAgent AsHarnessAgent(
|
||||
this IChatClient chatClient,
|
||||
int maxContextWindowTokens,
|
||||
int maxOutputTokens,
|
||||
HarnessAgentOptions? options = null,
|
||||
ILoggerFactory? loggerFactory = null,
|
||||
IServiceProvider? services = null) =>
|
||||
new(chatClient, maxContextWindowTokens, maxOutputTokens, options, loggerFactory, services);
|
||||
HarnessAgentOptions? options = null) =>
|
||||
new(chatClient, maxContextWindowTokens, maxOutputTokens, options);
|
||||
}
|
||||
|
||||
@@ -10,7 +10,6 @@ using Microsoft.Agents.AI.Compaction;
|
||||
using Microsoft.Agents.AI.Tools.Shell;
|
||||
#endif
|
||||
using Microsoft.Extensions.AI;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Shared.DiagnosticIds;
|
||||
using Microsoft.Shared.Diagnostics;
|
||||
|
||||
@@ -106,12 +105,6 @@ public sealed class HarnessAgent : DelegatingAIAgent
|
||||
/// additional context providers, and chat history provider.
|
||||
/// When <see langword="null"/>, the agent uses built-in default settings.
|
||||
/// </param>
|
||||
/// <param name="loggerFactory">
|
||||
/// Optional logger factory for creating loggers used by the agent and its components.
|
||||
/// </param>
|
||||
/// <param name="services">
|
||||
/// Optional service provider for resolving dependencies required by AI functions and other agent components.
|
||||
/// </param>
|
||||
/// <exception cref="ArgumentNullException">
|
||||
/// <paramref name="chatClient"/> is <see langword="null"/>.
|
||||
/// </exception>
|
||||
@@ -119,20 +112,18 @@ public sealed class HarnessAgent : DelegatingAIAgent
|
||||
/// <paramref name="maxContextWindowTokens"/> is not positive, or
|
||||
/// <paramref name="maxOutputTokens"/> is negative or greater than or equal to <paramref name="maxContextWindowTokens"/>.
|
||||
/// </exception>
|
||||
public HarnessAgent(IChatClient chatClient, int maxContextWindowTokens, int maxOutputTokens, HarnessAgentOptions? options = null, ILoggerFactory? loggerFactory = null, IServiceProvider? services = null)
|
||||
public HarnessAgent(IChatClient chatClient, int maxContextWindowTokens, int maxOutputTokens, HarnessAgentOptions? options = null)
|
||||
: base(BuildAgent(
|
||||
Throw.IfNull(chatClient),
|
||||
maxContextWindowTokens,
|
||||
maxOutputTokens,
|
||||
options,
|
||||
loggerFactory,
|
||||
services))
|
||||
options))
|
||||
{
|
||||
}
|
||||
|
||||
private static AIAgent BuildAgent(IChatClient chatClient, int maxContextWindowTokens, int maxOutputTokens, HarnessAgentOptions? options, ILoggerFactory? loggerFactory, IServiceProvider? services)
|
||||
private static AIAgent BuildAgent(IChatClient chatClient, int maxContextWindowTokens, int maxOutputTokens, HarnessAgentOptions? options)
|
||||
{
|
||||
ChatClientAgent innerAgent = BuildInnerAgent(chatClient, maxContextWindowTokens, maxOutputTokens, options, loggerFactory, services);
|
||||
ChatClientAgent innerAgent = BuildInnerAgent(chatClient, maxContextWindowTokens, maxOutputTokens, options);
|
||||
|
||||
AIAgentBuilder builder = innerAgent.AsBuilder();
|
||||
|
||||
@@ -146,10 +137,10 @@ public sealed class HarnessAgent : DelegatingAIAgent
|
||||
builder.UseOpenTelemetry(sourceName: options?.OpenTelemetrySourceName);
|
||||
}
|
||||
|
||||
return builder.Build(services);
|
||||
return builder.Build();
|
||||
}
|
||||
|
||||
private static ChatClientAgent BuildInnerAgent(IChatClient chatClient, int maxContextWindowTokens, int maxOutputTokens, HarnessAgentOptions? options, ILoggerFactory? loggerFactory, IServiceProvider? services)
|
||||
private static ChatClientAgent BuildInnerAgent(IChatClient chatClient, int maxContextWindowTokens, int maxOutputTokens, HarnessAgentOptions? options)
|
||||
{
|
||||
var compactionStrategy = new ContextWindowCompactionStrategy(
|
||||
maxContextWindowTokens: maxContextWindowTokens,
|
||||
@@ -174,13 +165,13 @@ public sealed class HarnessAgent : DelegatingAIAgent
|
||||
|
||||
ChatOptions chatOptions = BuildChatOptions(options, instructions, maxOutputTokens);
|
||||
|
||||
var compactionProvider = new CompactionProvider(compactionStrategy, loggerFactory: loggerFactory);
|
||||
var compactionProvider = new CompactionProvider(compactionStrategy);
|
||||
|
||||
IEnumerable<AIContextProvider> contextProviders = BuildContextProviders(options, loggerFactory);
|
||||
IEnumerable<AIContextProvider> contextProviders = BuildContextProviders(options);
|
||||
|
||||
return chatClient
|
||||
.AsBuilder()
|
||||
.UseFunctionInvocation(loggerFactory, configure: options?.MaximumIterationsPerRequest is int maxIterations
|
||||
.UseFunctionInvocation(configure: options?.MaximumIterationsPerRequest is int maxIterations
|
||||
? ficc => ficc.MaximumIterationsPerRequest = maxIterations
|
||||
: null)
|
||||
.UseMessageInjection()
|
||||
@@ -198,9 +189,7 @@ public sealed class HarnessAgent : DelegatingAIAgent
|
||||
RequirePerServiceCallChatHistoryPersistence = true,
|
||||
WarnOnChatHistoryProviderConflict = false,
|
||||
ThrowOnChatHistoryProviderConflict = false,
|
||||
},
|
||||
loggerFactory,
|
||||
services);
|
||||
});
|
||||
}
|
||||
|
||||
private static ChatOptions BuildChatOptions(HarnessAgentOptions? options, string instructions, int maxOutputTokens)
|
||||
@@ -226,7 +215,7 @@ public sealed class HarnessAgent : DelegatingAIAgent
|
||||
return result;
|
||||
}
|
||||
|
||||
private static List<AIContextProvider> BuildContextProviders(HarnessAgentOptions? options, ILoggerFactory? loggerFactory)
|
||||
private static List<AIContextProvider> BuildContextProviders(HarnessAgentOptions? options)
|
||||
{
|
||||
var providers = new List<AIContextProvider>();
|
||||
|
||||
@@ -266,8 +255,8 @@ public sealed class HarnessAgent : DelegatingAIAgent
|
||||
if (options?.DisableAgentSkillsProvider is not true)
|
||||
{
|
||||
AgentSkillsProvider skillsProvider = options?.AgentSkillsSource is AgentSkillsSource source
|
||||
? new AgentSkillsProvider(source, loggerFactory: loggerFactory)
|
||||
: new AgentSkillsProvider(Directory.GetCurrentDirectory(), loggerFactory: loggerFactory);
|
||||
? new AgentSkillsProvider(source)
|
||||
: new AgentSkillsProvider(Directory.GetCurrentDirectory());
|
||||
|
||||
providers.Add(skillsProvider);
|
||||
}
|
||||
|
||||
+1
-10
@@ -103,16 +103,7 @@ public static class AGUIEndpointRouteBuilderExtensions
|
||||
ArgumentNullException.ThrowIfNull(aiAgent);
|
||||
|
||||
var agentSessionStore = endpoints.ServiceProvider.GetKeyedService<AgentSessionStore>(aiAgent.Name);
|
||||
|
||||
// Ensure that we have an IsolationKeyScopedAgentSessionStore registered.
|
||||
var isolationKeyProvider = endpoints.ServiceProvider.GetService<SessionIsolationKeyProvider>();
|
||||
if (agentSessionStore?.GetService<IsolationKeyScopedAgentSessionStore>() is null)
|
||||
{
|
||||
agentSessionStore ??= new NoopAgentSessionStore();
|
||||
agentSessionStore = new IsolationKeyScopedAgentSessionStore(agentSessionStore, isolationKeyProvider, new() { Strict = isolationKeyProvider != null });
|
||||
}
|
||||
|
||||
var hostAgent = new AIHostAgent(aiAgent, agentSessionStore);
|
||||
var hostAgent = new AIHostAgent(aiAgent, agentSessionStore ?? new NoopAgentSessionStore());
|
||||
|
||||
return endpoints.MapPost(pattern, async ([FromBody] RunAgentInput? input, HttpContext context, CancellationToken cancellationToken) =>
|
||||
{
|
||||
|
||||
+1
-1
@@ -1,7 +1,7 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<!-- Preview while Microsoft.Agents.AI.Foundry is preview (blocked by Azure.AI.Projects 2.1.0-beta). Flip to IsReleased=true once that ships stable. -->
|
||||
<IsReleaseCandidate>true</IsReleaseCandidate>
|
||||
<NoWarn>$(NoWarn);MEAI001;OPENAI001</NoWarn>
|
||||
</PropertyGroup>
|
||||
|
||||
|
||||
+3
-5
@@ -1,7 +1,7 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<IsReleased>true</IsReleased>
|
||||
<IsReleaseCandidate>true</IsReleaseCandidate>
|
||||
<NoWarn>$(NoWarn);MEAI001;OPENAI001</NoWarn>
|
||||
</PropertyGroup>
|
||||
|
||||
@@ -13,11 +13,9 @@
|
||||
|
||||
<Import Project="$(RepoRoot)/dotnet/nuget/nuget-package.props" />
|
||||
|
||||
<!-- First Stable release after the RC milestone. Baseline against the latest
|
||||
published RC so package validation catches accidental breaking changes.
|
||||
Future releases should bump this to the previous stable version. -->
|
||||
<!-- Package not yet published to NuGet — disable baseline validation until first release -->
|
||||
<PropertyGroup>
|
||||
<PackageValidationBaselineVersion>1.8.0-rc1</PackageValidationBaselineVersion>
|
||||
<EnablePackageValidation>false</EnablePackageValidation>
|
||||
</PropertyGroup>
|
||||
|
||||
<PropertyGroup>
|
||||
|
||||
+1
-8
@@ -1,7 +1,7 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<IsReleased>true</IsReleased>
|
||||
<IsReleaseCandidate>true</IsReleaseCandidate>
|
||||
<NoWarn>$(NoWarn);MEAI001;OPENAI001</NoWarn>
|
||||
</PropertyGroup>
|
||||
|
||||
@@ -13,13 +13,6 @@
|
||||
|
||||
<Import Project="$(RepoRoot)/dotnet/nuget/nuget-package.props" />
|
||||
|
||||
<!-- First Stable release after the RC milestone. Baseline against the latest
|
||||
published RC so package validation catches accidental breaking changes.
|
||||
Future releases should bump this to the previous stable version. -->
|
||||
<PropertyGroup>
|
||||
<PackageValidationBaselineVersion>1.8.0-rc1</PackageValidationBaselineVersion>
|
||||
</PropertyGroup>
|
||||
|
||||
<PropertyGroup>
|
||||
<!-- NuGet Package Settings -->
|
||||
<Title>Microsoft Agent Framework Declarative Workflows</Title>
|
||||
|
||||
+6
-63
@@ -27,14 +27,6 @@ internal sealed class InvokeMcpToolExecutor(
|
||||
WorkflowFormulaState state) :
|
||||
DeclarativeActionExecutor<InvokeMcpTool>(model, state)
|
||||
{
|
||||
private const string ApprovalSnapshotStateKey = nameof(_approvalSnapshot);
|
||||
|
||||
/// <summary>
|
||||
/// Snapshot of evaluated parameters at approval-request time.
|
||||
/// Used to prevent TOCTOU attacks where state mutates during the approval window.
|
||||
/// </summary>
|
||||
private ApprovalSnapshot? _approvalSnapshot;
|
||||
|
||||
/// <summary>
|
||||
/// Step identifiers for the MCP tool invocation workflow.
|
||||
/// </summary>
|
||||
@@ -83,10 +75,6 @@ internal sealed class InvokeMcpToolExecutor(
|
||||
|
||||
if (requireApproval)
|
||||
{
|
||||
// Snapshot the evaluated parameters to prevent TOCTOU attacks.
|
||||
// If state mutates during the approval window, the approved values are used on resume.
|
||||
this._approvalSnapshot = new ApprovalSnapshot(serverUrl, serverLabel, toolName, arguments, connectionName);
|
||||
|
||||
// Create tool call content for approval request.
|
||||
// Transport headers (e.g. Authorization) are intentionally excluded from the
|
||||
// approval event: they must not cross into the externally-surfaced approval request.
|
||||
@@ -149,14 +137,13 @@ internal sealed class InvokeMcpToolExecutor(
|
||||
return;
|
||||
}
|
||||
|
||||
// Approved - use the snapshot from approval-request time to prevent TOCTOU attacks.
|
||||
// Headers are re-evaluated (they may contain auth secrets that should not be persisted).
|
||||
string serverUrl = this._approvalSnapshot?.ServerUrl ?? this.GetServerUrl();
|
||||
string? serverLabel = this._approvalSnapshot?.ServerLabel ?? this.GetServerLabel();
|
||||
string toolName = this._approvalSnapshot?.ToolName ?? this.GetToolName();
|
||||
Dictionary<string, object?>? arguments = this._approvalSnapshot?.Arguments ?? this.GetArguments();
|
||||
// Approved - now invoke the tool
|
||||
string serverUrl = this.GetServerUrl();
|
||||
string? serverLabel = this.GetServerLabel();
|
||||
string toolName = this.GetToolName();
|
||||
Dictionary<string, object?>? arguments = this.GetArguments();
|
||||
Dictionary<string, string>? headers = this.GetHeaders();
|
||||
string? connectionName = this._approvalSnapshot?.ConnectionName ?? this.GetConnectionName();
|
||||
string? connectionName = this.GetConnectionName();
|
||||
|
||||
McpServerToolResultContent resultContent = await mcpToolHandler.InvokeToolAsync(
|
||||
serverUrl,
|
||||
@@ -175,33 +162,9 @@ internal sealed class InvokeMcpToolExecutor(
|
||||
/// </summary>
|
||||
public async ValueTask CompleteAsync(IWorkflowContext context, ActionExecutorResult message, CancellationToken cancellationToken)
|
||||
{
|
||||
// Clear the approval snapshot after successful completion.
|
||||
this._approvalSnapshot = null;
|
||||
await ClearSnapshotStateAsync(context, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
await context.RaiseCompletionEventAsync(this.Model, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
/// <remarks>
|
||||
/// Persists the approval snapshot to workflow state so it survives checkpoint/restore cycles.
|
||||
/// </remarks>
|
||||
protected override async ValueTask OnCheckpointingAsync(IWorkflowContext context, CancellationToken cancellationToken = default)
|
||||
{
|
||||
await context.QueueStateUpdateAsync(ApprovalSnapshotStateKey, this._approvalSnapshot, null, cancellationToken).ConfigureAwait(false);
|
||||
await base.OnCheckpointingAsync(context, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
/// <remarks>
|
||||
/// Restores the approval snapshot from workflow state after a checkpoint restore.
|
||||
/// </remarks>
|
||||
protected override async ValueTask OnCheckpointRestoredAsync(IWorkflowContext context, CancellationToken cancellationToken = default)
|
||||
{
|
||||
await base.OnCheckpointRestoredAsync(context, cancellationToken).ConfigureAwait(false);
|
||||
this._approvalSnapshot = await context.ReadStateAsync<ApprovalSnapshot>(ApprovalSnapshotStateKey, null, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private async ValueTask ProcessResultAsync(IWorkflowContext context, McpServerToolResultContent resultContent, CancellationToken cancellationToken)
|
||||
{
|
||||
bool autoSend = this.GetAutoSendValue();
|
||||
@@ -402,24 +365,4 @@ internal sealed class InvokeMcpToolExecutor(
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Clears the persisted approval snapshot state after a successful tool invocation.
|
||||
/// </summary>
|
||||
private static async ValueTask ClearSnapshotStateAsync(IWorkflowContext context, CancellationToken cancellationToken)
|
||||
{
|
||||
await context.QueueStateUpdateAsync<ApprovalSnapshot?>(ApprovalSnapshotStateKey, null, null, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Stores the evaluated parameters at approval-request time so that
|
||||
/// <see cref="CaptureResponseAsync"/> uses the values the user reviewed,
|
||||
/// even if <see cref="WorkflowFormulaState"/> mutates during the approval window.
|
||||
/// </summary>
|
||||
internal sealed record ApprovalSnapshot(
|
||||
string ServerUrl,
|
||||
string? ServerLabel,
|
||||
string ToolName,
|
||||
Dictionary<string, object?>? Arguments,
|
||||
string? ConnectionName);
|
||||
}
|
||||
|
||||
@@ -6,11 +6,11 @@
|
||||
.DESCRIPTION
|
||||
The IT fixture targets stable, scenario-keyed agent names (e.g. it-happy-path) and only
|
||||
manages versions on each test run. The agent itself must already exist AND its managed
|
||||
identity must hold the Foundry User role on the project scope, otherwise inbound
|
||||
identity must hold the Azure AI User role on the project scope, otherwise inbound
|
||||
inference calls fail with HTTP 500 PermissionDenied.
|
||||
|
||||
This script idempotently creates each scenario agent (with a placeholder version) and
|
||||
grants Foundry User on the project to its managed identity. Re-run it safely; existing
|
||||
grants Azure AI User on the project to its managed identity. Re-run it safely; existing
|
||||
agents and role assignments are left in place.
|
||||
|
||||
.PARAMETER ProjectEndpoint
|
||||
@@ -135,20 +135,20 @@ foreach ($scenario in $Scenarios) {
|
||||
-Body $patchBody | Out-Null
|
||||
}
|
||||
|
||||
# 3. Grant Foundry User on the project scope to the agent MI (idempotent).
|
||||
# 3. Grant Azure AI User on the project scope to the agent MI (idempotent).
|
||||
$existing = az role assignment list --assignee $principalId --scope $projectScope `
|
||||
--query "[?roleDefinitionName=='Foundry User']" 2>$null | ConvertFrom-Json
|
||||
--query "[?roleDefinitionName=='Azure AI User']" 2>$null | ConvertFrom-Json
|
||||
if ($existing) {
|
||||
Write-Host " role already assigned"
|
||||
} else {
|
||||
Write-Host " granting Foundry User..."
|
||||
Write-Host " granting Azure AI User..."
|
||||
$maxAttempts = 12
|
||||
$granted = $false
|
||||
for ($i = 1; $i -le $maxAttempts; $i++) {
|
||||
$output = az role assignment create `
|
||||
--assignee-object-id $principalId `
|
||||
--assignee-principal-type ServicePrincipal `
|
||||
--role 'Foundry User' `
|
||||
--role 'Azure AI User' `
|
||||
--scope $projectScope 2>&1
|
||||
if ($LASTEXITCODE -eq 0) {
|
||||
$granted = $true
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"profiles": {
|
||||
"Microsoft.Agents.AI.DevUI.UnitTests": {
|
||||
"commandName": "Project",
|
||||
"launchBrowser": true,
|
||||
"environmentVariables": {
|
||||
"ASPNETCORE_ENVIRONMENT": "Development"
|
||||
},
|
||||
"applicationUrl": "https://localhost:63009;http://localhost:63010"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -9,7 +9,6 @@ using System.Threading.Tasks;
|
||||
using Microsoft.Agents.AI.Tools.Shell;
|
||||
#endif
|
||||
using Microsoft.Extensions.AI;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Moq;
|
||||
|
||||
namespace Microsoft.Agents.AI.UnitTests;
|
||||
@@ -1461,131 +1460,4 @@ public class HarnessAgentTests
|
||||
|
||||
#endregion
|
||||
#endif
|
||||
|
||||
#region LoggerFactory and ServiceProvider
|
||||
|
||||
/// <summary>
|
||||
/// Verify that the constructor succeeds when loggerFactory is provided.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void Constructor_SucceedsWithLoggerFactory()
|
||||
{
|
||||
// Arrange
|
||||
var chatClient = new Mock<IChatClient>().Object;
|
||||
var loggerFactory = new Mock<ILoggerFactory>().Object;
|
||||
|
||||
// Act
|
||||
var agent = new HarnessAgent(chatClient, TestMaxContextWindowTokens, TestMaxOutputTokens, CreateAllDisabledOptions(), loggerFactory);
|
||||
|
||||
// Assert
|
||||
Assert.NotNull(agent);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verify that the constructor succeeds when serviceProvider is provided.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void Constructor_SucceedsWithServiceProvider()
|
||||
{
|
||||
// Arrange
|
||||
var chatClient = new Mock<IChatClient>().Object;
|
||||
var services = new Mock<IServiceProvider>().Object;
|
||||
|
||||
// Act
|
||||
var agent = new HarnessAgent(chatClient, TestMaxContextWindowTokens, TestMaxOutputTokens, CreateAllDisabledOptions(), services: services);
|
||||
|
||||
// Assert
|
||||
Assert.NotNull(agent);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verify that the constructor succeeds when both loggerFactory and serviceProvider are provided.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void Constructor_SucceedsWithLoggerFactoryAndServiceProvider()
|
||||
{
|
||||
// Arrange
|
||||
var chatClient = new Mock<IChatClient>().Object;
|
||||
var loggerFactory = new Mock<ILoggerFactory>().Object;
|
||||
var services = new Mock<IServiceProvider>().Object;
|
||||
|
||||
// Act
|
||||
var agent = new HarnessAgent(chatClient, TestMaxContextWindowTokens, TestMaxOutputTokens, CreateAllDisabledOptions(), loggerFactory, services);
|
||||
|
||||
// Assert
|
||||
Assert.NotNull(agent);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verify that AsHarnessAgent extension method accepts loggerFactory and serviceProvider.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void AsHarnessAgent_SucceedsWithLoggerFactoryAndServiceProvider()
|
||||
{
|
||||
// Arrange
|
||||
var chatClient = new Mock<IChatClient>().Object;
|
||||
var loggerFactory = new Mock<ILoggerFactory>().Object;
|
||||
var services = new Mock<IServiceProvider>().Object;
|
||||
|
||||
// Act
|
||||
var agent = chatClient.AsHarnessAgent(TestMaxContextWindowTokens, TestMaxOutputTokens, CreateAllDisabledOptions(), loggerFactory, services);
|
||||
|
||||
// Assert
|
||||
Assert.NotNull(agent);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verify that ILoggerFactory is threaded to downstream components by confirming CreateLogger is called.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void Constructor_LoggerFactoryIsUsedByDownstreamComponents()
|
||||
{
|
||||
// Arrange
|
||||
var chatClient = new Mock<IChatClient>().Object;
|
||||
var mockLoggerFactory = new Mock<ILoggerFactory>();
|
||||
mockLoggerFactory
|
||||
.Setup(lf => lf.CreateLogger(It.IsAny<string>()))
|
||||
.Returns(new Mock<ILogger>().Object);
|
||||
|
||||
// Act — use options that leave CompactionProvider and AgentSkillsProvider enabled
|
||||
var options = new HarnessAgentOptions
|
||||
{
|
||||
DisableToolApproval = true,
|
||||
DisableOpenTelemetry = true,
|
||||
DisableFileMemory = true,
|
||||
DisableFileAccess = true,
|
||||
DisableWebSearch = true,
|
||||
DisableTodoProvider = true,
|
||||
DisableAgentModeProvider = true,
|
||||
};
|
||||
var agent = new HarnessAgent(chatClient, TestMaxContextWindowTokens, TestMaxOutputTokens, options, mockLoggerFactory.Object);
|
||||
|
||||
// Assert — CreateLogger should have been called by one or more downstream components
|
||||
Assert.NotNull(agent);
|
||||
mockLoggerFactory.Verify(lf => lf.CreateLogger(It.IsAny<string>()), Times.AtLeastOnce());
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verify that IServiceProvider is propagated through the agent pipeline by confirming
|
||||
/// it is queried during agent construction.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void Constructor_ServiceProviderIsQueriedDuringBuild()
|
||||
{
|
||||
// Arrange
|
||||
var chatClient = new Mock<IChatClient>().Object;
|
||||
var mockServices = new Mock<IServiceProvider>();
|
||||
mockServices
|
||||
.Setup(sp => sp.GetService(It.IsAny<Type>()))
|
||||
.Returns(null!);
|
||||
|
||||
// Act
|
||||
var agent = new HarnessAgent(chatClient, TestMaxContextWindowTokens, TestMaxOutputTokens, CreateAllDisabledOptions(), services: mockServices.Object);
|
||||
|
||||
// Assert — the service provider should have been queried during pipeline construction
|
||||
Assert.NotNull(agent);
|
||||
mockServices.Verify(sp => sp.GetService(It.IsAny<Type>()), Times.AtLeastOnce());
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"profiles": {
|
||||
"Microsoft.Agents.AI.Hosting.A2A.UnitTests": {
|
||||
"commandName": "Project",
|
||||
"launchBrowser": true,
|
||||
"environmentVariables": {
|
||||
"ASPNETCORE_ENVIRONMENT": "Development"
|
||||
},
|
||||
"applicationUrl": "https://localhost:52186;http://localhost:52187"
|
||||
}
|
||||
}
|
||||
}
|
||||
+12
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"profiles": {
|
||||
"Microsoft.Agents.AI.Hosting.OpenAI.UnitTests": {
|
||||
"commandName": "Project",
|
||||
"launchBrowser": true,
|
||||
"environmentVariables": {
|
||||
"ASPNETCORE_ENVIRONMENT": "Development"
|
||||
},
|
||||
"applicationUrl": "https://localhost:60491;http://localhost:60492"
|
||||
}
|
||||
}
|
||||
}
|
||||
-354
@@ -2,7 +2,6 @@
|
||||
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Reflection;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Agents.AI.Workflows.Declarative.Events;
|
||||
@@ -12,9 +11,7 @@ using Microsoft.Agents.AI.Workflows.Declarative.ObjectModel;
|
||||
using Microsoft.Agents.AI.Workflows.Declarative.PowerFx;
|
||||
using Microsoft.Agents.ObjectModel;
|
||||
using Microsoft.Extensions.AI;
|
||||
using Microsoft.PowerFx.Types;
|
||||
using Moq;
|
||||
using ApprovalSnapshot = Microsoft.Agents.AI.Workflows.Declarative.ObjectModel.InvokeMcpToolExecutor.ApprovalSnapshot;
|
||||
|
||||
namespace Microsoft.Agents.AI.Workflows.Declarative.UnitTests.ObjectModel;
|
||||
|
||||
@@ -845,313 +842,6 @@ public sealed class InvokeMcpToolExecutorTest(ITestOutputHelper output) : Workfl
|
||||
|
||||
#endregion
|
||||
|
||||
#region Approval Snapshot Security Tests
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that mutating the tool name variable after approval does not change
|
||||
/// which tool is actually invoked. The originally-approved tool name must be used.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task InvokeMcpToolCaptureResponseUsesApprovedToolNameNotMutatedAsync()
|
||||
{
|
||||
// Arrange
|
||||
const string ApprovedToolName = "safe_readonly_query";
|
||||
const string MutatedToolName = "dangerous_admin_tool";
|
||||
|
||||
this.State.Set("TargetTool", FormulaValue.New(ApprovedToolName));
|
||||
this.State.InitializeSystem();
|
||||
this.State.Bind();
|
||||
|
||||
InvokeMcpTool model = this.CreateModelWithVariableToolName(
|
||||
displayName: nameof(InvokeMcpToolCaptureResponseUsesApprovedToolNameNotMutatedAsync),
|
||||
serverUrl: TestServerUrl,
|
||||
variableName: "TargetTool");
|
||||
|
||||
string? capturedToolName = null;
|
||||
Mock<IMcpToolHandler> mockProvider = new();
|
||||
mockProvider.Setup(provider => provider.InvokeToolAsync(
|
||||
It.IsAny<string>(),
|
||||
It.IsAny<string?>(),
|
||||
It.IsAny<string>(),
|
||||
It.IsAny<IDictionary<string, object?>?>(),
|
||||
It.IsAny<IDictionary<string, string>?>(),
|
||||
It.IsAny<string?>(),
|
||||
It.IsAny<CancellationToken>()))
|
||||
.Callback<string, string?, string, IDictionary<string, object?>?, IDictionary<string, string>?, string?, CancellationToken>(
|
||||
(_, _, toolName, _, _, _, _) => capturedToolName = toolName)
|
||||
.ReturnsAsync(new McpServerToolResultContent("capture-call-id")
|
||||
{
|
||||
Outputs = [new TextContent("result")]
|
||||
});
|
||||
MockAgentProvider mockAgentProvider = new();
|
||||
InvokeMcpToolExecutor action = new(model, mockProvider.Object, mockAgentProvider.Object, this.State);
|
||||
|
||||
// Act - trigger ExecuteAsync to store the approval snapshot
|
||||
Mock<IWorkflowContext> mockContext = CreateMockWorkflowContext();
|
||||
await action.HandleAsync(new ActionExecutorResult(action.Id), mockContext.Object, CancellationToken.None);
|
||||
|
||||
// Simulate parallel branch mutating state during the approval window
|
||||
this.State.Set("TargetTool", FormulaValue.New(MutatedToolName));
|
||||
this.State.Bind();
|
||||
|
||||
// User clicks approve (they saw "safe_readonly_query" in the approval UI)
|
||||
McpServerToolCallContent toolCall = new(action.Id, ApprovedToolName, TestServerUrl);
|
||||
ToolApprovalRequestContent approvalRequest = new(action.Id, toolCall);
|
||||
ToolApprovalResponseContent approvalResponse = approvalRequest.CreateResponse(approved: true);
|
||||
ExternalInputResponse response = new(new ChatMessage(ChatRole.User, [approvalResponse]));
|
||||
|
||||
// Resume after approval
|
||||
await action.CaptureResponseAsync(mockContext.Object, response, CancellationToken.None);
|
||||
|
||||
// Assert - the originally-approved tool name must be used, not the mutated one
|
||||
Assert.NotNull(capturedToolName);
|
||||
Assert.Equal(ApprovedToolName, capturedToolName);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that mutating an argument variable after approval does not change
|
||||
/// the arguments actually passed to the MCP tool. The originally-approved arguments must be used.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task InvokeMcpToolCaptureResponseUsesApprovedArgumentsNotMutatedAsync()
|
||||
{
|
||||
// Arrange
|
||||
const string ApprovedQuery = "SELECT * FROM users LIMIT 10";
|
||||
const string MutatedQuery = "DROP TABLE users CASCADE; --";
|
||||
|
||||
this.State.Set("SqlQuery", FormulaValue.New(ApprovedQuery));
|
||||
this.State.InitializeSystem();
|
||||
this.State.Bind();
|
||||
|
||||
InvokeMcpTool model = this.CreateModelWithVariableArgument(
|
||||
displayName: nameof(InvokeMcpToolCaptureResponseUsesApprovedArgumentsNotMutatedAsync),
|
||||
serverUrl: TestServerUrl,
|
||||
toolName: TestToolName,
|
||||
argumentKey: "query",
|
||||
variableName: "SqlQuery");
|
||||
|
||||
IDictionary<string, object?>? capturedArguments = null;
|
||||
Mock<IMcpToolHandler> mockProvider = new();
|
||||
mockProvider.Setup(provider => provider.InvokeToolAsync(
|
||||
It.IsAny<string>(),
|
||||
It.IsAny<string?>(),
|
||||
It.IsAny<string>(),
|
||||
It.IsAny<IDictionary<string, object?>?>(),
|
||||
It.IsAny<IDictionary<string, string>?>(),
|
||||
It.IsAny<string?>(),
|
||||
It.IsAny<CancellationToken>()))
|
||||
.Callback<string, string?, string, IDictionary<string, object?>?, IDictionary<string, string>?, string?, CancellationToken>(
|
||||
(_, _, _, arguments, _, _, _) => capturedArguments = arguments)
|
||||
.ReturnsAsync(new McpServerToolResultContent("capture-call-id")
|
||||
{
|
||||
Outputs = [new TextContent("result")]
|
||||
});
|
||||
MockAgentProvider mockAgentProvider = new();
|
||||
InvokeMcpToolExecutor action = new(model, mockProvider.Object, mockAgentProvider.Object, this.State);
|
||||
|
||||
// Act - trigger ExecuteAsync to store the approval snapshot
|
||||
Mock<IWorkflowContext> mockContext = CreateMockWorkflowContext();
|
||||
await action.HandleAsync(new ActionExecutorResult(action.Id), mockContext.Object, CancellationToken.None);
|
||||
|
||||
// Simulate parallel branch mutating state during the approval window
|
||||
this.State.Set("SqlQuery", FormulaValue.New(MutatedQuery));
|
||||
this.State.Bind();
|
||||
|
||||
// User clicks approve
|
||||
McpServerToolCallContent toolCall = new(action.Id, TestToolName, TestServerUrl);
|
||||
ToolApprovalRequestContent approvalRequest = new(action.Id, toolCall);
|
||||
ToolApprovalResponseContent approvalResponse = approvalRequest.CreateResponse(approved: true);
|
||||
ExternalInputResponse response = new(new ChatMessage(ChatRole.User, [approvalResponse]));
|
||||
|
||||
// Resume after approval
|
||||
await action.CaptureResponseAsync(mockContext.Object, response, CancellationToken.None);
|
||||
|
||||
// Assert - the originally-approved argument must be used, not the mutated one
|
||||
Assert.NotNull(capturedArguments);
|
||||
Assert.Equal(ApprovedQuery, capturedArguments["query"]?.ToString());
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that mutating the server URL variable after approval does not redirect
|
||||
/// the MCP tool call to a different server. The originally-approved server URL must be used.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task InvokeMcpToolCaptureResponseUsesApprovedServerUrlNotMutatedAsync()
|
||||
{
|
||||
// Arrange
|
||||
const string ApprovedServerUrl = "https://internal-mcp.corp";
|
||||
const string MutatedServerUrl = "https://attacker.evil/steal";
|
||||
|
||||
this.State.Set("McpEndpoint", FormulaValue.New(ApprovedServerUrl));
|
||||
this.State.InitializeSystem();
|
||||
this.State.Bind();
|
||||
|
||||
InvokeMcpTool model = this.CreateModelWithVariableServerUrl(
|
||||
displayName: nameof(InvokeMcpToolCaptureResponseUsesApprovedServerUrlNotMutatedAsync),
|
||||
variableName: "McpEndpoint",
|
||||
toolName: TestToolName);
|
||||
|
||||
string? capturedServerUrl = null;
|
||||
Mock<IMcpToolHandler> mockProvider = new();
|
||||
mockProvider.Setup(provider => provider.InvokeToolAsync(
|
||||
It.IsAny<string>(),
|
||||
It.IsAny<string?>(),
|
||||
It.IsAny<string>(),
|
||||
It.IsAny<IDictionary<string, object?>?>(),
|
||||
It.IsAny<IDictionary<string, string>?>(),
|
||||
It.IsAny<string?>(),
|
||||
It.IsAny<CancellationToken>()))
|
||||
.Callback<string, string?, string, IDictionary<string, object?>?, IDictionary<string, string>?, string?, CancellationToken>(
|
||||
(serverUrl, _, _, _, _, _, _) => capturedServerUrl = serverUrl)
|
||||
.ReturnsAsync(new McpServerToolResultContent("capture-call-id")
|
||||
{
|
||||
Outputs = [new TextContent("result")]
|
||||
});
|
||||
MockAgentProvider mockAgentProvider = new();
|
||||
InvokeMcpToolExecutor action = new(model, mockProvider.Object, mockAgentProvider.Object, this.State);
|
||||
|
||||
// Act - trigger ExecuteAsync to store the approval snapshot
|
||||
Mock<IWorkflowContext> mockContext = CreateMockWorkflowContext();
|
||||
await action.HandleAsync(new ActionExecutorResult(action.Id), mockContext.Object, CancellationToken.None);
|
||||
|
||||
// Simulate parallel branch mutating state during the approval window
|
||||
this.State.Set("McpEndpoint", FormulaValue.New(MutatedServerUrl));
|
||||
this.State.Bind();
|
||||
|
||||
// User clicks approve
|
||||
McpServerToolCallContent toolCall = new(action.Id, TestToolName, ApprovedServerUrl);
|
||||
ToolApprovalRequestContent approvalRequest = new(action.Id, toolCall);
|
||||
ToolApprovalResponseContent approvalResponse = approvalRequest.CreateResponse(approved: true);
|
||||
ExternalInputResponse response = new(new ChatMessage(ChatRole.User, [approvalResponse]));
|
||||
|
||||
// Resume after approval
|
||||
await action.CaptureResponseAsync(mockContext.Object, response, CancellationToken.None);
|
||||
|
||||
// Assert - the originally-approved server URL must be used, not the mutated one
|
||||
Assert.NotNull(capturedServerUrl);
|
||||
Assert.Equal(ApprovedServerUrl, capturedServerUrl);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that the approval snapshot survives a checkpoint/restore cycle.
|
||||
/// After restore, the originally-approved tool name must still be used even if state was mutated.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task InvokeMcpToolCaptureResponseUsesSnapshotAfterCheckpointRestoreAsync()
|
||||
{
|
||||
// Arrange
|
||||
const string ApprovedToolName = "safe_readonly_query";
|
||||
const string MutatedToolName = "dangerous_admin_tool";
|
||||
|
||||
this.State.Set("TargetTool", FormulaValue.New(ApprovedToolName));
|
||||
this.State.InitializeSystem();
|
||||
this.State.Bind();
|
||||
|
||||
InvokeMcpTool model = this.CreateModelWithVariableToolName(
|
||||
displayName: nameof(InvokeMcpToolCaptureResponseUsesSnapshotAfterCheckpointRestoreAsync),
|
||||
serverUrl: TestServerUrl,
|
||||
variableName: "TargetTool");
|
||||
|
||||
string? capturedToolName = null;
|
||||
Mock<IMcpToolHandler> mockProvider = new();
|
||||
mockProvider.Setup(provider => provider.InvokeToolAsync(
|
||||
It.IsAny<string>(),
|
||||
It.IsAny<string?>(),
|
||||
It.IsAny<string>(),
|
||||
It.IsAny<IDictionary<string, object?>?>(),
|
||||
It.IsAny<IDictionary<string, string>?>(),
|
||||
It.IsAny<string?>(),
|
||||
It.IsAny<CancellationToken>()))
|
||||
.Callback<string, string?, string, IDictionary<string, object?>?, IDictionary<string, string>?, string?, CancellationToken>(
|
||||
(_, _, toolName, _, _, _, _) => capturedToolName = toolName)
|
||||
.ReturnsAsync(new McpServerToolResultContent("capture-call-id")
|
||||
{
|
||||
Outputs = [new TextContent("result")]
|
||||
});
|
||||
MockAgentProvider mockAgentProvider = new();
|
||||
InvokeMcpToolExecutor action = new(model, mockProvider.Object, mockAgentProvider.Object, this.State);
|
||||
|
||||
// Act - trigger ExecuteAsync to store the approval snapshot
|
||||
Mock<IWorkflowContext> mockContext = CreateMockWorkflowContextWithStateStore();
|
||||
await action.HandleAsync(new ActionExecutorResult(action.Id), mockContext.Object, CancellationToken.None);
|
||||
|
||||
// Simulate checkpoint: persist to state store
|
||||
await InvokeProtectedMethodAsync(action, "OnCheckpointingAsync", mockContext.Object, CancellationToken.None);
|
||||
|
||||
// Simulate restore on a "new" executor instance by clearing the in-memory field via reflection
|
||||
// (In production, a new executor instance would be created with _approvalSnapshot == null)
|
||||
typeof(InvokeMcpToolExecutor)
|
||||
.GetField("_approvalSnapshot", BindingFlags.NonPublic | BindingFlags.Instance)!
|
||||
.SetValue(action, null);
|
||||
|
||||
// Restore from state store
|
||||
await InvokeProtectedMethodAsync(action, "OnCheckpointRestoredAsync", mockContext.Object, CancellationToken.None);
|
||||
|
||||
// Mutate state after restore (simulating parallel branch)
|
||||
this.State.Set("TargetTool", FormulaValue.New(MutatedToolName));
|
||||
this.State.Bind();
|
||||
|
||||
// User clicks approve
|
||||
McpServerToolCallContent toolCall = new(action.Id, ApprovedToolName, TestServerUrl);
|
||||
ToolApprovalRequestContent approvalRequest = new(action.Id, toolCall);
|
||||
ToolApprovalResponseContent approvalResponse = approvalRequest.CreateResponse(approved: true);
|
||||
ExternalInputResponse response = new(new ChatMessage(ChatRole.User, [approvalResponse]));
|
||||
|
||||
// Resume after approval
|
||||
await action.CaptureResponseAsync(mockContext.Object, response, CancellationToken.None);
|
||||
|
||||
// Assert - the originally-approved tool name must be used, not the mutated one
|
||||
Assert.NotNull(capturedToolName);
|
||||
Assert.Equal(ApprovedToolName, capturedToolName);
|
||||
}
|
||||
|
||||
private static Mock<IWorkflowContext> CreateMockWorkflowContext()
|
||||
{
|
||||
Mock<IWorkflowContext> mockContext = new();
|
||||
mockContext.Setup(c => c.AddEventAsync(It.IsAny<WorkflowEvent>(), It.IsAny<CancellationToken>()))
|
||||
.Returns(default(ValueTask));
|
||||
mockContext.Setup(c => c.QueueStateUpdateAsync(It.IsAny<string>(), It.IsAny<object?>(), It.IsAny<string?>(), It.IsAny<CancellationToken>()))
|
||||
.Returns(default(ValueTask));
|
||||
mockContext.Setup(c => c.SendMessageAsync(It.IsAny<object>(), It.IsAny<string?>(), It.IsAny<CancellationToken>()))
|
||||
.Returns(default(ValueTask));
|
||||
return mockContext;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a mock workflow context that actually stores state values (for checkpoint/restore tests).
|
||||
/// </summary>
|
||||
private static Mock<IWorkflowContext> CreateMockWorkflowContextWithStateStore()
|
||||
{
|
||||
Dictionary<string, object?> stateStore = new();
|
||||
Mock<IWorkflowContext> mockContext = new();
|
||||
mockContext.Setup(c => c.AddEventAsync(It.IsAny<WorkflowEvent>(), It.IsAny<CancellationToken>()))
|
||||
.Returns(default(ValueTask));
|
||||
mockContext.Setup(c => c.QueueStateUpdateAsync(It.IsAny<string>(), It.IsAny<ApprovalSnapshot?>(), It.IsAny<string?>(), It.IsAny<CancellationToken>()))
|
||||
.Callback<string, ApprovalSnapshot?, string?, CancellationToken>((key, value, _, _) => stateStore[key] = value)
|
||||
.Returns(default(ValueTask));
|
||||
mockContext.Setup(c => c.SendMessageAsync(It.IsAny<object>(), It.IsAny<string?>(), It.IsAny<CancellationToken>()))
|
||||
.Returns(default(ValueTask));
|
||||
mockContext.Setup(c => c.ReadStateAsync<ApprovalSnapshot>(It.IsAny<string>(), It.IsAny<string?>(), It.IsAny<CancellationToken>()))
|
||||
.Returns<string, string?, CancellationToken>((key, _, _) =>
|
||||
new ValueTask<ApprovalSnapshot?>(stateStore.TryGetValue(key, out object? val) ? val as ApprovalSnapshot : null));
|
||||
mockContext.Setup(c => c.ReadStateKeysAsync(It.IsAny<string?>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(new HashSet<string>());
|
||||
return mockContext;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Invokes a protected method on an executor via reflection (for testing checkpoint hooks).
|
||||
/// </summary>
|
||||
private static async ValueTask InvokeProtectedMethodAsync(InvokeMcpToolExecutor action, string methodName, IWorkflowContext context, CancellationToken cancellationToken)
|
||||
{
|
||||
MethodInfo method = typeof(InvokeMcpToolExecutor)
|
||||
.GetMethod(methodName, BindingFlags.NonPublic | BindingFlags.Instance)!;
|
||||
ValueTask result = (ValueTask)method.Invoke(action, [context, cancellationToken])!;
|
||||
await result.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region CompleteAsync Tests
|
||||
|
||||
[Fact]
|
||||
@@ -1261,50 +951,6 @@ public sealed class InvokeMcpToolExecutorTest(ITestOutputHelper output) : Workfl
|
||||
return AssignParent<InvokeMcpTool>(builder);
|
||||
}
|
||||
|
||||
private InvokeMcpTool CreateModelWithVariableToolName(string displayName, string serverUrl, string variableName)
|
||||
{
|
||||
InvokeMcpTool.Builder builder = new()
|
||||
{
|
||||
Id = this.CreateActionId(),
|
||||
DisplayName = this.FormatDisplayName(displayName),
|
||||
ServerUrl = new StringExpression.Builder(StringExpression.Literal(serverUrl)),
|
||||
ToolName = new StringExpression.Builder(
|
||||
StringExpression.Variable(PropertyPath.TopicVariable(variableName))),
|
||||
RequireApproval = new BoolExpression.Builder(BoolExpression.Literal(true)),
|
||||
};
|
||||
return AssignParent<InvokeMcpTool>(builder);
|
||||
}
|
||||
|
||||
private InvokeMcpTool CreateModelWithVariableArgument(
|
||||
string displayName, string serverUrl, string toolName, string argumentKey, string variableName)
|
||||
{
|
||||
InvokeMcpTool.Builder builder = new()
|
||||
{
|
||||
Id = this.CreateActionId(),
|
||||
DisplayName = this.FormatDisplayName(displayName),
|
||||
ServerUrl = new StringExpression.Builder(StringExpression.Literal(serverUrl)),
|
||||
ToolName = new StringExpression.Builder(StringExpression.Literal(toolName)),
|
||||
RequireApproval = new BoolExpression.Builder(BoolExpression.Literal(true)),
|
||||
};
|
||||
builder.Arguments.Add(argumentKey,
|
||||
ValueExpression.Variable(PropertyPath.TopicVariable(variableName)));
|
||||
return AssignParent<InvokeMcpTool>(builder);
|
||||
}
|
||||
|
||||
private InvokeMcpTool CreateModelWithVariableServerUrl(string displayName, string variableName, string toolName)
|
||||
{
|
||||
InvokeMcpTool.Builder builder = new()
|
||||
{
|
||||
Id = this.CreateActionId(),
|
||||
DisplayName = this.FormatDisplayName(displayName),
|
||||
ServerUrl = new StringExpression.Builder(
|
||||
StringExpression.Variable(PropertyPath.TopicVariable(variableName))),
|
||||
ToolName = new StringExpression.Builder(StringExpression.Literal(toolName)),
|
||||
RequireApproval = new BoolExpression.Builder(BoolExpression.Literal(true)),
|
||||
};
|
||||
return AssignParent<InvokeMcpTool>(builder);
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Mock MCP Tool Provider
|
||||
|
||||
@@ -803,15 +803,6 @@ class RawAnthropicClient(
|
||||
}
|
||||
a_content.append(mcp_result)
|
||||
case "text_reasoning":
|
||||
if content.text is None:
|
||||
if (
|
||||
content.protected_data
|
||||
and a_content
|
||||
and a_content[-1].get("type") == "thinking"
|
||||
and "signature" not in a_content[-1]
|
||||
):
|
||||
a_content[-1]["signature"] = content.protected_data
|
||||
continue
|
||||
thinking_block: dict[str, Any] = {"type": "thinking", "thinking": content.text}
|
||||
if content.protected_data:
|
||||
thinking_block["signature"] = content.protected_data
|
||||
|
||||
@@ -485,48 +485,6 @@ def test_prepare_message_for_anthropic_text_reasoning_with_signature(
|
||||
assert result["content"][0]["signature"] == "sig_abc123"
|
||||
|
||||
|
||||
def test_prepare_message_for_anthropic_attaches_signature_only_reasoning(
|
||||
mock_anthropic_client: MagicMock,
|
||||
) -> None:
|
||||
client = create_test_anthropic_client(mock_anthropic_client)
|
||||
message = Message(
|
||||
role="assistant",
|
||||
contents=[
|
||||
Content.from_text_reasoning(text="Let me think about this..."),
|
||||
Content.from_text_reasoning(text=None, protected_data="sig_abc123"),
|
||||
],
|
||||
)
|
||||
|
||||
result = client._prepare_message_for_anthropic(message)
|
||||
|
||||
assert result["content"] == [
|
||||
{"type": "thinking", "thinking": "Let me think about this...", "signature": "sig_abc123"}
|
||||
]
|
||||
|
||||
|
||||
def test_prepare_message_for_anthropic_skips_orphan_signature_only_reasoning(
|
||||
mock_anthropic_client: MagicMock,
|
||||
) -> None:
|
||||
client = create_test_anthropic_client(mock_anthropic_client)
|
||||
message = Message(
|
||||
role="assistant",
|
||||
contents=[
|
||||
Content.from_text_reasoning(text=None, protected_data="sig_abc123"),
|
||||
Content.from_function_call(
|
||||
call_id="call_123",
|
||||
name="get_weather",
|
||||
arguments={"location": "San Francisco"},
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
result = client._prepare_message_for_anthropic(message)
|
||||
|
||||
assert len(result["content"]) == 1
|
||||
assert result["content"][0]["type"] == "tool_use"
|
||||
assert result["content"][0]["id"] == "call_123"
|
||||
|
||||
|
||||
def test_prepare_message_for_anthropic_mcp_server_tool_call(
|
||||
mock_anthropic_client: MagicMock,
|
||||
) -> None:
|
||||
|
||||
@@ -4,7 +4,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import copy
|
||||
import json
|
||||
import logging
|
||||
import sys
|
||||
@@ -37,7 +36,6 @@ from agent_framework.observability import ChatTelemetryLayer
|
||||
from boto3.session import Session as Boto3Session
|
||||
from botocore.client import BaseClient
|
||||
from botocore.config import Config as BotoConfig
|
||||
from botocore.exceptions import ClientError
|
||||
from pydantic import BaseModel
|
||||
|
||||
if sys.version_info >= (3, 13):
|
||||
@@ -117,20 +115,13 @@ class BedrockChatOptions(ChatOptions[ResponseModelT], Generic[ResponseModelT], t
|
||||
translates to ``toolConfig.tools``.
|
||||
tool_choice: How the model should use tools,
|
||||
translates to ``toolConfig.toolChoice``.
|
||||
response_format: Structured output format. Accepts a Pydantic BaseModel
|
||||
subclass or an OpenAI-style dict schema
|
||||
(``{"json_schema": {"name": ..., "schema": ...}}``).
|
||||
When provided, the Converse API request includes
|
||||
``outputConfig.textFormat`` with the schema serialized as a JSON
|
||||
string. ``ChatResponse.value`` will be populated with the parsed
|
||||
model instance. Only supported on models that support
|
||||
``outputConfig.textFormat``. Unsupported models raise a ValueError.
|
||||
|
||||
# Options not supported in Bedrock Converse API:
|
||||
seed: Not supported.
|
||||
frequency_penalty: Not supported.
|
||||
presence_penalty: Not supported.
|
||||
allow_multiple_tool_calls: Not supported (models handle parallel calls automatically).
|
||||
response_format: Not directly supported (use model-specific prompting).
|
||||
user: Not supported.
|
||||
store: Not supported.
|
||||
logit_bias: Not supported.
|
||||
@@ -170,6 +161,9 @@ class BedrockChatOptions(ChatOptions[ResponseModelT], Generic[ResponseModelT], t
|
||||
allow_multiple_tool_calls: None # type: ignore[misc]
|
||||
"""Not supported. Bedrock models handle parallel tool calls automatically."""
|
||||
|
||||
response_format: None # type: ignore[misc]
|
||||
"""Not directly supported. Use model-specific prompting for JSON output."""
|
||||
|
||||
user: None # type: ignore[misc]
|
||||
"""Not supported in Bedrock Converse API."""
|
||||
|
||||
@@ -330,28 +324,10 @@ class BedrockChatClient(
|
||||
return Boto3Session(**session_kwargs)
|
||||
|
||||
def _invoke_converse(self, request: Mapping[str, Any]) -> dict[str, Any]:
|
||||
try:
|
||||
response = self._bedrock_client.converse(**request)
|
||||
if not isinstance(response, Mapping):
|
||||
raise ChatClientInvalidResponseException("Bedrock converse response must be a mapping.")
|
||||
return response
|
||||
except ClientError as e:
|
||||
error_details = e.response.get("Error", {})
|
||||
error_code = error_details.get("Code", "")
|
||||
error_message = error_details.get("Message", "")
|
||||
# "outputConfig" in error_message catches cases where Bedrock explicitly
|
||||
# rejects the outputConfig field (unsupported model). Other ValidationExceptions
|
||||
# (e.g. malformed schema shape, invalid property values) will not mention
|
||||
# "outputConfig" and will bubble up as raw ClientError without being misdiagnosed.
|
||||
if error_code == "ValidationException" and (
|
||||
"outputconfig" in error_message.lower() or "outputconfig" in str(e).lower()
|
||||
):
|
||||
raise ValueError(
|
||||
f"Model '{self.model}' does not support structured output via outputConfig.textFormat. "
|
||||
"Check the model's Bedrock Converse outputConfig/textFormat support. "
|
||||
f"AWS error Code: {error_code}. AWS error Message: {error_message}"
|
||||
) from e
|
||||
raise
|
||||
response = self._bedrock_client.converse(**request)
|
||||
if not isinstance(response, Mapping):
|
||||
raise ChatClientInvalidResponseException("Bedrock converse response must be a mapping.")
|
||||
return response
|
||||
|
||||
@override
|
||||
def _inner_get_response(
|
||||
@@ -368,7 +344,7 @@ class BedrockChatClient(
|
||||
# Streaming mode - simulate streaming by yielding a single update
|
||||
async def _stream() -> AsyncIterable[ChatResponseUpdate]:
|
||||
response = await asyncio.to_thread(self._invoke_converse, request)
|
||||
parsed_response = self._process_converse_response(response, options)
|
||||
parsed_response = self._process_converse_response(response)
|
||||
contents = list(parsed_response.messages[0].contents if parsed_response.messages else [])
|
||||
if parsed_response.usage_details:
|
||||
contents.append(Content.from_usage(usage_details=parsed_response.usage_details)) # type: ignore[arg-type]
|
||||
@@ -384,12 +360,12 @@ class BedrockChatClient(
|
||||
raw_representation=parsed_response.raw_representation,
|
||||
)
|
||||
|
||||
return self._build_response_stream(_stream(), response_format=options.get("response_format"))
|
||||
return self._build_response_stream(_stream())
|
||||
|
||||
# Non-streaming mode
|
||||
async def _get_response() -> ChatResponse:
|
||||
raw_response = await asyncio.to_thread(self._invoke_converse, request)
|
||||
return self._process_converse_response(raw_response, options)
|
||||
return self._process_converse_response(raw_response)
|
||||
|
||||
return _get_response()
|
||||
|
||||
@@ -454,9 +430,6 @@ class BedrockChatClient(
|
||||
if tool_config:
|
||||
run_options["toolConfig"] = tool_config
|
||||
|
||||
if output_config := self._prepare_output_config(options.get("response_format")):
|
||||
run_options["outputConfig"] = output_config
|
||||
|
||||
return run_options
|
||||
|
||||
def _prepare_bedrock_messages(
|
||||
@@ -655,9 +628,7 @@ class BedrockChatClient(
|
||||
def _generate_tool_call_id() -> str:
|
||||
return f"tool-call-{uuid4().hex}"
|
||||
|
||||
def _process_converse_response(
|
||||
self, response: dict[str, Any], options: Mapping[str, Any] | None = None
|
||||
) -> ChatResponse:
|
||||
def _process_converse_response(self, response: dict[str, Any]) -> ChatResponse:
|
||||
"""Convert Bedrock Converse API response to ChatResponse."""
|
||||
output = response.get("output") or {}
|
||||
message = output.get("message") or {}
|
||||
@@ -675,7 +646,6 @@ class BedrockChatClient(
|
||||
usage_details=usage_details,
|
||||
model=model,
|
||||
finish_reason=finish_reason,
|
||||
response_format=options.get("response_format") if options else None,
|
||||
raw_representation=response,
|
||||
)
|
||||
|
||||
@@ -758,108 +728,6 @@ class BedrockChatClient(
|
||||
return None
|
||||
return FINISH_REASON_MAP.get(reason.lower())
|
||||
|
||||
def _prepare_output_config(self, response_format: Any | None) -> dict[str, Any] | None:
|
||||
"""Convert response_format into the AWS Bedrock outputConfig wire format.
|
||||
|
||||
Args:
|
||||
response_format: A Pydantic model class or a dict schema, or None.
|
||||
|
||||
Returns:
|
||||
A dict for the Converse API ``outputConfig`` parameter, or None if
|
||||
response_format is not set.
|
||||
"""
|
||||
if response_format is None:
|
||||
return None
|
||||
|
||||
if isinstance(response_format, Mapping):
|
||||
if "json_schema" in response_format:
|
||||
# Shape A — OpenAI-style wrapper
|
||||
json_schema_config = response_format["json_schema"]
|
||||
schema_src = json_schema_config.get("schema", {})
|
||||
name = json_schema_config.get("name", "output_schema")
|
||||
elif "schema" in response_format:
|
||||
# Shape B — inner shape directly {"name": ..., "schema": ...}
|
||||
schema_src = response_format["schema"]
|
||||
name = response_format.get("name", "output_schema")
|
||||
else:
|
||||
# Shape C — assume entire dict is the raw schema
|
||||
logger.warning(
|
||||
"response_format dict has no 'json_schema' or 'schema' key; "
|
||||
"treating entire dict as raw JSON schema."
|
||||
)
|
||||
schema_src = dict(response_format)
|
||||
name = "output_schema"
|
||||
|
||||
if isinstance(schema_src, str):
|
||||
schema_src = json.loads(schema_src)
|
||||
schema = copy.deepcopy(schema_src)
|
||||
else:
|
||||
if not isinstance(response_format, type) or not issubclass(response_format, BaseModel):
|
||||
raise TypeError(
|
||||
"response_format must be None, a dict JSON schema, "
|
||||
"or a Pydantic BaseModel subclass."
|
||||
)
|
||||
# response_format is a Pydantic model class
|
||||
schema = response_format.model_json_schema()
|
||||
name = response_format.__name__
|
||||
|
||||
self._set_additional_properties_false(schema)
|
||||
|
||||
json_schema: dict[str, Any] = {
|
||||
"name": name,
|
||||
"schema": json.dumps(schema),
|
||||
}
|
||||
|
||||
description = getattr(response_format, "__doc__", None) if not isinstance(response_format, Mapping) else None
|
||||
if description and isinstance(description, str) and description.strip():
|
||||
json_schema["description"] = description.strip()
|
||||
|
||||
return {
|
||||
"textFormat": {
|
||||
"type": "json_schema",
|
||||
"structure": {
|
||||
"jsonSchema": json_schema
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
def _set_additional_properties_false(self, schema: dict[str, Any]) -> None:
|
||||
"""Recursively set additionalProperties: false on all object types in a JSON schema.
|
||||
|
||||
AWS requires strict schema enforcement. This mirrors the approach used by
|
||||
AnthropicChatClient._prepare_response_format().
|
||||
|
||||
Args:
|
||||
schema: The JSON schema dict to modify in-place.
|
||||
"""
|
||||
visited: set[int] = set()
|
||||
|
||||
def walk(node: Any) -> None:
|
||||
if isinstance(node, dict):
|
||||
node_id = id(node)
|
||||
if node_id in visited:
|
||||
return
|
||||
visited.add(node_id)
|
||||
if node.get("type") == "object" or (
|
||||
"properties" in node and "type" not in node
|
||||
):
|
||||
existing = node.get("additionalProperties")
|
||||
if existing is None or existing is True:
|
||||
node["additionalProperties"] = False
|
||||
for value in node.values():
|
||||
if isinstance(value, (dict, list)):
|
||||
walk(value)
|
||||
elif isinstance(node, list):
|
||||
node_id = id(node)
|
||||
if node_id in visited:
|
||||
return
|
||||
visited.add(node_id)
|
||||
for item in node:
|
||||
if isinstance(item, (dict, list)):
|
||||
walk(item)
|
||||
|
||||
walk(schema)
|
||||
|
||||
def service_url(self) -> str:
|
||||
"""Returns the service URL for the Bedrock runtime in the configured AWS region.
|
||||
|
||||
|
||||
@@ -1,382 +0,0 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import json
|
||||
from typing import Any
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
from agent_framework import Content, Message
|
||||
from botocore.exceptions import ClientError
|
||||
from pydantic import BaseModel
|
||||
|
||||
from agent_framework_bedrock import BedrockChatClient
|
||||
|
||||
# region Test models
|
||||
|
||||
|
||||
class WeatherReport(BaseModel):
|
||||
city: str
|
||||
temperature: float
|
||||
summary: str
|
||||
|
||||
|
||||
class NestedAddress(BaseModel):
|
||||
street: str
|
||||
city: str
|
||||
zip_code: str
|
||||
|
||||
|
||||
class Person(BaseModel):
|
||||
name: str
|
||||
age: int
|
||||
address: NestedAddress
|
||||
|
||||
|
||||
# endregion
|
||||
|
||||
|
||||
# region Helpers
|
||||
|
||||
|
||||
class _StubBedrockRuntime:
|
||||
"""Stub that records calls and returns a canned response."""
|
||||
|
||||
def __init__(self, response_text: str = "Bedrock says hi") -> None:
|
||||
self.calls: list[dict[str, Any]] = []
|
||||
self._response_text = response_text
|
||||
|
||||
def converse(self, **kwargs: Any) -> dict[str, Any]:
|
||||
self.calls.append(kwargs)
|
||||
return {
|
||||
"modelId": kwargs["modelId"],
|
||||
"responseId": "resp-structured",
|
||||
"usage": {"inputTokens": 10, "outputTokens": 20, "totalTokens": 30},
|
||||
"output": {
|
||||
"completionReason": "end_turn",
|
||||
"message": {
|
||||
"id": "msg-structured",
|
||||
"role": "assistant",
|
||||
"content": [{"text": self._response_text}],
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _make_client(response_text: str = "Bedrock says hi") -> tuple[BedrockChatClient, _StubBedrockRuntime]:
|
||||
stub = _StubBedrockRuntime(response_text)
|
||||
client = BedrockChatClient(
|
||||
model="us.anthropic.claude-haiku-4-5-v1:0",
|
||||
region="us-east-1",
|
||||
client=stub,
|
||||
)
|
||||
return client, stub
|
||||
|
||||
|
||||
def _user_messages() -> list[Message]:
|
||||
return [Message(role="user", contents=[Content.from_text(text="Give me a weather report")])]
|
||||
|
||||
|
||||
# endregion
|
||||
|
||||
|
||||
# region Tests
|
||||
|
||||
|
||||
def test_prepare_output_config_correct_wire_shape() -> None:
|
||||
"""_prepare_output_config(WeatherReport) must produce the correct
|
||||
textFormat → structure → jsonSchema shape with type: 'json_schema'."""
|
||||
client, _ = _make_client()
|
||||
|
||||
output_config = client._prepare_output_config(WeatherReport)
|
||||
|
||||
assert output_config is not None
|
||||
text_format = output_config["textFormat"]
|
||||
assert text_format["type"] == "json_schema"
|
||||
assert "structure" in text_format
|
||||
json_schema = text_format["structure"]["jsonSchema"]
|
||||
assert json_schema["name"] == "WeatherReport"
|
||||
assert "schema" in json_schema
|
||||
|
||||
|
||||
def test_prepare_output_config_schema_is_json_string() -> None:
|
||||
"""The schema value inside jsonSchema must be a JSON string, not a dict."""
|
||||
client, _ = _make_client()
|
||||
|
||||
output_config = client._prepare_output_config(WeatherReport)
|
||||
|
||||
assert output_config is not None
|
||||
schema_value = output_config["textFormat"]["structure"]["jsonSchema"]["schema"]
|
||||
assert isinstance(schema_value, str), f"Expected str, got {type(schema_value)}"
|
||||
# Verify it's valid JSON
|
||||
parsed = json.loads(schema_value)
|
||||
assert isinstance(parsed, dict)
|
||||
assert parsed["type"] == "object"
|
||||
|
||||
|
||||
def test_additional_properties_false_set_recursively() -> None:
|
||||
"""additionalProperties: false must be set on all nested object types."""
|
||||
client, _ = _make_client()
|
||||
|
||||
output_config = client._prepare_output_config(Person)
|
||||
|
||||
assert output_config is not None
|
||||
schema_str = output_config["textFormat"]["structure"]["jsonSchema"]["schema"]
|
||||
schema = json.loads(schema_str)
|
||||
|
||||
# Top-level object
|
||||
assert schema.get("additionalProperties") is False
|
||||
|
||||
# Check $defs for NestedAddress
|
||||
defs = schema.get("$defs", {})
|
||||
assert "NestedAddress" in defs, "Expected NestedAddress to be present in $defs"
|
||||
assert defs["NestedAddress"].get("additionalProperties") is False, (
|
||||
"Expected additionalProperties=False on nested NestedAddress schema"
|
||||
)
|
||||
|
||||
|
||||
def test_no_output_config_when_response_format_none() -> None:
|
||||
"""When response_format is None, no outputConfig key should appear in the request."""
|
||||
client, stub = _make_client()
|
||||
messages = _user_messages()
|
||||
|
||||
request = client._prepare_options(messages, {"max_tokens": 100})
|
||||
|
||||
assert "outputConfig" not in request, (
|
||||
f"outputConfig should not be present when response_format is None, got: {request.get('outputConfig')}"
|
||||
)
|
||||
|
||||
|
||||
async def test_chat_response_value_populated() -> None:
|
||||
"""After a mocked response with response_format, .value should be a populated Pydantic model."""
|
||||
json_response = json.dumps({"city": "Seattle", "temperature": 72.5, "summary": "Sunny and warm"})
|
||||
client, stub = _make_client(response_text=json_response)
|
||||
messages = _user_messages()
|
||||
|
||||
response = await client.get_response(
|
||||
messages=messages,
|
||||
options={"max_tokens": 100, "response_format": WeatherReport},
|
||||
)
|
||||
|
||||
assert response.text == json_response
|
||||
assert response.value is not None
|
||||
assert isinstance(response.value, WeatherReport)
|
||||
assert response.value.city == "Seattle"
|
||||
assert response.value.temperature == 72.5
|
||||
assert response.value.summary == "Sunny and warm"
|
||||
|
||||
# Verify outputConfig was sent to the API
|
||||
assert len(stub.calls) == 1
|
||||
api_request = stub.calls[0]
|
||||
assert "outputConfig" in api_request
|
||||
assert api_request["outputConfig"]["textFormat"]["type"] == "json_schema"
|
||||
|
||||
|
||||
def test_dict_schema_response_format() -> None:
|
||||
"""_prepare_output_config should work when response_format is a dict, not just a Pydantic class."""
|
||||
client, _ = _make_client()
|
||||
|
||||
dict_schema = {
|
||||
"json_schema": {
|
||||
"name": "weather_output",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"city": {"type": "string"},
|
||||
"temp": {"type": "number"},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
output_config = client._prepare_output_config(dict_schema)
|
||||
|
||||
assert output_config is not None
|
||||
json_schema = output_config["textFormat"]["structure"]["jsonSchema"]
|
||||
assert json_schema["name"] == "weather_output"
|
||||
schema_parsed = json.loads(json_schema["schema"])
|
||||
assert schema_parsed["type"] == "object"
|
||||
assert "city" in schema_parsed["properties"]
|
||||
|
||||
|
||||
def test_prepare_output_config_none_returns_none() -> None:
|
||||
"""_prepare_output_config(None) must return None."""
|
||||
client, _ = _make_client()
|
||||
|
||||
result = client._prepare_output_config(None)
|
||||
|
||||
assert result is None
|
||||
|
||||
|
||||
async def test_chat_response_value_populated_streaming() -> None:
|
||||
"""In streaming mode, .value should also be populated on the final response."""
|
||||
json_response = json.dumps({"city": "Portland", "temperature": 68.0, "summary": "Cloudy"})
|
||||
client, stub = _make_client(response_text=json_response)
|
||||
messages = _user_messages()
|
||||
|
||||
stream = client.get_response(
|
||||
messages=messages,
|
||||
stream=True,
|
||||
options={"max_tokens": 100, "response_format": WeatherReport},
|
||||
)
|
||||
|
||||
# Consume stream and get final response
|
||||
async for _ in stream:
|
||||
pass
|
||||
response = await stream.get_final_response()
|
||||
|
||||
assert response.value is not None
|
||||
assert isinstance(response.value, WeatherReport)
|
||||
assert response.value.city == "Portland"
|
||||
|
||||
# Verify outputConfig was sent
|
||||
assert len(stub.calls) == 1
|
||||
assert "outputConfig" in stub.calls[0]
|
||||
|
||||
|
||||
async def test_unsupported_model_validation_exception() -> None:
|
||||
"""When a model doesn't support outputConfig, a clear error should be raised."""
|
||||
class _FailingStubBedrockRuntime:
|
||||
def converse(self, **kwargs: Any) -> dict[str, Any]:
|
||||
# Simulate botocore ClientError for ValidationException
|
||||
error_response = {"Error": {"Code": "ValidationException", "Message": "Invalid field outputConfig"}}
|
||||
raise ClientError(error_response, "Converse")
|
||||
|
||||
client = BedrockChatClient(
|
||||
model="us.anthropic.claude-v2",
|
||||
region="us-east-1",
|
||||
client=_FailingStubBedrockRuntime(),
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError) as exc:
|
||||
await client.get_response(
|
||||
messages=_user_messages(),
|
||||
options={"response_format": WeatherReport},
|
||||
)
|
||||
|
||||
assert "does not support structured output via outputConfig.textFormat" in str(exc.value)
|
||||
assert "Check the model's Bedrock Converse outputConfig/textFormat support." in str(exc.value)
|
||||
|
||||
|
||||
def test_invalid_response_format_type_raises() -> None:
|
||||
"""Non-dict, non-BaseModel response_format should raise TypeError."""
|
||||
client, _ = _make_client()
|
||||
with pytest.raises(TypeError, match="Pydantic BaseModel subclass"):
|
||||
client._prepare_output_config("not_a_valid_format")
|
||||
|
||||
|
||||
def test_mapping_response_format_accepted() -> None:
|
||||
"""A non-dict Mapping response_format must be accepted and produce
|
||||
correct outputConfig, not raise TypeError."""
|
||||
from collections.abc import MutableMapping
|
||||
|
||||
class _WrappedMapping(MutableMapping):
|
||||
def __init__(self, data):
|
||||
self._data = dict(data)
|
||||
|
||||
def __getitem__(self, key):
|
||||
return self._data[key]
|
||||
|
||||
def __setitem__(self, key, value):
|
||||
self._data[key] = value
|
||||
|
||||
def __delitem__(self, key):
|
||||
del self._data[key]
|
||||
|
||||
def __iter__(self):
|
||||
return iter(self._data)
|
||||
|
||||
def __len__(self):
|
||||
return len(self._data)
|
||||
|
||||
client, _ = _make_client()
|
||||
mapping_format = _WrappedMapping({
|
||||
"json_schema": {
|
||||
"name": "test_output",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"properties": {"result": {"type": "string"}},
|
||||
},
|
||||
}
|
||||
})
|
||||
|
||||
output_config = client._prepare_output_config(mapping_format)
|
||||
|
||||
assert output_config is not None
|
||||
json_schema = output_config["textFormat"]["structure"]["jsonSchema"]
|
||||
assert json_schema["name"] == "test_output"
|
||||
schema = json.loads(json_schema["schema"])
|
||||
assert schema.get("additionalProperties") is False
|
||||
|
||||
|
||||
def test_shape_b_dict_schema_wire_format() -> None:
|
||||
"""Dict response_format in Shape B (inner shape directly) should
|
||||
produce correct outputConfig."""
|
||||
client, _ = _make_client()
|
||||
|
||||
response_format = {
|
||||
"name": "weather_output",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"city": {"type": "string"},
|
||||
"temperature": {"type": "number"},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
output_config = client._prepare_output_config(response_format)
|
||||
|
||||
assert output_config is not None
|
||||
text_format = output_config["textFormat"]
|
||||
assert text_format["type"] == "json_schema"
|
||||
json_schema = text_format["structure"]["jsonSchema"]
|
||||
assert json_schema["name"] == "weather_output"
|
||||
schema = json.loads(json_schema["schema"])
|
||||
assert schema.get("additionalProperties") is False
|
||||
|
||||
|
||||
def test_dict_schema_not_mutated() -> None:
|
||||
"""Caller's dict schema must not be mutated by _prepare_output_config."""
|
||||
client, _ = _make_client()
|
||||
original_schema = {
|
||||
"json_schema": {
|
||||
"name": "test",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"properties": {"a": {"type": "string"}},
|
||||
},
|
||||
}
|
||||
}
|
||||
snapshot = copy.deepcopy(original_schema)
|
||||
client._prepare_output_config(original_schema)
|
||||
assert original_schema == snapshot, "Original dict schema was mutated"
|
||||
|
||||
|
||||
async def test_non_outputconfig_validation_exception_propagates() -> None:
|
||||
"""ValidationException unrelated to outputConfig must propagate
|
||||
as raw ClientError, not be caught and reclassified."""
|
||||
client, _ = _make_client()
|
||||
error_response = {
|
||||
"Error": {
|
||||
"Code": "ValidationException",
|
||||
"Message": "Invalid message format",
|
||||
}
|
||||
}
|
||||
with (
|
||||
patch.object(
|
||||
client,
|
||||
"_bedrock_client",
|
||||
**{"converse.side_effect": ClientError(error_response, "Converse")},
|
||||
),
|
||||
pytest.raises(ClientError),
|
||||
):
|
||||
await client.get_response(
|
||||
messages=_user_messages(),
|
||||
options={"max_tokens": 100},
|
||||
)
|
||||
|
||||
|
||||
# endregion
|
||||
@@ -56,7 +56,7 @@ agent_framework/
|
||||
- **`AgentMiddleware`** - Intercepts agent `run()` calls
|
||||
- **`ChatMiddleware`** - Intercepts chat client `get_response()` calls
|
||||
- **`FunctionMiddleware`** - Intercepts function/tool invocations
|
||||
- **`AgentContext`** / **`ChatContext`** / **`FunctionInvocationContext`** - Context objects passed through middleware. A tool can declare a `FunctionInvocationContext` parameter to receive it; `context.tools` is the live, mutable tools list for the run, and `context.add_tools(...)` / `context.remove_tools(...)` enable progressive tool exposure (changes apply on the next function-calling iteration).
|
||||
- **`AgentContext`** / **`ChatContext`** / **`FunctionInvocationContext`** - Context objects passed through middleware
|
||||
|
||||
### Sessions (`_sessions.py`)
|
||||
|
||||
@@ -76,19 +76,6 @@ agent_framework/
|
||||
- **`SkillScriptRunner`** - Protocol for file-based script execution. Any callable matching `(skill, script, args) -> Any` satisfies it. Code-defined scripts do not use a runner.
|
||||
- **`SkillsProvider`** - Context provider (extends `ContextProvider`) that discovers file-based skills from `SKILL.md` files and/or accepts code-defined `Skill` instances. Follows progressive disclosure: advertise → load → read resources / run scripts.
|
||||
|
||||
### Model Context Protocol (`_mcp.py`)
|
||||
|
||||
- **`MCPTool`** - Base wrapper that owns the MCP `ClientSession` and exposes the remote server's tools as `FunctionTool`s.
|
||||
- **`MCPStdioTool`** / **`MCPStreamableHTTPTool`** / **`MCPWebsocketTool`** - Transport-specific subclasses.
|
||||
- **`MCPTaskOptions`** (experimental, `MCP_LONG_RUNNING_TASKS` feature, **frozen**) - Per-tool-instance options controlling the SEP-2663 long-running task lifecycle. When the server advertises a tool with `execution.taskSupport == "required"`, `MCPTool.call_tool` transparently routes through `call_tool_as_task`, which sends an augmented `tools/call`, polls `tasks/get` until terminal, and reinterprets `tasks/result` as a normal `CallToolResult`. Instances are immutable; replace via `MCPTool.task_options = MCPTaskOptions(...)`. Fields:
|
||||
- `default_ttl: timedelta | None` — forwarded to the server as `params.task.ttl` (milliseconds). When `None`, the server's default applies.
|
||||
- `cancel_remote_task_on_local_cancellation: bool = True` — only gates the `CancelledError` path. Abandonment paths (see below) always cancel.
|
||||
- `max_task_wait: timedelta | None` — client-side deadline for the whole post-create lifecycle (poll + result fetch). When exceeded, raises `ToolExecutionException` and fires a best-effort `tasks/cancel`. `None` (default) means no client-side bound. Bounds sleeps, sends, AND reconnects via `asyncio.wait_for`.
|
||||
- **Permissive fallback**: servers that ignore the augmentation (return `CallToolResult` directly) or reject the unknown `task` field with `METHOD_NOT_FOUND` / `INVALID_PARAMS` fall back to the plain `session.call_tool(...)` path so legacy servers keep working. An unparseable success response (server accepted the augmented call but returned a payload that is neither `CreateTaskResult` nor `CallToolResult`) **does not** fall back — it raises `ToolExecutionException` to avoid double-executing a side-effecting tool.
|
||||
- **Submit-vs-track reconnect policy**: a dropped connection before a `task_id` is known raises `ToolExecutionException("connection lost; task state unknown")` without re-issuing the augmented `tools/call`, so a server that accepted the request but lost the response cannot be made to start the same operation twice; once a `task_id` exists, `tasks/get` / `tasks/result` reconnect once and retry against the same id (a shared `_send_with_one_reconnect` helper).
|
||||
- **Cancel-on-abandonment vs terminal failure**: any path where the remote task may still be running (max-wait exceeded, hard `McpError` in poll, malformed `tasks/get`, second connection loss in poll/fetch, reconnect failure) fires best-effort `tasks/cancel` before raising. Terminal failures (`failed`/`cancelled`/`input_required` server-side, `completed+isError`, malformed `tasks/result` after server completed) do **not** cancel — the server is already done. `_MCPTaskAbandoned` is the private marker distinguishing the two.
|
||||
- **Transient poll retry**: a slow `tasks/get` that surfaces as `McpError(code=408 REQUEST_TIMEOUT)` is retried (bounded by `max_task_wait`). All other non-connection `McpError`s during poll are treated as abandonment. `tasks/result` does not get transient retry — the server has already completed, so a slow payload fetch is anomalous.
|
||||
|
||||
### File Access Harness (`_harness/_file_access.py`)
|
||||
|
||||
- **`AgentFileStore`** - Abstract async store backing the file-access harness. Implementations expose `write_file`, `read_file`, `delete_file`, `list_files`, `file_exists`, `search_files`, and `create_directory` over forward-slash relative paths.
|
||||
|
||||
@@ -71,7 +71,6 @@ from ._evaluation import (
|
||||
Evaluator,
|
||||
ExpectedToolCall,
|
||||
LocalEvaluator,
|
||||
RubricScore,
|
||||
evaluate_agent,
|
||||
evaluate_workflow,
|
||||
evaluator,
|
||||
@@ -124,7 +123,7 @@ from ._harness._todo import (
|
||||
TodoSessionStore,
|
||||
TodoStore,
|
||||
)
|
||||
from ._mcp import MCPStdioTool, MCPStreamableHTTPTool, MCPTaskOptions, MCPWebsocketTool
|
||||
from ._mcp import MCPStdioTool, MCPStreamableHTTPTool, MCPWebsocketTool
|
||||
from ._middleware import (
|
||||
AgentContext,
|
||||
AgentMiddleware,
|
||||
@@ -168,9 +167,6 @@ from ._skills import (
|
||||
InlineSkillResource,
|
||||
InlineSkillScript,
|
||||
InMemorySkillsSource,
|
||||
MCPSkill,
|
||||
MCPSkillResource,
|
||||
MCPSkillsSource,
|
||||
Skill,
|
||||
SkillFrontmatter,
|
||||
SkillResource,
|
||||
@@ -444,12 +440,8 @@ __all__ = [
|
||||
"InlineSkillResource",
|
||||
"InlineSkillScript",
|
||||
"LocalEvaluator",
|
||||
"MCPSkill",
|
||||
"MCPSkillResource",
|
||||
"MCPSkillsSource",
|
||||
"MCPStdioTool",
|
||||
"MCPStreamableHTTPTool",
|
||||
"MCPTaskOptions",
|
||||
"MCPWebsocketTool",
|
||||
"MemoryContextProvider",
|
||||
"MemoryFileStore",
|
||||
@@ -468,7 +460,6 @@ __all__ = [
|
||||
"ResponseStream",
|
||||
"Role",
|
||||
"RoleLiteral",
|
||||
"RubricScore",
|
||||
"RunContext",
|
||||
"Runner",
|
||||
"RunnerContext",
|
||||
|
||||
@@ -311,15 +311,12 @@ class EvalScoreResult:
|
||||
score: Numeric score from the evaluator.
|
||||
passed: Whether the item passed this evaluator's threshold.
|
||||
sample: Optional raw evaluator output (rationale, metadata).
|
||||
dimensions: Per-dimension scores when this evaluator is a rubric
|
||||
evaluator. ``None`` for non-rubric (e.g. built-in) evaluators.
|
||||
"""
|
||||
|
||||
name: str
|
||||
score: float
|
||||
passed: bool | None = None
|
||||
sample: dict[str, Any] | None = None
|
||||
dimensions: list[RubricScore] | None = None
|
||||
|
||||
|
||||
@experimental(feature_id=ExperimentalFeature.EVALS)
|
||||
@@ -499,179 +496,6 @@ class EvalResults:
|
||||
detail += f" Errored items: {', '.join(summaries)}."
|
||||
raise EvalNotPassedError(detail)
|
||||
|
||||
def assert_score_at_least(
|
||||
self,
|
||||
min_score: float,
|
||||
*,
|
||||
evaluator: str | None = None,
|
||||
msg: str | None = None,
|
||||
) -> None:
|
||||
"""Assert every item's score (optionally filtered by evaluator) is ``>= min_score``.
|
||||
|
||||
Designed for CI gates on generated rubric evaluators (e.g.
|
||||
``results.assert_score_at_least(0.80)``). Includes any
|
||||
sub-results from workflow evaluations.
|
||||
|
||||
Args:
|
||||
min_score: Minimum acceptable score (inclusive).
|
||||
evaluator: When set, only check scores from the evaluator
|
||||
whose ``EvalScoreResult.name`` matches.
|
||||
msg: Optional custom failure message.
|
||||
|
||||
Raises:
|
||||
EvalNotPassedError: When any matching score is below the threshold.
|
||||
"""
|
||||
offenders: list[str] = []
|
||||
|
||||
def _check(results: EvalResults) -> None:
|
||||
for item in results.items:
|
||||
for score in item.scores:
|
||||
if evaluator is not None and score.name != evaluator:
|
||||
continue
|
||||
if score.score < min_score:
|
||||
offenders.append(f"{item.item_id}/{score.name}={score.score:.3f}")
|
||||
for sub in results.sub_results.values():
|
||||
_check(sub)
|
||||
|
||||
_check(self)
|
||||
if offenders:
|
||||
detail = msg or (
|
||||
f"{len(offenders)} score(s) below threshold {min_score}"
|
||||
f"{' for ' + evaluator if evaluator else ''}: {', '.join(offenders[:5])}"
|
||||
+ (f" (+{len(offenders) - 5} more)" if len(offenders) > 5 else "")
|
||||
)
|
||||
raise EvalNotPassedError(detail)
|
||||
|
||||
def assert_dimension_score_at_least(
|
||||
self,
|
||||
dimension_id: str,
|
||||
min_score: float,
|
||||
*,
|
||||
evaluator: str | None = None,
|
||||
require_applicable: bool = False,
|
||||
msg: str | None = None,
|
||||
) -> None:
|
||||
"""Assert every item's score for a rubric *dimension* is ``>= min_score``.
|
||||
|
||||
Walks ``EvalScoreResult.dimensions`` looking for the named
|
||||
dimension across all items (and sub-results). Non-applicable
|
||||
dimensions are skipped by default; pass
|
||||
``require_applicable=True`` to fail when no applicable score is
|
||||
produced.
|
||||
|
||||
Args:
|
||||
dimension_id: Dimension id (matches the rubric definition).
|
||||
min_score: Minimum acceptable dimension score (inclusive).
|
||||
evaluator: When set, only consider scores from the evaluator
|
||||
whose ``EvalScoreResult.name`` matches.
|
||||
require_applicable: When ``True``, missing or non-applicable
|
||||
dimension scores raise. Defaults to ``False`` (skip).
|
||||
msg: Optional custom failure message.
|
||||
|
||||
Raises:
|
||||
EvalNotPassedError: When the dimension fails the threshold.
|
||||
"""
|
||||
offenders: list[str] = []
|
||||
missing_items: list[str] = []
|
||||
|
||||
def _check(results: EvalResults) -> None:
|
||||
for item in results.items:
|
||||
found_applicable = False
|
||||
for score in item.scores:
|
||||
if evaluator is not None and score.name != evaluator:
|
||||
continue
|
||||
if not score.dimensions:
|
||||
continue
|
||||
for rs in score.dimensions:
|
||||
if rs.id != dimension_id:
|
||||
continue
|
||||
if not rs.applicable:
|
||||
continue
|
||||
found_applicable = True
|
||||
if rs.score is None or rs.score < min_score:
|
||||
offenders.append(
|
||||
f"{item.item_id}/{score.name}/{dimension_id}="
|
||||
f"{rs.score if rs.score is not None else 'None'}"
|
||||
)
|
||||
if require_applicable and not found_applicable:
|
||||
missing_items.append(item.item_id)
|
||||
for sub in results.sub_results.values():
|
||||
_check(sub)
|
||||
|
||||
_check(self)
|
||||
problems: list[str] = []
|
||||
if offenders:
|
||||
problems.append(
|
||||
f"{len(offenders)} dimension score(s) for '{dimension_id}' below {min_score}: "
|
||||
f"{', '.join(offenders[:5])}" + (f" (+{len(offenders) - 5} more)" if len(offenders) > 5 else "")
|
||||
)
|
||||
if missing_items:
|
||||
problems.append(
|
||||
f"Dimension '{dimension_id}' not applicable on {len(missing_items)} item(s): "
|
||||
f"{', '.join(missing_items[:5])}"
|
||||
)
|
||||
if problems:
|
||||
raise EvalNotPassedError(msg or "; ".join(problems))
|
||||
|
||||
def assert_no_failed_items(self, msg: str | None = None) -> None:
|
||||
"""Assert no item ended in ``fail`` or ``error`` status.
|
||||
|
||||
Includes any sub-results from workflow evaluations.
|
||||
|
||||
Args:
|
||||
msg: Optional custom failure message.
|
||||
|
||||
Raises:
|
||||
EvalNotPassedError: When any item failed or errored.
|
||||
"""
|
||||
bad: list[str] = []
|
||||
|
||||
def _check(results: EvalResults) -> None:
|
||||
for item in results.items:
|
||||
if item.is_failed or item.is_error:
|
||||
bad.append(f"{item.item_id}:{item.status}")
|
||||
for sub in results.sub_results.values():
|
||||
_check(sub)
|
||||
|
||||
_check(self)
|
||||
if bad:
|
||||
detail = msg or (
|
||||
f"{len(bad)} item(s) failed or errored: {', '.join(bad[:5])}"
|
||||
+ (f" (+{len(bad) - 5} more)" if len(bad) > 5 else "")
|
||||
)
|
||||
raise EvalNotPassedError(detail)
|
||||
|
||||
|
||||
# endregion
|
||||
|
||||
# region Generated rubric evaluators
|
||||
|
||||
|
||||
@experimental(feature_id=ExperimentalFeature.EVALS)
|
||||
@dataclass(frozen=True)
|
||||
class RubricScore:
|
||||
"""A single dimension's score from a rubric-based evaluator run.
|
||||
|
||||
Rubric evaluators emit one ``RubricScore`` per dimension per item.
|
||||
Attached to :class:`EvalScoreResult` as a typed view of the raw
|
||||
``properties.rubric_scores`` payload returned by providers such as
|
||||
Foundry's generated rubric evaluators.
|
||||
|
||||
Attributes:
|
||||
id: Dimension id (matches the rubric definition).
|
||||
score: Numeric score, or ``None`` when the dimension was marked
|
||||
non-applicable for this item.
|
||||
applicable: Whether the dimension applied to this item.
|
||||
weight: Dimension weight (mirrors the rubric definition).
|
||||
reason: Short rationale produced by the evaluator.
|
||||
"""
|
||||
|
||||
id: str
|
||||
score: int | None
|
||||
applicable: bool
|
||||
weight: int
|
||||
reason: str
|
||||
|
||||
|
||||
# endregion
|
||||
|
||||
|
||||
@@ -58,9 +58,6 @@ class ExperimentalFeature(str, Enum):
|
||||
FOUNDRY_PREVIEW_TOOLS = "FOUNDRY_PREVIEW_TOOLS"
|
||||
FUNCTIONAL_WORKFLOWS = "FUNCTIONAL_WORKFLOWS"
|
||||
HARNESS = "HARNESS"
|
||||
MCP_LONG_RUNNING_TASKS = "MCP_LONG_RUNNING_TASKS"
|
||||
MCP_SKILLS = "MCP_SKILLS"
|
||||
PROGRESSIVE_TOOLS = "PROGRESSIVE_TOOLS"
|
||||
SKILLS = "SKILLS"
|
||||
TO_PROMPT_AGENT = "TO_PROMPT_AGENT"
|
||||
|
||||
|
||||
@@ -4,7 +4,6 @@ from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import contextlib
|
||||
import contextvars
|
||||
import json
|
||||
import logging
|
||||
@@ -13,14 +12,12 @@ import sys
|
||||
from abc import abstractmethod
|
||||
from collections.abc import Callable, Collection, Coroutine, Mapping, Sequence
|
||||
from contextlib import AsyncExitStack, _AsyncGeneratorContextManager # type: ignore
|
||||
from dataclasses import dataclass
|
||||
from datetime import timedelta
|
||||
from functools import partial
|
||||
from typing import TYPE_CHECKING, Any, Literal, TypedDict, cast
|
||||
|
||||
from opentelemetry import propagate
|
||||
|
||||
from ._feature_stage import ExperimentalFeature, experimental
|
||||
from ._tools import FunctionTool
|
||||
from ._types import (
|
||||
ChatOptions,
|
||||
@@ -152,73 +149,6 @@ def _url_origin(url: Any) -> tuple[str, str, int | None]:
|
||||
return (url.scheme, url.host or "", port)
|
||||
|
||||
|
||||
# Internal polling bounds for MCP long-running tasks. Not user-tunable today;
|
||||
# promote to MCPTaskOptions if a concrete need arises.
|
||||
_MCP_TASK_MIN_POLL_INTERVAL = timedelta(milliseconds=500)
|
||||
_MCP_TASK_MAX_POLL_INTERVAL = timedelta(seconds=5)
|
||||
_MCP_TASK_CANCEL_TIMEOUT = timedelta(seconds=5)
|
||||
_MCP_TASK_TERMINAL_STATUSES: frozenset[str] = frozenset({"completed", "failed", "cancelled", "input_required"})
|
||||
|
||||
# Total send attempts for a Phase 2 request (initial try + one reconnect-and-retry).
|
||||
# A single transient disconnect should not abort a long-running task; sustained outages
|
||||
# surface as ``_MCPTaskAbandoned`` after the second failure.
|
||||
_MCP_RECONNECT_ATTEMPTS = 2
|
||||
|
||||
|
||||
class _MCPTaskAbandoned(ToolExecutionException):
|
||||
"""Raised when the remote MCP task may still be running and must be cancelled.
|
||||
|
||||
Subclass of ToolExecutionException so callers see a normal tool failure.
|
||||
"""
|
||||
|
||||
|
||||
class _MCPDeadlineExpired(Exception):
|
||||
"""Internal marker for ``max_task_wait`` expiry; distinct from inner TimeoutError."""
|
||||
|
||||
|
||||
@experimental(feature_id=ExperimentalFeature.MCP_LONG_RUNNING_TASKS)
|
||||
@dataclass(frozen=True)
|
||||
class MCPTaskOptions:
|
||||
"""Options controlling how MCPTool drives the MCP long-running task lifecycle.
|
||||
|
||||
When an MCP server advertises a tool with ``execution.taskSupport == "required"``,
|
||||
the framework transparently drives the SEP-2663 ``tools/call`` → ``tasks/get``
|
||||
(polled) → ``tasks/result`` lifecycle so the agent sees a normal tool result.
|
||||
|
||||
Instances are immutable; replace the whole object via
|
||||
``MCPTool.task_options = MCPTaskOptions(...)`` to change behavior.
|
||||
|
||||
Attributes:
|
||||
default_ttl: Optional task-record retention time forwarded to the server as
|
||||
``params.task.ttl`` (milliseconds, integer). The server keeps the task
|
||||
record around this long after the task reaches a terminal status so the
|
||||
client can still call ``tasks/get`` / ``tasks/result``; it does not
|
||||
cancel a running task. When ``None``, the server applies its own default.
|
||||
Must be positive if set (zero would expire the record before any client
|
||||
could read it).
|
||||
cancel_remote_task_on_local_cancellation: If True (default), a local
|
||||
cancellation of the awaiting coroutine triggers a best-effort
|
||||
``tasks/cancel`` on the server before re-raising ``CancelledError``.
|
||||
Only gates ``CancelledError``; abandonment paths (max-wait,
|
||||
unrecoverable poll errors, lost connection after task_id is known)
|
||||
always cancel regardless of this flag.
|
||||
max_task_wait: Optional client-side deadline for the whole post-create
|
||||
lifecycle (poll + result fetch). When exceeded, raises
|
||||
``ToolExecutionException`` and fires a best-effort ``tasks/cancel``.
|
||||
``None`` (default) means no client-side bound. Must be positive if set.
|
||||
"""
|
||||
|
||||
default_ttl: timedelta | None = None
|
||||
cancel_remote_task_on_local_cancellation: bool = True
|
||||
max_task_wait: timedelta | None = None
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if self.default_ttl is not None and self.default_ttl.total_seconds() <= 0:
|
||||
raise ValueError("MCPTaskOptions.default_ttl must be positive.")
|
||||
if self.max_task_wait is not None and self.max_task_wait.total_seconds() <= 0:
|
||||
raise ValueError("MCPTaskOptions.max_task_wait must be positive.")
|
||||
|
||||
|
||||
def streamable_http_client(*args: Any, **kwargs: Any) -> _AsyncGeneratorContextManager[Any, None]:
|
||||
"""Lazily import the MCP streamable HTTP transport."""
|
||||
try:
|
||||
@@ -287,7 +217,6 @@ class MCPTool:
|
||||
request_timeout: int | None = None,
|
||||
client: SupportsChatGetResponse | None = None,
|
||||
additional_properties: dict[str, Any] | None = None,
|
||||
task_options: MCPTaskOptions | None = None,
|
||||
) -> None:
|
||||
"""Initialize the MCP Tool base.
|
||||
|
||||
@@ -319,9 +248,6 @@ class MCPTool:
|
||||
request_timeout: Timeout in seconds for MCP requests.
|
||||
client: A chat client for sampling callbacks.
|
||||
additional_properties: Additional properties for the tool.
|
||||
task_options: Options controlling how long-running MCP tasks are driven for
|
||||
tools that advertise ``execution.taskSupport == "required"``. When ``None``,
|
||||
the defaults from :class:`MCPTaskOptions` are used.
|
||||
"""
|
||||
self.name = name
|
||||
self.description = description or ""
|
||||
@@ -333,10 +259,6 @@ class MCPTool:
|
||||
self.parse_tool_results = parse_tool_results
|
||||
self.load_prompts_flag = load_prompts
|
||||
self.parse_prompt_results = parse_prompt_results
|
||||
# Defer constructing the default MCPTaskOptions so the experimental warning
|
||||
# only fires when LRO is actually engaged (lazy-resolved by _effective_task_options).
|
||||
self._task_options_explicit: MCPTaskOptions | None = task_options
|
||||
self._task_options_default: MCPTaskOptions | None = None
|
||||
self._exit_stack = AsyncExitStack()
|
||||
self._lifecycle_lock = asyncio.Lock()
|
||||
self._lifecycle_request_lock = asyncio.Lock()
|
||||
@@ -348,7 +270,6 @@ class MCPTool:
|
||||
self.client = client
|
||||
self._functions: list[FunctionTool] = []
|
||||
self._tool_call_meta_by_name: dict[str, dict[str, Any]] = {}
|
||||
self._tool_task_support_by_name: dict[str, str] = {}
|
||||
self.is_connected: bool = False
|
||||
self._tools_loaded: bool = False
|
||||
self._prompts_loaded: bool = False
|
||||
@@ -1210,7 +1131,6 @@ class MCPTool:
|
||||
# Track existing function names to prevent duplicates
|
||||
existing_names = {func.name for func in self._functions}
|
||||
tool_call_meta_by_name: dict[str, dict[str, Any]] = {}
|
||||
tool_task_support_by_name: dict[str, str] = {}
|
||||
|
||||
params: types.PaginatedRequestParams | None = None
|
||||
while True:
|
||||
@@ -1248,10 +1168,6 @@ class MCPTool:
|
||||
if tool.meta is not None:
|
||||
tool_call_meta_by_name[tool.name] = dict(tool.meta)
|
||||
|
||||
task_support = getattr(getattr(tool, "execution", None), "taskSupport", None)
|
||||
if task_support is not None:
|
||||
tool_task_support_by_name[tool.name] = task_support
|
||||
|
||||
normalized_name = _normalize_mcp_name(tool.name)
|
||||
local_name = _build_prefixed_mcp_name(normalized_name, self.tool_name_prefix)
|
||||
|
||||
@@ -1300,7 +1216,6 @@ class MCPTool:
|
||||
params = types.PaginatedRequestParams(cursor=tool_list.nextCursor)
|
||||
|
||||
self._tool_call_meta_by_name = tool_call_meta_by_name
|
||||
self._tool_task_support_by_name = tool_task_support_by_name
|
||||
|
||||
async def _close_on_owner(self) -> None:
|
||||
# Cancel any pending reload tasks before tearing down the session.
|
||||
@@ -1377,29 +1292,6 @@ class MCPTool:
|
||||
inner_exception=ex,
|
||||
) from ex
|
||||
|
||||
def _effective_task_options(self) -> MCPTaskOptions:
|
||||
"""Return the effective MCPTaskOptions, lazily constructing defaults on first use.
|
||||
|
||||
Defers the implicit ``MCPTaskOptions()`` so the experimental warning only
|
||||
fires when LRO is actually engaged (server advertises ``taskSupport=required``).
|
||||
"""
|
||||
explicit = self._task_options_explicit
|
||||
if explicit is not None:
|
||||
return explicit
|
||||
if self._task_options_default is None:
|
||||
self._task_options_default = MCPTaskOptions()
|
||||
return self._task_options_default
|
||||
|
||||
@property
|
||||
def task_options(self) -> MCPTaskOptions:
|
||||
"""The effective MCPTaskOptions for this tool (lazy defaults)."""
|
||||
return self._effective_task_options()
|
||||
|
||||
@task_options.setter
|
||||
def task_options(self, value: MCPTaskOptions | None) -> None:
|
||||
self._task_options_explicit = value
|
||||
self._task_options_default = None
|
||||
|
||||
async def call_tool(self, tool_name: str, **kwargs: Any) -> str | list[Content]:
|
||||
"""Call a tool with the given arguments.
|
||||
|
||||
@@ -1430,12 +1322,47 @@ class MCPTool:
|
||||
"Tools are not loaded for this server, please set load_tools=True in the constructor."
|
||||
)
|
||||
|
||||
# Tools advertising taskSupport == "required" cannot complete via plain tools/call;
|
||||
# route through the long-running task lifecycle transparently.
|
||||
if self._tool_task_support_by_name.get(tool_name) == "required":
|
||||
return await self.call_tool_as_task(tool_name, **kwargs)
|
||||
raw_user_meta: object | None = kwargs.get("_meta")
|
||||
user_meta: dict[str, Any] | None = None
|
||||
if raw_user_meta is not None and not isinstance(raw_user_meta, dict):
|
||||
raise ToolExecutionException("MCP tool metadata provided via _meta must be a dict.")
|
||||
if isinstance(raw_user_meta, dict):
|
||||
raw_user_meta_dict = cast(Mapping[object, object], raw_user_meta)
|
||||
user_meta = {}
|
||||
for key, value in raw_user_meta_dict.items():
|
||||
if not isinstance(key, str):
|
||||
raise ToolExecutionException("MCP tool metadata provided via _meta must use string keys.")
|
||||
user_meta[key] = value
|
||||
|
||||
filtered_kwargs, meta = self._prepare_call_kwargs(tool_name, kwargs)
|
||||
# Filter out framework kwargs that cannot be serialized by the MCP SDK.
|
||||
# These are internal objects passed through the function invocation pipeline
|
||||
# that should not be forwarded to external MCP servers.
|
||||
# conversation_id is an internal tracking ID used by services like Azure AI.
|
||||
# options contains metadata/store used by AG-UI for Azure AI client requirements.
|
||||
# response_format is a Pydantic model class used for structured output (not serializable).
|
||||
filtered_kwargs = {
|
||||
k: v
|
||||
for k, v in kwargs.items()
|
||||
if k
|
||||
not in {
|
||||
"chat_options",
|
||||
"tools",
|
||||
"tool_choice",
|
||||
"session",
|
||||
"thread",
|
||||
"conversation_id",
|
||||
"options",
|
||||
"response_format",
|
||||
"_meta",
|
||||
}
|
||||
}
|
||||
|
||||
# Some MCP proxies require their tools/list metadata to be echoed on tools/call.
|
||||
tool_meta = self._tool_call_meta_by_name.get(tool_name)
|
||||
request_meta = dict(tool_meta) if tool_meta is not None else None
|
||||
if user_meta is not None:
|
||||
request_meta = {**(request_meta or {}), **user_meta}
|
||||
meta = _inject_otel_into_mcp_meta(request_meta)
|
||||
|
||||
parser = self.parse_tool_results or self._parse_tool_result_from_mcp
|
||||
# Try the operation, reconnecting once if the connection is closed
|
||||
@@ -1484,479 +1411,6 @@ class MCPTool:
|
||||
raise ToolExecutionException(f"Failed to call tool '{tool_name}'.", inner_exception=ex) from ex
|
||||
raise ToolExecutionException(f"Failed to call tool '{tool_name}' after retries.")
|
||||
|
||||
def _prepare_call_kwargs(
|
||||
self, tool_name: str, kwargs: dict[str, Any]
|
||||
) -> tuple[dict[str, Any], dict[str, Any] | None]:
|
||||
"""Filter framework-only kwargs and build the merged MCP request metadata."""
|
||||
raw_user_meta: object | None = kwargs.get("_meta")
|
||||
user_meta: dict[str, Any] | None = None
|
||||
if raw_user_meta is not None and not isinstance(raw_user_meta, dict):
|
||||
raise ToolExecutionException("MCP tool metadata provided via _meta must be a dict.")
|
||||
if isinstance(raw_user_meta, dict):
|
||||
raw_user_meta_dict = cast(Mapping[object, object], raw_user_meta)
|
||||
user_meta = {}
|
||||
for key, value in raw_user_meta_dict.items():
|
||||
if not isinstance(key, str):
|
||||
raise ToolExecutionException("MCP tool metadata provided via _meta must use string keys.")
|
||||
user_meta[key] = value
|
||||
|
||||
# Filter out framework kwargs that cannot be serialized by the MCP SDK.
|
||||
# These are internal objects passed through the function invocation pipeline
|
||||
# that should not be forwarded to external MCP servers.
|
||||
# conversation_id is an internal tracking ID used by services like Azure AI.
|
||||
# options contains metadata/store used by AG-UI for Azure AI client requirements.
|
||||
# response_format is a Pydantic model class used for structured output (not serializable).
|
||||
filtered_kwargs = {
|
||||
k: v
|
||||
for k, v in kwargs.items()
|
||||
if k
|
||||
not in {
|
||||
"chat_options",
|
||||
"tools",
|
||||
"tool_choice",
|
||||
"session",
|
||||
"thread",
|
||||
"conversation_id",
|
||||
"options",
|
||||
"response_format",
|
||||
"_meta",
|
||||
}
|
||||
}
|
||||
|
||||
# Some MCP proxies require their tools/list metadata to be echoed on tools/call.
|
||||
tool_meta = self._tool_call_meta_by_name.get(tool_name)
|
||||
request_meta = dict(tool_meta) if tool_meta is not None else None
|
||||
if user_meta is not None:
|
||||
request_meta = {**(request_meta or {}), **user_meta}
|
||||
meta = _inject_otel_into_mcp_meta(request_meta)
|
||||
return filtered_kwargs, meta
|
||||
|
||||
async def call_tool_as_task(self, tool_name: str, **kwargs: Any) -> str | list[Content]:
|
||||
"""Call an MCP tool via the long-running task lifecycle (SEP-2663).
|
||||
|
||||
Issues an augmented ``tools/call`` with ``params.task`` set from
|
||||
``self.task_options``, then polls ``tasks/get`` until the server reports a
|
||||
terminal status. On ``completed`` the payload is fetched via ``tasks/result``,
|
||||
validated as a ``CallToolResult`` and parsed identically to :meth:`call_tool`.
|
||||
|
||||
Local cancellation triggers a best-effort ``tasks/cancel`` (controlled by
|
||||
:attr:`MCPTaskOptions.cancel_remote_task_on_local_cancellation`) before
|
||||
``asyncio.CancelledError`` is re-raised.
|
||||
|
||||
Args:
|
||||
tool_name: The remote MCP tool name.
|
||||
|
||||
Keyword Args:
|
||||
kwargs: Arguments forwarded to the tool. See :meth:`call_tool` for the
|
||||
framework kwargs that are filtered out.
|
||||
|
||||
Returns:
|
||||
A list of Content items (or a string when a custom ``parse_tool_results``
|
||||
callback is configured).
|
||||
"""
|
||||
from anyio import ClosedResourceError
|
||||
from mcp.shared.exceptions import McpError
|
||||
|
||||
if not self.load_tools_flag:
|
||||
raise ToolExecutionException(
|
||||
"Tools are not loaded for this server, please set load_tools=True in the constructor."
|
||||
)
|
||||
|
||||
filtered_kwargs, meta = self._prepare_call_kwargs(tool_name, kwargs)
|
||||
parser = self.parse_tool_results or self._parse_tool_result_from_mcp
|
||||
|
||||
# Submit the task: issue augmented tools/call. Do NOT retry on connection loss here:
|
||||
# the server may have accepted the request and created a task before the
|
||||
# response was lost, so retrying could start the long-running operation twice.
|
||||
# Reconnect-and-retry is only safe after the task_id is known.
|
||||
try:
|
||||
task_id, fallback_result = await self._call_tool_as_task_create(tool_name, filtered_kwargs, meta)
|
||||
except (ClosedResourceError, McpError) as ex:
|
||||
if not self._is_connection_lost(ex):
|
||||
error_message = ex.error.message if isinstance(ex, McpError) else str(ex)
|
||||
raise ToolExecutionException(error_message, inner_exception=ex) from ex
|
||||
raise ToolExecutionException(
|
||||
f"Failed to call tool '{tool_name}' - connection lost; task state unknown.",
|
||||
inner_exception=ex,
|
||||
) from ex
|
||||
except ToolExecutionException:
|
||||
raise
|
||||
except Exception as ex:
|
||||
raise ToolExecutionException(f"Failed to call tool '{tool_name}'.", inner_exception=ex) from ex
|
||||
|
||||
# Server returned a CallToolResult (no task created) or fell back to plain tools/call.
|
||||
if fallback_result is not None:
|
||||
if fallback_result.isError:
|
||||
parsed = parser(fallback_result)
|
||||
text = (
|
||||
"\n".join(c.text for c in parsed if c.type == "text" and c.text)
|
||||
if isinstance(parsed, list)
|
||||
else str(parsed)
|
||||
)
|
||||
raise ToolExecutionException(text or str(parsed))
|
||||
return parser(fallback_result)
|
||||
|
||||
if task_id is None:
|
||||
raise ToolExecutionException(
|
||||
f"MCP server did not return a task_id or fallback result for '{tool_name}'."
|
||||
)
|
||||
|
||||
# Track to completion: poll until terminal, then fetch payload. Never re-issue
|
||||
# tools/call past this point; reconnect-and-retry only against the same task_id.
|
||||
opts = self._effective_task_options()
|
||||
max_wait_s = opts.max_task_wait.total_seconds() if opts.max_task_wait is not None else None
|
||||
|
||||
async def _await_task_completion() -> str | list[Content]:
|
||||
terminal = await self._poll_task_until_terminal(task_id)
|
||||
return await self._handle_terminal_task(tool_name, task_id, terminal, parser)
|
||||
|
||||
try:
|
||||
if max_wait_s is not None:
|
||||
try:
|
||||
result = await self._await_with_deadline(_await_task_completion(), max_wait_s)
|
||||
return cast("str | list[Content]", result)
|
||||
except _MCPDeadlineExpired as ex:
|
||||
self._spawn_best_effort_cancel(task_id)
|
||||
raise ToolExecutionException(
|
||||
f"MCP task '{task_id}' exceeded max_task_wait of {max_wait_s}s.",
|
||||
inner_exception=ex,
|
||||
) from ex
|
||||
else:
|
||||
return await _await_task_completion()
|
||||
except asyncio.CancelledError:
|
||||
if opts.cancel_remote_task_on_local_cancellation:
|
||||
self._spawn_best_effort_cancel(task_id)
|
||||
raise
|
||||
except _MCPTaskAbandoned:
|
||||
# Pre-terminal abandonment (hard poll error, malformed get, second
|
||||
# disconnect, reconnect failure): cancel + re-raise as plain
|
||||
# ToolExecutionException to the function-calling loop.
|
||||
self._spawn_best_effort_cancel(task_id)
|
||||
raise
|
||||
# Plain ToolExecutionException from terminal failures (failed/cancelled/
|
||||
# input_required, completed+isError, malformed result post-completion)
|
||||
# propagates without cancel — server is already done.
|
||||
|
||||
async def _call_tool_as_task_create(
|
||||
self, tool_name: str, arguments: dict[str, Any], meta: dict[str, Any] | None
|
||||
) -> tuple[str | None, types.CallToolResult | None]:
|
||||
"""Send the augmented tools/call.
|
||||
|
||||
Returns ``(task_id, None)`` when the server created a task,
|
||||
``(None, CallToolResult)`` when it returned a non-task result, falling back
|
||||
to plain ``tools/call`` if the server rejects the ``task`` field outright.
|
||||
"""
|
||||
from mcp import types
|
||||
from mcp.shared.exceptions import McpError
|
||||
from pydantic import ValidationError
|
||||
|
||||
opts = self._effective_task_options()
|
||||
ttl_ms: int | None = None
|
||||
if opts.default_ttl is not None:
|
||||
ttl_ms = int(opts.default_ttl.total_seconds() * 1000)
|
||||
# Always send TaskMetadata to mark the call as task-augmented; ttl may be omitted.
|
||||
task_metadata = types.TaskMetadata(ttl=ttl_ms)
|
||||
|
||||
request_meta = types.RequestParams.Meta(**meta) if meta else None
|
||||
params = types.CallToolRequestParams(
|
||||
name=tool_name,
|
||||
arguments=arguments,
|
||||
task=task_metadata,
|
||||
_meta=request_meta, # type: ignore[call-arg]
|
||||
)
|
||||
request = types.ClientRequest(types.CallToolRequest(params=params))
|
||||
|
||||
# Use the lenient Result type so we can extract the task_id even when
|
||||
# the strict CreateTaskResult schema rejects the payload (the MCP Python
|
||||
# SDK requires Task.ttl, but servers may legitimately omit it).
|
||||
try:
|
||||
lenient = await self.session.send_request( # type: ignore[union-attr]
|
||||
request,
|
||||
types.Result,
|
||||
)
|
||||
except McpError as ex:
|
||||
if ex.error.code not in (types.METHOD_NOT_FOUND, types.INVALID_PARAMS):
|
||||
raise
|
||||
logger.debug(
|
||||
"Server rejected augmented tools/call for '%s' (code=%s); falling back.",
|
||||
tool_name,
|
||||
ex.error.code,
|
||||
)
|
||||
fallback = await self.session.call_tool(tool_name, arguments=arguments, meta=meta) # type: ignore[union-attr]
|
||||
return None, fallback
|
||||
|
||||
# Inspect the raw payload: a CreateTaskResult carries `task.taskId`;
|
||||
# a legacy CallToolResult carries `content` and/or `isError`.
|
||||
raw: dict[str, Any] = lenient.model_dump(by_alias=True, exclude_none=True)
|
||||
raw.pop("_meta", None)
|
||||
|
||||
task_field = raw.get("task")
|
||||
if isinstance(task_field, dict):
|
||||
task_id_val = cast(dict[str, Any], task_field).get("taskId")
|
||||
if isinstance(task_id_val, str):
|
||||
return task_id_val, None
|
||||
|
||||
try:
|
||||
legacy = types.CallToolResult.model_validate(raw)
|
||||
except ValidationError as ex:
|
||||
# Augmented call succeeded server-side; re-issuing a plain tools/call
|
||||
# could double-execute a side-effecting tool.
|
||||
raise ToolExecutionException(
|
||||
f"MCP server returned an unparseable response to augmented tools/call "
|
||||
f"for '{tool_name}'; cannot safely retry (server may have started the operation).",
|
||||
inner_exception=ex,
|
||||
) from ex
|
||||
|
||||
return None, legacy
|
||||
|
||||
async def _poll_task_until_terminal(self, task_id: str) -> types.GetTaskResult:
|
||||
"""Poll ``tasks/get`` until the task reaches a terminal status."""
|
||||
import httpx
|
||||
from mcp import types
|
||||
from mcp.shared.exceptions import McpError
|
||||
|
||||
# SDK raises McpError(code=httpx.REQUEST_TIMEOUT=408) on session read timeout.
|
||||
transient_codes: frozenset[int] = frozenset({int(httpx.codes.REQUEST_TIMEOUT)})
|
||||
|
||||
while True:
|
||||
request = types.ClientRequest(
|
||||
types.GetTaskRequest(params=types.GetTaskRequestParams(taskId=task_id))
|
||||
)
|
||||
try:
|
||||
# GetTaskResult.ttl is required-but-Optional in the SDK; coerce below.
|
||||
lenient = await self._send_with_one_reconnect(
|
||||
request, types.Result, operation="tasks/get", task_id=task_id
|
||||
)
|
||||
except McpError as ex:
|
||||
if ex.error.code in transient_codes:
|
||||
logger.debug(
|
||||
"Transient %s on tasks/get for '%s'; will retry.", ex.error.code, task_id
|
||||
)
|
||||
await asyncio.sleep(_MCP_TASK_MIN_POLL_INTERVAL.total_seconds())
|
||||
continue
|
||||
# Hard server error mid-poll: task may still be running.
|
||||
raise _MCPTaskAbandoned(ex.error.message, inner_exception=ex) from ex
|
||||
|
||||
try:
|
||||
snapshot = self._coerce_get_task_result(lenient, task_id)
|
||||
except ToolExecutionException as ex:
|
||||
# Malformed tasks/get response; task may still be running.
|
||||
raise _MCPTaskAbandoned(str(ex), inner_exception=ex) from ex
|
||||
|
||||
if snapshot.status in _MCP_TASK_TERMINAL_STATUSES:
|
||||
return snapshot
|
||||
|
||||
await asyncio.sleep(self._compute_poll_delay(snapshot.pollInterval).total_seconds())
|
||||
|
||||
@staticmethod
|
||||
def _coerce_get_task_result(lenient: types.Result, task_id: str) -> types.GetTaskResult:
|
||||
"""Coerce a lenient Result into GetTaskResult, defaulting ``ttl`` when absent."""
|
||||
from mcp import types
|
||||
|
||||
raw = lenient.model_dump(by_alias=True, exclude_none=True)
|
||||
raw.pop("_meta", None)
|
||||
raw.setdefault("ttl", None)
|
||||
try:
|
||||
return types.GetTaskResult.model_validate(raw)
|
||||
except Exception as ex:
|
||||
raise ToolExecutionException(
|
||||
f"MCP server returned a malformed tasks/get response for task '{task_id}'.",
|
||||
inner_exception=ex,
|
||||
) from ex
|
||||
|
||||
@staticmethod
|
||||
def _compute_poll_delay(server_interval_ms: int | None) -> timedelta:
|
||||
"""Clamp the server-suggested poll interval to ``[min, max]``."""
|
||||
if server_interval_ms is None or server_interval_ms <= 0:
|
||||
return _MCP_TASK_MIN_POLL_INTERVAL
|
||||
suggested = timedelta(milliseconds=server_interval_ms)
|
||||
if suggested < _MCP_TASK_MIN_POLL_INTERVAL:
|
||||
return _MCP_TASK_MIN_POLL_INTERVAL
|
||||
if suggested > _MCP_TASK_MAX_POLL_INTERVAL:
|
||||
return _MCP_TASK_MAX_POLL_INTERVAL
|
||||
return suggested
|
||||
|
||||
async def _handle_terminal_task(
|
||||
self,
|
||||
tool_name: str,
|
||||
task_id: str,
|
||||
snapshot: types.GetTaskResult,
|
||||
parser: Callable[[types.CallToolResult], str | list[Content]],
|
||||
) -> str | list[Content]:
|
||||
"""Map a terminal task snapshot to either a parsed result or an exception."""
|
||||
status = snapshot.status
|
||||
if status == "completed":
|
||||
payload = await self._fetch_task_result(task_id)
|
||||
if payload.isError:
|
||||
parsed = parser(payload)
|
||||
text = (
|
||||
"\n".join(c.text for c in parsed if c.type == "text" and c.text)
|
||||
if isinstance(parsed, list)
|
||||
else str(parsed)
|
||||
)
|
||||
raise ToolExecutionException(text or str(parsed))
|
||||
return parser(payload)
|
||||
|
||||
# Non-completed terminal statuses surface as ToolExecutionException so the
|
||||
# function-calling loop sees a normal failure for tool_name.
|
||||
message = snapshot.statusMessage or f"MCP task ended with status '{status}'."
|
||||
if status == "input_required":
|
||||
# Spec-non-terminal; treated as terminal here because the framework does
|
||||
# not implement the interactive input flow.
|
||||
message = snapshot.statusMessage or "MCP task requires additional input and cannot continue."
|
||||
raise ToolExecutionException(f"Tool '{tool_name}' task {status}: {message}")
|
||||
|
||||
async def _fetch_task_result(self, task_id: str) -> types.CallToolResult:
|
||||
"""Send ``tasks/result`` and reinterpret the open-typed payload as a CallToolResult."""
|
||||
from mcp import types
|
||||
from mcp.shared.exceptions import McpError
|
||||
from pydantic import ValidationError
|
||||
|
||||
request = types.ClientRequest(
|
||||
types.GetTaskPayloadRequest(params=types.GetTaskPayloadRequestParams(taskId=task_id))
|
||||
)
|
||||
# Connection-loss retry only via the helper; no transient-code retry — server
|
||||
# has already completed the task, so a slow payload fetch is anomalous.
|
||||
try:
|
||||
payload = await self._send_with_one_reconnect(
|
||||
request, types.GetTaskPayloadResult, operation="tasks/result", task_id=task_id
|
||||
)
|
||||
except McpError as ex:
|
||||
# Server reported completed; a hard fetch error is a plain failure (no cancel).
|
||||
raise ToolExecutionException(ex.error.message, inner_exception=ex) from ex
|
||||
|
||||
# GetTaskPayloadResult carries the tool result via extra fields; reinterpret as CallToolResult.
|
||||
payload_dict = payload.model_dump(by_alias=True, exclude_none=True)
|
||||
payload_dict.pop("_meta", None)
|
||||
try:
|
||||
return types.CallToolResult.model_validate(payload_dict)
|
||||
except ValidationError as ex:
|
||||
# Server reported completed; malformed payload is a plain failure (no cancel needed).
|
||||
raise ToolExecutionException(
|
||||
f"MCP task '{task_id}' result payload could not be parsed as a CallToolResult.",
|
||||
inner_exception=ex,
|
||||
) from ex
|
||||
|
||||
async def _send_with_one_reconnect(
|
||||
self,
|
||||
request: types.ClientRequest,
|
||||
result_type: type[Any],
|
||||
*,
|
||||
operation: str,
|
||||
task_id: str,
|
||||
) -> Any:
|
||||
"""Send ``request`` with one reconnect-and-retry on connection loss.
|
||||
|
||||
After a second loss (or reconnect failure), raise ``_MCPTaskAbandoned``.
|
||||
Non-connection errors propagate unchanged.
|
||||
"""
|
||||
from anyio import ClosedResourceError
|
||||
from mcp.shared.exceptions import McpError
|
||||
|
||||
for attempt in range(_MCP_RECONNECT_ATTEMPTS):
|
||||
try:
|
||||
return await self.session.send_request(request, result_type) # type: ignore[union-attr]
|
||||
except (ClosedResourceError, McpError) as ex:
|
||||
if not self._is_connection_lost(ex):
|
||||
raise
|
||||
if attempt < _MCP_RECONNECT_ATTEMPTS - 1:
|
||||
logger.info(
|
||||
"MCP connection lost during %s; reconnecting (task_id=%s).", operation, task_id
|
||||
)
|
||||
try:
|
||||
await self.connect(reset=True)
|
||||
except Exception as reconn_ex:
|
||||
# Reconnect failure: task may still be running.
|
||||
raise _MCPTaskAbandoned(
|
||||
"Failed to reconnect to MCP server.", inner_exception=reconn_ex
|
||||
) from reconn_ex
|
||||
continue
|
||||
# Final attempt also lost the connection: task may still be running.
|
||||
raise _MCPTaskAbandoned(
|
||||
f"MCP connection lost; task state unknown (task_id={task_id}).",
|
||||
inner_exception=ex,
|
||||
) from ex
|
||||
raise AssertionError(f"unreachable: {operation} for {task_id}") # pragma: no cover
|
||||
|
||||
@staticmethod
|
||||
async def _await_with_deadline(coro: Coroutine[Any, Any, Any], timeout_s: float) -> Any:
|
||||
"""Await ``coro`` with a deadline; raise ``_MCPDeadlineExpired`` only on deadline.
|
||||
|
||||
Unlike ``asyncio.wait_for``, an ``asyncio.TimeoutError`` raised by ``coro``
|
||||
itself propagates unchanged so callers can distinguish their own deadline
|
||||
from a stray inner timeout.
|
||||
"""
|
||||
inner = asyncio.ensure_future(coro)
|
||||
try:
|
||||
done, _pending = await asyncio.wait({inner}, timeout=timeout_s)
|
||||
except BaseException:
|
||||
# Outer caller cancelled (or another exception): cancel inner + drain.
|
||||
inner.cancel()
|
||||
with contextlib.suppress(BaseException):
|
||||
await inner
|
||||
raise
|
||||
if inner in done:
|
||||
return inner.result()
|
||||
# Deadline fired before inner finished.
|
||||
inner.cancel()
|
||||
with contextlib.suppress(BaseException):
|
||||
await inner
|
||||
raise _MCPDeadlineExpired
|
||||
|
||||
def _spawn_best_effort_cancel(self, task_id: str) -> None:
|
||||
"""Fire-and-forget ``tasks/cancel`` so local cancellation propagates server-side."""
|
||||
try:
|
||||
loop = asyncio.get_running_loop()
|
||||
except RuntimeError:
|
||||
return
|
||||
cancel_task = loop.create_task(self._try_cancel_task(task_id))
|
||||
# Reuse pending-reload bookkeeping so close-on-owner waits/cancels these too.
|
||||
self._pending_reload_tasks.add(cancel_task)
|
||||
cancel_task.add_done_callback(self._pending_reload_tasks.discard)
|
||||
|
||||
async def _try_cancel_task(self, task_id: str) -> None:
|
||||
"""Send ``tasks/cancel``; bounded by ``_MCP_TASK_CANCEL_TIMEOUT``.
|
||||
|
||||
Failures log at warning so unattributed orphan tasks are debuggable.
|
||||
"""
|
||||
from mcp import types
|
||||
|
||||
request = types.ClientRequest(
|
||||
types.CancelTaskRequest(params=types.CancelTaskRequestParams(taskId=task_id))
|
||||
)
|
||||
try:
|
||||
await asyncio.wait_for(
|
||||
self.session.send_request(request, types.CancelTaskResult), # type: ignore[union-attr]
|
||||
timeout=_MCP_TASK_CANCEL_TIMEOUT.total_seconds(),
|
||||
)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except asyncio.TimeoutError:
|
||||
logger.warning(
|
||||
"Best-effort tasks/cancel for '%s' timed out after %.1fs; "
|
||||
"remote task may still be running.",
|
||||
task_id,
|
||||
_MCP_TASK_CANCEL_TIMEOUT.total_seconds(),
|
||||
)
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"Best-effort tasks/cancel for '%s' failed; remote task may still be running.",
|
||||
task_id,
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _is_connection_lost(ex: BaseException) -> bool:
|
||||
"""Return True if *ex* indicates the MCP transport was torn down."""
|
||||
from anyio import ClosedResourceError
|
||||
from mcp.shared.exceptions import McpError
|
||||
|
||||
if isinstance(ex, ClosedResourceError):
|
||||
return True
|
||||
if isinstance(ex, McpError):
|
||||
return "session terminated" in ex.error.message.lower()
|
||||
return False
|
||||
|
||||
async def get_prompt(self, prompt_name: str, **kwargs: Any) -> str:
|
||||
"""Call a prompt with the given arguments.
|
||||
|
||||
@@ -2100,7 +1554,6 @@ class MCPStdioTool(MCPTool):
|
||||
encoding: str | None = None,
|
||||
client: SupportsChatGetResponse | None = None,
|
||||
additional_properties: dict[str, Any] | None = None,
|
||||
task_options: MCPTaskOptions | None = None,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
"""Initialize the MCP stdio tool.
|
||||
@@ -2145,8 +1598,6 @@ class MCPStdioTool(MCPTool):
|
||||
env: The environment variables to set for the command.
|
||||
encoding: The encoding to use for the command output.
|
||||
client: The chat client to use for sampling.
|
||||
task_options: Options for tools that advertise
|
||||
``execution.taskSupport == "required"``. See :class:`MCPTaskOptions`.
|
||||
kwargs: Any extra arguments to pass to the stdio client.
|
||||
"""
|
||||
super().__init__(
|
||||
@@ -2163,7 +1614,6 @@ class MCPStdioTool(MCPTool):
|
||||
load_prompts=load_prompts,
|
||||
parse_prompt_results=parse_prompt_results,
|
||||
request_timeout=request_timeout,
|
||||
task_options=task_options,
|
||||
)
|
||||
self.command = command
|
||||
self.args = args or []
|
||||
@@ -2237,7 +1687,6 @@ class MCPStreamableHTTPTool(MCPTool):
|
||||
additional_properties: dict[str, Any] | None = None,
|
||||
http_client: AsyncClient | None = None,
|
||||
header_provider: Callable[[dict[str, Any]], dict[str, str]] | None = None,
|
||||
task_options: MCPTaskOptions | None = None,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
"""Initialize the MCP streamable HTTP tool.
|
||||
@@ -2290,8 +1739,6 @@ class MCPStreamableHTTPTool(MCPTool):
|
||||
of HTTP headers to inject into every outbound request to the MCP server.
|
||||
Use this to forward per-request context (e.g. authentication tokens set in
|
||||
agent middleware) without creating a separate ``httpx.AsyncClient``.
|
||||
task_options: Options for tools that advertise
|
||||
``execution.taskSupport == "required"``. See :class:`MCPTaskOptions`.
|
||||
kwargs: Additional keyword arguments (accepted for backward compatibility but not used).
|
||||
"""
|
||||
super().__init__(
|
||||
@@ -2308,7 +1755,6 @@ class MCPStreamableHTTPTool(MCPTool):
|
||||
load_prompts=load_prompts,
|
||||
parse_prompt_results=parse_prompt_results,
|
||||
request_timeout=request_timeout,
|
||||
task_options=task_options,
|
||||
)
|
||||
self.url = url
|
||||
self.terminate_on_close = terminate_on_close
|
||||
@@ -2416,7 +1862,6 @@ class MCPWebsocketTool(MCPTool):
|
||||
allowed_tools: Collection[str] | None = None,
|
||||
client: SupportsChatGetResponse | None = None,
|
||||
additional_properties: dict[str, Any] | None = None,
|
||||
task_options: MCPTaskOptions | None = None,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
"""Initialize the MCP WebSocket tool.
|
||||
@@ -2459,8 +1904,6 @@ class MCPWebsocketTool(MCPTool):
|
||||
allowed_tools: A list of tools that are allowed to use this tool.
|
||||
additional_properties: Additional properties.
|
||||
client: The chat client to use for sampling.
|
||||
task_options: Options for tools that advertise
|
||||
``execution.taskSupport == "required"``. See :class:`MCPTaskOptions`.
|
||||
kwargs: Any extra arguments to pass to the WebSocket client.
|
||||
"""
|
||||
super().__init__(
|
||||
@@ -2477,7 +1920,6 @@ class MCPWebsocketTool(MCPTool):
|
||||
load_prompts=load_prompts,
|
||||
parse_prompt_results=parse_prompt_results,
|
||||
request_timeout=request_timeout,
|
||||
task_options=task_options,
|
||||
)
|
||||
self.url = url
|
||||
self._client_kwargs = kwargs
|
||||
|
||||
@@ -11,7 +11,6 @@ from enum import Enum
|
||||
from typing import TYPE_CHECKING, Any, Generic, Literal, TypeAlias, cast, overload
|
||||
|
||||
from ._clients import SupportsChatGetResponse
|
||||
from ._feature_stage import ExperimentalFeature, experimental
|
||||
from ._types import (
|
||||
AgentResponse,
|
||||
AgentResponseUpdate,
|
||||
@@ -215,12 +214,6 @@ class FunctionInvocationContext:
|
||||
result: Function execution result. Can be observed after calling ``call_next()``
|
||||
to see the actual execution result or can be set to override the execution result.
|
||||
kwargs: Additional runtime keyword arguments forwarded to the function invocation.
|
||||
tools: The live, mutable list of tools available to the model for the current
|
||||
agent run, or ``None`` when the function is invoked outside of a
|
||||
function-calling loop (for example via ``FunctionTool.invoke`` directly).
|
||||
Tools can add or remove tools during execution using :meth:`add_tools`
|
||||
and :meth:`remove_tools` (progressive tool exposure). Mutations take
|
||||
effect on the **next** model iteration, not the in-flight batch.
|
||||
|
||||
Examples:
|
||||
.. code-block:: python
|
||||
@@ -239,18 +232,6 @@ class FunctionInvocationContext:
|
||||
|
||||
# Continue execution
|
||||
await call_next()
|
||||
|
||||
Progressive tool exposure from inside a tool:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
from agent_framework import FunctionInvocationContext, tool
|
||||
|
||||
|
||||
@tool(approval_mode="never_require")
|
||||
def load_math_tools(ctx: FunctionInvocationContext) -> str:
|
||||
ctx.add_tools([factorial, fibonacci])
|
||||
return "Math tools are now available."
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@@ -261,7 +242,6 @@ class FunctionInvocationContext:
|
||||
metadata: Mapping[str, Any] | None = None,
|
||||
result: Any = None,
|
||||
kwargs: Mapping[str, Any] | None = None,
|
||||
tools: list[ToolTypes] | None = None,
|
||||
) -> None:
|
||||
"""Initialize the FunctionInvocationContext.
|
||||
|
||||
@@ -272,9 +252,6 @@ class FunctionInvocationContext:
|
||||
metadata: Metadata dictionary for sharing data between function middleware.
|
||||
result: Function execution result.
|
||||
kwargs: Additional runtime keyword arguments forwarded to the function invocation.
|
||||
tools: The live, mutable list of tools for the current agent run. When provided,
|
||||
this is the same list object the model sees on the next iteration, so
|
||||
appending or removing tools changes the model's available tools.
|
||||
"""
|
||||
self.function = function
|
||||
self.arguments = arguments
|
||||
@@ -282,96 +259,6 @@ class FunctionInvocationContext:
|
||||
self.metadata: dict[str, Any] = dict(metadata) if metadata is not None else {}
|
||||
self.result = result
|
||||
self.kwargs: dict[str, Any] = dict(kwargs) if kwargs is not None else {}
|
||||
self.tools = tools
|
||||
|
||||
@experimental(feature_id=ExperimentalFeature.PROGRESSIVE_TOOLS)
|
||||
def add_tools(
|
||||
self,
|
||||
tools: ToolTypes | Callable[..., Any] | Sequence[ToolTypes | Callable[..., Any]],
|
||||
) -> None:
|
||||
"""Add one or more tools to the current agent run (progressive tool exposure).
|
||||
|
||||
Callable inputs are converted to :class:`FunctionTool`, and tool collections are
|
||||
flattened, using the same normalization as the rest of the framework. Added tools
|
||||
become available to the model on the **next** iteration of the function-calling
|
||||
loop; they do not affect tool calls already requested in the in-flight batch.
|
||||
|
||||
Adding a tool whose name already exists is a no-op when it is the same object, and
|
||||
raises ``ValueError`` when it is a different object with a duplicate name.
|
||||
|
||||
Args:
|
||||
tools: A single tool/callable or a sequence of tools/callables to add.
|
||||
|
||||
Raises:
|
||||
RuntimeError: If the context has no live tools list (for example when the
|
||||
function is invoked outside of a function-calling loop).
|
||||
ValueError: If a different tool with a duplicate name is added.
|
||||
"""
|
||||
from ._tools import _append_unique_tools, normalize_tools # type: ignore[reportPrivateUsage]
|
||||
|
||||
if self.tools is None:
|
||||
raise RuntimeError(
|
||||
"Cannot add tools: this FunctionInvocationContext is not bound to a live "
|
||||
"agent run. add_tools is only available for functions invoked within an "
|
||||
"agent's function-calling loop."
|
||||
)
|
||||
# Validate the whole batch against a throwaway copy first, so a duplicate-name
|
||||
# clash partway through the batch raises before the live tool list is mutated
|
||||
# (all-or-nothing semantics).
|
||||
merged = _append_unique_tools(list(self.tools), normalize_tools(tools))
|
||||
self.tools[:] = merged
|
||||
|
||||
@experimental(feature_id=ExperimentalFeature.PROGRESSIVE_TOOLS)
|
||||
def remove_tools(
|
||||
self,
|
||||
tools: ToolTypes | Callable[..., Any] | Sequence[ToolTypes | Callable[..., Any]] | str | Sequence[str],
|
||||
) -> None:
|
||||
"""Remove one or more tools from the current agent run (progressive tool exposure).
|
||||
|
||||
Tools may be specified by name, by tool object, or by the original callable. Names
|
||||
that are not currently present are ignored. Removals take effect on the **next**
|
||||
iteration of the function-calling loop; tool calls already requested in the
|
||||
in-flight batch still execute.
|
||||
|
||||
Args:
|
||||
tools: A tool name, tool/callable, or a sequence of any of these to remove.
|
||||
|
||||
Raises:
|
||||
RuntimeError: If the context has no live tools list (for example when the
|
||||
function is invoked outside of a function-calling loop).
|
||||
"""
|
||||
from ._tools import _get_tool_name, normalize_tools # type: ignore[reportPrivateUsage]
|
||||
|
||||
if self.tools is None:
|
||||
raise RuntimeError(
|
||||
"Cannot remove tools: this FunctionInvocationContext is not bound to a live "
|
||||
"agent run. remove_tools is only available for functions invoked within an "
|
||||
"agent's function-calling loop."
|
||||
)
|
||||
|
||||
names_to_remove: set[str] = set()
|
||||
raw_items: list[Any]
|
||||
if isinstance(tools, str):
|
||||
raw_items = [tools]
|
||||
elif isinstance(tools, Sequence) and not isinstance(tools, (bytes, bytearray)):
|
||||
raw_items = list(cast("Sequence[Any]", tools))
|
||||
else:
|
||||
raw_items = [tools]
|
||||
for item in raw_items:
|
||||
if isinstance(item, str):
|
||||
names_to_remove.add(item)
|
||||
continue
|
||||
for normalized in normalize_tools(item):
|
||||
if name := _get_tool_name(normalized): # type: ignore[reportPrivateUsage]
|
||||
names_to_remove.add(name)
|
||||
|
||||
if not names_to_remove:
|
||||
return
|
||||
self.tools[:] = [
|
||||
tool
|
||||
for tool in self.tools
|
||||
if _get_tool_name(tool) not in names_to_remove # type: ignore[reportPrivateUsage]
|
||||
]
|
||||
|
||||
|
||||
class ChatContext:
|
||||
|
||||
@@ -7,8 +7,6 @@ import json
|
||||
import logging
|
||||
import re
|
||||
from collections.abc import Mapping, MutableMapping
|
||||
from dataclasses import asdict, is_dataclass
|
||||
from datetime import date, datetime
|
||||
from typing import Any, ClassVar, Protocol, TypeVar, runtime_checkable
|
||||
|
||||
logger = logging.getLogger("agent_framework")
|
||||
@@ -616,46 +614,3 @@ class SerializationMixin:
|
||||
# Fallback and default
|
||||
# Convert class name to snake_case
|
||||
return _CAMEL_TO_SNAKE_PATTERN.sub("_", cls.__name__).lower()
|
||||
|
||||
|
||||
def make_json_safe(obj: Any) -> Any:
|
||||
"""Recursively convert an object to a JSON-serializable form.
|
||||
|
||||
Handles dataclasses, Pydantic models, objects with ``to_dict``/``dict``/``__dict__``,
|
||||
datetimes, lists, dicts, and primitives. Falls back to ``str()`` for any remaining
|
||||
non-serializable value so that ``json.dumps`` never raises a ``TypeError``.
|
||||
|
||||
Args:
|
||||
obj: Object to make JSON safe.
|
||||
|
||||
Returns:
|
||||
A JSON-serializable version of the object.
|
||||
"""
|
||||
if obj is None or isinstance(obj, (str, int, float, bool)):
|
||||
return obj
|
||||
if isinstance(obj, (datetime, date)):
|
||||
return obj.isoformat()
|
||||
if is_dataclass(obj) and not isinstance(obj, type):
|
||||
return make_json_safe(asdict(obj)) # type: ignore[arg-type]
|
||||
if callable(getattr(obj, "model_dump", None)):
|
||||
try:
|
||||
return make_json_safe(obj.model_dump()) # type: ignore[no-any-return]
|
||||
except TypeError:
|
||||
pass
|
||||
if callable(getattr(obj, "to_dict", None)):
|
||||
try:
|
||||
return make_json_safe(obj.to_dict()) # type: ignore[no-any-return]
|
||||
except TypeError:
|
||||
pass
|
||||
if callable(getattr(obj, "dict", None)):
|
||||
try:
|
||||
return make_json_safe(obj.dict()) # type: ignore[no-any-return]
|
||||
except TypeError:
|
||||
pass
|
||||
if isinstance(obj, dict):
|
||||
return {str(key): make_json_safe(value) for key, value in obj.items()} # type: ignore[misc]
|
||||
if isinstance(obj, (list, tuple)):
|
||||
return [make_json_safe(item) for item in obj] # type: ignore[misc]
|
||||
if hasattr(obj, "__dict__"):
|
||||
return {key: make_json_safe(value) for key, value in vars(obj).items()} # type: ignore[misc]
|
||||
return str(obj)
|
||||
|
||||
@@ -44,7 +44,6 @@ Only use skills from trusted sources.
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import inspect
|
||||
import json
|
||||
import logging
|
||||
@@ -61,10 +60,6 @@ from ._sessions import ContextProvider
|
||||
from ._tools import FunctionTool
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from mcp.client.session import ClientSession
|
||||
from mcp.types import ReadResourceResult
|
||||
from pydantic import AnyUrl
|
||||
|
||||
from ._agents import SupportsAgentRun
|
||||
from ._sessions import AgentSession, SessionContext
|
||||
|
||||
@@ -3290,443 +3285,4 @@ class AggregatingSkillsSource(SkillsSource):
|
||||
return result
|
||||
|
||||
|
||||
# region MCP Skills
|
||||
|
||||
|
||||
def _mcp_any_url(uri: str) -> AnyUrl:
|
||||
"""Convert a string URI to a :class:`pydantic.AnyUrl` for MCP client calls."""
|
||||
from pydantic import AnyUrl as _AnyUrl
|
||||
|
||||
return _AnyUrl(uri)
|
||||
|
||||
|
||||
def _is_mcp_resource_not_found(ex: Exception) -> bool:
|
||||
"""Return ``True`` when *ex* is an :class:`McpError` indicating a missing resource.
|
||||
|
||||
Two codes are treated as "not found":
|
||||
|
||||
* ``-32002`` — the MCP-spec "Resource not found" code returned by a
|
||||
compliant server when the URI does not exist. Not exported as a
|
||||
constant from ``mcp.types`` but defined by the resources subprotocol.
|
||||
* ``METHOD_NOT_FOUND`` (``-32601``) — the server does not implement
|
||||
``resources/read`` at all, which for the skills source is functionally
|
||||
equivalent to "no skills available."
|
||||
|
||||
All other codes — ``INVALID_PARAMS``, ``INTERNAL_ERROR``, ``PARSE_ERROR``,
|
||||
``CONNECTION_CLOSED``, auth rejections, and generic handler errors
|
||||
(code ``0``) — are treated as real failures so that a misconfigured
|
||||
token or crashing server is not silently mistaken for "the server has no
|
||||
skills."
|
||||
"""
|
||||
from mcp.shared.exceptions import McpError as _McpError
|
||||
|
||||
if not isinstance(ex, _McpError):
|
||||
return False
|
||||
from mcp.types import METHOD_NOT_FOUND as _METHOD_NOT_FOUND
|
||||
|
||||
return ex.error.code in {-32002, _METHOD_NOT_FOUND}
|
||||
|
||||
|
||||
def _mcp_join_text(result: ReadResourceResult) -> str:
|
||||
"""Join all :class:`TextResourceContents` items in a result into a single string."""
|
||||
from mcp.types import TextResourceContents as _TextResourceContents
|
||||
|
||||
return "\n".join(c.text for c in result.contents if isinstance(c, _TextResourceContents))
|
||||
|
||||
|
||||
class _McpSkillIndexEntry: # noqa: B903
|
||||
"""A single entry in the ``skill://index.json`` discovery document.
|
||||
|
||||
All fields are optional to support lenient deserialization; callers
|
||||
validate required fields before use.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
name: str | None = None,
|
||||
type: str | None = None,
|
||||
description: str | None = None,
|
||||
url: str | None = None,
|
||||
digest: str | None = None,
|
||||
) -> None:
|
||||
self.name = name
|
||||
self.type = type
|
||||
self.description = description
|
||||
self.url = url
|
||||
self.digest = digest
|
||||
|
||||
|
||||
class _McpSkillIndex:
|
||||
"""DTO for the ``skill://index.json`` discovery document.
|
||||
|
||||
Represents the Agent Skills Discovery v0.2.0 schema as bound to MCP
|
||||
by SEP-2640.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
schema: str | None = None,
|
||||
skills: list[_McpSkillIndexEntry] | None = None,
|
||||
) -> None:
|
||||
self.schema = schema
|
||||
self.skills: list[_McpSkillIndexEntry] = skills if skills is not None else []
|
||||
|
||||
|
||||
def _parse_mcp_skill_index(text: str) -> _McpSkillIndex:
|
||||
"""Parse a JSON string into a :class:`_McpSkillIndex`.
|
||||
|
||||
Args:
|
||||
text: Raw JSON text from ``skill://index.json``.
|
||||
|
||||
Returns:
|
||||
A populated :class:`_McpSkillIndex` instance.
|
||||
|
||||
Raises:
|
||||
json.JSONDecodeError: If the text is not valid JSON.
|
||||
ValueError: If the top-level value is not a JSON object.
|
||||
"""
|
||||
raw: dict[str, Any] = json.loads(text)
|
||||
|
||||
if not isinstance(raw, dict):
|
||||
raise ValueError("skill://index.json must be a JSON object")
|
||||
|
||||
entries: list[_McpSkillIndexEntry] = []
|
||||
|
||||
raw_skills: list[Any] = raw.get("skills") or []
|
||||
|
||||
for item in raw_skills:
|
||||
if isinstance(item, dict):
|
||||
d = cast(dict[str, Any], item)
|
||||
|
||||
entries.append(
|
||||
_McpSkillIndexEntry(
|
||||
name=d.get("name"),
|
||||
type=d.get("type"),
|
||||
description=d.get("description"),
|
||||
url=d.get("url"),
|
||||
digest=d.get("digest"),
|
||||
)
|
||||
)
|
||||
|
||||
return _McpSkillIndex(schema=raw.get("$schema"), skills=entries)
|
||||
|
||||
|
||||
@experimental(feature_id=ExperimentalFeature.MCP_SKILLS)
|
||||
class MCPSkillResource(SkillResource):
|
||||
"""A :class:`SkillResource` backed by content fetched from an MCP server.
|
||||
|
||||
The :class:`~mcp.types.ReadResourceResult` is fetched eagerly by
|
||||
:meth:`MCPSkill.get_resource` at construction time; :meth:`read`
|
||||
extracts text or binary content from the result.
|
||||
"""
|
||||
|
||||
def __init__(self, *, name: str, result: ReadResourceResult) -> None:
|
||||
"""Initialize an MCPSkillResource.
|
||||
|
||||
Args:
|
||||
name: The resource name (e.g. a relative path or identifier).
|
||||
result: The result returned by the MCP server's ``resources/read`` request.
|
||||
"""
|
||||
super().__init__(name=name)
|
||||
self._result = result
|
||||
|
||||
async def read(self, **kwargs: Any) -> Any:
|
||||
"""Read the resource content.
|
||||
|
||||
Returns:
|
||||
A ``bytes`` object when the resource contains binary content,
|
||||
a ``str`` when it contains text, or ``None`` when the server
|
||||
returned no content blocks.
|
||||
"""
|
||||
from mcp.types import BlobResourceContents, TextResourceContents
|
||||
|
||||
for content in self._result.contents:
|
||||
if isinstance(content, BlobResourceContents):
|
||||
blob = content.blob
|
||||
# Strip data-URI prefix if present (some MCP servers send
|
||||
# full data URIs instead of raw base64).
|
||||
if blob.startswith("data:"):
|
||||
blob = blob.split(",", 1)[-1]
|
||||
return base64.b64decode(blob)
|
||||
|
||||
text = "\n".join(c.text for c in self._result.contents if isinstance(c, TextResourceContents))
|
||||
return text if text else None
|
||||
|
||||
|
||||
@experimental(feature_id=ExperimentalFeature.MCP_SKILLS)
|
||||
class MCPSkill(Skill):
|
||||
"""A :class:`Skill` discovered from an MCP server exposing the Agent Skills convention.
|
||||
|
||||
The skill is constructed from ``skill://index.json`` discovery metadata;
|
||||
:meth:`get_content` fetches the full ``SKILL.md`` content from the MCP
|
||||
server on demand via ``resources/read``.
|
||||
|
||||
Per SEP-2640, resources referenced inside SKILL.md are fetched on demand
|
||||
via the originating MCP server: :meth:`get_resource` resolves a relative
|
||||
resource name against the skill's root URI, issues a ``resources/read``
|
||||
request, and returns an :class:`MCPSkillResource` with pre-fetched content.
|
||||
"""
|
||||
|
||||
_SKILL_MD_SUFFIX: Final[str] = "SKILL.md"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
frontmatter: SkillFrontmatter,
|
||||
skill_md_uri: str,
|
||||
client: ClientSession,
|
||||
) -> None:
|
||||
"""Initialize an MCPSkill.
|
||||
|
||||
Args:
|
||||
frontmatter: The parsed frontmatter metadata for this skill.
|
||||
skill_md_uri: The full MCP resource URI of the ``SKILL.md`` resource
|
||||
(e.g. ``skill://unit-converter/SKILL.md``). The skill's root URI
|
||||
is derived by stripping the trailing ``SKILL.md`` segment.
|
||||
client: The MCP client session used to fetch resources on demand.
|
||||
"""
|
||||
self._frontmatter = frontmatter
|
||||
self._skill_md_uri = skill_md_uri
|
||||
self._skill_root_uri = self._compute_skill_root_uri(skill_md_uri)
|
||||
self._client = client
|
||||
self._content: str | None = None
|
||||
|
||||
@property
|
||||
def frontmatter(self) -> SkillFrontmatter:
|
||||
"""The L1 discovery metadata for this skill."""
|
||||
return self._frontmatter
|
||||
|
||||
async def get_content(self) -> str:
|
||||
"""Get the full SKILL.md content from the MCP server.
|
||||
|
||||
Fetches the content via ``resources/read`` on the first call and
|
||||
caches the result for subsequent calls.
|
||||
|
||||
Returns:
|
||||
The SKILL.md content string.
|
||||
|
||||
Raises:
|
||||
ValueError: If the MCP server returned no text content for the
|
||||
SKILL.md resource.
|
||||
"""
|
||||
if self._content is not None:
|
||||
return self._content
|
||||
|
||||
result = await self._client.read_resource(_mcp_any_url(self._skill_md_uri))
|
||||
text = _mcp_join_text(result)
|
||||
if not text:
|
||||
raise ValueError(
|
||||
f"The MCP server returned no text content for SKILL.md resource '{self._skill_md_uri}'."
|
||||
)
|
||||
self._content = text
|
||||
return text
|
||||
|
||||
async def get_resource(self, name: str) -> SkillResource | None:
|
||||
"""Get a sibling resource by name from the MCP server.
|
||||
|
||||
Resolves *name* as a relative path against the skill's root URI,
|
||||
issues a ``resources/read`` request to the MCP server, and returns
|
||||
an :class:`MCPSkillResource` with the pre-fetched content.
|
||||
|
||||
Args:
|
||||
name: The resource name (e.g. ``references/checklist.md``).
|
||||
|
||||
Returns:
|
||||
An :class:`MCPSkillResource`, or ``None`` when the name is empty
|
||||
or the resource does not exist on the server.
|
||||
"""
|
||||
if not name or not name.strip():
|
||||
return None
|
||||
|
||||
normalized = self._validate_resource_name(name)
|
||||
if normalized is None:
|
||||
return None
|
||||
|
||||
uri = self._skill_root_uri + normalized
|
||||
try:
|
||||
result = await self._client.read_resource(_mcp_any_url(uri))
|
||||
except Exception as ex:
|
||||
if _is_mcp_resource_not_found(ex):
|
||||
logger.debug("MCP resource '%s' not available: %s", uri, ex)
|
||||
return None
|
||||
raise
|
||||
|
||||
return MCPSkillResource(name=name, result=result)
|
||||
|
||||
@staticmethod
|
||||
def _validate_resource_name(name: str) -> str | None:
|
||||
"""Validate a resource name and return the normalized form.
|
||||
|
||||
Defense in depth: refuses names that could escape the skill root
|
||||
(absolute paths, embedded URI schemes, parent-traversal segments).
|
||||
The MCP server is the authority on URI resolution, but rejecting
|
||||
obviously unsafe shapes client-side avoids leaking escape attempts
|
||||
upstream.
|
||||
|
||||
Args:
|
||||
name: The raw resource name to validate.
|
||||
|
||||
Returns:
|
||||
The normalized name with backslashes replaced by forward slashes,
|
||||
or ``None`` if the name is unsafe.
|
||||
"""
|
||||
normalized = name.replace("\\", "/")
|
||||
if (
|
||||
normalized.startswith("/")
|
||||
or "://" in normalized
|
||||
or any(seg == ".." for seg in normalized.split("/"))
|
||||
):
|
||||
logger.debug("Rejecting resource name with unsafe path components: %r", name)
|
||||
return None
|
||||
return normalized
|
||||
|
||||
@staticmethod
|
||||
def _compute_skill_root_uri(skill_md_uri: str) -> str:
|
||||
"""Strip the trailing ``SKILL.md`` from the URI to produce the skill root.
|
||||
|
||||
If the URI doesn't end with ``SKILL.md``, ensures it ends with a
|
||||
trailing slash.
|
||||
"""
|
||||
if skill_md_uri.endswith(MCPSkill._SKILL_MD_SUFFIX):
|
||||
return skill_md_uri[: -len(MCPSkill._SKILL_MD_SUFFIX)]
|
||||
if skill_md_uri.endswith("/"):
|
||||
return skill_md_uri
|
||||
return skill_md_uri + "/"
|
||||
|
||||
|
||||
@experimental(feature_id=ExperimentalFeature.MCP_SKILLS)
|
||||
class MCPSkillsSource(SkillsSource):
|
||||
"""A :class:`SkillsSource` that discovers Agent Skills served over MCP.
|
||||
|
||||
Discovery follows the SEP-2640 recommended approach: the source reads
|
||||
the well-known ``skill://index.json`` resource and constructs one
|
||||
:class:`MCPSkill` per ``skill-md`` entry directly from the entry's
|
||||
``name``, ``description``, and ``url`` fields.
|
||||
|
||||
The referenced ``SKILL.md`` resource is **not** read during discovery;
|
||||
the host fetches its body on demand via ``resources/read`` when the
|
||||
skill content is needed.
|
||||
|
||||
Only index entries of type ``skill-md`` are supported; entries of any
|
||||
other type are silently skipped.
|
||||
|
||||
If ``skill://index.json`` is absent, unreadable, empty, or fails to
|
||||
parse, this source returns an empty list.
|
||||
|
||||
Examples:
|
||||
.. code-block:: python
|
||||
|
||||
from mcp.client.session import ClientSession
|
||||
|
||||
source = MCPSkillsSource(client=session)
|
||||
skills = await source.get_skills()
|
||||
"""
|
||||
|
||||
_INDEX_URI: Final[str] = "skill://index.json"
|
||||
_SKILL_MD_TYPE: Final[str] = "skill-md"
|
||||
|
||||
def __init__(self, client: ClientSession) -> None:
|
||||
"""Initialize an MCPSkillsSource.
|
||||
|
||||
Args:
|
||||
client: An MCP client session connected to a server that
|
||||
exposes Agent Skills resources.
|
||||
"""
|
||||
self._client = client
|
||||
|
||||
async def get_skills(self) -> list[Skill]:
|
||||
"""Discover and return skills from the MCP server.
|
||||
|
||||
Reads ``skill://index.json``, parses it, and creates an
|
||||
:class:`MCPSkill` for each valid ``skill-md`` entry.
|
||||
|
||||
Returns:
|
||||
A list of discovered :class:`MCPSkill` instances.
|
||||
"""
|
||||
index = await self._try_read_index()
|
||||
if index is None:
|
||||
return []
|
||||
|
||||
skills: list[Skill] = []
|
||||
for entry in index.skills:
|
||||
result = self._try_create_skill(entry)
|
||||
if result is not None:
|
||||
skills.append(result)
|
||||
logger.info("Loaded MCP skill: %s", result.frontmatter.name)
|
||||
else:
|
||||
logger.debug(
|
||||
"Skipping skill index entry '%s'",
|
||||
entry.name or "(unnamed)",
|
||||
)
|
||||
|
||||
logger.info("Successfully loaded %d skills from MCP server", len(skills))
|
||||
return skills
|
||||
|
||||
async def _try_read_index(self) -> _McpSkillIndex | None:
|
||||
"""Attempt to read and parse ``skill://index.json`` from the MCP server.
|
||||
|
||||
Returns:
|
||||
A parsed :class:`_McpSkillIndex`, or ``None`` if the index is
|
||||
absent, empty, or malformed.
|
||||
"""
|
||||
try:
|
||||
result = await self._client.read_resource(_mcp_any_url(self._INDEX_URI))
|
||||
except Exception as ex:
|
||||
if _is_mcp_resource_not_found(ex):
|
||||
logger.debug("No skill://index.json resource available on MCP server: %s", ex)
|
||||
return None
|
||||
logger.warning("Failed to read skill://index.json from MCP server.", exc_info=True)
|
||||
raise
|
||||
|
||||
index_text = _mcp_join_text(result)
|
||||
if not index_text:
|
||||
logger.debug("skill://index.json on MCP server returned empty/non-text contents")
|
||||
return None
|
||||
|
||||
try:
|
||||
return _parse_mcp_skill_index(index_text)
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
logger.warning("Failed to parse skill://index.json JSON document.", exc_info=True)
|
||||
return None
|
||||
|
||||
def _try_create_skill(self, entry: _McpSkillIndexEntry) -> MCPSkill | None:
|
||||
"""Attempt to create an :class:`MCPSkill` from an index entry.
|
||||
|
||||
Args:
|
||||
entry: A single entry from the skill index.
|
||||
|
||||
Returns:
|
||||
An :class:`MCPSkill` if the entry is valid, or ``None`` if the
|
||||
entry should be skipped.
|
||||
"""
|
||||
if entry.type != self._SKILL_MD_TYPE:
|
||||
logger.debug(
|
||||
"Skipping entry '%s': unsupported type '%s'",
|
||||
entry.name or "(unnamed)",
|
||||
entry.type or "(none)",
|
||||
)
|
||||
return None
|
||||
|
||||
if not entry.name or not entry.name.strip():
|
||||
logger.debug("Skipping entry: missing required 'name' field")
|
||||
return None
|
||||
|
||||
if not entry.description or not entry.description.strip():
|
||||
logger.debug("Skipping entry '%s': missing required 'description' field", entry.name)
|
||||
return None
|
||||
|
||||
if not entry.url or not entry.url.strip():
|
||||
logger.debug("Skipping entry '%s': missing required 'url' field", entry.name)
|
||||
return None
|
||||
|
||||
try:
|
||||
fm = SkillFrontmatter(name=entry.name, description=entry.description)
|
||||
except ValueError as ex:
|
||||
logger.debug("Skipping entry '%s': invalid metadata: %s", entry.name, ex)
|
||||
return None
|
||||
|
||||
return MCPSkill(frontmatter=fm, skill_md_uri=entry.url, client=self._client)
|
||||
|
||||
|
||||
# endregion
|
||||
|
||||
@@ -1418,7 +1418,6 @@ async def _auto_invoke_function(
|
||||
sequence_index: int | None = None,
|
||||
request_index: int | None = None,
|
||||
middleware_pipeline: FunctionMiddlewarePipeline | None = None,
|
||||
live_tools: list[ToolTypes] | None = None,
|
||||
) -> Content:
|
||||
"""Invoke a function call requested by the agent, applying middleware that is defined.
|
||||
|
||||
@@ -1433,8 +1432,6 @@ async def _auto_invoke_function(
|
||||
sequence_index: The index of the function call in the sequence.
|
||||
request_index: The index of the request iteration.
|
||||
middleware_pipeline: Optional middleware pipeline to apply during execution.
|
||||
live_tools: The live, mutable tools list for the current agent run, exposed on
|
||||
the FunctionInvocationContext so tools can add/remove tools at runtime.
|
||||
|
||||
Returns:
|
||||
The function result content.
|
||||
@@ -1526,7 +1523,6 @@ async def _auto_invoke_function(
|
||||
arguments=args,
|
||||
session=invocation_session,
|
||||
kwargs=runtime_kwargs.copy(),
|
||||
tools=live_tools,
|
||||
)
|
||||
function_result = await tool.invoke(
|
||||
arguments=args,
|
||||
@@ -1541,10 +1537,6 @@ async def _auto_invoke_function(
|
||||
except UserInputRequiredException:
|
||||
raise
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
f"Function '{tool.name}' raised an exception; returning an error result to the "
|
||||
f"model. Set include_detailed_errors=True for the full detail. Exception: {exc!r}"
|
||||
)
|
||||
message = "Error: Function failed."
|
||||
if config.get("include_detailed_errors", False):
|
||||
message = f"{message} Exception: {exc}"
|
||||
@@ -1560,7 +1552,6 @@ async def _auto_invoke_function(
|
||||
arguments=args,
|
||||
session=invocation_session,
|
||||
kwargs=runtime_kwargs.copy(),
|
||||
tools=live_tools,
|
||||
)
|
||||
|
||||
call_id = function_call_content.call_id
|
||||
@@ -1617,10 +1608,6 @@ async def _auto_invoke_function(
|
||||
except UserInputRequiredException:
|
||||
raise
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
f"Function '{tool.name}' raised an exception; returning an error result to the "
|
||||
f"model. Set include_detailed_errors=True for the full detail. Exception: {exc!r}"
|
||||
)
|
||||
message = "Error: Function failed."
|
||||
if config.get("include_detailed_errors", False):
|
||||
message = f"{message} Exception: {exc}"
|
||||
@@ -1672,9 +1659,6 @@ async def _try_execute_function_calls(
|
||||
from ._types import Content
|
||||
|
||||
tool_map = _get_tool_map(tools)
|
||||
# The live tools list (when tools is the run-local list) is exposed on the
|
||||
# FunctionInvocationContext so tools can add/remove tools during the run.
|
||||
live_tools: list[ToolTypes] | None = cast("list[ToolTypes]", tools) if isinstance(tools, list) else None
|
||||
approval_tools = [tool_name for tool_name, tool in tool_map.items() if tool.approval_mode == "always_require"]
|
||||
logger.debug(
|
||||
"_try_execute_function_calls: tool_map keys=%s, approval_tools=%s",
|
||||
@@ -1749,7 +1733,6 @@ async def _try_execute_function_calls(
|
||||
request_index=attempt_idx,
|
||||
middleware_pipeline=middleware_pipeline,
|
||||
config=config,
|
||||
live_tools=live_tools,
|
||||
)
|
||||
return (result, False)
|
||||
except MiddlewareTermination as exc:
|
||||
@@ -2388,13 +2371,6 @@ class FunctionInvocationLayer(Generic[OptionsCoT]):
|
||||
function_invocation_kwargs=function_invocation_kwargs,
|
||||
client_kwargs=filtered_kwargs,
|
||||
)
|
||||
# Establish a single, run-local mutable tools list so that tools can add or remove
|
||||
# tools during the run (progressive tool exposure). A fresh list is created via
|
||||
# normalize_tools so the caller's original tools container is never mutated, while
|
||||
# the same list object is shared with the model (options["tools"]) and the tool map
|
||||
# rebuilt on every loop iteration.
|
||||
if mutable_options.get("tools"):
|
||||
mutable_options["tools"] = normalize_tools(mutable_options["tools"])
|
||||
if not stream:
|
||||
|
||||
async def _get_response() -> ChatResponse[Any]:
|
||||
|
||||
@@ -12,7 +12,6 @@ from datetime import datetime, timezone
|
||||
from typing import TYPE_CHECKING, Any, ClassVar, Literal, cast, overload
|
||||
|
||||
from .._agents import BaseAgent
|
||||
from .._serialization import make_json_safe
|
||||
from .._sessions import (
|
||||
AgentSession,
|
||||
ContextProvider,
|
||||
@@ -62,7 +61,7 @@ class WorkflowAgent(BaseAgent):
|
||||
data: Any
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return {"request_id": self.request_id, "data": make_json_safe(self.data)}
|
||||
return {"request_id": self.request_id, "data": self.data}
|
||||
|
||||
def to_json(self) -> str:
|
||||
return json.dumps(self.to_dict())
|
||||
|
||||
@@ -47,7 +47,6 @@ from copy import deepcopy
|
||||
from typing import Any, Generic, Literal, TypeVar, overload
|
||||
|
||||
from .._feature_stage import ExperimentalFeature, experimental
|
||||
from .._serialization import make_json_safe
|
||||
from .._types import AgentResponse, AgentResponseUpdate, ResponseStream
|
||||
from ..observability import OtelAttr, capture_exception, create_workflow_span
|
||||
from ._checkpoint import CheckpointStorage, WorkflowCheckpoint
|
||||
@@ -1516,7 +1515,7 @@ class FunctionalWorkflowAgent:
|
||||
function_call = Content.from_function_call(
|
||||
call_id=request_id,
|
||||
name=self.REQUEST_INFO_FUNCTION_NAME,
|
||||
arguments={"request_id": request_id, "data": make_json_safe(event.data)},
|
||||
arguments={"request_id": request_id, "data": event.data},
|
||||
)
|
||||
return Content.from_function_approval_request(
|
||||
id=request_id,
|
||||
|
||||
@@ -34,7 +34,6 @@ _IMPORTS: dict[str, tuple[str, str]] = {
|
||||
"FoundryLocalChatOptions": ("agent_framework_foundry_local", "agent-framework-foundry-local"),
|
||||
"FoundryLocalClient": ("agent_framework_foundry_local", "agent-framework-foundry-local"),
|
||||
"FoundryLocalSettings": ("agent_framework_foundry_local", "agent-framework-foundry-local"),
|
||||
"GeneratedEvaluatorRef": ("agent_framework_foundry", "agent-framework-foundry"),
|
||||
"RawAnthropicFoundryClient": ("agent_framework_anthropic", "agent-framework-anthropic"),
|
||||
"RawFoundryAgent": ("agent_framework_foundry", "agent-framework-foundry"),
|
||||
"RawFoundryAgentChatClient": ("agent_framework_foundry", "agent-framework-foundry"),
|
||||
|
||||
@@ -20,7 +20,6 @@ from agent_framework_foundry import (
|
||||
FoundryEmbeddingSettings,
|
||||
FoundryEvals,
|
||||
FoundryMemoryProvider,
|
||||
GeneratedEvaluatorRef,
|
||||
RawFoundryAgent,
|
||||
RawFoundryAgentChatClient,
|
||||
RawFoundryChatClient,
|
||||
@@ -53,7 +52,6 @@ __all__ = [
|
||||
"FoundryLocalClient",
|
||||
"FoundryLocalSettings",
|
||||
"FoundryMemoryProvider",
|
||||
"GeneratedEvaluatorRef",
|
||||
"RawAnthropicFoundryClient",
|
||||
"RawFoundryAgent",
|
||||
"RawFoundryAgentChatClient",
|
||||
|
||||
@@ -498,34 +498,14 @@ def _get_exporters_from_env(
|
||||
# Get base endpoint
|
||||
base_endpoint = os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT")
|
||||
|
||||
# Get signal-specific endpoints (these override base endpoint and are used verbatim)
|
||||
traces_endpoint_specific = os.getenv("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT")
|
||||
metrics_endpoint_specific = os.getenv("OTEL_EXPORTER_OTLP_METRICS_ENDPOINT")
|
||||
logs_endpoint_specific = os.getenv("OTEL_EXPORTER_OTLP_LOGS_ENDPOINT")
|
||||
# Get signal-specific endpoints (these override base endpoint)
|
||||
traces_endpoint = os.getenv("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT") or base_endpoint
|
||||
metrics_endpoint = os.getenv("OTEL_EXPORTER_OTLP_METRICS_ENDPOINT") or base_endpoint
|
||||
logs_endpoint = os.getenv("OTEL_EXPORTER_OTLP_LOGS_ENDPOINT") or base_endpoint
|
||||
|
||||
# Get protocol (default is grpc)
|
||||
protocol = os.getenv("OTEL_EXPORTER_OTLP_PROTOCOL", "grpc").lower()
|
||||
|
||||
# Per the OTel spec, OTEL_EXPORTER_OTLP_ENDPOINT is a *base* URL for HTTP — the SDK
|
||||
# auto-appends /v1/{traces,metrics,logs} when it reads the env var directly. The
|
||||
# signal-specific endpoint env vars are *full* URLs used verbatim. Because we read
|
||||
# the env vars here and forward them as the ``endpoint=`` constructor argument
|
||||
# (which the SDK always treats as a full URL), we must replicate the auto-append
|
||||
# ourselves for HTTP when falling back to the base endpoint. For gRPC, the base
|
||||
# endpoint is used as-is.
|
||||
traces_endpoint: str | None
|
||||
metrics_endpoint: str | None
|
||||
logs_endpoint: str | None
|
||||
if protocol in ("http/protobuf", "http") and base_endpoint:
|
||||
base_for_http = base_endpoint.rstrip("/")
|
||||
traces_endpoint = traces_endpoint_specific or f"{base_for_http}/v1/traces"
|
||||
metrics_endpoint = metrics_endpoint_specific or f"{base_for_http}/v1/metrics"
|
||||
logs_endpoint = logs_endpoint_specific or f"{base_for_http}/v1/logs"
|
||||
else:
|
||||
traces_endpoint = traces_endpoint_specific or base_endpoint
|
||||
metrics_endpoint = metrics_endpoint_specific or base_endpoint
|
||||
logs_endpoint = logs_endpoint_specific or base_endpoint
|
||||
|
||||
# Get base headers
|
||||
base_headers_str = os.getenv("OTEL_EXPORTER_OTLP_HEADERS", "")
|
||||
base_headers = _parse_headers(base_headers_str)
|
||||
|
||||
@@ -3975,425 +3975,3 @@ async def test_user_input_request_empty_contents_returns_fallback(chat_client_ba
|
||||
]
|
||||
assert len(function_results) >= 1
|
||||
assert any("user input" in (fr.result or "").lower() for fr in function_results)
|
||||
|
||||
|
||||
# region Progressive tool exposure (FunctionInvocationContext.add_tools / remove_tools)
|
||||
|
||||
|
||||
def _pte_function_call_response(call_id: str, name: str, arguments: str = "{}") -> ChatResponse:
|
||||
return ChatResponse(
|
||||
messages=Message(
|
||||
role="assistant",
|
||||
contents=[Content.from_function_call(call_id=call_id, name=name, arguments=arguments)],
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _pte_text_response(text: str = "done") -> ChatResponse:
|
||||
return ChatResponse(messages=Message(role="assistant", contents=[text]))
|
||||
|
||||
|
||||
@tool(name="factorial", approval_mode="never_require")
|
||||
def _pte_factorial(n: int) -> int:
|
||||
"""Compute the factorial of n."""
|
||||
result = 1
|
||||
for value in range(2, n + 1):
|
||||
result *= value
|
||||
return result
|
||||
|
||||
|
||||
async def test_context_exposes_live_tools(chat_client_base: SupportsChatGetResponse):
|
||||
from agent_framework import FunctionTool
|
||||
|
||||
seen_names: list[str] = []
|
||||
|
||||
@tool(name="inspect_tools", approval_mode="never_require")
|
||||
def inspect_tools(ctx: FunctionInvocationContext) -> str:
|
||||
assert ctx.tools is not None
|
||||
seen_names.extend(t.name for t in ctx.tools if isinstance(t, FunctionTool))
|
||||
return "inspected"
|
||||
|
||||
chat_client_base.run_responses = [
|
||||
_pte_function_call_response("1", "inspect_tools"),
|
||||
_pte_text_response(),
|
||||
]
|
||||
await chat_client_base.get_response(
|
||||
[Message(role="user", contents=["hi"])],
|
||||
options={"tool_choice": "auto", "tools": [inspect_tools]},
|
||||
)
|
||||
assert "inspect_tools" in seen_names
|
||||
|
||||
|
||||
async def test_add_tools_available_next_iteration(chat_client_base: SupportsChatGetResponse):
|
||||
exec_counter = 0
|
||||
|
||||
@tool(name="factorial", approval_mode="never_require")
|
||||
def factorial(n: int) -> int:
|
||||
nonlocal exec_counter
|
||||
exec_counter += 1
|
||||
return 120
|
||||
|
||||
@tool(name="load_math", approval_mode="never_require")
|
||||
def load_math(ctx: FunctionInvocationContext) -> str:
|
||||
ctx.add_tools(factorial)
|
||||
return "math tools loaded"
|
||||
|
||||
chat_client_base.function_invocation_configuration["max_iterations"] = 3 # type: ignore[attr-defined]
|
||||
chat_client_base.run_responses = [
|
||||
_pte_function_call_response("1", "load_math"),
|
||||
_pte_function_call_response("2", "factorial", '{"n": 5}'),
|
||||
_pte_text_response(),
|
||||
]
|
||||
response = await chat_client_base.get_response(
|
||||
[Message(role="user", contents=["compute 5!"])],
|
||||
options={"tool_choice": "auto", "tools": [load_math]},
|
||||
)
|
||||
assert exec_counter == 1
|
||||
assert response.messages[-1].text == "done"
|
||||
|
||||
|
||||
async def test_add_tools_model_sees_added_tools_in_options(chat_client_base: SupportsChatGetResponse):
|
||||
from agent_framework import FunctionTool
|
||||
|
||||
recorded: list[list[str]] = []
|
||||
client_cls = type(chat_client_base)
|
||||
original = client_cls._get_non_streaming_response
|
||||
|
||||
async def recording(self: Any, *, messages: Any, options: dict[str, Any], **kwargs: Any) -> ChatResponse:
|
||||
tools = options.get("tools") or []
|
||||
recorded.append([t.name for t in tools if isinstance(t, FunctionTool)])
|
||||
return await original(self, messages=messages, options=options, **kwargs)
|
||||
|
||||
@tool(name="load_math", approval_mode="never_require")
|
||||
def load_math(ctx: FunctionInvocationContext) -> str:
|
||||
ctx.add_tools(_pte_factorial)
|
||||
return "loaded"
|
||||
|
||||
chat_client_base.function_invocation_configuration["max_iterations"] = 3 # type: ignore[attr-defined]
|
||||
chat_client_base.run_responses = [
|
||||
_pte_function_call_response("1", "load_math"),
|
||||
_pte_function_call_response("2", "factorial", '{"n": 5}'),
|
||||
_pte_text_response(),
|
||||
]
|
||||
monkey = pytest.MonkeyPatch()
|
||||
monkey.setattr(client_cls, "_get_non_streaming_response", recording)
|
||||
try:
|
||||
await chat_client_base.get_response(
|
||||
[Message(role="user", contents=["compute 5!"])],
|
||||
options={"tool_choice": "auto", "tools": [load_math]},
|
||||
)
|
||||
finally:
|
||||
monkey.undo()
|
||||
|
||||
assert recorded[0] == ["load_math"]
|
||||
assert "factorial" in recorded[1]
|
||||
|
||||
|
||||
async def test_remove_tools_next_iteration(chat_client_base: SupportsChatGetResponse):
|
||||
from agent_framework import FunctionTool
|
||||
|
||||
recorded: list[list[str]] = []
|
||||
client_cls = type(chat_client_base)
|
||||
original = client_cls._get_non_streaming_response
|
||||
|
||||
async def recording(self: Any, *, messages: Any, options: dict[str, Any], **kwargs: Any) -> ChatResponse:
|
||||
tools = options.get("tools") or []
|
||||
recorded.append([t.name for t in tools if isinstance(t, FunctionTool)])
|
||||
return await original(self, messages=messages, options=options, **kwargs)
|
||||
|
||||
@tool(name="get_weather", approval_mode="never_require")
|
||||
def get_weather(location: str) -> str:
|
||||
return "sunny"
|
||||
|
||||
@tool(name="drop_weather", approval_mode="never_require")
|
||||
def drop_weather(ctx: FunctionInvocationContext) -> str:
|
||||
ctx.remove_tools("get_weather")
|
||||
return "removed"
|
||||
|
||||
chat_client_base.function_invocation_configuration["max_iterations"] = 3 # type: ignore[attr-defined]
|
||||
chat_client_base.run_responses = [
|
||||
_pte_function_call_response("1", "drop_weather"),
|
||||
_pte_text_response(),
|
||||
]
|
||||
monkey = pytest.MonkeyPatch()
|
||||
monkey.setattr(client_cls, "_get_non_streaming_response", recording)
|
||||
try:
|
||||
await chat_client_base.get_response(
|
||||
[Message(role="user", contents=["hi"])],
|
||||
options={"tool_choice": "auto", "tools": [get_weather, drop_weather]},
|
||||
)
|
||||
finally:
|
||||
monkey.undo()
|
||||
|
||||
assert set(recorded[0]) == {"get_weather", "drop_weather"}
|
||||
assert "get_weather" not in recorded[1]
|
||||
|
||||
|
||||
async def test_add_tools_does_not_mutate_caller_tools_list(chat_client_base: SupportsChatGetResponse):
|
||||
@tool(name="load_math", approval_mode="never_require")
|
||||
def load_math(ctx: FunctionInvocationContext) -> str:
|
||||
ctx.add_tools(_pte_factorial)
|
||||
return "loaded"
|
||||
|
||||
original_tools: list[Any] = [load_math]
|
||||
chat_client_base.function_invocation_configuration["max_iterations"] = 3 # type: ignore[attr-defined]
|
||||
chat_client_base.run_responses = [
|
||||
_pte_function_call_response("1", "load_math"),
|
||||
_pte_text_response(),
|
||||
]
|
||||
await chat_client_base.get_response(
|
||||
[Message(role="user", contents=["hi"])],
|
||||
options={"tool_choice": "auto", "tools": original_tools},
|
||||
)
|
||||
assert original_tools == [load_math]
|
||||
|
||||
|
||||
async def test_add_tools_persists_across_iterations(chat_client_base: SupportsChatGetResponse):
|
||||
from agent_framework import FunctionTool
|
||||
|
||||
recorded: list[list[str]] = []
|
||||
client_cls = type(chat_client_base)
|
||||
original = client_cls._get_non_streaming_response
|
||||
|
||||
async def recording(self: Any, *, messages: Any, options: dict[str, Any], **kwargs: Any) -> ChatResponse:
|
||||
tools = options.get("tools") or []
|
||||
recorded.append([t.name for t in tools if isinstance(t, FunctionTool)])
|
||||
return await original(self, messages=messages, options=options, **kwargs)
|
||||
|
||||
@tool(name="load_math", approval_mode="never_require")
|
||||
def load_math(ctx: FunctionInvocationContext) -> str:
|
||||
ctx.add_tools(_pte_factorial)
|
||||
return "loaded"
|
||||
|
||||
chat_client_base.function_invocation_configuration["max_iterations"] = 4 # type: ignore[attr-defined]
|
||||
chat_client_base.run_responses = [
|
||||
_pte_function_call_response("1", "load_math"),
|
||||
_pte_function_call_response("2", "factorial", '{"n": 5}'),
|
||||
_pte_function_call_response("3", "factorial", '{"n": 3}'),
|
||||
_pte_text_response(),
|
||||
]
|
||||
monkey = pytest.MonkeyPatch()
|
||||
monkey.setattr(client_cls, "_get_non_streaming_response", recording)
|
||||
try:
|
||||
await chat_client_base.get_response(
|
||||
[Message(role="user", contents=["hi"])],
|
||||
options={"tool_choice": "auto", "tools": [load_math]},
|
||||
)
|
||||
finally:
|
||||
monkey.undo()
|
||||
|
||||
assert "factorial" in recorded[1]
|
||||
assert "factorial" in recorded[2]
|
||||
|
||||
|
||||
async def test_add_tools_through_function_middleware(chat_client_base: SupportsChatGetResponse):
|
||||
exec_counter = 0
|
||||
|
||||
class PassthroughMiddleware(FunctionMiddleware):
|
||||
async def process(self, context: FunctionInvocationContext, call_next: Any) -> None:
|
||||
await call_next()
|
||||
|
||||
@tool(name="factorial", approval_mode="never_require")
|
||||
def factorial(n: int) -> int:
|
||||
nonlocal exec_counter
|
||||
exec_counter += 1
|
||||
return 120
|
||||
|
||||
@tool(name="load_math", approval_mode="never_require")
|
||||
def load_math(ctx: FunctionInvocationContext) -> str:
|
||||
ctx.add_tools(factorial)
|
||||
return "loaded"
|
||||
|
||||
chat_client_base.function_invocation_configuration["max_iterations"] = 3 # type: ignore[attr-defined]
|
||||
chat_client_base.run_responses = [
|
||||
_pte_function_call_response("1", "load_math"),
|
||||
_pte_function_call_response("2", "factorial", '{"n": 5}'),
|
||||
_pte_text_response(),
|
||||
]
|
||||
await chat_client_base.get_response(
|
||||
[Message(role="user", contents=["hi"])],
|
||||
options={"tool_choice": "auto", "tools": [load_math]},
|
||||
middleware=[PassthroughMiddleware()],
|
||||
)
|
||||
assert exec_counter == 1
|
||||
|
||||
|
||||
async def test_add_tools_with_approval_required_tool(chat_client_base: SupportsChatGetResponse):
|
||||
@tool(name="secure_tool", approval_mode="always_require")
|
||||
def secure_tool(value: str) -> str:
|
||||
return f"secure: {value}"
|
||||
|
||||
@tool(name="load_secure", approval_mode="never_require")
|
||||
def load_secure(ctx: FunctionInvocationContext) -> str:
|
||||
ctx.add_tools(secure_tool)
|
||||
return "loaded"
|
||||
|
||||
chat_client_base.function_invocation_configuration["max_iterations"] = 3 # type: ignore[attr-defined]
|
||||
chat_client_base.run_responses = [
|
||||
_pte_function_call_response("1", "load_secure"),
|
||||
_pte_function_call_response("2", "secure_tool", '{"value": "x"}'),
|
||||
_pte_text_response(),
|
||||
]
|
||||
response = await chat_client_base.get_response(
|
||||
[Message(role="user", contents=["hi"])],
|
||||
options={"tool_choice": "auto", "tools": [load_secure]},
|
||||
)
|
||||
assert any(item.type == "function_approval_request" for msg in response.messages for item in msg.contents)
|
||||
|
||||
|
||||
async def test_add_tools_accepts_plain_callable(chat_client_base: SupportsChatGetResponse):
|
||||
exec_counter = 0
|
||||
|
||||
def plain_factorial(n: int) -> int:
|
||||
"""Compute factorial."""
|
||||
nonlocal exec_counter
|
||||
exec_counter += 1
|
||||
return 120
|
||||
|
||||
@tool(name="load_math", approval_mode="never_require")
|
||||
def load_math(ctx: FunctionInvocationContext) -> str:
|
||||
ctx.add_tools(plain_factorial)
|
||||
return "loaded"
|
||||
|
||||
chat_client_base.function_invocation_configuration["max_iterations"] = 3 # type: ignore[attr-defined]
|
||||
chat_client_base.run_responses = [
|
||||
_pte_function_call_response("1", "load_math"),
|
||||
_pte_function_call_response("2", "plain_factorial", '{"n": 5}'),
|
||||
_pte_text_response(),
|
||||
]
|
||||
await chat_client_base.get_response(
|
||||
[Message(role="user", contents=["hi"])],
|
||||
options={"tool_choice": "auto", "tools": [load_math]},
|
||||
)
|
||||
assert exec_counter == 1
|
||||
|
||||
|
||||
async def test_add_tools_streaming(chat_client_base: SupportsChatGetResponse):
|
||||
exec_counter = 0
|
||||
|
||||
@tool(name="factorial", approval_mode="never_require")
|
||||
def factorial(n: int) -> int:
|
||||
nonlocal exec_counter
|
||||
exec_counter += 1
|
||||
return 120
|
||||
|
||||
@tool(name="load_math", approval_mode="never_require")
|
||||
def load_math(ctx: FunctionInvocationContext) -> str:
|
||||
ctx.add_tools(factorial)
|
||||
return "loaded"
|
||||
|
||||
chat_client_base.function_invocation_configuration["max_iterations"] = 3 # type: ignore[attr-defined]
|
||||
chat_client_base.streaming_responses = [
|
||||
[
|
||||
ChatResponseUpdate(
|
||||
contents=[Content.from_function_call(call_id="1", name="load_math", arguments="{}")],
|
||||
role="assistant",
|
||||
)
|
||||
],
|
||||
[
|
||||
ChatResponseUpdate(
|
||||
contents=[Content.from_function_call(call_id="2", name="factorial", arguments='{"n": 5}')],
|
||||
role="assistant",
|
||||
)
|
||||
],
|
||||
[ChatResponseUpdate(contents=[Content.from_text("done")], role="assistant", finish_reason="stop")],
|
||||
]
|
||||
async for _ in chat_client_base.get_response(
|
||||
[Message(role="user", contents=["hi"])],
|
||||
stream=True,
|
||||
options={"tool_choice": "auto", "tools": [load_math]},
|
||||
):
|
||||
pass
|
||||
assert exec_counter == 1
|
||||
|
||||
|
||||
def test_add_tools_duplicate_same_object_is_noop():
|
||||
@tool(name="dup", approval_mode="never_require")
|
||||
def dup(x: int) -> int:
|
||||
return x
|
||||
|
||||
ctx = FunctionInvocationContext(function=dup, arguments={}, tools=[dup])
|
||||
ctx.add_tools(dup)
|
||||
assert ctx.tools is not None
|
||||
assert len(ctx.tools) == 1
|
||||
|
||||
|
||||
def test_add_tools_duplicate_name_different_object_raises():
|
||||
@tool(name="dup", approval_mode="never_require")
|
||||
def dup_a(x: int) -> int:
|
||||
return x
|
||||
|
||||
@tool(name="dup", approval_mode="never_require")
|
||||
def dup_b(x: int) -> int:
|
||||
return x
|
||||
|
||||
ctx = FunctionInvocationContext(function=dup_a, arguments={}, tools=[dup_a])
|
||||
with pytest.raises(ValueError):
|
||||
ctx.add_tools(dup_b)
|
||||
|
||||
|
||||
def test_add_tools_batch_with_duplicate_is_atomic():
|
||||
"""A duplicate-name clash partway through a batch must leave the live list unchanged."""
|
||||
|
||||
@tool(name="existing", approval_mode="never_require")
|
||||
def existing(x: int) -> int:
|
||||
return x
|
||||
|
||||
@tool(name="fresh", approval_mode="never_require")
|
||||
def fresh(x: int) -> int:
|
||||
return x
|
||||
|
||||
@tool(name="existing", approval_mode="never_require")
|
||||
def clashing(x: int) -> int:
|
||||
return x
|
||||
|
||||
ctx = FunctionInvocationContext(function=existing, arguments={}, tools=[existing])
|
||||
with pytest.raises(ValueError):
|
||||
ctx.add_tools([fresh, clashing])
|
||||
assert ctx.tools is not None
|
||||
# The valid "fresh" tool must not have been committed before the clash raised.
|
||||
assert ctx.tools == [existing]
|
||||
|
||||
|
||||
def test_remove_tools_by_name_and_object():
|
||||
@tool(name="a", approval_mode="never_require")
|
||||
def a(x: int) -> int:
|
||||
return x
|
||||
|
||||
@tool(name="b", approval_mode="never_require")
|
||||
def b(x: int) -> int:
|
||||
return x
|
||||
|
||||
ctx = FunctionInvocationContext(function=a, arguments={}, tools=[a, b])
|
||||
ctx.remove_tools("a")
|
||||
assert ctx.tools is not None
|
||||
assert [t.name for t in ctx.tools] == ["b"]
|
||||
ctx.remove_tools(b)
|
||||
assert ctx.tools == []
|
||||
|
||||
|
||||
def test_remove_tools_unknown_name_is_noop():
|
||||
@tool(name="a", approval_mode="never_require")
|
||||
def a(x: int) -> int:
|
||||
return x
|
||||
|
||||
ctx = FunctionInvocationContext(function=a, arguments={}, tools=[a])
|
||||
ctx.remove_tools("nonexistent")
|
||||
assert ctx.tools is not None
|
||||
assert [t.name for t in ctx.tools] == ["a"]
|
||||
|
||||
|
||||
def test_progressive_tools_helpers_raise_without_live_tools():
|
||||
@tool(name="a", approval_mode="never_require")
|
||||
def a(x: int) -> int:
|
||||
return x
|
||||
|
||||
ctx = FunctionInvocationContext(function=a, arguments={})
|
||||
assert ctx.tools is None
|
||||
with pytest.raises(RuntimeError):
|
||||
ctx.add_tools(a)
|
||||
with pytest.raises(RuntimeError):
|
||||
ctx.remove_tools("a")
|
||||
|
||||
|
||||
# endregion
|
||||
|
||||
@@ -11,13 +11,8 @@ import pytest
|
||||
from agent_framework._evaluation import (
|
||||
CheckResult,
|
||||
EvalItem,
|
||||
EvalItemResult,
|
||||
EvalNotPassedError,
|
||||
EvalResults,
|
||||
EvalScoreResult,
|
||||
ExpectedToolCall,
|
||||
LocalEvaluator,
|
||||
RubricScore,
|
||||
_coerce_result,
|
||||
evaluator,
|
||||
keyword_check,
|
||||
@@ -1015,101 +1010,19 @@ class TestAllPassedSubResults:
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Rubric assertions (EvalResults.assert_*)
|
||||
# r5 review: _build_overall_item with empty outputs
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _rubric_results(*scores_per_item: list[EvalScoreResult]) -> EvalResults:
|
||||
items = [
|
||||
EvalItemResult(item_id=f"item-{i}", status="pass", scores=scores) for i, scores in enumerate(scores_per_item)
|
||||
]
|
||||
return EvalResults(
|
||||
provider="test",
|
||||
eval_id="ev1",
|
||||
run_id="run1",
|
||||
result_counts={"passed": len(items), "failed": 0, "errored": 0, "total": len(items)},
|
||||
items=items,
|
||||
)
|
||||
class TestBuildOverallItemEmpty:
|
||||
"""Test _build_overall_item returns None for empty workflow outputs."""
|
||||
|
||||
def test_returns_none_for_empty_outputs(self):
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
class TestRubricAssertions:
|
||||
"""Tests for EvalResults.assert_dimension_score_at_least."""
|
||||
from agent_framework._evaluation import _build_overall_item
|
||||
|
||||
def test_dimension_at_or_above_threshold_passes(self) -> None:
|
||||
results = _rubric_results(
|
||||
[
|
||||
EvalScoreResult(
|
||||
name="policy",
|
||||
score=0.9,
|
||||
dimensions=[RubricScore(id="clarity", score=4, applicable=True, weight=1, reason="")],
|
||||
)
|
||||
],
|
||||
)
|
||||
# Should not raise.
|
||||
results.assert_dimension_score_at_least("clarity", 3)
|
||||
|
||||
def test_dimension_below_threshold_raises(self) -> None:
|
||||
results = _rubric_results(
|
||||
[
|
||||
EvalScoreResult(
|
||||
name="policy",
|
||||
score=0.5,
|
||||
dimensions=[RubricScore(id="clarity", score=2, applicable=True, weight=1, reason="")],
|
||||
)
|
||||
],
|
||||
)
|
||||
with pytest.raises(EvalNotPassedError):
|
||||
results.assert_dimension_score_at_least("clarity", 3)
|
||||
|
||||
def test_non_applicable_skipped_by_default(self) -> None:
|
||||
results = _rubric_results(
|
||||
[
|
||||
EvalScoreResult(
|
||||
name="policy",
|
||||
score=1.0,
|
||||
dimensions=[RubricScore(id="clarity", score=None, applicable=False, weight=1, reason="n/a")],
|
||||
)
|
||||
],
|
||||
)
|
||||
# No applicable scores; default behaviour is to skip silently.
|
||||
results.assert_dimension_score_at_least("clarity", 3)
|
||||
|
||||
def test_require_applicable_raises_when_dimension_absent(self) -> None:
|
||||
results = _rubric_results(
|
||||
[EvalScoreResult(name="policy", score=1.0, dimensions=[])],
|
||||
)
|
||||
with pytest.raises(EvalNotPassedError, match="not applicable"):
|
||||
results.assert_dimension_score_at_least("clarity", 3, require_applicable=True)
|
||||
|
||||
def test_require_applicable_raises_when_filtered_evaluator_missing(self) -> None:
|
||||
# Regression: previously the (not evaluator or found_any) guard caused
|
||||
# this case to silently pass even with require_applicable=True.
|
||||
results = _rubric_results(
|
||||
[
|
||||
EvalScoreResult(
|
||||
name="other",
|
||||
score=0.9,
|
||||
dimensions=[RubricScore(id="clarity", score=4, applicable=True, weight=1, reason="")],
|
||||
)
|
||||
],
|
||||
)
|
||||
with pytest.raises(EvalNotPassedError, match="not applicable"):
|
||||
results.assert_dimension_score_at_least("clarity", 3, evaluator="policy", require_applicable=True)
|
||||
|
||||
def test_evaluator_filter_isolates_offenders(self) -> None:
|
||||
results = _rubric_results(
|
||||
[
|
||||
EvalScoreResult(
|
||||
name="other",
|
||||
score=0.1,
|
||||
dimensions=[RubricScore(id="clarity", score=1, applicable=True, weight=1, reason="")],
|
||||
),
|
||||
EvalScoreResult(
|
||||
name="policy",
|
||||
score=0.9,
|
||||
dimensions=[RubricScore(id="clarity", score=4, applicable=True, weight=1, reason="")],
|
||||
),
|
||||
],
|
||||
)
|
||||
# The low-scoring "other" evaluator is filtered out; "policy" passes.
|
||||
results.assert_dimension_score_at_least("clarity", 3, evaluator="policy")
|
||||
mock_result = MagicMock()
|
||||
mock_result.get_outputs.return_value = []
|
||||
item = _build_overall_item("Hello", mock_result)
|
||||
assert item is None
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,667 +0,0 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
"""Tests for MCP-based skills (MCPSkillsSource, MCPSkill, MCPSkillResource)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import json
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import pytest
|
||||
from mcp.shared.exceptions import McpError
|
||||
from mcp.types import (
|
||||
BlobResourceContents,
|
||||
ErrorData,
|
||||
ReadResourceResult,
|
||||
TextResourceContents,
|
||||
)
|
||||
from pydantic import AnyUrl
|
||||
|
||||
from agent_framework import MCPSkill, MCPSkillResource, MCPSkillsSource
|
||||
from agent_framework._skills import _parse_mcp_skill_index
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fixtures & helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
SAMPLE_SKILL_MD = """\
|
||||
---
|
||||
name: unit-converter
|
||||
description: Convert between common units.
|
||||
---
|
||||
# Unit Converter
|
||||
|
||||
Body content here.
|
||||
"""
|
||||
|
||||
SAMPLE_SKILL_INDEX = json.dumps(
|
||||
{
|
||||
"$schema": "https://schemas.agentskills.io/discovery/0.2.0/schema.json",
|
||||
"skills": [
|
||||
{
|
||||
"name": "unit-converter",
|
||||
"type": "skill-md",
|
||||
"description": "Convert between common units.",
|
||||
"url": "skill://unit-converter/SKILL.md",
|
||||
}
|
||||
],
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _make_text_result(text: str, uri: str = "skill://test") -> ReadResourceResult:
|
||||
"""Create a ReadResourceResult with a single TextResourceContents."""
|
||||
return ReadResourceResult(
|
||||
contents=[TextResourceContents(uri=AnyUrl(uri), text=text, mimeType="text/markdown")]
|
||||
)
|
||||
|
||||
|
||||
def _make_blob_result(
|
||||
data: bytes,
|
||||
uri: str = "skill://test",
|
||||
mime_type: str = "application/octet-stream",
|
||||
) -> ReadResourceResult:
|
||||
"""Create a ReadResourceResult with a single BlobResourceContents."""
|
||||
return ReadResourceResult(
|
||||
contents=[BlobResourceContents(uri=AnyUrl(uri), blob=base64.b64encode(data).decode(), mimeType=mime_type)]
|
||||
)
|
||||
|
||||
|
||||
def _make_empty_result() -> ReadResourceResult:
|
||||
"""Create a ReadResourceResult with no contents."""
|
||||
return ReadResourceResult(contents=[])
|
||||
|
||||
|
||||
def _make_client(**read_resource_responses: ReadResourceResult) -> AsyncMock:
|
||||
"""Create a mock ClientSession whose read_resource returns different results per URI.
|
||||
|
||||
Args:
|
||||
**read_resource_responses: Mapping of URI string to ReadResourceResult.
|
||||
Any URI not in this mapping raises McpError with the MCP-spec
|
||||
"Resource not found" code (-32002).
|
||||
"""
|
||||
client = AsyncMock()
|
||||
|
||||
async def _read_resource(uri: AnyUrl) -> ReadResourceResult:
|
||||
uri_str = str(uri)
|
||||
if uri_str in read_resource_responses:
|
||||
return read_resource_responses[uri_str]
|
||||
raise McpError(error=ErrorData(code=-32002, message=f"Resource not found: {uri_str}"))
|
||||
|
||||
client.read_resource = AsyncMock(side_effect=_read_resource)
|
||||
return client
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _parse_mcp_skill_index tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestParseMCPSkillIndex:
|
||||
"""Tests for the _parse_mcp_skill_index helper."""
|
||||
|
||||
def test_parses_valid_index(self) -> None:
|
||||
index = _parse_mcp_skill_index(SAMPLE_SKILL_INDEX)
|
||||
assert index.schema == "https://schemas.agentskills.io/discovery/0.2.0/schema.json"
|
||||
assert len(index.skills) == 1
|
||||
assert index.skills[0].name == "unit-converter"
|
||||
assert index.skills[0].type == "skill-md"
|
||||
assert index.skills[0].url == "skill://unit-converter/SKILL.md"
|
||||
|
||||
def test_parses_empty_skills_array(self) -> None:
|
||||
index = _parse_mcp_skill_index('{"$schema": "test", "skills": []}')
|
||||
assert index.skills == []
|
||||
|
||||
def test_parses_missing_skills_key(self) -> None:
|
||||
index = _parse_mcp_skill_index('{"$schema": "test"}')
|
||||
assert index.skills == []
|
||||
|
||||
def test_raises_on_non_object(self) -> None:
|
||||
with pytest.raises(ValueError, match="must be a JSON object"):
|
||||
_parse_mcp_skill_index("[]")
|
||||
|
||||
def test_raises_on_invalid_json(self) -> None:
|
||||
with pytest.raises(json.JSONDecodeError):
|
||||
_parse_mcp_skill_index("not json")
|
||||
|
||||
def test_skips_non_dict_entries(self) -> None:
|
||||
index = _parse_mcp_skill_index('{"skills": ["not-a-dict", {"name": "ok", "type": "skill-md"}]}')
|
||||
assert len(index.skills) == 1
|
||||
assert index.skills[0].name == "ok"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# MCPSkillResource tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestMCPSkillResource:
|
||||
"""Tests for MCPSkillResource."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_read_text_content(self) -> None:
|
||||
result = _make_text_result("hello world")
|
||||
resource = MCPSkillResource(name="test.md", result=result)
|
||||
content = await resource.read()
|
||||
assert content == "hello world"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_read_binary_content(self) -> None:
|
||||
data = bytes([0x01, 0x02, 0x03, 0x04])
|
||||
result = _make_blob_result(data)
|
||||
resource = MCPSkillResource(name="icon.bin", result=result)
|
||||
content = await resource.read()
|
||||
assert content == data
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_read_empty_returns_none(self) -> None:
|
||||
result = _make_empty_result()
|
||||
resource = MCPSkillResource(name="empty", result=result)
|
||||
content = await resource.read()
|
||||
assert content is None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_read_multiple_text_contents_joined(self) -> None:
|
||||
result = ReadResourceResult(
|
||||
contents=[
|
||||
TextResourceContents(uri=AnyUrl("skill://a"), text="line1", mimeType="text/plain"),
|
||||
TextResourceContents(uri=AnyUrl("skill://b"), text="line2", mimeType="text/plain"),
|
||||
]
|
||||
)
|
||||
resource = MCPSkillResource(name="multi", result=result)
|
||||
content = await resource.read()
|
||||
assert content == "line1\nline2"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_binary_takes_precedence_over_text(self) -> None:
|
||||
data = b"\xff\xfe"
|
||||
result = ReadResourceResult(
|
||||
contents=[
|
||||
TextResourceContents(uri=AnyUrl("skill://a"), text="text", mimeType="text/plain"),
|
||||
BlobResourceContents(
|
||||
uri=AnyUrl("skill://b"),
|
||||
blob=base64.b64encode(data).decode(),
|
||||
mimeType="application/octet-stream",
|
||||
),
|
||||
]
|
||||
)
|
||||
resource = MCPSkillResource(name="mixed", result=result)
|
||||
content = await resource.read()
|
||||
# The implementation iterates all contents checking for BlobResourceContents
|
||||
# first, so when both text and binary are present, binary is returned.
|
||||
assert content == data
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# MCPSkill tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestMCPSkill:
|
||||
"""Tests for MCPSkill."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_content_fetches_and_caches(self) -> None:
|
||||
client = _make_client(**{"skill://unit-converter/SKILL.md": _make_text_result(SAMPLE_SKILL_MD)})
|
||||
from agent_framework import SkillFrontmatter
|
||||
|
||||
fm = SkillFrontmatter(name="unit-converter", description="Convert between common units.")
|
||||
skill = MCPSkill(frontmatter=fm, skill_md_uri="skill://unit-converter/SKILL.md", client=client)
|
||||
|
||||
content1 = await skill.get_content()
|
||||
content2 = await skill.get_content()
|
||||
|
||||
assert "Body content here." in content1
|
||||
assert content1 == content2
|
||||
# Only one MCP call should be made (cached)
|
||||
assert client.read_resource.call_count == 1
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_content_raises_on_empty(self) -> None:
|
||||
client = _make_client(**{"skill://empty/SKILL.md": _make_empty_result()})
|
||||
from agent_framework import SkillFrontmatter
|
||||
|
||||
fm = SkillFrontmatter(name="empty-skill", description="Empty skill.")
|
||||
skill = MCPSkill(frontmatter=fm, skill_md_uri="skill://empty/SKILL.md", client=client)
|
||||
|
||||
with pytest.raises(ValueError, match="no text content"):
|
||||
await skill.get_content()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_resource_text(self) -> None:
|
||||
client = _make_client(
|
||||
**{
|
||||
"skill://unit-converter/SKILL.md": _make_text_result(SAMPLE_SKILL_MD),
|
||||
"skill://unit-converter/references/checklist.md": _make_text_result("- check thing 1\n- check thing 2"),
|
||||
}
|
||||
)
|
||||
from agent_framework import SkillFrontmatter
|
||||
|
||||
fm = SkillFrontmatter(name="unit-converter", description="Convert between common units.")
|
||||
skill = MCPSkill(frontmatter=fm, skill_md_uri="skill://unit-converter/SKILL.md", client=client)
|
||||
|
||||
resource = await skill.get_resource("references/checklist.md")
|
||||
assert resource is not None
|
||||
content = await resource.read()
|
||||
assert content == "- check thing 1\n- check thing 2"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_resource_binary(self) -> None:
|
||||
data = bytes([0x01, 0x02, 0x03, 0x04])
|
||||
client = _make_client(
|
||||
**{
|
||||
"skill://unit-converter/SKILL.md": _make_text_result(SAMPLE_SKILL_MD),
|
||||
"skill://unit-converter/assets/icon.bin": _make_blob_result(data),
|
||||
}
|
||||
)
|
||||
from agent_framework import SkillFrontmatter
|
||||
|
||||
fm = SkillFrontmatter(name="unit-converter", description="Convert between common units.")
|
||||
skill = MCPSkill(frontmatter=fm, skill_md_uri="skill://unit-converter/SKILL.md", client=client)
|
||||
|
||||
resource = await skill.get_resource("assets/icon.bin")
|
||||
assert resource is not None
|
||||
content = await resource.read()
|
||||
assert content == data
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_resource_unknown_returns_none(self) -> None:
|
||||
client = _make_client(**{"skill://unit-converter/SKILL.md": _make_text_result(SAMPLE_SKILL_MD)})
|
||||
from agent_framework import SkillFrontmatter
|
||||
|
||||
fm = SkillFrontmatter(name="unit-converter", description="Convert between common units.")
|
||||
skill = MCPSkill(frontmatter=fm, skill_md_uri="skill://unit-converter/SKILL.md", client=client)
|
||||
|
||||
resource = await skill.get_resource("references/does-not-exist.md")
|
||||
assert resource is None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"name",
|
||||
[
|
||||
"../escape.md",
|
||||
"references/../../escape.md",
|
||||
"..",
|
||||
"..\\escape.md",
|
||||
"/etc/passwd",
|
||||
"http://attacker.example.com/payload",
|
||||
],
|
||||
)
|
||||
async def test_get_resource_path_traversal_returns_none(self, name: str) -> None:
|
||||
# Register a permissive mock that would happily return content for any URI,
|
||||
# so the test fails unless the client-side validation rejects the name
|
||||
# before issuing the read.
|
||||
client = AsyncMock()
|
||||
client.read_resource = AsyncMock(return_value=_make_text_result("should never be returned"))
|
||||
|
||||
from agent_framework import SkillFrontmatter
|
||||
|
||||
fm = SkillFrontmatter(name="unit-converter", description="Convert between common units.")
|
||||
skill = MCPSkill(frontmatter=fm, skill_md_uri="skill://unit-converter/SKILL.md", client=client)
|
||||
|
||||
resource = await skill.get_resource(name)
|
||||
assert resource is None
|
||||
client.read_resource.assert_not_called()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_resource_empty_name_returns_none(self) -> None:
|
||||
client = _make_client()
|
||||
from agent_framework import SkillFrontmatter
|
||||
|
||||
fm = SkillFrontmatter(name="test-skill", description="Test.")
|
||||
skill = MCPSkill(frontmatter=fm, skill_md_uri="skill://test/SKILL.md", client=client)
|
||||
|
||||
assert await skill.get_resource("") is None
|
||||
assert await skill.get_resource(" ") is None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_script_returns_none(self) -> None:
|
||||
client = _make_client()
|
||||
from agent_framework import SkillFrontmatter
|
||||
|
||||
fm = SkillFrontmatter(name="test-skill", description="Test.")
|
||||
skill = MCPSkill(frontmatter=fm, skill_md_uri="skill://test/SKILL.md", client=client)
|
||||
|
||||
assert await skill.get_script("anything") is None
|
||||
|
||||
def test_compute_skill_root_uri_strips_suffix(self) -> None:
|
||||
assert MCPSkill._compute_skill_root_uri("skill://unit-converter/SKILL.md") == "skill://unit-converter/"
|
||||
|
||||
def test_compute_skill_root_uri_trailing_slash(self) -> None:
|
||||
assert MCPSkill._compute_skill_root_uri("skill://unit-converter/") == "skill://unit-converter/"
|
||||
|
||||
def test_compute_skill_root_uri_no_suffix_adds_slash(self) -> None:
|
||||
assert MCPSkill._compute_skill_root_uri("skill://unit-converter") == "skill://unit-converter/"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# MCPSkillsSource tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestMCPSkillsSource:
|
||||
"""Tests for MCPSkillsSource."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_index_based_discovery_returns_skill(self) -> None:
|
||||
client = _make_client(
|
||||
**{
|
||||
"skill://index.json": _make_text_result(SAMPLE_SKILL_INDEX, uri="skill://index.json"),
|
||||
"skill://unit-converter/SKILL.md": _make_text_result(SAMPLE_SKILL_MD),
|
||||
}
|
||||
)
|
||||
source = MCPSkillsSource(client=client)
|
||||
skills = await source.get_skills()
|
||||
|
||||
assert len(skills) == 1
|
||||
assert skills[0].frontmatter.name == "unit-converter"
|
||||
assert skills[0].frontmatter.description == "Convert between common units."
|
||||
|
||||
# Content is fetched on demand, not during discovery
|
||||
content = await skills[0].get_content()
|
||||
assert "Body content here." in content
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_no_index_returns_empty(self) -> None:
|
||||
client = _make_client() # No resources at all
|
||||
source = MCPSkillsSource(client=client)
|
||||
skills = await source.get_skills()
|
||||
assert skills == []
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_does_not_read_skill_md_during_discovery(self) -> None:
|
||||
# Index points to a skill, but SKILL.md is not registered on the server.
|
||||
# Discovery should succeed because it only reads the index.
|
||||
client = _make_client(
|
||||
**{"skill://index.json": _make_text_result(SAMPLE_SKILL_INDEX, uri="skill://index.json")}
|
||||
)
|
||||
source = MCPSkillsSource(client=client)
|
||||
skills = await source.get_skills()
|
||||
|
||||
assert len(skills) == 1
|
||||
assert skills[0].frontmatter.name == "unit-converter"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_invalid_name_is_skipped(self) -> None:
|
||||
index_json = json.dumps(
|
||||
{
|
||||
"$schema": "https://schemas.agentskills.io/discovery/0.2.0/schema.json",
|
||||
"skills": [
|
||||
{
|
||||
"name": "UnitConverter", # Invalid: uppercase
|
||||
"type": "skill-md",
|
||||
"description": "Convert between common units.",
|
||||
"url": "skill://UnitConverter/SKILL.md",
|
||||
}
|
||||
],
|
||||
}
|
||||
)
|
||||
client = _make_client(**{"skill://index.json": _make_text_result(index_json, uri="skill://index.json")})
|
||||
source = MCPSkillsSource(client=client)
|
||||
skills = await source.get_skills()
|
||||
assert skills == []
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_missing_required_fields_is_skipped(self) -> None:
|
||||
index_json = json.dumps(
|
||||
{
|
||||
"$schema": "https://schemas.agentskills.io/discovery/0.2.0/schema.json",
|
||||
"skills": [
|
||||
{
|
||||
"name": "unit-converter",
|
||||
"type": "skill-md",
|
||||
# Missing description and url
|
||||
}
|
||||
],
|
||||
}
|
||||
)
|
||||
client = _make_client(**{"skill://index.json": _make_text_result(index_json, uri="skill://index.json")})
|
||||
source = MCPSkillsSource(client=client)
|
||||
skills = await source.get_skills()
|
||||
assert skills == []
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unsupported_type_is_skipped(self) -> None:
|
||||
index_json = json.dumps(
|
||||
{
|
||||
"$schema": "https://schemas.agentskills.io/discovery/0.2.0/schema.json",
|
||||
"skills": [
|
||||
{
|
||||
"name": "some-skill",
|
||||
"type": "archive",
|
||||
"description": "Packaged skill.",
|
||||
"url": "skill://some-skill.tar.gz",
|
||||
}
|
||||
],
|
||||
}
|
||||
)
|
||||
client = _make_client(**{"skill://index.json": _make_text_result(index_json, uri="skill://index.json")})
|
||||
source = MCPSkillsSource(client=client)
|
||||
skills = await source.get_skills()
|
||||
assert skills == []
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_template_type_is_skipped(self) -> None:
|
||||
index_json = json.dumps(
|
||||
{
|
||||
"$schema": "https://schemas.agentskills.io/discovery/0.2.0/schema.json",
|
||||
"skills": [
|
||||
{
|
||||
"type": "mcp-resource-template",
|
||||
"description": "Per-product documentation skill",
|
||||
"url": "skill://docs/{product}/SKILL.md",
|
||||
}
|
||||
],
|
||||
}
|
||||
)
|
||||
client = _make_client(**{"skill://index.json": _make_text_result(index_json, uri="skill://index.json")})
|
||||
source = MCPSkillsSource(client=client)
|
||||
skills = await source.get_skills()
|
||||
assert skills == []
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_empty_index_returns_empty(self) -> None:
|
||||
client = _make_client(
|
||||
**{"skill://index.json": _make_text_result('{"skills": []}', uri="skill://index.json")}
|
||||
)
|
||||
source = MCPSkillsSource(client=client)
|
||||
skills = await source.get_skills()
|
||||
assert skills == []
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_malformed_index_json_returns_empty(self) -> None:
|
||||
client = _make_client(
|
||||
**{"skill://index.json": _make_text_result("not valid json", uri="skill://index.json")}
|
||||
)
|
||||
source = MCPSkillsSource(client=client)
|
||||
skills = await source.get_skills()
|
||||
assert skills == []
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sibling_text_resource(self) -> None:
|
||||
client = _make_client(
|
||||
**{
|
||||
"skill://index.json": _make_text_result(SAMPLE_SKILL_INDEX, uri="skill://index.json"),
|
||||
"skill://unit-converter/SKILL.md": _make_text_result(SAMPLE_SKILL_MD),
|
||||
"skill://unit-converter/references/checklist.md": _make_text_result("- check thing 1\n- check thing 2"),
|
||||
}
|
||||
)
|
||||
source = MCPSkillsSource(client=client)
|
||||
skill = (await source.get_skills())[0]
|
||||
resource = await skill.get_resource("references/checklist.md")
|
||||
assert resource is not None
|
||||
content = await resource.read()
|
||||
assert content == "- check thing 1\n- check thing 2"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sibling_binary_resource(self) -> None:
|
||||
data = bytes([0x01, 0x02, 0x03, 0x04])
|
||||
client = _make_client(
|
||||
**{
|
||||
"skill://index.json": _make_text_result(SAMPLE_SKILL_INDEX, uri="skill://index.json"),
|
||||
"skill://unit-converter/SKILL.md": _make_text_result(SAMPLE_SKILL_MD),
|
||||
"skill://unit-converter/assets/icon.bin": _make_blob_result(data),
|
||||
}
|
||||
)
|
||||
source = MCPSkillsSource(client=client)
|
||||
skill = (await source.get_skills())[0]
|
||||
resource = await skill.get_resource("assets/icon.bin")
|
||||
assert resource is not None
|
||||
content = await resource.read()
|
||||
assert content == data
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# McpError code branching tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestMCPSkillsSourceErrorCodeBranching:
|
||||
"""Tests that MCPSkillsSource and MCPSkill branch on McpError.error.code.
|
||||
|
||||
Only "not found" codes (RESOURCE_NOT_FOUND -32002, METHOD_NOT_FOUND -32601)
|
||||
should be silently swallowed as "no skills available." Other McpError codes
|
||||
and non-McpError exceptions must propagate so that auth failures, server
|
||||
crashes, and connection drops are visible.
|
||||
"""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_index_method_not_found_returns_empty(self) -> None:
|
||||
"""METHOD_NOT_FOUND (-32601) -> server doesn't support resources/read."""
|
||||
client = AsyncMock()
|
||||
client.read_resource = AsyncMock(side_effect=McpError(error=ErrorData(code=-32601, message="Method not found")))
|
||||
source = MCPSkillsSource(client=client)
|
||||
skills = await source.get_skills()
|
||||
assert skills == []
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_index_resource_not_found_returns_empty(self) -> None:
|
||||
"""MCP-spec "Resource not found" (-32002) -> server has no index."""
|
||||
client = AsyncMock()
|
||||
client.read_resource = AsyncMock(
|
||||
side_effect=McpError(error=ErrorData(code=-32002, message="Resource not found"))
|
||||
)
|
||||
source = MCPSkillsSource(client=client)
|
||||
skills = await source.get_skills()
|
||||
assert skills == []
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_index_invalid_params_propagates(self) -> None:
|
||||
"""INVALID_PARAMS (-32602) is a real bug, must propagate (not "not found")."""
|
||||
client = AsyncMock()
|
||||
client.read_resource = AsyncMock(side_effect=McpError(error=ErrorData(code=-32602, message="Invalid params")))
|
||||
source = MCPSkillsSource(client=client)
|
||||
with pytest.raises(McpError):
|
||||
await source.get_skills()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_index_internal_error_propagates(self) -> None:
|
||||
"""INTERNAL_ERROR (-32603) must propagate, not silently return empty."""
|
||||
client = AsyncMock()
|
||||
client.read_resource = AsyncMock(side_effect=McpError(error=ErrorData(code=-32603, message="Internal error")))
|
||||
source = MCPSkillsSource(client=client)
|
||||
with pytest.raises(McpError):
|
||||
await source.get_skills()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_index_connection_closed_propagates(self) -> None:
|
||||
"""CONNECTION_CLOSED (-32000) must propagate."""
|
||||
client = AsyncMock()
|
||||
client.read_resource = AsyncMock(
|
||||
side_effect=McpError(error=ErrorData(code=-32000, message="Connection closed"))
|
||||
)
|
||||
source = MCPSkillsSource(client=client)
|
||||
with pytest.raises(McpError):
|
||||
await source.get_skills()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_index_generic_error_code_propagates(self) -> None:
|
||||
"""Generic handler error (code 0) must propagate."""
|
||||
client = AsyncMock()
|
||||
client.read_resource = AsyncMock(side_effect=McpError(error=ErrorData(code=0, message="Some handler error")))
|
||||
source = MCPSkillsSource(client=client)
|
||||
with pytest.raises(McpError):
|
||||
await source.get_skills()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_index_non_mcp_error_propagates(self) -> None:
|
||||
"""Non-McpError exceptions (connection drop, timeout) must propagate."""
|
||||
client = AsyncMock()
|
||||
client.read_resource = AsyncMock(side_effect=ConnectionError("connection lost"))
|
||||
source = MCPSkillsSource(client=client)
|
||||
with pytest.raises(ConnectionError):
|
||||
await source.get_skills()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_resource_internal_error_propagates(self) -> None:
|
||||
"""McpError with INTERNAL_ERROR on get_resource must propagate."""
|
||||
from agent_framework import SkillFrontmatter
|
||||
|
||||
client = AsyncMock()
|
||||
client.read_resource = AsyncMock(side_effect=McpError(error=ErrorData(code=-32603, message="Server crashed")))
|
||||
fm = SkillFrontmatter(name="test-skill", description="Test.")
|
||||
skill = MCPSkill(frontmatter=fm, skill_md_uri="skill://test/SKILL.md", client=client)
|
||||
with pytest.raises(McpError):
|
||||
await skill.get_resource("references/file.md")
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_resource_not_found_returns_none(self) -> None:
|
||||
"""McpError with RESOURCE_NOT_FOUND (-32002) on get_resource returns None."""
|
||||
from agent_framework import SkillFrontmatter
|
||||
|
||||
client = AsyncMock()
|
||||
client.read_resource = AsyncMock(
|
||||
side_effect=McpError(error=ErrorData(code=-32002, message="Resource not found"))
|
||||
)
|
||||
fm = SkillFrontmatter(name="test-skill", description="Test.")
|
||||
skill = MCPSkill(frontmatter=fm, skill_md_uri="skill://test/SKILL.md", client=client)
|
||||
result = await skill.get_resource("references/file.md")
|
||||
assert result is None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_resource_connection_error_propagates(self) -> None:
|
||||
"""A plain ConnectionError on get_resource must propagate, not return None."""
|
||||
from agent_framework import SkillFrontmatter
|
||||
|
||||
client = AsyncMock()
|
||||
client.read_resource = AsyncMock(side_effect=ConnectionError("connection lost"))
|
||||
fm = SkillFrontmatter(name="test-skill", description="Test.")
|
||||
skill = MCPSkill(frontmatter=fm, skill_md_uri="skill://test/SKILL.md", client=client)
|
||||
with pytest.raises(ConnectionError):
|
||||
await skill.get_resource("references/file.md")
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_resource_timeout_error_propagates(self) -> None:
|
||||
"""A TimeoutError on get_resource must propagate, not return None."""
|
||||
from agent_framework import SkillFrontmatter
|
||||
|
||||
client = AsyncMock()
|
||||
client.read_resource = AsyncMock(side_effect=TimeoutError("read timed out"))
|
||||
fm = SkillFrontmatter(name="test-skill", description="Test.")
|
||||
skill = MCPSkill(frontmatter=fm, skill_md_uri="skill://test/SKILL.md", client=client)
|
||||
with pytest.raises(TimeoutError):
|
||||
await skill.get_resource("references/file.md")
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_resource_generic_mcp_error_propagates(self) -> None:
|
||||
"""McpError with a generic code (0) on get_resource must propagate."""
|
||||
from agent_framework import SkillFrontmatter
|
||||
|
||||
client = AsyncMock()
|
||||
client.read_resource = AsyncMock(
|
||||
side_effect=McpError(error=ErrorData(code=0, message="Handler error"))
|
||||
)
|
||||
fm = SkillFrontmatter(name="test-skill", description="Test.")
|
||||
skill = MCPSkill(frontmatter=fm, skill_md_uri="skill://test/SKILL.md", client=client)
|
||||
with pytest.raises(McpError):
|
||||
await skill.get_resource("references/file.md")
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_index_timeout_error_propagates(self) -> None:
|
||||
"""A TimeoutError reading skill://index.json must propagate."""
|
||||
client = AsyncMock()
|
||||
client.read_resource = AsyncMock(side_effect=TimeoutError("read timed out"))
|
||||
source = MCPSkillsSource(client=client)
|
||||
with pytest.raises(TimeoutError):
|
||||
await source.get_skills()
|
||||
@@ -761,115 +761,6 @@ def test_get_exporters_from_env_missing_grpc_dependency(monkeypatch):
|
||||
_get_exporters_from_env()
|
||||
|
||||
|
||||
# region Test OTLP endpoint computation (base-URL auto-append for HTTP)
|
||||
|
||||
|
||||
def test_get_exporters_from_env_http_base_endpoint_appends_signal_paths(monkeypatch):
|
||||
"""OTEL_EXPORTER_OTLP_ENDPOINT is a base URL for HTTP; SDK auto-appends
|
||||
/v1/{traces,metrics,logs}. Because we read the env var and forward it as the
|
||||
constructor ``endpoint=`` arg (which the SDK treats as a full URL), we must
|
||||
replicate the auto-append ourselves.
|
||||
"""
|
||||
from unittest.mock import patch
|
||||
|
||||
from agent_framework import observability
|
||||
|
||||
monkeypatch.setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://localhost:4318")
|
||||
monkeypatch.setenv("OTEL_EXPORTER_OTLP_PROTOCOL", "http/protobuf")
|
||||
for key in (
|
||||
"OTEL_EXPORTER_OTLP_TRACES_ENDPOINT",
|
||||
"OTEL_EXPORTER_OTLP_METRICS_ENDPOINT",
|
||||
"OTEL_EXPORTER_OTLP_LOGS_ENDPOINT",
|
||||
):
|
||||
monkeypatch.delenv(key, raising=False)
|
||||
|
||||
with patch.object(observability, "_create_otlp_exporters", return_value=[]) as create:
|
||||
observability._get_exporters_from_env()
|
||||
|
||||
kwargs = create.call_args.kwargs
|
||||
assert kwargs["protocol"] == "http/protobuf"
|
||||
assert kwargs["traces_endpoint"] == "http://localhost:4318/v1/traces"
|
||||
assert kwargs["metrics_endpoint"] == "http://localhost:4318/v1/metrics"
|
||||
assert kwargs["logs_endpoint"] == "http://localhost:4318/v1/logs"
|
||||
|
||||
|
||||
def test_get_exporters_from_env_http_base_endpoint_trailing_slash(monkeypatch):
|
||||
"""A trailing slash on the base endpoint should not produce a doubled slash."""
|
||||
from unittest.mock import patch
|
||||
|
||||
from agent_framework import observability
|
||||
|
||||
monkeypatch.setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://localhost:4318/")
|
||||
monkeypatch.setenv("OTEL_EXPORTER_OTLP_PROTOCOL", "http/protobuf")
|
||||
for key in (
|
||||
"OTEL_EXPORTER_OTLP_TRACES_ENDPOINT",
|
||||
"OTEL_EXPORTER_OTLP_METRICS_ENDPOINT",
|
||||
"OTEL_EXPORTER_OTLP_LOGS_ENDPOINT",
|
||||
):
|
||||
monkeypatch.delenv(key, raising=False)
|
||||
|
||||
with patch.object(observability, "_create_otlp_exporters", return_value=[]) as create:
|
||||
observability._get_exporters_from_env()
|
||||
|
||||
kwargs = create.call_args.kwargs
|
||||
assert kwargs["traces_endpoint"] == "http://localhost:4318/v1/traces"
|
||||
assert kwargs["metrics_endpoint"] == "http://localhost:4318/v1/metrics"
|
||||
assert kwargs["logs_endpoint"] == "http://localhost:4318/v1/logs"
|
||||
|
||||
|
||||
def test_get_exporters_from_env_http_signal_specific_used_verbatim(monkeypatch):
|
||||
"""Signal-specific endpoint env vars are full URLs and must be used verbatim,
|
||||
even when a base endpoint is also set.
|
||||
"""
|
||||
from unittest.mock import patch
|
||||
|
||||
from agent_framework import observability
|
||||
|
||||
monkeypatch.setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://localhost:4318")
|
||||
monkeypatch.setenv("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", "http://traces.example.com/custom/path")
|
||||
monkeypatch.setenv("OTEL_EXPORTER_OTLP_PROTOCOL", "http/protobuf")
|
||||
for key in (
|
||||
"OTEL_EXPORTER_OTLP_METRICS_ENDPOINT",
|
||||
"OTEL_EXPORTER_OTLP_LOGS_ENDPOINT",
|
||||
):
|
||||
monkeypatch.delenv(key, raising=False)
|
||||
|
||||
with patch.object(observability, "_create_otlp_exporters", return_value=[]) as create:
|
||||
observability._get_exporters_from_env()
|
||||
|
||||
kwargs = create.call_args.kwargs
|
||||
# Signal-specific is verbatim — no path appended
|
||||
assert kwargs["traces_endpoint"] == "http://traces.example.com/custom/path"
|
||||
# Others fall back to base, with path appended
|
||||
assert kwargs["metrics_endpoint"] == "http://localhost:4318/v1/metrics"
|
||||
assert kwargs["logs_endpoint"] == "http://localhost:4318/v1/logs"
|
||||
|
||||
|
||||
def test_get_exporters_from_env_grpc_base_endpoint_unchanged(monkeypatch):
|
||||
"""For gRPC, the base endpoint applies to all signals as-is (no path append)."""
|
||||
from unittest.mock import patch
|
||||
|
||||
from agent_framework import observability
|
||||
|
||||
monkeypatch.setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://localhost:4317")
|
||||
monkeypatch.setenv("OTEL_EXPORTER_OTLP_PROTOCOL", "grpc")
|
||||
for key in (
|
||||
"OTEL_EXPORTER_OTLP_TRACES_ENDPOINT",
|
||||
"OTEL_EXPORTER_OTLP_METRICS_ENDPOINT",
|
||||
"OTEL_EXPORTER_OTLP_LOGS_ENDPOINT",
|
||||
):
|
||||
monkeypatch.delenv(key, raising=False)
|
||||
|
||||
with patch.object(observability, "_create_otlp_exporters", return_value=[]) as create:
|
||||
observability._get_exporters_from_env()
|
||||
|
||||
kwargs = create.call_args.kwargs
|
||||
assert kwargs["protocol"] == "grpc"
|
||||
assert kwargs["traces_endpoint"] == "http://localhost:4317"
|
||||
assert kwargs["metrics_endpoint"] == "http://localhost:4317"
|
||||
assert kwargs["logs_endpoint"] == "http://localhost:4317"
|
||||
|
||||
|
||||
# region Test create_resource
|
||||
|
||||
|
||||
@@ -1800,65 +1691,6 @@ def test_to_otel_part_function_call():
|
||||
}
|
||||
|
||||
|
||||
def test_to_otel_part_function_call_reuses_prepared_arguments():
|
||||
"""Test _to_otel_part does not re-serialize function-call arguments in the observability hot path."""
|
||||
from agent_framework import Content
|
||||
from agent_framework.observability import _to_otel_part
|
||||
|
||||
arguments = {"payload": object()}
|
||||
content = Content(type="function_call", call_id="call_789", name="handoff", arguments=arguments)
|
||||
result = _to_otel_part(content)
|
||||
|
||||
assert result is not None
|
||||
assert result["arguments"] is arguments
|
||||
|
||||
|
||||
def test_make_json_safe_non_callable_method_attribute():
|
||||
"""Test make_json_safe handles objects where model_dump/to_dict/dict are non-callable attributes."""
|
||||
from agent_framework._serialization import make_json_safe
|
||||
|
||||
class ObjWithNonCallableModelDump:
|
||||
model_dump = 42 # not callable
|
||||
|
||||
obj = ObjWithNonCallableModelDump()
|
||||
result = make_json_safe(obj)
|
||||
assert result == {}
|
||||
|
||||
|
||||
def test_make_json_safe_callable_method_type_error_falls_through():
|
||||
"""Test make_json_safe falls through when serializer-like methods require arguments."""
|
||||
from agent_framework._serialization import make_json_safe
|
||||
|
||||
class ObjWithRequiredArgModelDump:
|
||||
def __init__(self) -> None:
|
||||
self.value = "fallback"
|
||||
|
||||
def model_dump(self, required: str) -> dict[str, str]:
|
||||
return {"required": required}
|
||||
|
||||
obj = ObjWithRequiredArgModelDump()
|
||||
result = make_json_safe(obj)
|
||||
assert result == {"value": "fallback"}
|
||||
|
||||
|
||||
def test_make_json_safe_dict_with_non_string_keys():
|
||||
"""Test make_json_safe converts non-primitive dict keys to strings."""
|
||||
import json
|
||||
from datetime import datetime
|
||||
|
||||
from agent_framework._serialization import make_json_safe
|
||||
|
||||
dt_key = datetime(2024, 1, 1)
|
||||
obj = {dt_key: "value", 42: "num_value", "str_key": "normal"}
|
||||
result = make_json_safe(obj)
|
||||
# json.dumps must not raise TypeError
|
||||
serialized = json.dumps(result)
|
||||
parsed = json.loads(serialized)
|
||||
assert parsed[str(dt_key)] == "value"
|
||||
assert parsed["42"] == "num_value"
|
||||
assert parsed["str_key"] == "normal"
|
||||
|
||||
|
||||
def test_to_otel_part_function_result():
|
||||
"""Test _to_otel_part with function_result content."""
|
||||
from agent_framework import Content
|
||||
@@ -3187,49 +3019,6 @@ async def test_system_instructions_preserves_non_ascii_characters(span_exporter:
|
||||
assert [msg.get("role") for msg in input_messages] == ["user"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("enable_sensitive_data", [True], indirect=True)
|
||||
def test_capture_messages_with_prepared_request_info_function_call_arguments(span_exporter: InMemorySpanExporter):
|
||||
"""Test _capture_messages handles request-info function-call arguments prepared at Content creation."""
|
||||
import dataclasses
|
||||
import json
|
||||
|
||||
from opentelemetry import trace
|
||||
|
||||
from agent_framework import WorkflowAgent
|
||||
|
||||
@dataclasses.dataclass
|
||||
class HandoffRequest:
|
||||
target_agent: str
|
||||
reason: str
|
||||
|
||||
arguments = WorkflowAgent.RequestInfoFunctionArgs(
|
||||
request_id="call_dc",
|
||||
data=HandoffRequest(target_agent="helper", reason="overflow"),
|
||||
).to_dict()
|
||||
msg = Message(
|
||||
role="assistant",
|
||||
contents=[
|
||||
Content(
|
||||
type="function_call",
|
||||
call_id="call_dc",
|
||||
name="request_info",
|
||||
arguments=arguments,
|
||||
)
|
||||
],
|
||||
)
|
||||
span_exporter.clear()
|
||||
tracer = trace.get_tracer("test")
|
||||
with tracer.start_as_current_span("test_span") as span:
|
||||
_capture_messages(span=span, provider_name="test_provider", messages=[msg])
|
||||
|
||||
spans = span_exporter.get_finished_spans()
|
||||
span = spans[0]
|
||||
input_messages = json.loads(span.attributes[OtelAttr.INPUT_MESSAGES])
|
||||
tool_part = input_messages[0]["parts"][0]
|
||||
assert tool_part["type"] == "tool_call"
|
||||
assert tool_part["arguments"]["data"] == {"target_agent": "helper", "reason": "overflow"}
|
||||
|
||||
|
||||
def test_capture_messages_keeps_framework_instructions_out_of_logs_and_span_messages(
|
||||
span_exporter: InMemorySpanExporter,
|
||||
):
|
||||
|
||||
@@ -5,7 +5,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
from collections.abc import Iterator
|
||||
from contextlib import contextmanager
|
||||
@@ -1643,37 +1642,6 @@ class TestFunctionalWorkflowAgentHITL:
|
||||
break
|
||||
assert approval_found, "expected FunctionApprovalRequestContent in agent response"
|
||||
|
||||
async def test_request_info_dataclass_arguments_are_serialized_for_agent(self):
|
||||
@dataclass
|
||||
class HandoffRequest:
|
||||
target_agent: str
|
||||
reason: str
|
||||
|
||||
@workflow
|
||||
async def wf(x: str, ctx: RunContext) -> str:
|
||||
answer = await ctx.request_info(
|
||||
HandoffRequest(target_agent=x, reason="overflow"),
|
||||
response_type=str,
|
||||
request_id="rid-1",
|
||||
)
|
||||
return f"got:{answer}"
|
||||
|
||||
agent = wf.as_agent()
|
||||
response = await agent.run("helper")
|
||||
|
||||
function_call_arguments = None
|
||||
for message in response.messages:
|
||||
for content in message.contents:
|
||||
if getattr(content, "type", None) == "function_approval_request" and content.function_call is not None:
|
||||
function_call_arguments = content.function_call.arguments
|
||||
break
|
||||
|
||||
assert function_call_arguments == {
|
||||
"request_id": "rid-1",
|
||||
"data": {"target_agent": "helper", "reason": "overflow"},
|
||||
}
|
||||
assert json.loads(json.dumps(function_call_arguments)) == function_call_arguments
|
||||
|
||||
async def test_resume_via_agent_responses_kwarg(self):
|
||||
@workflow
|
||||
async def wf(x: str, ctx: RunContext) -> str:
|
||||
|
||||
@@ -1,9 +1,7 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
import json
|
||||
import uuid
|
||||
from collections.abc import Awaitable, Sequence
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Literal, overload
|
||||
|
||||
import pytest
|
||||
@@ -25,7 +23,6 @@ from agent_framework import (
|
||||
WorkflowAgent,
|
||||
WorkflowBuilder,
|
||||
WorkflowContext,
|
||||
WorkflowEvent,
|
||||
executor,
|
||||
handler,
|
||||
response_handler,
|
||||
@@ -296,33 +293,6 @@ class TestWorkflowAgent:
|
||||
# Verify cleanup - pending requests should be cleared after function response handling
|
||||
assert len(agent.pending_requests) == 0
|
||||
|
||||
def test_request_info_dataclass_arguments_are_serialized_when_content_is_created(self) -> None:
|
||||
"""Test WorkflowAgent prepares request_info arguments before observability captures messages."""
|
||||
|
||||
@dataclass
|
||||
class HandoffRequest:
|
||||
target_agent: str
|
||||
reason: str
|
||||
|
||||
executor = SimpleExecutor(id="executor1", response_text="Response")
|
||||
workflow = WorkflowBuilder(start_executor=executor).build()
|
||||
agent = WorkflowAgent(workflow=workflow, name="Request Test Agent")
|
||||
event = WorkflowEvent.request_info(
|
||||
request_id="request_123",
|
||||
source_executor_id="executor1",
|
||||
request_data=HandoffRequest(target_agent="helper", reason="overflow"),
|
||||
response_type=str,
|
||||
)
|
||||
|
||||
function_call, approval_request = agent._process_request_info_event(event) # pyright: ignore[reportPrivateUsage]
|
||||
|
||||
assert function_call.arguments == {
|
||||
"request_id": "request_123",
|
||||
"data": {"target_agent": "helper", "reason": "overflow"},
|
||||
}
|
||||
assert approval_request.function_call is function_call
|
||||
assert json.loads(json.dumps(function_call.arguments)) == function_call.arguments
|
||||
|
||||
def test_workflow_as_agent_method(self) -> None:
|
||||
"""Test that Workflow.as_agent() creates a properly configured WorkflowAgent."""
|
||||
# Create a simple workflow
|
||||
|
||||
@@ -12,7 +12,6 @@ from ._embedding_client import (
|
||||
)
|
||||
from ._foundry_evals import (
|
||||
FoundryEvals,
|
||||
GeneratedEvaluatorRef,
|
||||
evaluate_foundry_target,
|
||||
evaluate_traces,
|
||||
)
|
||||
@@ -34,7 +33,6 @@ __all__ = [
|
||||
"FoundryEmbeddingSettings",
|
||||
"FoundryEvals",
|
||||
"FoundryMemoryProvider",
|
||||
"GeneratedEvaluatorRef",
|
||||
"RawFoundryAgent",
|
||||
"RawFoundryAgentChatClient",
|
||||
"RawFoundryChatClient",
|
||||
|
||||
@@ -351,7 +351,6 @@ class RawFoundryAgentChatClient( # type: ignore[misc]
|
||||
if _uses_foundry_agent_session(conversation_id):
|
||||
run_options.pop("previous_response_id", None)
|
||||
run_options.pop("conversation", None)
|
||||
run_options.pop("model", None)
|
||||
extra_body["agent_session_id"] = conversation_id
|
||||
# Non-preview Prompt/Hosted Agent calls need agent_reference in the request body to
|
||||
# tell the Responses API which Foundry agent (and version) is in use, since ``model``
|
||||
@@ -367,6 +366,7 @@ class RawFoundryAgentChatClient( # type: ignore[misc]
|
||||
# Strip tools from request body - Foundry API rejects requests with both
|
||||
# agent endpoint and tools present. FunctionTools are invoked client-side
|
||||
# by the function invocation layer, not sent to the service.
|
||||
run_options.pop("model", None)
|
||||
if not self.allow_preview:
|
||||
run_options.pop("tools", None)
|
||||
run_options.pop("tool_choice", None)
|
||||
|
||||
@@ -28,9 +28,8 @@ from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from collections.abc import Iterable, Sequence
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Any, cast
|
||||
from collections.abc import Sequence
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from agent_framework._evaluation import (
|
||||
AgentEvalConverter,
|
||||
@@ -40,7 +39,6 @@ from agent_framework._evaluation import (
|
||||
EvalItemResult,
|
||||
EvalResults,
|
||||
EvalScoreResult,
|
||||
RubricScore,
|
||||
)
|
||||
from agent_framework._feature_stage import ExperimentalFeature, experimental
|
||||
from openai import AsyncOpenAI
|
||||
@@ -53,54 +51,6 @@ if TYPE_CHECKING:
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# region Generated rubric evaluator references
|
||||
|
||||
|
||||
@experimental(feature_id=ExperimentalFeature.EVALS)
|
||||
@dataclass(frozen=True)
|
||||
class GeneratedEvaluatorRef:
|
||||
"""A reference to a rubric evaluator that already exists in Foundry.
|
||||
|
||||
Pass instances of this class to :class:`FoundryEvals` to score items
|
||||
with a pre-existing rubric evaluator (manually authored or
|
||||
auto-generated through the Foundry portal). agent-framework is a
|
||||
consumer here: it does not create or modify the evaluator definition;
|
||||
it only references the persisted version by name.
|
||||
|
||||
Pinning ``version`` is strongly recommended so evaluation runs are
|
||||
reproducible. ``version=None`` resolves to whichever version is
|
||||
current at execution time; :class:`FoundryEvals` emits a warning when
|
||||
a versionless reference is used. CI gates should always pass a
|
||||
concrete version.
|
||||
|
||||
Attributes:
|
||||
name: Evaluator name as stored in the Foundry project (for
|
||||
example ``"reservation-policy-rubric"``). Distinct from
|
||||
built-in evaluators such as ``"builtin.relevance"``.
|
||||
version: Pinned evaluator version. ``None`` means "latest" —
|
||||
this is discouraged for CI/repro and :class:`FoundryEvals`
|
||||
will emit a warning when used.
|
||||
display_name: Optional human-readable name used in result
|
||||
summaries. Defaults to ``name`` when unset.
|
||||
"""
|
||||
|
||||
name: str
|
||||
version: str | None = None
|
||||
display_name: str | None = None
|
||||
|
||||
@classmethod
|
||||
def latest(cls, name: str, *, display_name: str | None = None) -> GeneratedEvaluatorRef:
|
||||
"""Construct a versionless reference (resolves to the latest version at run time).
|
||||
|
||||
Discouraged for reproducible runs. Prefer the constructor with
|
||||
an explicit ``version`` so CI and replay evaluations stay stable
|
||||
when the evaluator is updated in Foundry.
|
||||
"""
|
||||
return cls(name=name, version=None, display_name=display_name)
|
||||
|
||||
|
||||
# endregion
|
||||
# Agent evaluators that accept query/response as conversation arrays.
|
||||
# Maintained manually — check https://learn.microsoft.com/en-us/azure/ai-studio/how-to/develop/evaluate-sdk
|
||||
# for the latest evaluator list. These are the evaluators that need conversation-format input.
|
||||
@@ -216,7 +166,7 @@ def _resolve_evaluator(name: str) -> str:
|
||||
|
||||
|
||||
def _build_testing_criteria(
|
||||
evaluators: Sequence[str | GeneratedEvaluatorRef],
|
||||
evaluators: Sequence[str],
|
||||
model: str,
|
||||
*,
|
||||
include_data_mapping: bool = False,
|
||||
@@ -225,9 +175,7 @@ def _build_testing_criteria(
|
||||
"""Build ``testing_criteria`` for ``evals.create()``.
|
||||
|
||||
Args:
|
||||
evaluators: Evaluator names (built-in shorts / fully-qualified
|
||||
``builtin.*`` names) or :class:`GeneratedEvaluatorRef`
|
||||
instances for generated rubric evaluators.
|
||||
evaluators: Evaluator names.
|
||||
model: Model deployment for the LLM judge.
|
||||
include_data_mapping: Whether to include field-level data mapping
|
||||
(required for the JSONL data source, not needed for response-based).
|
||||
@@ -235,38 +183,7 @@ def _build_testing_criteria(
|
||||
definitions.
|
||||
"""
|
||||
criteria: list[dict[str, Any]] = []
|
||||
for entry_spec in evaluators:
|
||||
if isinstance(entry_spec, GeneratedEvaluatorRef):
|
||||
short = entry_spec.display_name or entry_spec.name
|
||||
ref_entry: dict[str, Any] = {
|
||||
"type": "azure_ai_evaluator",
|
||||
"name": short,
|
||||
"evaluator_name": entry_spec.name,
|
||||
"initialization_parameters": {"deployment_name": model},
|
||||
}
|
||||
if entry_spec.version is not None:
|
||||
ref_entry["evaluator_version"] = entry_spec.version
|
||||
else:
|
||||
logger.warning(
|
||||
"GeneratedEvaluatorRef '%s' has no pinned version; the eval run "
|
||||
"will resolve to whichever version is current at execution time. "
|
||||
"Pin the version for reproducible runs.",
|
||||
entry_spec.name,
|
||||
)
|
||||
if include_data_mapping:
|
||||
# Rubric evaluators accept conversation arrays like agent
|
||||
# evaluators, plus tool_definitions when items are tool-aware.
|
||||
ref_mapping: dict[str, str] = {
|
||||
"query": "{{item.query_messages}}",
|
||||
"response": "{{item.response_messages}}",
|
||||
}
|
||||
if include_tool_definitions:
|
||||
ref_mapping["tool_definitions"] = "{{item.tool_definitions}}"
|
||||
ref_entry["data_mapping"] = ref_mapping
|
||||
criteria.append(ref_entry)
|
||||
continue
|
||||
|
||||
name = entry_spec
|
||||
for name in evaluators:
|
||||
qualified = _resolve_evaluator(name)
|
||||
short = name if not name.startswith("builtin.") else name.split(".")[-1]
|
||||
|
||||
@@ -330,9 +247,9 @@ def _build_item_schema(
|
||||
|
||||
|
||||
def _resolve_default_evaluators(
|
||||
evaluators: Sequence[str | GeneratedEvaluatorRef] | None,
|
||||
evaluators: Sequence[str] | None,
|
||||
items: Sequence[EvalItem | dict[str, Any]] | None = None,
|
||||
) -> list[str | GeneratedEvaluatorRef]:
|
||||
) -> list[str]:
|
||||
"""Resolve evaluators, applying defaults when ``None``.
|
||||
|
||||
Defaults to relevance + coherence + task_adherence. Automatically adds
|
||||
@@ -341,7 +258,7 @@ def _resolve_default_evaluators(
|
||||
if evaluators is not None:
|
||||
return list(evaluators)
|
||||
|
||||
result: list[str | GeneratedEvaluatorRef] = list(_DEFAULT_EVALUATORS)
|
||||
result = list(_DEFAULT_EVALUATORS)
|
||||
if items is not None:
|
||||
has_tools = any((item.tools if isinstance(item, EvalItem) else item.get("tool_definitions")) for item in items)
|
||||
if has_tools:
|
||||
@@ -350,24 +267,14 @@ def _resolve_default_evaluators(
|
||||
|
||||
|
||||
def _filter_tool_evaluators(
|
||||
evaluators: list[str | GeneratedEvaluatorRef],
|
||||
evaluators: list[str],
|
||||
items: Sequence[EvalItem | dict[str, Any]],
|
||||
) -> list[str | GeneratedEvaluatorRef]:
|
||||
"""Remove tool evaluators if no items have tool definitions.
|
||||
|
||||
Generated rubric evaluators are tool-aware but not tool-required; they
|
||||
are preserved regardless of whether items carry tool definitions.
|
||||
"""
|
||||
) -> list[str]:
|
||||
"""Remove tool evaluators if no items have tool definitions."""
|
||||
has_tools = any((item.tools if isinstance(item, EvalItem) else item.get("tool_definitions")) for item in items)
|
||||
if has_tools:
|
||||
return evaluators
|
||||
|
||||
def _is_tool_only(spec: str | GeneratedEvaluatorRef) -> bool:
|
||||
if isinstance(spec, GeneratedEvaluatorRef):
|
||||
return False
|
||||
return _resolve_evaluator(spec) in _TOOL_EVALUATORS
|
||||
|
||||
filtered = [e for e in evaluators if not _is_tool_only(e)]
|
||||
filtered = [e for e in evaluators if _resolve_evaluator(e) not in _TOOL_EVALUATORS]
|
||||
if not filtered:
|
||||
raise ValueError(
|
||||
f"All requested evaluators {evaluators} require tool definitions, "
|
||||
@@ -375,7 +282,7 @@ def _filter_tool_evaluators(
|
||||
"or choose evaluators that do not require tools."
|
||||
)
|
||||
if len(filtered) < len(evaluators):
|
||||
removed = [e for e in evaluators if _is_tool_only(e)]
|
||||
removed = [e for e in evaluators if _resolve_evaluator(e) in _TOOL_EVALUATORS]
|
||||
logger.info("Removed tool evaluators %s (no items have tools)", removed)
|
||||
return filtered
|
||||
|
||||
@@ -447,114 +354,6 @@ def _extract_per_evaluator(run: RunRetrieveResponse) -> dict[str, dict[str, int]
|
||||
return per_eval
|
||||
|
||||
|
||||
_RUBRIC_DIMENSION_KEYS: tuple[str, ...] = ("dimension_scores", "rubric_scores")
|
||||
"""Property keys that may carry per-dimension rubric breakdowns.
|
||||
|
||||
The published Foundry rubric-evaluator output format uses
|
||||
``properties.dimension_scores`` (see the Microsoft Learn "Rubric
|
||||
evaluators" reference). Earlier preview builds and some SDK shapes
|
||||
used ``rubric_scores``; we accept both for defensive forward/backward
|
||||
compatibility.
|
||||
"""
|
||||
|
||||
|
||||
def _parse_dimension_entries(raw: Any) -> list[RubricScore]:
|
||||
"""Parse a raw list-like payload into ``RubricScore`` instances.
|
||||
|
||||
Returns an empty list when ``raw`` is falsy, not iterable, or
|
||||
contains no well-formed entries.
|
||||
"""
|
||||
if not raw:
|
||||
return []
|
||||
try:
|
||||
raw_iter: Iterable[Any] = iter(raw)
|
||||
except TypeError:
|
||||
return []
|
||||
|
||||
parsed: list[RubricScore] = []
|
||||
for raw_entry in raw_iter:
|
||||
entry: Any = raw_entry
|
||||
try:
|
||||
rid: Any
|
||||
score_val: Any
|
||||
applicable: Any
|
||||
weight: Any
|
||||
reason: Any
|
||||
if isinstance(entry, dict):
|
||||
entry_any = cast("dict[str, Any]", entry)
|
||||
rid = entry_any.get("id")
|
||||
score_val = entry_any.get("score")
|
||||
applicable = entry_any.get("applicable")
|
||||
weight = entry_any.get("weight")
|
||||
reason = entry_any.get("reason", "")
|
||||
else:
|
||||
rid = getattr(entry, "id", None)
|
||||
score_val = getattr(entry, "score", None)
|
||||
applicable = getattr(entry, "applicable", None)
|
||||
weight = getattr(entry, "weight", None)
|
||||
reason = getattr(entry, "reason", "") or ""
|
||||
if rid is None or weight is None or applicable is None:
|
||||
continue
|
||||
parsed.append(
|
||||
RubricScore(
|
||||
id=str(rid),
|
||||
score=int(score_val) if isinstance(score_val, (int, float)) else None,
|
||||
applicable=bool(applicable),
|
||||
weight=int(weight),
|
||||
reason=str(reason) if reason is not None else "",
|
||||
)
|
||||
)
|
||||
except (TypeError, ValueError):
|
||||
logger.debug("Skipping malformed rubric dimension entry: %s", cast("Any", entry), exc_info=True)
|
||||
return parsed
|
||||
|
||||
|
||||
def _extract_rubric_scores(sample: Any) -> list[RubricScore] | None:
|
||||
"""Extract typed ``RubricScore`` instances from an evaluator's raw sample payload.
|
||||
|
||||
Foundry rubric evaluators include a per-dimension breakdown under
|
||||
``properties.dimension_scores`` on each result (preview builds used
|
||||
``rubric_scores``; both keys are accepted, with the canonical
|
||||
``dimension_scores`` taking priority). The exact location may
|
||||
vary across SDK versions, so this helper accepts a few shapes:
|
||||
|
||||
* The SDK ``sample`` object exposes
|
||||
``properties.dimension_scores`` / ``properties.rubric_scores``.
|
||||
* The ``sample`` is a dict containing the same under
|
||||
``properties.<key>``.
|
||||
* The ``sample`` is a dict with ``dimension_scores`` /
|
||||
``rubric_scores`` at the top level.
|
||||
|
||||
Returns ``None`` when no rubric scores are present (i.e. the
|
||||
evaluator was not a rubric evaluator).
|
||||
"""
|
||||
if sample is None:
|
||||
return None
|
||||
|
||||
containers: list[Any] = []
|
||||
properties: Any = getattr(sample, "properties", None)
|
||||
if properties is not None:
|
||||
containers.append(properties)
|
||||
if isinstance(sample, dict):
|
||||
sample_any = cast("dict[str, Any]", sample)
|
||||
props_dict: Any = sample_any.get("properties")
|
||||
if props_dict is not None and props_dict is not properties:
|
||||
containers.append(props_dict)
|
||||
containers.append(sample_any)
|
||||
|
||||
for container in containers:
|
||||
for key in _RUBRIC_DIMENSION_KEYS:
|
||||
raw: Any = None
|
||||
if isinstance(container, dict):
|
||||
raw = cast("dict[str, Any]", container).get(key)
|
||||
elif hasattr(container, key):
|
||||
raw = getattr(container, key, None)
|
||||
parsed = _parse_dimension_entries(raw)
|
||||
if parsed:
|
||||
return parsed
|
||||
return None
|
||||
|
||||
|
||||
async def _fetch_output_items(
|
||||
client: AsyncOpenAI,
|
||||
eval_id: str,
|
||||
@@ -578,15 +377,12 @@ async def _fetch_output_items(
|
||||
# Extract per-evaluator scores
|
||||
scores: list[EvalScoreResult] = []
|
||||
for r in oi.results or []:
|
||||
sample = r.sample
|
||||
dimensions = _extract_rubric_scores(sample)
|
||||
scores.append(
|
||||
EvalScoreResult(
|
||||
name=r.name,
|
||||
score=r.score,
|
||||
passed=r.passed,
|
||||
sample=sample,
|
||||
dimensions=dimensions,
|
||||
sample=r.sample,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -598,18 +394,15 @@ async def _fetch_output_items(
|
||||
output_text: str | None = None
|
||||
response_id: str | None = None
|
||||
|
||||
# mypy infers oi.sample as dict[str, object] | None, but the
|
||||
# OpenAI SDK actually returns a typed Sample model. Cast to Any so
|
||||
# both type checkers accept the attribute access pattern.
|
||||
oi_sample: Any = oi.sample
|
||||
if oi_sample is not None:
|
||||
err = oi_sample.error
|
||||
if err is not None and (err.code or err.message):
|
||||
sample = oi.sample
|
||||
if sample is not None: # pyright: ignore[reportUnnecessaryComparison]
|
||||
err = sample.error
|
||||
if err is not None and (err.code or err.message): # pyright: ignore[reportUnnecessaryComparison]
|
||||
error_code = err.code or None
|
||||
error_message = err.message or None
|
||||
|
||||
usage = oi_sample.usage
|
||||
if usage is not None and usage.total_tokens:
|
||||
usage = sample.usage
|
||||
if usage is not None and usage.total_tokens: # pyright: ignore[reportUnnecessaryComparison]
|
||||
token_usage = {
|
||||
"prompt_tokens": usage.prompt_tokens,
|
||||
"completion_tokens": usage.completion_tokens,
|
||||
@@ -618,13 +411,13 @@ async def _fetch_output_items(
|
||||
}
|
||||
|
||||
# Extract input/output text
|
||||
if oi_sample.input:
|
||||
parts = [si.content for si in oi_sample.input if si.role == "user"]
|
||||
if sample.input:
|
||||
parts = [si.content for si in sample.input if si.role == "user"]
|
||||
if parts:
|
||||
input_text = " ".join(parts)
|
||||
|
||||
if oi_sample.output:
|
||||
parts = [so.content or "" for so in oi_sample.output if so.role == "assistant"]
|
||||
if sample.output:
|
||||
parts = [so.content or "" for so in sample.output if so.role == "assistant"]
|
||||
if parts:
|
||||
output_text = " ".join(parts)
|
||||
|
||||
@@ -679,7 +472,7 @@ async def _evaluate_via_responses_impl(
|
||||
*,
|
||||
client: AsyncOpenAI,
|
||||
response_ids: Sequence[str],
|
||||
evaluators: list[str | GeneratedEvaluatorRef],
|
||||
evaluators: list[str],
|
||||
model: str,
|
||||
eval_name: str,
|
||||
poll_interval: float,
|
||||
@@ -780,11 +573,8 @@ class FoundryEvals:
|
||||
(from ``azure.ai.projects.aio``). Provide this or *client*.
|
||||
model: Model deployment name for the evaluator LLM judge.
|
||||
Resolved from ``client.model`` when omitted.
|
||||
evaluators: Evaluator specifications. Entries may be built-in
|
||||
short names (e.g. ``"relevance"``), fully-qualified
|
||||
``"builtin.*"`` names, or :class:`GeneratedEvaluatorRef`
|
||||
instances for previously generated rubric evaluators. When
|
||||
``None`` (default), uses smart defaults based on item data.
|
||||
evaluators: Evaluator names (e.g. ``["relevance", "tool_call_accuracy"]``).
|
||||
When ``None`` (default), uses smart defaults based on item data.
|
||||
conversation_split: How to split multi-turn conversations into
|
||||
query/response halves. Defaults to ``LAST_TURN``. Pass a
|
||||
``ConversationSplit`` enum value or a custom callable — see
|
||||
@@ -833,7 +623,7 @@ class FoundryEvals:
|
||||
client: FoundryChatClient | None = None,
|
||||
project_client: AIProjectClient | None = None,
|
||||
model: str | None = None,
|
||||
evaluators: Sequence[str | GeneratedEvaluatorRef] | None = None,
|
||||
evaluators: Sequence[str] | None = None,
|
||||
conversation_split: ConversationSplitter = ConversationSplit.LAST_TURN,
|
||||
poll_interval: float = 5.0,
|
||||
timeout: float = 180.0,
|
||||
@@ -852,9 +642,7 @@ class FoundryEvals:
|
||||
"Model is required. Pass model= explicitly or use a FoundryChatClient that has a model configured."
|
||||
)
|
||||
self._model = resolved_model
|
||||
self._evaluators: list[str | GeneratedEvaluatorRef] | None = (
|
||||
list(evaluators) if evaluators is not None else None
|
||||
)
|
||||
self._evaluators = list(evaluators) if evaluators is not None else None
|
||||
self._conversation_split = conversation_split
|
||||
self._poll_interval = poll_interval
|
||||
self._timeout = timeout
|
||||
@@ -890,7 +678,7 @@ class FoundryEvals:
|
||||
async def _evaluate_via_dataset(
|
||||
self,
|
||||
items: Sequence[EvalItem],
|
||||
evaluators: list[str | GeneratedEvaluatorRef],
|
||||
evaluators: list[str],
|
||||
eval_name: str,
|
||||
) -> EvalResults:
|
||||
"""Evaluate using JSONL dataset upload path."""
|
||||
|
||||
@@ -203,7 +203,7 @@ async def test_raw_foundry_agent_chat_client_prepare_options_accepts_function_to
|
||||
|
||||
|
||||
async def test_raw_foundry_agent_chat_client_prepare_options_strips_client_side_fields() -> None:
|
||||
"""Test that _prepare_options strips tool-loop fields but preserves model for non-session requests."""
|
||||
"""Test that _prepare_options strips model and tool-loop fields from run_options."""
|
||||
|
||||
mock_project = MagicMock()
|
||||
mock_openai = MagicMock()
|
||||
@@ -235,49 +235,16 @@ async def test_raw_foundry_agent_chat_client_prepare_options_strips_client_side_
|
||||
options={"tools": [my_func]},
|
||||
)
|
||||
|
||||
# model is preserved for non-session (PromptAgent) requests
|
||||
assert result["model"] == "gpt-4.1"
|
||||
assert "model" not in result
|
||||
assert "tools" not in result
|
||||
assert "tool_choice" not in result
|
||||
assert "parallel_tool_calls" not in result
|
||||
# agent_reference is required so the Responses API can resolve model server-side; see #5582.
|
||||
assert result == {
|
||||
"model": "gpt-4.1",
|
||||
"extra_body": {"agent_reference": {"name": "test-agent", "type": "agent_reference"}},
|
||||
}
|
||||
|
||||
|
||||
async def test_raw_foundry_agent_chat_client_prepare_options_strips_model_for_hosted_session() -> None:
|
||||
"""Test that model is stripped when using a hosted agent session (not a PromptAgent)."""
|
||||
|
||||
mock_project = MagicMock()
|
||||
mock_openai = MagicMock()
|
||||
mock_project.get_openai_client.return_value = mock_openai
|
||||
|
||||
client = RawFoundryAgentChatClient(
|
||||
project_client=mock_project,
|
||||
agent_name="test-agent",
|
||||
)
|
||||
|
||||
with patch(
|
||||
"agent_framework_openai._chat_client.RawOpenAIChatClient._prepare_options",
|
||||
new_callable=AsyncMock,
|
||||
return_value={
|
||||
"model": "gpt-4.1",
|
||||
"previous_response_id": "resp_abc",
|
||||
},
|
||||
):
|
||||
result = await client._prepare_options(
|
||||
messages=[Message(role="user", contents="hi")],
|
||||
options={"conversation_id": "agent-session-123"},
|
||||
)
|
||||
|
||||
assert "model" not in result
|
||||
assert "previous_response_id" not in result
|
||||
assert result["extra_body"]["agent_session_id"] == "agent-session-123"
|
||||
assert result["extra_body"]["agent_reference"] == {"name": "test-agent", "type": "agent_reference"}
|
||||
|
||||
|
||||
async def test_raw_foundry_agent_chat_client_prepare_options_injects_agent_reference_first_turn() -> None:
|
||||
"""First-turn (no conversation_id) Prompt Agent calls must carry agent_reference in extra_body.
|
||||
|
||||
@@ -305,6 +272,7 @@ async def test_raw_foundry_agent_chat_client_prepare_options_injects_agent_refer
|
||||
options={},
|
||||
)
|
||||
|
||||
assert "model" not in result
|
||||
assert result["extra_body"] == {
|
||||
"agent_reference": {"name": "test-agent", "type": "agent_reference", "version": "2"},
|
||||
}
|
||||
@@ -365,8 +333,7 @@ async def test_raw_foundry_agent_chat_client_prepare_options_skips_agent_referen
|
||||
options={},
|
||||
)
|
||||
|
||||
# model is preserved for non-session requests (platform tolerates it for hosted agents)
|
||||
assert result["model"] == "gpt-4.1"
|
||||
assert "model" not in result
|
||||
# No extra_body at all is the cleanest signal — agent_reference must not be injected here.
|
||||
assert "extra_body" not in result
|
||||
|
||||
@@ -396,39 +363,6 @@ async def test_raw_foundry_agent_chat_client_prepare_options_respects_caller_age
|
||||
assert result["extra_body"]["agent_reference"] == caller_reference
|
||||
|
||||
|
||||
async def test_raw_foundry_agent_chat_client_prepare_options_preserves_model_for_resp_continuation() -> None:
|
||||
"""Test that model is preserved when conversation_id is a resp_* continuation (HostedAgent v1 / v2-no-session)."""
|
||||
|
||||
mock_project = MagicMock()
|
||||
mock_openai = MagicMock()
|
||||
mock_project.get_openai_client.return_value = mock_openai
|
||||
|
||||
client = RawFoundryAgentChatClient(
|
||||
project_client=mock_project,
|
||||
agent_name="test-agent",
|
||||
)
|
||||
|
||||
with patch(
|
||||
"agent_framework_openai._chat_client.RawOpenAIChatClient._prepare_options",
|
||||
new_callable=AsyncMock,
|
||||
return_value={
|
||||
"model": "gpt-4.1",
|
||||
"previous_response_id": "resp_abc123",
|
||||
},
|
||||
):
|
||||
result = await client._prepare_options(
|
||||
messages=[Message(role="user", contents="hi")],
|
||||
options={"conversation_id": "resp_abc123"},
|
||||
)
|
||||
|
||||
# model preserved — resp_* is standard Responses API continuity, not a hosted session
|
||||
assert result["model"] == "gpt-4.1"
|
||||
# previous_response_id preserved — not stripped outside hosted session path
|
||||
assert result["previous_response_id"] == "resp_abc123"
|
||||
# no agent_session_id injected
|
||||
assert "extra_body" not in result or "agent_session_id" not in result.get("extra_body", {})
|
||||
|
||||
|
||||
async def test_raw_foundry_agent_chat_client_prepare_options_maps_agent_session_id_to_extra_body() -> None:
|
||||
"""Test that service_session_id is forwarded as agent_session_id for hosted sessions."""
|
||||
|
||||
|
||||
@@ -25,25 +25,16 @@ from agent_framework._evaluation import (
|
||||
from agent_framework._workflows._workflow import WorkflowRunResult
|
||||
from openai import AsyncOpenAI
|
||||
|
||||
from agent_framework_foundry import GeneratedEvaluatorRef
|
||||
from agent_framework_foundry._foundry_evals import (
|
||||
_AGENT_EVALUATORS,
|
||||
_BUILTIN_EVALUATORS,
|
||||
_TOOL_EVALUATORS,
|
||||
FoundryEvals,
|
||||
_build_item_schema,
|
||||
_build_testing_criteria,
|
||||
_extract_per_evaluator,
|
||||
_extract_result_counts,
|
||||
_extract_rubric_scores,
|
||||
_fetch_output_items,
|
||||
_filter_tool_evaluators,
|
||||
_poll_eval_run,
|
||||
_resolve_default_evaluators,
|
||||
_resolve_evaluator,
|
||||
_resolve_openai_client,
|
||||
evaluate_foundry_target,
|
||||
evaluate_traces,
|
||||
)
|
||||
|
||||
|
||||
@@ -815,67 +806,6 @@ class TestBuildTestingCriteria:
|
||||
for c in criteria:
|
||||
assert "tool_definitions" in c["data_mapping"], f"{c['name']} missing tool_definitions"
|
||||
|
||||
def test_generated_evaluator_ref_pinned_version(self) -> None:
|
||||
|
||||
ref = GeneratedEvaluatorRef(name="my-rubric", version="1")
|
||||
criteria = _build_testing_criteria([ref], "gpt-4o", include_data_mapping=True)
|
||||
|
||||
assert len(criteria) == 1
|
||||
c = criteria[0]
|
||||
assert c["type"] == "azure_ai_evaluator"
|
||||
assert c["evaluator_name"] == "my-rubric"
|
||||
assert c["evaluator_version"] == "1"
|
||||
assert c["name"] == "my-rubric"
|
||||
assert c["initialization_parameters"] == {"deployment_name": "gpt-4o"}
|
||||
assert c["data_mapping"] == {
|
||||
"query": "{{item.query_messages}}",
|
||||
"response": "{{item.response_messages}}",
|
||||
}
|
||||
|
||||
def test_generated_evaluator_ref_display_name_used_as_short(self) -> None:
|
||||
|
||||
ref = GeneratedEvaluatorRef(name="my-rubric", version="2", display_name="My Rubric")
|
||||
criteria = _build_testing_criteria([ref], "gpt-4o")
|
||||
|
||||
assert criteria[0]["name"] == "My Rubric"
|
||||
assert criteria[0]["evaluator_name"] == "my-rubric"
|
||||
|
||||
def test_generated_evaluator_ref_tool_definitions_added(self) -> None:
|
||||
|
||||
ref = GeneratedEvaluatorRef(name="my-rubric", version="1")
|
||||
criteria = _build_testing_criteria(
|
||||
[ref],
|
||||
"gpt-4o",
|
||||
include_data_mapping=True,
|
||||
include_tool_definitions=True,
|
||||
)
|
||||
|
||||
assert criteria[0]["data_mapping"]["tool_definitions"] == "{{item.tool_definitions}}"
|
||||
|
||||
def test_generated_evaluator_ref_unpinned_warns(self, caplog: pytest.LogCaptureFixture) -> None:
|
||||
import logging
|
||||
|
||||
ref = GeneratedEvaluatorRef.latest("my-rubric")
|
||||
with caplog.at_level(logging.WARNING, logger="agent_framework_foundry._foundry_evals"):
|
||||
criteria = _build_testing_criteria([ref], "gpt-4o")
|
||||
|
||||
assert "evaluator_version" not in criteria[0]
|
||||
assert any("no pinned version" in r.message for r in caplog.records)
|
||||
|
||||
def test_generated_evaluator_ref_mixed_with_builtins(self) -> None:
|
||||
|
||||
ref = GeneratedEvaluatorRef(name="my-rubric", version="1")
|
||||
criteria = _build_testing_criteria(
|
||||
["relevance", ref, "task_adherence"],
|
||||
"gpt-4o",
|
||||
include_data_mapping=True,
|
||||
)
|
||||
|
||||
assert [c["name"] for c in criteria] == ["relevance", "my-rubric", "task_adherence"]
|
||||
assert criteria[0]["evaluator_name"] == "builtin.relevance"
|
||||
assert criteria[1]["evaluator_name"] == "my-rubric"
|
||||
assert criteria[2]["evaluator_name"] == "builtin.task_adherence"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _build_item_schema
|
||||
@@ -1333,29 +1263,6 @@ class TestFilterToolEvaluators:
|
||||
items,
|
||||
)
|
||||
|
||||
def test_preserves_generated_ref_when_no_tools(self) -> None:
|
||||
|
||||
ref = GeneratedEvaluatorRef(name="rubric", version="1")
|
||||
items = [
|
||||
EvalItem(conversation=[Message("user", ["q"]), Message("assistant", ["r"])]),
|
||||
]
|
||||
result = _filter_tool_evaluators(
|
||||
["relevance", ref, "tool_call_accuracy"],
|
||||
items,
|
||||
)
|
||||
assert "relevance" in result
|
||||
assert ref in result
|
||||
assert "tool_call_accuracy" not in result
|
||||
|
||||
def test_generated_ref_alone_does_not_raise(self) -> None:
|
||||
|
||||
ref = GeneratedEvaluatorRef(name="rubric", version="1")
|
||||
items = [
|
||||
EvalItem(conversation=[Message("user", ["q"]), Message("assistant", ["r"])]),
|
||||
]
|
||||
result = _filter_tool_evaluators([ref], items)
|
||||
assert result == [ref]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# EvalResults
|
||||
@@ -2360,6 +2267,7 @@ class TestEvalResultsWithItems:
|
||||
|
||||
class TestFetchOutputItems:
|
||||
async def test_fetches_and_converts_output_items(self) -> None:
|
||||
from agent_framework_foundry._foundry_evals import _fetch_output_items
|
||||
|
||||
# Build mock output items matching the OpenAI SDK schema
|
||||
mock_result = MagicMock()
|
||||
@@ -2421,6 +2329,7 @@ class TestFetchOutputItems:
|
||||
assert item.error_code is None
|
||||
|
||||
async def test_handles_errored_item(self) -> None:
|
||||
from agent_framework_foundry._foundry_evals import _fetch_output_items
|
||||
|
||||
mock_error = MagicMock()
|
||||
mock_error.code = "QueryExtractionError"
|
||||
@@ -2452,6 +2361,7 @@ class TestFetchOutputItems:
|
||||
assert len(item.scores) == 0
|
||||
|
||||
async def test_handles_api_failure_gracefully(self) -> None:
|
||||
from agent_framework_foundry._foundry_evals import _fetch_output_items
|
||||
|
||||
mock_client = MagicMock()
|
||||
mock_client.evals.runs.output_items.list = AsyncMock(side_effect=TypeError("API error"))
|
||||
@@ -2459,166 +2369,6 @@ class TestFetchOutputItems:
|
||||
items = await _fetch_output_items(mock_client, "eval_1", "run_1")
|
||||
assert items == []
|
||||
|
||||
async def test_extracts_rubric_scores_from_dict_sample(self) -> None:
|
||||
|
||||
mock_result = MagicMock()
|
||||
mock_result.name = "my-rubric"
|
||||
mock_result.score = 0.85
|
||||
mock_result.passed = True
|
||||
mock_result.sample = {
|
||||
"properties": {
|
||||
"rubric_scores": [
|
||||
{"id": "policy", "score": 4, "applicable": True, "weight": 1, "reason": "ok"},
|
||||
{"id": "safety", "score": None, "applicable": False, "weight": 1, "reason": "n/a"},
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
mock_oi = MagicMock()
|
||||
mock_oi.id = "oi_1"
|
||||
mock_oi.status = "pass"
|
||||
mock_oi.results = [mock_result]
|
||||
mock_oi.sample = None
|
||||
mock_oi.datasource_item = {}
|
||||
|
||||
mock_client = MagicMock()
|
||||
mock_client.evals.runs.output_items.list = AsyncMock(return_value=_AsyncPage([mock_oi]))
|
||||
|
||||
items = await _fetch_output_items(mock_client, "eval_1", "run_1")
|
||||
|
||||
assert len(items) == 1
|
||||
scores = items[0].scores
|
||||
assert len(scores) == 1
|
||||
assert scores[0].dimensions is not None
|
||||
assert len(scores[0].dimensions) == 2
|
||||
policy = next(d for d in scores[0].dimensions if d.id == "policy")
|
||||
assert policy.score == 4
|
||||
assert policy.applicable is True
|
||||
assert policy.weight == 1
|
||||
assert policy.reason == "ok"
|
||||
safety = next(d for d in scores[0].dimensions if d.id == "safety")
|
||||
assert safety.score is None
|
||||
assert safety.applicable is False
|
||||
|
||||
async def test_no_rubric_scores_when_absent(self) -> None:
|
||||
|
||||
mock_result = MagicMock()
|
||||
mock_result.name = "relevance"
|
||||
mock_result.score = 0.85
|
||||
mock_result.passed = True
|
||||
mock_result.sample = None
|
||||
|
||||
mock_oi = MagicMock()
|
||||
mock_oi.id = "oi_2"
|
||||
mock_oi.status = "pass"
|
||||
mock_oi.results = [mock_result]
|
||||
mock_oi.sample = None
|
||||
mock_oi.datasource_item = {}
|
||||
|
||||
mock_client = MagicMock()
|
||||
mock_client.evals.runs.output_items.list = AsyncMock(return_value=_AsyncPage([mock_oi]))
|
||||
|
||||
items = await _fetch_output_items(mock_client, "eval_1", "run_1")
|
||||
|
||||
assert items[0].scores[0].dimensions is None
|
||||
|
||||
|
||||
class TestExtractRubricScores:
|
||||
def test_handles_attribute_style_properties(self) -> None:
|
||||
|
||||
rs = MagicMock()
|
||||
rs.id = "policy"
|
||||
rs.score = 5
|
||||
rs.applicable = True
|
||||
rs.weight = 2
|
||||
rs.reason = "ok"
|
||||
|
||||
sample = MagicMock()
|
||||
sample.properties = MagicMock()
|
||||
sample.properties.rubric_scores = [rs]
|
||||
|
||||
result = _extract_rubric_scores(sample)
|
||||
assert result is not None
|
||||
assert result[0].id == "policy"
|
||||
assert result[0].score == 5
|
||||
assert result[0].weight == 2
|
||||
|
||||
def test_top_level_rubric_scores_in_dict(self) -> None:
|
||||
|
||||
sample = {"rubric_scores": [{"id": "a", "score": 3, "applicable": True, "weight": 1, "reason": "r"}]}
|
||||
result = _extract_rubric_scores(sample)
|
||||
assert result is not None
|
||||
assert result[0].id == "a"
|
||||
|
||||
def test_returns_none_when_missing(self) -> None:
|
||||
|
||||
assert _extract_rubric_scores(None) is None
|
||||
assert _extract_rubric_scores({}) is None
|
||||
assert _extract_rubric_scores({"properties": {}}) is None
|
||||
|
||||
def test_skips_malformed_entries(self) -> None:
|
||||
|
||||
sample = {
|
||||
"properties": {
|
||||
"rubric_scores": [
|
||||
{"id": "good", "score": 3, "applicable": True, "weight": 1, "reason": "ok"},
|
||||
{"id": "bad-no-weight", "score": 2, "applicable": True, "reason": "x"},
|
||||
]
|
||||
}
|
||||
}
|
||||
result = _extract_rubric_scores(sample)
|
||||
assert result is not None
|
||||
assert len(result) == 1
|
||||
assert result[0].id == "good"
|
||||
|
||||
def test_canonical_dimension_scores_key_from_docs(self) -> None:
|
||||
"""Per the Microsoft Learn docs, runtime output uses ``properties.dimension_scores``."""
|
||||
|
||||
sample = {
|
||||
"properties": {
|
||||
"dimension_scores": [
|
||||
{
|
||||
"id": "intent_recognition",
|
||||
"score": 5,
|
||||
"applicable": True,
|
||||
"weight": 9,
|
||||
"reason": "Identified correctly.",
|
||||
},
|
||||
{
|
||||
"id": "general_quality",
|
||||
"score": 4,
|
||||
"applicable": True,
|
||||
"weight": 5,
|
||||
"reason": "Strong overall.",
|
||||
},
|
||||
]
|
||||
}
|
||||
}
|
||||
result = _extract_rubric_scores(sample)
|
||||
assert result is not None
|
||||
assert [r.id for r in result] == ["intent_recognition", "general_quality"]
|
||||
assert [r.score for r in result] == [5, 4]
|
||||
assert [r.weight for r in result] == [9, 5]
|
||||
|
||||
def test_dimension_scores_via_attribute(self) -> None:
|
||||
"""Canonical key also resolves when properties exposes ``dimension_scores`` as an attr."""
|
||||
|
||||
rs = MagicMock()
|
||||
rs.id = "policy_enforcement"
|
||||
rs.score = 1
|
||||
rs.applicable = True
|
||||
rs.weight = 5
|
||||
rs.reason = "violated"
|
||||
|
||||
sample = MagicMock()
|
||||
sample.properties = MagicMock(spec=["dimension_scores"])
|
||||
sample.properties.dimension_scores = [rs]
|
||||
|
||||
result = _extract_rubric_scores(sample)
|
||||
assert result is not None
|
||||
assert result[0].id == "policy_enforcement"
|
||||
assert result[0].score == 1
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _poll_eval_run — timeout / failed / canceled paths
|
||||
@@ -2628,6 +2378,7 @@ class TestExtractRubricScores:
|
||||
class TestPollEvalRun:
|
||||
async def test_timeout_returns_timeout_status(self) -> None:
|
||||
"""Poll timeout returns EvalResults with status='timeout'."""
|
||||
from agent_framework_foundry._foundry_evals import _poll_eval_run
|
||||
|
||||
mock_client = MagicMock()
|
||||
mock_pending = MagicMock()
|
||||
@@ -2641,6 +2392,7 @@ class TestPollEvalRun:
|
||||
|
||||
async def test_failed_run_returns_error(self) -> None:
|
||||
"""Failed run returns EvalResults with error message."""
|
||||
from agent_framework_foundry._foundry_evals import _poll_eval_run
|
||||
|
||||
mock_client = MagicMock()
|
||||
mock_failed = MagicMock()
|
||||
@@ -2658,6 +2410,7 @@ class TestPollEvalRun:
|
||||
|
||||
async def test_canceled_run_returns_canceled_status(self) -> None:
|
||||
"""Canceled run returns EvalResults with status='canceled'."""
|
||||
from agent_framework_foundry._foundry_evals import _poll_eval_run
|
||||
|
||||
mock_client = MagicMock()
|
||||
mock_canceled = MagicMock()
|
||||
@@ -2682,6 +2435,7 @@ class TestPollEvalRun:
|
||||
class TestEvaluateTraces:
|
||||
async def test_raises_without_required_args(self) -> None:
|
||||
"""Raises ValueError when no response_ids, trace_ids, or agent_id given."""
|
||||
from agent_framework_foundry._foundry_evals import evaluate_traces
|
||||
|
||||
mock_client = MagicMock()
|
||||
with pytest.raises(ValueError, match="Provide at least one of"):
|
||||
@@ -2692,6 +2446,7 @@ class TestEvaluateTraces:
|
||||
|
||||
async def test_response_ids_path(self) -> None:
|
||||
"""evaluate_traces with response_ids uses the responses API path."""
|
||||
from agent_framework_foundry._foundry_evals import evaluate_traces
|
||||
|
||||
mock_client = MagicMock()
|
||||
|
||||
@@ -2739,6 +2494,7 @@ class TestEvaluateTraces:
|
||||
|
||||
async def test_trace_ids_path(self) -> None:
|
||||
"""evaluate_traces with trace_ids builds azure_ai_traces data source."""
|
||||
from agent_framework_foundry._foundry_evals import evaluate_traces
|
||||
|
||||
mock_client = MagicMock()
|
||||
|
||||
@@ -2778,6 +2534,7 @@ class TestEvaluateTraces:
|
||||
class TestEvaluateFoundryTarget:
|
||||
async def test_happy_path(self) -> None:
|
||||
"""evaluate_foundry_target creates eval + run and polls to completion."""
|
||||
from agent_framework_foundry._foundry_evals import evaluate_foundry_target
|
||||
|
||||
mock_client = MagicMock()
|
||||
|
||||
@@ -2913,11 +2670,13 @@ class TestEvaluatorSetConsistency:
|
||||
"""Verify that _AGENT_EVALUATORS and _TOOL_EVALUATORS are subsets of _BUILTIN_EVALUATORS."""
|
||||
|
||||
def test_agent_evaluators_subset(self):
|
||||
from agent_framework_foundry._foundry_evals import _AGENT_EVALUATORS, _BUILTIN_EVALUATORS
|
||||
|
||||
diff = _AGENT_EVALUATORS - set(_BUILTIN_EVALUATORS.values())
|
||||
assert not diff, f"_AGENT_EVALUATORS has names not in _BUILTIN_EVALUATORS: {diff}"
|
||||
|
||||
def test_tool_evaluators_subset(self):
|
||||
from agent_framework_foundry._foundry_evals import _BUILTIN_EVALUATORS, _TOOL_EVALUATORS
|
||||
|
||||
diff = _TOOL_EVALUATORS - set(_BUILTIN_EVALUATORS.values())
|
||||
assert not diff, f"_TOOL_EVALUATORS has names not in _BUILTIN_EVALUATORS: {diff}"
|
||||
@@ -2931,6 +2690,7 @@ class TestEvaluatorSetConsistency:
|
||||
class TestEvaluateTracesAgentId:
|
||||
async def test_agent_id_only_path(self) -> None:
|
||||
"""evaluate_traces with agent_id only builds azure_ai_traces data source."""
|
||||
from agent_framework_foundry._foundry_evals import evaluate_traces
|
||||
|
||||
mock_client = MagicMock()
|
||||
|
||||
@@ -2988,6 +2748,7 @@ class TestFilterToolEvaluatorsRaises:
|
||||
class TestEvaluateFoundryTargetValidation:
|
||||
async def test_target_without_type_raises(self) -> None:
|
||||
"""target dict without 'type' key raises ValueError."""
|
||||
from agent_framework_foundry._foundry_evals import evaluate_foundry_target
|
||||
|
||||
mock_client = MagicMock()
|
||||
with pytest.raises(ValueError, match="'type' key"):
|
||||
|
||||
@@ -9,7 +9,7 @@ import logging
|
||||
import os
|
||||
import tempfile
|
||||
import threading
|
||||
from collections.abc import AsyncIterable, AsyncIterator, Generator, Mapping, Sequence
|
||||
from collections.abc import AsyncIterable, AsyncIterator, Generator, Sequence
|
||||
from contextlib import AbstractAsyncContextManager, AsyncExitStack, suppress
|
||||
from dataclasses import asdict, is_dataclass
|
||||
from pathlib import Path
|
||||
@@ -472,12 +472,14 @@ class ResponsesHostServer(ResponsesAgentServerHost):
|
||||
# Run the agent in non-streaming mode
|
||||
response = await self._agent.run(stream=False, **run_kwargs) # type: ignore[reportUnknownMemberType]
|
||||
|
||||
async for item in _to_outputs_for_messages(
|
||||
response_event_stream,
|
||||
response.messages,
|
||||
approval_storage=self._approval_storage,
|
||||
):
|
||||
yield item
|
||||
for message in response.messages:
|
||||
for content in message.contents:
|
||||
async for item in _to_outputs(
|
||||
response_event_stream,
|
||||
content,
|
||||
approval_storage=self._approval_storage,
|
||||
):
|
||||
yield item
|
||||
yield response_event_stream.emit_completed()
|
||||
else:
|
||||
if tracker is None: # pragma: no cover - defensive, set above
|
||||
@@ -618,8 +620,10 @@ class ResponsesHostServer(ResponsesAgentServerHost):
|
||||
checkpoint_storage=write_storage,
|
||||
)
|
||||
|
||||
async for item in _to_outputs_for_messages(response_event_stream, response.messages):
|
||||
yield item
|
||||
for message in response.messages:
|
||||
for content in message.contents:
|
||||
async for item in _to_outputs(response_event_stream, content):
|
||||
yield item
|
||||
|
||||
await self._delete_not_latest_checkpoints(write_storage, self._agent.workflow.name)
|
||||
yield response_event_stream.emit_completed()
|
||||
@@ -725,7 +729,7 @@ class _OutputItemTracker:
|
||||
yield self._fc_builder.emit_arguments_delta(args_str)
|
||||
|
||||
elif content.type == "mcp_server_tool_call" and content.tool_name:
|
||||
key = content.call_id or f"{content.server_name or 'default'}::{content.tool_name}"
|
||||
key = f"{content.server_name or 'default'}::{content.tool_name}"
|
||||
if self._active_type != "mcp_server_tool_call" or self._active_id != key:
|
||||
yield from self._close()
|
||||
yield from self._open_mcp_call(content)
|
||||
@@ -734,24 +738,6 @@ class _OutputItemTracker:
|
||||
if self._mcp_builder is not None:
|
||||
yield self._mcp_builder.emit_arguments_delta(args_str)
|
||||
|
||||
elif (
|
||||
content.type == "mcp_server_tool_result"
|
||||
and self._active_type == "mcp_server_tool_call"
|
||||
and self._mcp_builder is not None
|
||||
and content.call_id is not None
|
||||
and content.call_id == self._mcp_builder.item_id
|
||||
):
|
||||
accumulated = "".join(self._accumulated)
|
||||
yield self._mcp_builder.emit_arguments_done(accumulated)
|
||||
yield self._mcp_builder.emit_completed()
|
||||
yield self._mcp_builder.emit_done(output=_stringify_mcp_output(content.output))
|
||||
self._mcp_builder = None
|
||||
self._active_type = None
|
||||
self._active_id = None
|
||||
self._accumulated.clear()
|
||||
self.needs_async = False
|
||||
return
|
||||
|
||||
else:
|
||||
yield from self._close()
|
||||
self.needs_async = True
|
||||
@@ -791,10 +777,9 @@ class _OutputItemTracker:
|
||||
self._mcp_builder = self._stream.add_output_item_mcp_call(
|
||||
server_label=content.server_name or "default",
|
||||
name=content.tool_name or "",
|
||||
item_id=content.call_id,
|
||||
)
|
||||
self._active_type = "mcp_server_tool_call"
|
||||
self._active_id = content.call_id or f"{content.server_name or 'default'}::{content.tool_name}"
|
||||
self._active_id = f"{content.server_name or 'default'}::{content.tool_name}"
|
||||
yield self._mcp_builder.emit_added()
|
||||
|
||||
def _close(self) -> Generator[ResponseStreamEvent]:
|
||||
@@ -942,19 +927,16 @@ async def _item_to_message(item: Item, *, approval_storage: ApprovalStorage | No
|
||||
|
||||
if item.type == "mcp_call":
|
||||
mcp = cast(ItemMcpToolCall, item)
|
||||
contents = [
|
||||
Content.from_mcp_server_tool_call(
|
||||
mcp.id,
|
||||
mcp.name,
|
||||
server_name=mcp.server_label,
|
||||
arguments=mcp.arguments,
|
||||
)
|
||||
]
|
||||
if getattr(mcp, "output", None) is not None:
|
||||
contents.append(Content.from_mcp_server_tool_result(call_id=mcp.id, output=mcp.output))
|
||||
return Message(
|
||||
role="assistant",
|
||||
contents=contents,
|
||||
contents=[
|
||||
Content.from_mcp_server_tool_call(
|
||||
mcp.id,
|
||||
mcp.name,
|
||||
server_name=mcp.server_label,
|
||||
arguments=mcp.arguments,
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
if item.type == "mcp_approval_request":
|
||||
@@ -1215,19 +1197,16 @@ async def _output_item_to_message(item: OutputItem, *, approval_storage: Approva
|
||||
|
||||
if item.type == "mcp_call":
|
||||
mcp = cast(OutputItemMcpToolCall, item)
|
||||
contents = [
|
||||
Content.from_mcp_server_tool_call(
|
||||
mcp.id,
|
||||
mcp.name,
|
||||
server_name=mcp.server_label,
|
||||
arguments=mcp.arguments,
|
||||
)
|
||||
]
|
||||
if getattr(mcp, "output", None) is not None:
|
||||
contents.append(Content.from_mcp_server_tool_result(call_id=mcp.id, output=mcp.output))
|
||||
return Message(
|
||||
role="assistant",
|
||||
contents=contents,
|
||||
contents=[
|
||||
Content.from_mcp_server_tool_call(
|
||||
mcp.id,
|
||||
mcp.name,
|
||||
server_name=mcp.server_label,
|
||||
arguments=mcp.arguments,
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
if item.type == "mcp_approval_request":
|
||||
@@ -1604,7 +1583,6 @@ async def _to_outputs(
|
||||
mcp_call = stream.add_output_item_mcp_call(
|
||||
server_label=content.server_name or "default",
|
||||
name=content.tool_name or "",
|
||||
item_id=content.call_id,
|
||||
)
|
||||
yield mcp_call.emit_added()
|
||||
async for event in mcp_call.aarguments(_arguments_to_str(content.arguments)):
|
||||
@@ -1679,91 +1657,4 @@ async def _to_outputs(
|
||||
logger.warning(f"Content type '{content.type}' is not supported yet. This is usually safe to ignore.")
|
||||
|
||||
|
||||
def _stringify_mcp_output(output: Any) -> str:
|
||||
"""Convert hosted MCP output payloads into the string shape expected by mcp_call.output."""
|
||||
if output is None:
|
||||
return ""
|
||||
if isinstance(output, str):
|
||||
return output
|
||||
if isinstance(output, Mapping):
|
||||
text = cast(Any, output).get("text")
|
||||
if isinstance(text, str):
|
||||
return text
|
||||
return json.dumps(output, default=str)
|
||||
if isinstance(output, Sequence) and not isinstance(output, (str, bytes, bytearray)):
|
||||
parts: list[str] = []
|
||||
entries = cast(Sequence[object], output)
|
||||
for entry in entries:
|
||||
if isinstance(entry, Content) and entry.type == "text":
|
||||
parts.append(entry.text or "")
|
||||
continue
|
||||
parts.append(_stringify_mcp_output(entry))
|
||||
return "".join(parts)
|
||||
return str(output)
|
||||
|
||||
|
||||
def _emit_completed_mcp_call(
|
||||
stream: ResponseEventStream,
|
||||
call_content: Content,
|
||||
*,
|
||||
arguments: str,
|
||||
output: str,
|
||||
) -> Generator[ResponseStreamEvent]:
|
||||
"""Emit a single completed MCP call item carrying both arguments and output."""
|
||||
mcp_call = stream.add_output_item_mcp_call(
|
||||
server_label=call_content.server_name or "default",
|
||||
name=call_content.tool_name or "",
|
||||
item_id=call_content.call_id,
|
||||
)
|
||||
yield mcp_call.emit_added()
|
||||
yield mcp_call.emit_arguments_done(arguments)
|
||||
yield mcp_call.emit_completed()
|
||||
yield mcp_call.emit_done(output=output)
|
||||
|
||||
|
||||
async def _to_outputs_for_messages(
|
||||
stream: ResponseEventStream,
|
||||
messages: Sequence[Message],
|
||||
*,
|
||||
approval_storage: ApprovalStorage | None = None,
|
||||
) -> AsyncIterator[ResponseStreamEvent]:
|
||||
"""Convert messages to output events with hosted-MCP call/result coalescing.
|
||||
|
||||
Parse once in message/content order and emit either:
|
||||
- a single canonical completed ``mcp_call`` when adjacent hosted MCP
|
||||
call/result content are encountered, or
|
||||
- standard output items for all other content types.
|
||||
"""
|
||||
pending_mcp_call: Content | None = None
|
||||
|
||||
for message in messages:
|
||||
for content in message.contents:
|
||||
if pending_mcp_call is not None:
|
||||
if content.type == "mcp_server_tool_result" and content.call_id == pending_mcp_call.call_id:
|
||||
for event in _emit_completed_mcp_call(
|
||||
stream,
|
||||
pending_mcp_call,
|
||||
arguments=_arguments_to_str(pending_mcp_call.arguments),
|
||||
output=_stringify_mcp_output(content.output),
|
||||
):
|
||||
yield event
|
||||
pending_mcp_call = None
|
||||
continue
|
||||
|
||||
async for event in _to_outputs(stream, pending_mcp_call, approval_storage=approval_storage):
|
||||
yield event
|
||||
pending_mcp_call = None
|
||||
|
||||
if content.type == "mcp_server_tool_call" and content.call_id:
|
||||
pending_mcp_call = content
|
||||
continue
|
||||
|
||||
async for event in _to_outputs(stream, content, approval_storage=approval_storage):
|
||||
yield event
|
||||
|
||||
if pending_mcp_call is not None:
|
||||
async for event in _to_outputs(stream, pending_mcp_call, approval_storage=approval_storage):
|
||||
yield event
|
||||
|
||||
|
||||
# endregion
|
||||
|
||||
@@ -25,7 +25,7 @@ classifiers = [
|
||||
dependencies = [
|
||||
"agent-framework-core>=1.7.0,<2",
|
||||
"azure-ai-agentserver-core>=2.0.0b3,<3",
|
||||
"azure-ai-agentserver-responses>=1.0.0b7,<2",
|
||||
"azure-ai-agentserver-responses>=1.0.0b5,<2",
|
||||
"azure-ai-agentserver-invocations>=1.0.0b3,<2",
|
||||
]
|
||||
|
||||
|
||||
@@ -260,50 +260,6 @@ class TestNonStreaming:
|
||||
assert "function_call_output" in types
|
||||
assert "message" in types
|
||||
|
||||
async def test_hosted_mcp_call_and_result_persist_as_single_mcp_call(self) -> None:
|
||||
agent = _make_agent(
|
||||
response=AgentResponse(
|
||||
messages=[
|
||||
Message(
|
||||
role="assistant",
|
||||
contents=[
|
||||
Content.from_mcp_server_tool_call(
|
||||
call_id="mcp_abc123",
|
||||
tool_name="search",
|
||||
server_name="api_specs",
|
||||
arguments='{"q": "cats"}',
|
||||
)
|
||||
],
|
||||
),
|
||||
Message(
|
||||
role="tool",
|
||||
contents=[
|
||||
Content.from_mcp_server_tool_result(
|
||||
call_id="mcp_abc123",
|
||||
output=[Content.from_text(text="found 10 cats")],
|
||||
)
|
||||
],
|
||||
),
|
||||
Message(role="assistant", contents=[Content.from_text("I found 10 cats!")]),
|
||||
]
|
||||
)
|
||||
)
|
||||
server = _make_server(agent)
|
||||
resp = await _post(server, stream=False)
|
||||
|
||||
assert resp.status_code == 200
|
||||
body = resp.json()
|
||||
assert body["status"] == "completed"
|
||||
|
||||
types = [item["type"] for item in body["output"]]
|
||||
assert "mcp_call" in types
|
||||
assert "custom_tool_call_output" not in types
|
||||
|
||||
mcp_items = [item for item in body["output"] if item["type"] == "mcp_call"]
|
||||
assert len(mcp_items) == 1
|
||||
assert mcp_items[0]["id"] == "mcp_abc123"
|
||||
assert mcp_items[0]["output"] == "found 10 cats"
|
||||
|
||||
async def test_reasoning_content(self) -> None:
|
||||
agent = _make_agent(
|
||||
response=AgentResponse(
|
||||
@@ -661,53 +617,6 @@ class TestStreaming:
|
||||
assert "response.output_item.added" in types
|
||||
assert "response.output_item.done" in types
|
||||
|
||||
async def test_mcp_tool_call_and_result_streaming_emit_single_completed_mcp_call(self) -> None:
|
||||
agent = _make_agent(
|
||||
stream_updates=[
|
||||
AgentResponseUpdate(
|
||||
contents=[
|
||||
Content.from_mcp_server_tool_call(
|
||||
call_id="mcp_abc123",
|
||||
tool_name="search",
|
||||
server_name="api_specs",
|
||||
arguments='{"q":',
|
||||
)
|
||||
],
|
||||
role="assistant",
|
||||
),
|
||||
AgentResponseUpdate(
|
||||
contents=[
|
||||
Content.from_mcp_server_tool_call(
|
||||
call_id="mcp_abc123",
|
||||
tool_name="search",
|
||||
server_name="api_specs",
|
||||
arguments=' "cats"}',
|
||||
)
|
||||
],
|
||||
role="assistant",
|
||||
),
|
||||
AgentResponseUpdate(
|
||||
contents=[
|
||||
Content.from_mcp_server_tool_result(
|
||||
call_id="mcp_abc123",
|
||||
output=[Content.from_text(text="found 10 cats")],
|
||||
)
|
||||
],
|
||||
role="tool",
|
||||
),
|
||||
]
|
||||
)
|
||||
server = _make_server(agent)
|
||||
resp = await _post(server, stream=True)
|
||||
|
||||
assert resp.status_code == 200
|
||||
events = _parse_sse_events(resp.text)
|
||||
done_events = [e for e in events if e["event"] == "response.output_item.done"]
|
||||
assert len(done_events) == 1
|
||||
assert done_events[0]["data"]["item"]["type"] == "mcp_call"
|
||||
assert done_events[0]["data"]["item"]["id"] == "mcp_abc123"
|
||||
assert done_events[0]["data"]["item"]["output"] == "found 10 cats"
|
||||
|
||||
|
||||
# endregion
|
||||
|
||||
@@ -811,24 +720,6 @@ class TestOutputItemToMessage:
|
||||
assert msg.contents[0].server_name == "my_server"
|
||||
assert msg.contents[0].tool_name == "search"
|
||||
|
||||
async def test_mcp_call_with_output_reconstructs_mcp_result_content(self) -> None:
|
||||
from azure.ai.agentserver.responses.models import OutputItemMcpToolCall
|
||||
|
||||
item = OutputItemMcpToolCall({
|
||||
"type": "mcp_call",
|
||||
"id": "mcp-1",
|
||||
"server_label": "my_server",
|
||||
"name": "search",
|
||||
"arguments": '{"q": "test"}',
|
||||
"output": "found 10 cats",
|
||||
})
|
||||
msg = await _output_item_to_message(item)
|
||||
assert msg.role == "assistant"
|
||||
assert len(msg.contents) == 2
|
||||
assert msg.contents[0].type == "mcp_server_tool_call"
|
||||
assert msg.contents[1].type == "mcp_server_tool_result"
|
||||
assert msg.contents[1].output == "found 10 cats"
|
||||
|
||||
async def test_mcp_approval_request(self) -> None:
|
||||
from azure.ai.agentserver.responses.models import OutputItemMcpApprovalRequest
|
||||
|
||||
@@ -1298,25 +1189,6 @@ class TestItemToMessage:
|
||||
assert msg.contents[0].server_name == "my_server"
|
||||
assert msg.contents[0].tool_name == "search"
|
||||
|
||||
async def test_mcp_call_with_output_reconstructs_mcp_result_content(self) -> None:
|
||||
from azure.ai.agentserver.responses.models import ItemMcpToolCall
|
||||
|
||||
item = ItemMcpToolCall({
|
||||
"type": "mcp_call",
|
||||
"id": "mcp-1",
|
||||
"server_label": "my_server",
|
||||
"name": "search",
|
||||
"arguments": '{"q": "test"}',
|
||||
"output": "found 10 cats",
|
||||
})
|
||||
msg = await _item_to_message(item)
|
||||
assert msg is not None
|
||||
assert msg.role == "assistant"
|
||||
assert len(msg.contents) == 2
|
||||
assert msg.contents[0].type == "mcp_server_tool_call"
|
||||
assert msg.contents[1].type == "mcp_server_tool_result"
|
||||
assert msg.contents[1].output == "found 10 cats"
|
||||
|
||||
async def test_mcp_approval_request(self) -> None:
|
||||
from azure.ai.agentserver.responses.models import ItemMcpApprovalRequest
|
||||
|
||||
@@ -2065,75 +1937,6 @@ class TestMultiTurnMixedContent:
|
||||
assert len(fc_contents) >= 1
|
||||
assert fc_contents[0].name == "search"
|
||||
|
||||
async def test_hosted_mcp_call_round_trip_does_not_orphan_function_call_output(self) -> None:
|
||||
"""Turn 1 produces hosted MCP call + result, turn 2 must replay both without orphaning output."""
|
||||
agent = _make_multi_response_agent([
|
||||
AgentResponse(
|
||||
messages=[
|
||||
Message(
|
||||
role="assistant",
|
||||
contents=[
|
||||
Content.from_mcp_server_tool_call(
|
||||
call_id="mcp_abc123",
|
||||
tool_name="search",
|
||||
server_name="api_specs",
|
||||
arguments='{"q": "cats"}',
|
||||
)
|
||||
],
|
||||
),
|
||||
Message(
|
||||
role="tool",
|
||||
contents=[
|
||||
Content.from_mcp_server_tool_result(
|
||||
call_id="mcp_abc123",
|
||||
output=[Content.from_text(text="found 10 cats")],
|
||||
)
|
||||
],
|
||||
),
|
||||
Message(role="assistant", contents=[Content.from_text("I found 10 cats!")]),
|
||||
]
|
||||
),
|
||||
AgentResponse(messages=[Message(role="assistant", contents=[Content.from_text("Here are more details")])]),
|
||||
])
|
||||
server = _make_server(agent)
|
||||
|
||||
resp1 = await _post(server, input_text="Search for cats", stream=False)
|
||||
assert resp1.status_code == 200
|
||||
response_id = resp1.json()["id"]
|
||||
|
||||
types1 = [item["type"] for item in resp1.json()["output"]]
|
||||
assert "mcp_call" in types1
|
||||
assert "custom_tool_call_output" not in types1
|
||||
|
||||
resp2 = await _post_json(
|
||||
server,
|
||||
{
|
||||
"model": "test-model",
|
||||
"input": "Tell me more",
|
||||
"stream": False,
|
||||
"previous_response_id": response_id,
|
||||
},
|
||||
)
|
||||
assert resp2.status_code == 200
|
||||
assert resp2.json()["status"] == "completed"
|
||||
|
||||
second_call_messages = agent.run.call_args_list[1].kwargs["messages"]
|
||||
mcp_call_contents = [
|
||||
c for m in second_call_messages for c in m.contents if c.type == "mcp_server_tool_call"
|
||||
]
|
||||
mcp_result_contents = [
|
||||
c for m in second_call_messages for c in m.contents if c.type == "mcp_server_tool_result"
|
||||
]
|
||||
function_result_contents = [
|
||||
c for m in second_call_messages for c in m.contents if c.type == "function_result"
|
||||
]
|
||||
|
||||
assert len(mcp_call_contents) >= 1
|
||||
assert len(mcp_result_contents) >= 1
|
||||
assert all((c.call_id or "") != "mcp_abc123" for c in function_result_contents)
|
||||
assert any((c.call_id or "") == "mcp_abc123" for c in mcp_call_contents)
|
||||
assert any((c.call_id or "") == "mcp_abc123" for c in mcp_result_contents)
|
||||
|
||||
async def test_multi_turn_reasoning_in_history(self) -> None:
|
||||
"""Turn 1 produces reasoning + text, turn 2 sees them in history."""
|
||||
agent = _make_multi_response_agent([
|
||||
|
||||
@@ -13,12 +13,9 @@ The Model Context Protocol (MCP) is an open standard for connecting AI agents to
|
||||
| **Agent as MCP Server** | [`agent_as_mcp_server.py`](agent_as_mcp_server.py) | Shows how to expose an Agent Framework agent as an MCP server that other AI applications can connect to |
|
||||
| **API Key Authentication** | [`mcp_api_key_auth.py`](mcp_api_key_auth.py) | Demonstrates API key authentication with MCP servers using `header_provider`, runtime invocation kwargs, and a command-line API key argument |
|
||||
| **GitHub Integration with PAT** | [`mcp_github_pat.py`](mcp_github_pat.py) | Demonstrates connecting to GitHub's MCP server using Personal Access Token (PAT) authentication |
|
||||
| **Long-Running Task** | [`mcp_long_running_task.py`](mcp_long_running_task.py) | Demonstrates transparent SEP-2663 long-running task handling for MCP tools that advertise `taskSupport=required`. Self-spawns a stdio MCP child server |
|
||||
|
||||
## Prerequisites
|
||||
|
||||
Most samples in this folder use OpenAI:
|
||||
|
||||
- `OPENAI_API_KEY` environment variable
|
||||
- `OPENAI_CHAT_MODEL` environment variable
|
||||
|
||||
@@ -26,8 +23,3 @@ Run `mcp_api_key_auth.py` with the MCP API key as the first command-line argumen
|
||||
|
||||
For `mcp_github_pat.py`:
|
||||
- `GITHUB_PAT` - Your GitHub Personal Access Token (create at https://github.com/settings/tokens)
|
||||
|
||||
For `mcp_long_running_task.py` (uses Azure OpenAI via Entra-ID):
|
||||
- Run `az login` once
|
||||
- `AZURE_OPENAI_ENDPOINT` - your Azure OpenAI resource endpoint, e.g. `https://<resource>.openai.azure.com/`
|
||||
- `AZURE_OPENAI_CHAT_MODEL` (or `AZURE_OPENAI_MODEL`) - the deployment name (e.g. `gpt-4o-mini`)
|
||||
|
||||
@@ -1,181 +0,0 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
"""
|
||||
MCP Long-Running Task (SEP-2663) Example
|
||||
|
||||
Demonstrates that ``MCPStdioTool`` transparently drives the MCP long-running
|
||||
task lifecycle for tools that advertise ``execution.taskSupport == "required"``.
|
||||
The agent observes a single function-call result; the framework handles the
|
||||
``tools/call`` → ``tasks/get`` (polled) → ``tasks/result`` sequence in the
|
||||
background.
|
||||
|
||||
Run it as a single file. The script doubles as both the client and the stdio
|
||||
MCP child server (the child branch is selected via ``--server``):
|
||||
|
||||
python mcp_long_running_task.py
|
||||
|
||||
Requirements:
|
||||
- Azure CLI sign-in (``az login``) — used for Entra-ID auth against Azure OpenAI.
|
||||
- ``AZURE_OPENAI_ENDPOINT`` — your Azure OpenAI resource endpoint, e.g.
|
||||
``https://<resource>.openai.azure.com/``.
|
||||
- ``AZURE_OPENAI_CHAT_MODEL`` (or ``AZURE_OPENAI_MODEL``) — the deployment name,
|
||||
e.g. ``gpt-4o-mini``.
|
||||
|
||||
This sample uses the lower-level ``mcp.server.lowlevel.Server`` so it can:
|
||||
1. Advertise a tool with ``execution=ToolExecution(taskSupport="required")``.
|
||||
2. Enable the SDK's experimental task support for the ``tasks/*`` lifecycle.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import sys
|
||||
from datetime import timedelta
|
||||
from typing import Any
|
||||
|
||||
from agent_framework import Agent, MCPStdioTool, MCPTaskOptions
|
||||
from agent_framework.openai import OpenAIChatClient
|
||||
from azure.identity import AzureCliCredential
|
||||
from dotenv import load_dotenv
|
||||
|
||||
load_dotenv()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# MCP stdio server (child-process branch)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def _run_server() -> None:
|
||||
"""Run a minimal stdio MCP server exposing one long-running tool."""
|
||||
import mcp.types as types
|
||||
from mcp.server.lowlevel import Server
|
||||
from mcp.server.stdio import stdio_server
|
||||
|
||||
server: Server[Any, Any] = Server("mcp-long-running-task-demo")
|
||||
# Auto-registers handlers for tasks/get, tasks/result, tasks/cancel, tasks/list
|
||||
# backed by an in-memory store.
|
||||
server.experimental.enable_tasks()
|
||||
|
||||
@server.list_tools()
|
||||
async def _list_tools() -> list[types.Tool]: # pyright: ignore[reportUnusedFunction]
|
||||
return [
|
||||
types.Tool(
|
||||
name="slow_summary",
|
||||
description=(
|
||||
"Produces a short summary of the supplied text after simulating several seconds of expensive work."
|
||||
),
|
||||
inputSchema={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"text": {
|
||||
"type": "string",
|
||||
"description": "Text to summarize.",
|
||||
}
|
||||
},
|
||||
"required": ["text"],
|
||||
},
|
||||
# Advertise that this tool MUST be invoked via the task lifecycle.
|
||||
execution=types.ToolExecution(taskSupport="required"),
|
||||
)
|
||||
]
|
||||
|
||||
@server.call_tool()
|
||||
async def _call_tool(name: str, arguments: dict[str, Any]) -> Any: # pyright: ignore[reportUnusedFunction]
|
||||
if name != "slow_summary":
|
||||
raise ValueError(f"Unknown tool: {name}")
|
||||
|
||||
ctx = server.request_context
|
||||
|
||||
async def _work(task: Any) -> types.CallToolResult:
|
||||
await task.update_status("Thinking...")
|
||||
await asyncio.sleep(15.0)
|
||||
text: str = (arguments.get("text") or "").strip()
|
||||
words = text.split()
|
||||
preview = " ".join(words[:6]) + ("..." if len(words) > 6 else "")
|
||||
summary = (
|
||||
f"Summarized {len(words)} word(s). First few words: '{preview}'."
|
||||
if words
|
||||
else "No input text was provided."
|
||||
)
|
||||
return types.CallToolResult(
|
||||
content=[types.TextContent(type="text", text=summary)],
|
||||
isError=False,
|
||||
)
|
||||
|
||||
if not ctx.experimental.is_task:
|
||||
# Client invoked the tool without task augmentation. Return a hard
|
||||
# error so a misconfigured client surfaces the problem clearly.
|
||||
return types.CallToolResult(
|
||||
content=[
|
||||
types.TextContent(
|
||||
type="text",
|
||||
text="'slow_summary' must be invoked as a task.",
|
||||
)
|
||||
],
|
||||
isError=True,
|
||||
)
|
||||
|
||||
return await ctx.experimental.run_task(_work)
|
||||
|
||||
async with stdio_server() as (read_stream, write_stream):
|
||||
await server.run(read_stream, write_stream, server.create_initialization_options())
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Agent client (default branch)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def _run_client() -> None:
|
||||
mcp_tool = MCPStdioTool(
|
||||
name="LongRunningDemo",
|
||||
description="Demo MCP server exposing a tool that advertises taskSupport=required.",
|
||||
command=sys.executable,
|
||||
args=[__file__, "--server"],
|
||||
# Optional: cap individual tasks at two minutes. The server may apply its
|
||||
# own default if this is omitted.
|
||||
task_options=MCPTaskOptions(default_ttl=timedelta(minutes=2)),
|
||||
)
|
||||
|
||||
async with Agent(
|
||||
client=OpenAIChatClient(credential=AzureCliCredential()),
|
||||
name="LROAgent",
|
||||
instructions=(
|
||||
"You are a helpful assistant. Use the slow_summary tool when the user "
|
||||
"asks for a summary. Wait for the result and present it directly."
|
||||
),
|
||||
tools=mcp_tool,
|
||||
) as agent:
|
||||
prompt = (
|
||||
"Please summarize the following text using your slow_summary tool: "
|
||||
"'The Model Context Protocol lets language models talk to external "
|
||||
"tools and resources through a small JSON-RPC surface.'"
|
||||
)
|
||||
|
||||
print("=== run() ===")
|
||||
print(f"User: {prompt}")
|
||||
response = await agent.run(prompt)
|
||||
print(f"Agent: {response.text}\n")
|
||||
|
||||
print("=== run(stream=True) ===")
|
||||
print(f"User: {prompt}")
|
||||
print("Agent: ", end="", flush=True)
|
||||
async for update in agent.run(prompt, stream=True):
|
||||
if update.text:
|
||||
print(update.text, end="", flush=True)
|
||||
print()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Entry point
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def main() -> None:
|
||||
if len(sys.argv) > 1 and sys.argv[1] == "--server":
|
||||
asyncio.run(_run_server())
|
||||
return
|
||||
asyncio.run(_run_client())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -27,7 +27,6 @@ This folder contains Azure AI Foundry and Foundry Local samples for Agent Framew
|
||||
| [`foundry_chat_client_with_local_mcp.py`](foundry_chat_client_with_local_mcp.py) | Foundry Chat Client with local MCP |
|
||||
| [`foundry_chat_client_with_session.py`](foundry_chat_client_with_session.py) | Foundry Chat Client with session management |
|
||||
| [`foundry_chat_client_with_toolbox.py`](foundry_chat_client_with_toolbox.py) | Foundry Chat Client connected to a toolbox via its MCP endpoint using `MCPStreamableHTTPTool` |
|
||||
| [`foundry_chat_client_with_toolbox_skills.py`](foundry_chat_client_with_toolbox_skills.py) | Foundry Chat Client that discovers MCP-based skills from a Foundry Toolbox endpoint via `MCPSkillsSource` (uses an Azure AD bearer token and the toolbox preview header) |
|
||||
|
||||
## FoundryLocalClient Samples
|
||||
|
||||
|
||||
@@ -1,87 +0,0 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
from collections.abc import Generator
|
||||
|
||||
import httpx
|
||||
from agent_framework import Agent, MCPSkillsSource, SkillsProvider
|
||||
from agent_framework.foundry import FoundryChatClient
|
||||
from azure.core.credentials import TokenCredential
|
||||
from azure.identity import AzureCliCredential, get_bearer_token_provider
|
||||
from dotenv import load_dotenv
|
||||
from mcp.client.session import ClientSession
|
||||
from mcp.client.streamable_http import streamable_http_client
|
||||
|
||||
# Load environment variables from .env file
|
||||
load_dotenv()
|
||||
|
||||
"""
|
||||
Foundry Chat Client with Toolbox-Hosted Skills
|
||||
|
||||
Discover Agent Skills served by a Microsoft Foundry Toolbox MCP endpoint
|
||||
and inject them into a ``FoundryChatClient`` agent via ``MCPSkillsSource``.
|
||||
The toolbox's discovery document (``skill://index.json``) is read once at
|
||||
startup; SKILL.md bodies are fetched on demand as the agent uses them.
|
||||
|
||||
Prerequisites:
|
||||
- A Microsoft Foundry project with a toolbox that exposes
|
||||
``skill://index.json`` with ``skill-md`` entries
|
||||
- FOUNDRY_PROJECT_ENDPOINT and FOUNDRY_MODEL environment variables set
|
||||
- FOUNDRY_TOOLBOX_MCP_SERVER_URL: the toolbox's MCP endpoint URL, e.g.
|
||||
``https://<account>.services.ai.azure.com/api/projects/<project>/toolboxes/<name>/mcp?api-version=v1``
|
||||
- Azure CLI authentication (``az login``)
|
||||
"""
|
||||
|
||||
|
||||
class _BearerAuth(httpx.Auth):
|
||||
"""Attach a fresh Foundry bearer token to every request."""
|
||||
|
||||
def __init__(self, credential: TokenCredential) -> None:
|
||||
self._get_token = get_bearer_token_provider(credential, "https://ai.azure.com/.default")
|
||||
|
||||
def auth_flow(self, request: httpx.Request) -> Generator[httpx.Request, httpx.Response, None]:
|
||||
request.headers["Authorization"] = f"Bearer {self._get_token()}"
|
||||
yield request
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
"""Example showing toolbox-hosted MCP skills for a Foundry Chat Client agent."""
|
||||
credential = AzureCliCredential()
|
||||
|
||||
# HTTP client that signs every request with a fresh Foundry bearer token
|
||||
# and advertises the toolbox preview feature flag, plus the MCP streamable
|
||||
# HTTP transport that uses it.
|
||||
async with (
|
||||
httpx.AsyncClient(
|
||||
auth=_BearerAuth(credential),
|
||||
headers={"Foundry-Features": "Toolboxes=V1Preview"},
|
||||
timeout=httpx.Timeout(30.0, read=300.0),
|
||||
follow_redirects=True,
|
||||
) as http_client,
|
||||
streamable_http_client(
|
||||
url=os.environ["FOUNDRY_TOOLBOX_MCP_SERVER_URL"],
|
||||
http_client=http_client,
|
||||
) as (read, write, _),
|
||||
ClientSession(read, write) as session,
|
||||
):
|
||||
await session.initialize()
|
||||
|
||||
# Discover skills served by the toolbox and inject them as a context provider.
|
||||
skills_provider = SkillsProvider(MCPSkillsSource(client=session))
|
||||
|
||||
async with Agent(
|
||||
client=FoundryChatClient(credential=credential),
|
||||
name="ToolboxMCPSkillsAgent",
|
||||
instructions="You are a helpful assistant. Use available skills to answer the user.",
|
||||
context_providers=[skills_provider],
|
||||
) as agent:
|
||||
query = input("User: ").strip() # noqa: ASYNC250
|
||||
if not query:
|
||||
return
|
||||
response = await agent.run(query)
|
||||
print(f"Assistant: {response.text}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -12,7 +12,6 @@ Start with file-based or code-defined skills, then explore combining them and ad
|
||||
| [**code_defined_skill**](code_defined_skill/) | Define skills entirely in Python code using `Skill`, `@skill.resource`, and `@skill.script` decorators. Uses a code-defined unit-converter skill. |
|
||||
| [**class_based_skill**](class_based_skill/) | Define skills as Python classes using `ClassSkill` with `@ClassSkill.resource` and `@ClassSkill.script` decorators for auto-discovery. Uses a class-based unit-converter skill. |
|
||||
| [**mixed_skills**](mixed_skills/) | Combine code-defined, class-based, and file-based skills in a single agent. Uses a code-defined volume-converter, a class-based temperature-converter, and a file-based unit-converter. |
|
||||
| [**mcp_based_skill**](mcp_based_skill/) | Discover skills served over the [Model Context Protocol (MCP)](https://modelcontextprotocol.io) via `MCPSkillsSource`. Connects to a remote MCP server that exposes skills as `skill://...` resources following the SEP-2640 convention. |
|
||||
| [**script_approval**](script_approval/) | Require human-in-the-loop approval before executing skill scripts |
|
||||
|
||||
## Key Concepts
|
||||
|
||||
@@ -1,51 +0,0 @@
|
||||
# MCP-Based Agent Skills Sample
|
||||
|
||||
This sample demonstrates how to discover **Agent Skills served over MCP** with an `Agent`.
|
||||
|
||||
## What it demonstrates
|
||||
|
||||
- Connecting to a remote MCP server (over streamable HTTP) that exposes skill
|
||||
resources following the SEP-2640 convention.
|
||||
- Building a `SkillsProvider` from an `MCPSkillsSource`, which reads
|
||||
`skill://index.json` (SEP-2640 canonical discovery) and constructs skills from
|
||||
the index entries.
|
||||
- The progressive disclosure pattern across MCP: advertise → load → read
|
||||
resources, exactly as for filesystem-backed skills.
|
||||
|
||||
## Running the Sample
|
||||
|
||||
### Prerequisites
|
||||
|
||||
- Python 3.10+
|
||||
- An [Azure AI Foundry](https://ai.azure.com/) project with a deployed model
|
||||
- Azure CLI authentication (`az login`)
|
||||
- A running MCP server that hosts SEP-2640 skill resources (see "Providing
|
||||
an MCP server" below)
|
||||
|
||||
### Setup
|
||||
|
||||
Set the following environment variables (in a `.env` file or your shell):
|
||||
|
||||
```powershell
|
||||
$env:FOUNDRY_PROJECT_ENDPOINT="https://your-endpoint.services.ai.azure.com/api/projects/your-project"
|
||||
$env:FOUNDRY_MODEL="gpt-4o-mini"
|
||||
$env:MCP_SKILLS_SERVER_URL="https://your-mcp-server.example.com/mcp"
|
||||
```
|
||||
|
||||
### Run
|
||||
|
||||
```powershell
|
||||
python mcp_based_skill.py
|
||||
```
|
||||
|
||||
## Providing an MCP server
|
||||
|
||||
This sample is a **consumer**: it does not host an MCP server itself. To try
|
||||
it end-to-end you need an MCP server that exposes the SEP-2640 skill
|
||||
resources (`skill://index.json` plus per-skill `SKILL.md`).
|
||||
|
||||
- See [`samples/02-agents/mcp/agent_as_mcp_server.py`](../../mcp/agent_as_mcp_server.py)
|
||||
for an example of hosting an MCP server via the Agent Framework.
|
||||
- The Model Context Protocol working group maintains reference MCP-skills
|
||||
servers at
|
||||
[`modelcontextprotocol/experimental-ext-skills`](https://github.com/modelcontextprotocol/experimental-ext-skills).
|
||||
@@ -1,75 +0,0 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
|
||||
# Uncomment this filter to suppress the experimental Skills warning before
|
||||
# using the sample's Skills APIs.
|
||||
# import warnings
|
||||
# warnings.filterwarnings("ignore", message=r"\[SKILLS\].*", category=FutureWarning)
|
||||
from agent_framework import Agent, MCPSkillsSource, SkillsProvider
|
||||
from agent_framework.foundry import FoundryChatClient
|
||||
from azure.identity import AzureCliCredential
|
||||
from dotenv import load_dotenv
|
||||
from mcp.client.session import ClientSession
|
||||
from mcp.client.streamable_http import streamable_http_client
|
||||
|
||||
"""
|
||||
MCP-Based Agent Skills
|
||||
|
||||
This sample demonstrates how to discover Agent Skills served over the
|
||||
Model Context Protocol (MCP) using :class:`MCPSkillsSource`.
|
||||
|
||||
The sample connects to a remote MCP server that exposes skill resources
|
||||
under the ``skill://`` URI scheme:
|
||||
|
||||
* ``skill://index.json`` — discovery document listing all skills
|
||||
* ``skill://<skill-name>/SKILL.md`` — the skill instructions
|
||||
|
||||
To run, set ``MCP_SKILLS_SERVER_URL`` to the streamable HTTP endpoint of an
|
||||
MCP server that hosts the skill resources.
|
||||
"""
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
"""Connect to a remote MCP skills server and run the agent."""
|
||||
load_dotenv()
|
||||
|
||||
endpoint = os.environ["FOUNDRY_PROJECT_ENDPOINT"]
|
||||
deployment = os.environ.get("FOUNDRY_MODEL", "gpt-4o-mini")
|
||||
mcp_url = os.environ["MCP_SKILLS_SERVER_URL"]
|
||||
|
||||
print("Discovering MCP-based skills")
|
||||
print("-" * 60)
|
||||
|
||||
# 1. Connect to the MCP server over streamable HTTP.
|
||||
async with streamable_http_client(url=mcp_url) as (read, write, _), ClientSession(read, write) as session:
|
||||
await session.initialize()
|
||||
|
||||
# 2. Build a SkillsProvider that discovers skills over MCP.
|
||||
# MCPSkillsSource reads skill://index.json and creates one
|
||||
# MCPSkill per skill-md entry; SKILL.md bodies are fetched
|
||||
# on demand via resources/read.
|
||||
skills_provider = SkillsProvider(MCPSkillsSource(client=session))
|
||||
|
||||
# 3. Run the agent.
|
||||
client = FoundryChatClient(
|
||||
project_endpoint=endpoint,
|
||||
model=deployment,
|
||||
credential=AzureCliCredential(),
|
||||
)
|
||||
|
||||
async with Agent(
|
||||
client=client,
|
||||
instructions="You are a helpful assistant. Use available skills to answer the user.",
|
||||
context_providers=[skills_provider],
|
||||
) as agent:
|
||||
query = input("User: ").strip() # noqa: ASYNC250
|
||||
if not query:
|
||||
return
|
||||
response = await agent.run(query)
|
||||
print(f"Agent: {response}\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -1,75 +0,0 @@
|
||||
# Tools
|
||||
|
||||
Samples that show how to define, configure, and control function tools for an
|
||||
agent — from basic declarations to approvals, invocation limits, session
|
||||
injection, and dynamic (progressive) tool exposure.
|
||||
|
||||
## Function tools
|
||||
|
||||
| File | Demonstrates |
|
||||
|------|--------------|
|
||||
| [`function_tool_with_explicit_schema.py`](function_tool_with_explicit_schema.py) | Defining a tool with an explicit JSON schema. |
|
||||
| [`function_tool_declaration_only.py`](function_tool_declaration_only.py) | A declaration-only tool (schema without a local implementation). |
|
||||
| [`function_tool_with_kwargs.py`](function_tool_with_kwargs.py) | Passing extra keyword arguments into a tool. |
|
||||
| [`function_tool_from_dict_with_dependency_injection.py`](function_tool_from_dict_with_dependency_injection.py) | Dependency injection into a tool defined from a dict. |
|
||||
| [`function_tool_with_session_injection.py`](function_tool_with_session_injection.py) | Injecting the session into a tool. |
|
||||
| [`tool_in_class.py`](tool_in_class.py) | Using a method on a class as a tool. |
|
||||
| [`agent_as_tool_with_session_propagation.py`](agent_as_tool_with_session_propagation.py) | Exposing an agent as a tool with session propagation. |
|
||||
|
||||
## Approvals & invocation control
|
||||
|
||||
| File | Demonstrates |
|
||||
|------|--------------|
|
||||
| [`function_tool_with_approval.py`](function_tool_with_approval.py) | Requiring human approval before a tool runs. |
|
||||
| [`function_tool_with_approval_and_sessions.py`](function_tool_with_approval_and_sessions.py) | Tool approvals combined with sessions. |
|
||||
| [`function_invocation_configuration.py`](function_invocation_configuration.py) | Configuring function-invocation settings (e.g. max iterations). |
|
||||
| [`control_total_tool_executions.py`](control_total_tool_executions.py) | All the ways to cap how many times tools run. |
|
||||
| [`function_tool_with_max_invocations.py`](function_tool_with_max_invocations.py) | Limiting the number of invocations per tool. |
|
||||
| [`function_tool_with_max_exceptions.py`](function_tool_with_max_exceptions.py) | Limiting the number of exceptions a tool may raise. |
|
||||
| [`function_tool_recover_from_failures.py`](function_tool_recover_from_failures.py) | Returning errors so the agent can recover from tool failures. |
|
||||
|
||||
## Progressive tool exposure (dynamic loading)
|
||||
|
||||
| File | Demonstrates |
|
||||
|------|--------------|
|
||||
| [`dynamic_tool_exposure.py`](dynamic_tool_exposure.py) | A "loader" tool that adds more tools at runtime via `FunctionInvocationContext`. |
|
||||
|
||||
Frontloading a model with hundreds of tools hurts tool-selection accuracy,
|
||||
bloats context, and raises cost. Instead, start with a small set of loader
|
||||
tools and let the model pull in more on demand. Inside a tool, the injected
|
||||
`ctx: FunctionInvocationContext` exposes a live `ctx.tools` list plus
|
||||
`ctx.add_tools(...)` / `ctx.remove_tools(...)` helpers. Tools added or removed
|
||||
take effect on the **next iteration** of the function-calling loop.
|
||||
|
||||
> [!NOTE]
|
||||
> Progressive tool exposure applies to the standard function-calling loop. It
|
||||
> does **not** apply to CodeAct providers (`agent-framework-monty`,
|
||||
> `agent-framework-hyperlight`). In CodeAct the model only sees a single
|
||||
> `execute_code` tool, and host tools are exposed *inside the sandbox* as typed
|
||||
> Python functions rather than as model tool-schemas. Host tools there are
|
||||
> invoked without a `FunctionInvocationContext`, so `ctx.add_tools()` is not
|
||||
> available; the helpers fail fast with a clear `RuntimeError` instead of
|
||||
> silently doing nothing. To change a CodeAct agent's tool set, use the
|
||||
> provider's own `add_tools` / `remove_tool` / `clear_tools` methods (applied
|
||||
> between runs). The recommended provider-driven path for Monty and Hyperlight
|
||||
> is shown in [`../context_providers/code_act/`](../context_providers/code_act/)
|
||||
> ([`code_act.py`](../context_providers/code_act/code_act.py) for Hyperlight,
|
||||
> [`monty_code_act.py`](../context_providers/code_act/monty_code_act.py) for
|
||||
> Monty).
|
||||
|
||||
## Local shell & code interpreters
|
||||
|
||||
| Path | Demonstrates |
|
||||
|------|--------------|
|
||||
| [`local_shell_with_allowlist.py`](local_shell_with_allowlist.py) | `LocalShellTool` restricted by a strict command allow-list. |
|
||||
| [`local_shell_with_environment_provider.py`](local_shell_with_environment_provider.py) | `LocalShellTool` wired with a `ShellEnvironmentProvider`. |
|
||||
| [`local_code_interpreter/`](local_code_interpreter/) | Hyperlight-backed sandboxed code interpreter (standalone tool — *extra* pattern). |
|
||||
| [`monty_code_interpreter/`](monty_code_interpreter/) | Monty-backed sandboxed code interpreter (standalone tool — *extra* pattern). |
|
||||
|
||||
> [!TIP]
|
||||
> The `local_code_interpreter/` and `monty_code_interpreter/` samples show the
|
||||
> standalone-tool wiring and are provided as *extra* reference. For most
|
||||
> Monty/Hyperlight use cases the **recommended** path is the provider-driven
|
||||
> CodeAct setup in
|
||||
> [`../context_providers/code_act/`](../context_providers/code_act/), which adds
|
||||
> dynamic tool / capability management.
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user