Compare commits

..
Author SHA1 Message Date
Jacob Alber 8a1ad0199a fix: whitespace formatting 2026-05-07 16:44:17 +00:00
Jacob Alber cbc471dc18 fix sample project 2026-05-07 16:41:41 +00:00
Jacob Alber 6ddb74f8b3 fix: Update for breaking changes in Github.Copilot.SDK 2026-05-07 16:25:15 +00:00
2f93971a59 Fix formatting
Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>
2026-05-07 12:05:32 -04:00
Jacob Alber 93ccf52c39 feat: Update Github Copilot SDK to 1.0.0-beta.2 2026-05-07 11:53:19 -04:00
51 changed files with 287 additions and 4798 deletions
+1 -84
View File
@@ -273,8 +273,6 @@ jobs:
-c ${{ matrix.configuration }} `
--no-build -v Normal `
--report-xunit-trx `
--report-junit `
--results-directory ../IntegrationTestResults/ `
--ignore-exit-code 8 `
--filter-not-trait "Category=IntegrationDisabled" `
--filter-not-trait "Category=FoundryHostedAgents" `
@@ -296,10 +294,6 @@ jobs:
AZURE_AI_PROJECT_ENDPOINT: ${{ vars.AZURE_AI_PROJECT_ENDPOINT }}
AZURE_AI_MODEL_DEPLOYMENT_NAME: ${{ vars.AZURE_AI_MODEL_DEPLOYMENT_NAME }}
AZURE_AI_BING_CONNECTION_ID: ${{ vars.AZURE_AI_BING_CONNECTION_ID }}
# Anthropic Models
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
ANTHROPIC_CHAT_MODEL_NAME: ${{ vars.ANTHROPIC_CHAT_MODEL_NAME }}
ANTHROPIC_REASONING_MODEL_NAME: ${{ vars.ANTHROPIC_REASONING_MODEL_NAME }}
# Generate test reports and check coverage
- name: Generate test reports
@@ -322,14 +316,6 @@ jobs:
shell: pwsh
run: ./dotnet/eng/scripts/dotnet-check-coverage.ps1 -JsonReportPath "TestResults/Reports/Summary.json" -CoverageThreshold $env:COVERAGE_THRESHOLD
- name: Upload integration test results
if: always() && github.event_name != 'pull_request' && matrix.integration-tests
uses: actions/upload-artifact@v7
with:
name: dotnet-test-results-${{ matrix.targetFramework }}-${{ matrix.os }}
path: IntegrationTestResults/**/*.junit
if-no-files-found: ignore
# The Foundry hosted-agent IT is costly (it builds a container, pushes to ACR, and provisions
# live agents on a separate Foundry project). Running it in its own job keeps the overall
# workflow time roughly flat: it executes in parallel to dotnet-build and dotnet-test and is
@@ -393,14 +379,6 @@ jobs:
# We rebuild and push the test container image on every IT run so framework code changes
# are picked up; the image tag is content-hashed across the test container source AND its
# framework project references, so identical content is a no-op push.
#
# `-UsePrebuiltProjectReferences` opts into the no-rebuild fast path: publish skips
# rebuilding ProjectReferences and consumes the DLLs the prior "Build Foundry hosted IT
# (and its deps)" step already produced. This avoids MSB3026 ("file is being used by
# another process") collisions caused by the previous build's shared-compilation server
# still holding file handles to those DLLs. Safe in CI because the prebuild step ran in
# the same job against the same source. Do not remove the prebuild step (the subsequent
# `dotnet test --no-build` step depends on it too).
- name: Build and push Foundry Hosted Agents test container
id: build-foundry-hosted-image
shell: pwsh
@@ -410,7 +388,7 @@ jobs:
if ([string]::IsNullOrWhiteSpace($registry)) {
throw "IT_HOSTED_AGENT_REGISTRY not set in the integration environment."
}
& "${{ github.workspace }}/dotnet/tests/Foundry.Hosting.IntegrationTests/scripts/it-build-image.ps1" -Registry $registry -UsePrebuiltProjectReferences | Tee-Object -FilePath $env:GITHUB_ENV -Append
& "${{ github.workspace }}/dotnet/tests/Foundry.Hosting.IntegrationTests/scripts/it-build-image.ps1" -Registry $registry | Tee-Object -FilePath $env:GITHUB_ENV -Append
- name: Run Foundry Hosted Agents Integration Tests
shell: pwsh
@@ -470,64 +448,3 @@ jobs:
uses: actions/github-script@v8
with:
script: core.setFailed('Integration Tests Cancelled!')
# Integration test trend report (aggregates JUnit XML results from dotnet test jobs)
dotnet-integration-test-report:
name: Integration Test Report
if: >
always() &&
github.event_name != 'pull_request' &&
(contains(join(needs.*.result, ','), 'success') ||
contains(join(needs.*.result, ','), 'failure'))
needs: [dotnet-test]
runs-on: ubuntu-latest
defaults:
run:
working-directory: python
steps:
- uses: actions/checkout@v6
with:
persist-credentials: false
sparse-checkout: |
.github/actions/python-setup
python
- name: Set up python and install the project
uses: ./.github/actions/python-setup
with:
python-version: "3.13"
os: ${{ runner.os }}
- name: Download all test results from current run
uses: actions/download-artifact@v4
with:
pattern: dotnet-test-results-*
path: dotnet-test-results/
- name: Restore report history cache
uses: actions/cache/restore@v4
with:
path: python/dotnet-integration-report-history.json
key: dotnet-integration-report-history-${{ github.run_id }}
restore-keys: |
dotnet-integration-report-history-
- name: Generate trend report
run: >
uv run python scripts/integration_test_report/aggregate.py
../dotnet-test-results/
dotnet-integration-report-history.json
dotnet-integration-test-report.md
- name: Post to Job Summary
if: always()
run: cat dotnet-integration-test-report.md >> $GITHUB_STEP_SUMMARY
- name: Save report history cache
if: always()
uses: actions/cache/save@v4
with:
path: python/dotnet-integration-report-history.json
key: dotnet-integration-report-history-${{ github.run_id }}
- name: Upload trend report
if: always()
uses: actions/upload-artifact@v7
with:
name: dotnet-integration-test-report
path: |
python/dotnet-integration-test-report.md
python/dotnet-integration-report-history.json
-1
View File
@@ -33,4 +33,3 @@ Console.WriteLine(await agent.RunAsync("Write a haiku about Microsoft Agent Fram
- [Design Documents](../docs/design)
- [Architectural Decision Records](../docs/decisions)
- [MSFT Learn Docs](https://learn.microsoft.com/agent-framework/overview/agent-framework-overview)
@@ -50,12 +50,12 @@ Console.WriteLine(await agent.RunAsync("My name is Ruaidhrí", session));
Console.WriteLine(await agent.RunAsync("I am 20 years old", session));
// We can serialize the session. The serialized state will include the state of the memory component.
JsonElement sessionElement = await agent.SerializeSessionAsync(session);
JsonElement sesionElement = await agent.SerializeSessionAsync(session);
Console.WriteLine("\n>> Use deserialized session with previously created memories\n");
// Later we can deserialize the session and continue the conversation with the previous memory component state.
var deserializedSession = await agent.DeserializeSessionAsync(sessionElement);
var deserializedSession = await agent.DeserializeSessionAsync(sesionElement);
Console.WriteLine(await agent.RunAsync("What is my name and age?", deserializedSession));
Console.WriteLine("\n>> Read memories using memory component\n");
@@ -190,7 +190,7 @@ internal static class InputConverter
private static ChatMessage ConvertFunctionCallOutput(FunctionCallOutputItemParam funcOutput)
{
var output = DecodeFunctionResultPayload(funcOutput.Output);
var output = funcOutput.Output?.ToString() ?? string.Empty;
return new ChatMessage(
ChatRole.Tool,
[new FunctionResultContent(funcOutput.CallId, output)]);
@@ -482,54 +482,9 @@ internal static class InputConverter
private static ChatMessage ConvertFunctionToolCallOutput(OutputItemFunctionToolCallOutput funcOutput)
{
var output = DecodeFunctionResultPayload(funcOutput.Output);
return new ChatMessage(
ChatRole.Tool,
[new FunctionResultContent(funcOutput.CallId, output)]);
}
/// <summary>
/// Decodes the wire payload of a <c>function_call_output.output</c> field back into the
/// underlying tool-result text suitable for replay as <see cref="FunctionResultContent.Result"/>.
/// </summary>
/// <remarks>
/// Mirrors <c>OutputConverter.EncodeFunctionResultAsJsonStringPayload</c>. Per the OpenAI
/// Responses spec, <c>output</c> is a JSON string; we extract its underlying value. Legacy
/// producers that emitted raw JSON values (arrays/objects) are tolerated by passing the raw
/// bytes through unchanged.
/// </remarks>
private static string DecodeFunctionResultPayload(BinaryData? rawOutput)
{
if (rawOutput is null)
{
return string.Empty;
}
var raw = rawOutput.ToString();
if (string.IsNullOrEmpty(raw))
{
return string.Empty;
}
try
{
using var doc = JsonDocument.Parse(raw);
if (doc.RootElement.ValueKind == JsonValueKind.String)
{
return doc.RootElement.GetString() ?? string.Empty;
}
// Legacy/non-conforming producers may have emitted a raw JSON value
// (array/object/number/bool/null). Pass the raw text through as the
// payload so the replayed FunctionResultContent.Result preserves the
// original tool output shape.
return raw;
}
catch (JsonException)
{
// Not valid JSON — treat the bytes as a literal string payload.
return raw;
}
[new FunctionResultContent(funcOutput.CallId, funcOutput.Output)]);
}
private static ChatRole ConvertMessageRole(MessageRole role)
@@ -279,7 +279,12 @@ internal static class OutputConverter
accumulatedText = null;
previousMessageId = null;
var outputText = EncodeFunctionResultAsJsonStringPayload(functionResult.Result);
var outputText = functionResult.Result switch
{
null => string.Empty,
string s => s,
_ => JsonSerializer.Serialize(functionResult.Result),
};
var itemId = GenerateItemId("fc");
var outputItem = new OutputItemFunctionToolCallOutput(
@@ -443,44 +448,4 @@ internal static class OutputConverter
var body = Convert.ToHexString(bytes); // 50 hex chars, uppercase
return $"{prefix}_{body}";
}
/// <summary>
/// Encodes a <see cref="FunctionResultContent.Result"/> value into the wire payload for
/// the OpenAI Responses <c>function_call_output.output</c> field.
/// </summary>
/// <remarks>
/// The OpenAI Responses spec requires <c>output</c> to be a JSON string. The Responses
/// SDK's <see cref="OutputItemFunctionToolCallOutput"/> accepts a <see cref="BinaryData"/>
/// containing the *raw JSON value* for the field, so the returned text is always a JSON
/// string literal (quoted, with escapes). This avoids two bugs:
/// <list type="bullet">
/// <item>Complex results (e.g. <c>List&lt;TodoItem&gt;</c>) landing on the wire as an
/// unquoted JSON array, which the strict-parsing OpenAI .NET client
/// (<c>FunctionCallOutputResponseItem</c>) rejects with
/// "requires an element of type 'String', but the target element has type 'Array'".</item>
/// <item>Numeric- or JSON-shaped string results (e.g. <c>"42"</c> or <c>"{\"k\":1}"</c>)
/// silently changing type on the wire because <c>BinaryData</c> auto-detects JSON.</item>
/// </list>
/// <see cref="JsonElement"/> / <see cref="JsonDocument"/> values are unwrapped first so
/// a string-kind element does not get double-encoded into <c>"\"value\""</c>.
/// </remarks>
[UnconditionalSuppressMessage("Trimming", "IL2026", Justification = "Serializing function call result payload.")]
[UnconditionalSuppressMessage("AOT", "IL3050", Justification = "Serializing function call result payload.")]
private static string EncodeFunctionResultAsJsonStringPayload(object? result)
{
string innerText = result switch
{
null => string.Empty,
string s => s,
JsonElement je => je.ValueKind == JsonValueKind.String
? (je.GetString() ?? string.Empty)
: je.GetRawText(),
JsonDocument jd => jd.RootElement.ValueKind == JsonValueKind.String
? (jd.RootElement.GetString() ?? string.Empty)
: jd.RootElement.GetRawText(),
_ => JsonSerializer.Serialize(result),
};
return JsonSerializer.Serialize(innerText);
}
}
@@ -13,5 +13,5 @@ internal sealed class SequenceNumber
/// Gets the next sequence number.
/// </summary>
/// <returns>The next sequence number.</returns>
public int Increment() => System.Threading.Interlocked.Increment(ref this._sequenceNumber) - 1;
public int Increment() => this._sequenceNumber++;
}
@@ -1,7 +1,6 @@
// Copyright (c) Microsoft. All rights reserved.
using System.Text.Json.Serialization;
using Microsoft.Agents.AI.Workflows.Specialized.Magentic;
namespace Microsoft.Agents.AI.Workflows;
@@ -11,7 +10,6 @@ namespace Microsoft.Agents.AI.Workflows;
[JsonDerivedType(typeof(ExecutorInvokedEvent))]
[JsonDerivedType(typeof(ExecutorCompletedEvent))]
[JsonDerivedType(typeof(ExecutorFailedEvent))]
[JsonDerivedType(typeof(MagenticOrchestratorEvent))]
public class ExecutorEvent(string executorId, object? data) : WorkflowEvent(data)
{
/// <summary>
@@ -1,44 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
using System.Collections.Generic;
using System.Linq;
using Microsoft.Extensions.AI;
namespace Microsoft.Agents.AI.Workflows;
/// <summary>
/// Request for human review of a proposed plan.
/// </summary>
/// <param name="Plan">The proposed plan.</param>
/// <param name="CurrentProgress">The current progress ledger, if available. During the initial plan review,
/// this will be <see langword="null"/>. In subsequent reviews after replanning (due to stalls), this will
/// contain the latest progress ledger that determined that no progress has been made or the workflow was in
/// a loop.</param>
/// <param name="IsStalled">Whether the workflow is currently stalled.</param>
public record MagenticPlanReviewRequest(ChatMessage Plan, MagenticProgressLedger? CurrentProgress, bool IsStalled)
{
/// <summary>
/// Create an approving <see cref="MagenticPlanReviewResponse"/>.
/// </summary>
/// <returns></returns>
public MagenticPlanReviewResponse Approve() => new([]);
/// <summary>
/// Create a <see cref="MagenticPlanReviewResponse"/> with revisions.
/// </summary>
/// <returns></returns>
public MagenticPlanReviewResponse Revise(string message) => new([new(ChatRole.User, message)]);
/// <summary>
/// Create a <see cref="MagenticPlanReviewResponse"/> with revisions.
/// </summary>
/// <returns></returns>
public MagenticPlanReviewResponse Revise(ChatMessage message) => new([message]);
/// <summary>
/// Create a <see cref="MagenticPlanReviewResponse"/> with revisions.
/// </summary>
/// <returns></returns>
public MagenticPlanReviewResponse Revise(IEnumerable<ChatMessage> messages)
=> new(messages is List<ChatMessage> messageList ? messageList : messages.ToList());
}
@@ -1,18 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
using System.Collections.Generic;
using Microsoft.Extensions.AI;
namespace Microsoft.Agents.AI.Workflows;
/// <summary>
/// Review feedback for a proposed plan, including any revisions if the plan is not approved as-is. An
/// empty list of review messages indicates approval of the proposed plan without any revisions.
/// </summary>
/// <param name="Review">
/// Review feedback for a generated plan. Empty if the plan is approved as-is and changes are requested.
/// </param>
public record MagenticPlanReviewResponse(List<ChatMessage> Review)
{
internal bool IsApproved => this.Review.Count == 0;
}
@@ -1,269 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using System.Linq;
using System.Text;
using System.Text.Json;
using System.Text.Json.Serialization;
using System.Text.Json.Serialization.Metadata;
namespace Microsoft.Agents.AI.Workflows;
/// <summary>
/// Maintains a ledger of progress made by the Magentic workflow.
/// </summary>
public class MagenticProgressLedger
{
internal static readonly BooleanProgressLedgerSlot IsRequestSatisfiedSlot = new("is_request_satisfied",
"Is the request fully satisfied? (True if complete, or False if the original request has yet to be SUCCESSFULLY and FULLY addressed)");
internal static readonly BooleanProgressLedgerSlot IsInLoopSlot = new("is_in_loop",
"Are we in a loop where we are repeating the same requests and or getting the same responses as before? " +
"Loops can span multiple turns, and can include repeated actions like scrolling up or down more than a handful of times.");
internal static readonly BooleanProgressLedgerSlot IsProgressBeingMadeSlot = new("is_progress_being_made",
"Are we making forward progress? (True if just starting, or recent messages are adding value. False if recent " +
"messages show evidence of being stuck in a loop or if there is evidence of significant barriers to success " +
"such as the inability to read from a required file)");
internal readonly StringProgressLedgerSlot NextSpeakerSlot;
internal static readonly StringProgressLedgerSlot InstructionOrQuestionSlot = new("instruction_or_question",
"What instruction or question would you give this team member? (Phrase as if speaking directly to them, and " +
"include any specific information they may need)");
internal MagenticProgressLedger(string teamNames, IEnumerable<ProgressLedgerSlot> additionalQuestions, JsonElement? state = null)
{
this.NextSpeakerSlot = new("next_speaker", $"Who should speak next? (select from: {teamNames})");
this.AdditionalQuestions = additionalQuestions as ProgressLedgerSlot[] ?? additionalQuestions.ToArray();
if (state != null)
{
this.TryUpdateState(state.Value);
}
}
internal ProgressLedgerSlot[] AdditionalQuestions { get; }
internal bool TryUpdateState(JsonElement element)
{
// In principle all of these should be inlineable, but the CodeAnalysis fails to properly chain through the and-chain to realize that
// all must be true for `requiredQuestionsAnswered` to be true, meaning all of the out parameters would be initialized properly.
bool isInLoop = false;
bool isProgressBeingMade = false;
string? nextSpeaker = string.Empty;
string? instructionOrQuestion = string.Empty;
bool requiredQuestionsAnswered =
IsRequestSatisfiedSlot.TryGetValueFrom(element, out bool isRequestSatisfied) &&
IsInLoopSlot.TryGetValueFrom(element, out isInLoop) &&
IsProgressBeingMadeSlot.TryGetValueFrom(element, out isProgressBeingMade) &&
this.NextSpeakerSlot.TryGetValueFrom(element, out nextSpeaker) &&
InstructionOrQuestionSlot.TryGetValueFrom(element, out instructionOrQuestion);
if (requiredQuestionsAnswered)
{
this.State = element;
this.IsRequestSatisfied = isRequestSatisfied;
this.IsInLoop = isInLoop;
this.IsProgressBeingMade = isProgressBeingMade;
this.NextSpeaker = nextSpeaker!;
this.InstructionOrQuestion = instructionOrQuestion!;
}
// TODO: To what extent do we want to enforce that the additional questions are also answered?
return requiredQuestionsAnswered;
}
[JsonInclude]
internal JsonElement? State;
/// <summary>
/// Specifies whether plan execution has started.
/// </summary>
[JsonIgnore]
public bool IsStarted => this.State != null;
/// <summary>
/// Specifies whether the task has been fully satisfied.
/// </summary>
[JsonIgnore]
public bool IsRequestSatisfied { get; private set; }
/// <summary>
/// Specifies whether the team is in a loop.
/// </summary>
[JsonIgnore]
public bool IsInLoop { get; private set; }
/// <summary>
/// Specifies whether the team is making progress on the task.
/// </summary>
[JsonIgnore]
public bool IsProgressBeingMade { get; private set; }
/// <summary>
/// Gets the next team member to take a turn.
/// </summary>
[JsonIgnore]
public string NextSpeaker { get; private set; } = string.Empty;
/// <summary>
/// Gets the instruction or question to send to the next team member.
/// </summary>
[JsonIgnore]
public string InstructionOrQuestion { get; private set; } = string.Empty;
[JsonIgnore]
internal IEnumerable<ProgressLedgerSlot> Slots =>
[
IsRequestSatisfiedSlot,
IsInLoopSlot,
IsProgressBeingMadeSlot,
this.NextSpeakerSlot,
InstructionOrQuestionSlot,
.. this.AdditionalQuestions
];
internal bool TryGetCurrentSlotValue<T>(ProgressLedgerSlot<T> slot, [NotNullWhen(true)] out T? value)
{
if (!this.State.HasValue)
{
value = default;
return false;
}
return slot.TryGetValueFrom(this.State.Value, out value);
}
private (string QuestionBlock, string AnswerSchema)? _questionFormatCache;
internal (string QuestionBlock, string AnswerSchema) FormatQuestions()
{
if (!this._questionFormatCache.HasValue)
{
StringBuilder questionBuilder = new(), schemaBuilder = new();
schemaBuilder.AppendLine("{");
foreach (ProgressLedgerSlot slot in this.Slots)
{
questionBuilder.AppendLine(slot.FormattedQuestion);
schemaBuilder.AppendLine($"\"{slot.Key}\": {{")
.AppendLine($" \"{ProgressLedgerSlot.ValueKey}\": {slot.SchemaType}{slot.SuffixString},")
.AppendLine($" \"{ProgressLedgerSlot.ReasonKey}\": string")
.AppendLine("}");
}
schemaBuilder.AppendLine("}");
this._questionFormatCache = (questionBuilder.ToString(), schemaBuilder.ToString());
}
return this._questionFormatCache.Value;
}
}
internal abstract record ProgressLedgerSlot(string Key, string Question, string? SchemaTypeSuffix = null)
{
public const string ValueKey = "answer";
public const string ReasonKey = "reason";
internal string SuffixString => this.SchemaTypeSuffix == null ? string.Empty : $"({this.SchemaTypeSuffix})";
protected internal abstract string SchemaType { get; }
public string FormattedQuestion
{
get
{
if (field == null)
{
IEnumerable<string> questionLines = this.Question.Split(['\r', '\n'], StringSplitOptions.RemoveEmptyEntries)
.Select(line => line.TrimEnd());
field = $" - {string.Join("\n ", questionLines)}";
}
return field;
}
}
}
internal abstract record ProgressLedgerSlot<T>(string Key, string Question, string? SchemaTypeSuffix = null, JsonSerializerOptions? SerializerOptions = null)
: ProgressLedgerSlot(Key, Question, SchemaTypeSuffix)
{
protected internal virtual JsonTypeInfo<T> GetJsonTypeInfo() =>
((this.SerializerOptions ?? WorkflowsJsonUtilities.DefaultOptions).TryGetTypeInfo(typeof(T), out JsonTypeInfo? typeInfo)
? typeInfo as JsonTypeInfo<T> : null)
?? throw new InvalidOperationException($"Cannot get TypeInfo for {typeof(T)} from {(this.SerializerOptions == null ? "provided" : "default")} SerializationOptions.");
public bool TryGetValueFrom(JsonElement answers, [NotNullWhen(true)] out T? value)
{
if (answers.TryGetProperty(this.Key, out JsonElement slotElement) &&
slotElement.ValueKind != JsonValueKind.Null &&
slotElement.TryGetProperty(ValueKey, out JsonElement answerValue))
{
try
{
T? result = answerValue.Deserialize(this.GetJsonTypeInfo());
if (result != null)
{
value = result;
return true;
}
}
catch
{
}
}
value = default;
return false;
}
public bool TryGetReasonFrom(JsonElement answers, [NotNullWhen(true)] out string? value)
{
if (answers.TryGetProperty(this.Key, out JsonElement slotElement) &&
slotElement.ValueKind != JsonValueKind.Null &&
slotElement.TryGetProperty(ReasonKey, out JsonElement reasonValue))
{
try
{
string? result = reasonValue.Deserialize(WorkflowsJsonUtilities.JsonContext.Default.String);
if (result != null)
{
value = result;
return true;
}
}
catch
{
}
}
value = default;
return false;
}
}
internal sealed record BooleanProgressLedgerSlot(string Key, string Question, string? SchemaTypeSuffix = null) : ProgressLedgerSlot<bool>(Key, Question, SchemaTypeSuffix)
{
// Since we know the type statically, we can directly return the JsonTypeInfo for string from our JsonContext,
// which is more efficient than looking it up via the options.
protected internal override JsonTypeInfo<bool> GetJsonTypeInfo() => WorkflowsJsonUtilities.JsonContext.Default.Boolean;
protected internal override string SchemaType => "boolean";
}
internal sealed record StringProgressLedgerSlot(string Key, string Question, string? SchemaTypeSuffix = null) : ProgressLedgerSlot<string>(Key, Question, SchemaTypeSuffix)
{
// Since we know the type statically, we can directly return the JsonTypeInfo for string from our JsonContext,
// which is more efficient than looking it up via the options.
protected internal override JsonTypeInfo<string> GetJsonTypeInfo() => WorkflowsJsonUtilities.JsonContext.Default.String;
protected internal override string SchemaType => "string";
}
@@ -1,158 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
using System.Collections.Generic;
using System.Threading.Tasks;
using Microsoft.Agents.AI.Workflows.Specialized.Magentic;
using ExecutorFactoryFunc = System.Func<Microsoft.Agents.AI.Workflows.ExecutorConfig<Microsoft.Agents.AI.Workflows.ExecutorOptions>,
string,
System.Threading.Tasks.ValueTask<Microsoft.Agents.AI.Workflows.Specialized.Magentic.MagenticOrchestrator>>;
namespace Microsoft.Agents.AI.Workflows;
/// <summary>
/// Fluent builder for creating Magentic One multi-agent orchestration workflows.
///
/// Magentic One workflows use an LLM-powered manager to coordinate multiple agents through dynamic task planning, progress tracking,
/// and adaptive replanning.The manager creates plans, selects agents, monitors progress, and determines when to replan or complete.
///
/// The builder provides a fluent API for configuring participants, the manager, optional plan review, checkpointing, and event
/// callbacks.
///
/// Human-in-the-loop Support: Magentic provides specialized HITL mechanisms via:
/// - `RequirePlanSignoff` - Review and approve/revise plans before execution
/// - Tool approval via `function_approval_request`: Approve individual tool calls on participating agents. Note that tool calls are
/// not supported on the ManagerAgent.
/// </summary>
/// <param name="managerAgent"></param>
public class MagenticWorkflowBuilder(AIAgent managerAgent)
{
private readonly List<AIAgent> _team = new();
private string? _name;
private string? _description;
private int _maxStalls = TaskLimits.DefaultMaxStallCount;
private int? _maxRounds;
private int? _maxResets;
private bool _requirePlanSignoff = true;
/// <inheritdoc cref="GroupChatWorkflowBuilder.AddParticipants(IEnumerable{AIAgent})"/>
public MagenticWorkflowBuilder AddParticipants(params IEnumerable<AIAgent> agents)
{
this._team.AddRange(agents);
return this;
}
/// <inheritdoc cref="WorkflowBuilder.WithName(string)"/>
public MagenticWorkflowBuilder WithName(string name)
{
this._name = name;
return this;
}
/// <inheritdoc cref="WorkflowBuilder.WithDescription(string)"/>
public MagenticWorkflowBuilder WithDescription(string description)
{
this._description = description;
return this;
}
/// <summary>
/// Set the maximum number of coordination rounds. <see langword="null"/> means unlimited.
/// </summary>
/// <returns></returns>
public MagenticWorkflowBuilder WithMaxRounds(int? maxRounds = null)
{
this._maxRounds = maxRounds;
return this;
}
/// <summary>
/// Set the maximum number ofnumber of resets allowed. <see langword="null"/> means unlimited.
/// </summary>
/// <returns></returns>
public MagenticWorkflowBuilder WithMaxResets(int? maxResets = null)
{
this._maxResets = maxResets;
return this;
}
/// <summary>
/// Set the maximum number of consecutive rounds without progress before replan (default 3).
/// </summary>
/// <returns></returns>
public MagenticWorkflowBuilder WithMaxStalls(int maxStalls = TaskLimits.DefaultMaxStallCount)
{
this._maxStalls = maxStalls;
return this;
}
/// <summary>
/// If <see langword="true"/>, requires human approval of the initial plan or any updates before proceeding. True by default.
/// </summary>
/// <param name="requirePlanSignoff"></param>
/// <returns></returns>
public MagenticWorkflowBuilder RequirePlanSignoff(bool requirePlanSignoff = true)
{
this._requirePlanSignoff = requirePlanSignoff;
return this;
}
private WorkflowBuilder ReduceToWorkflowBuilder()
{
// Create a copy of the team so that improper modifications by using the builder after .Build() do not affect the
// workflow in unexpected ways.
List<AIAgent> team = [.. this._team];
ExecutorBinding orchestrator = CreateOrchestratorBinding(managerAgent, team, this.Limits, this._requirePlanSignoff);
WorkflowBuilder result = new(orchestrator);
AIAgentHostOptions options = new()
{
ReassignOtherAgentsAsUsers = true,
ForwardIncomingMessages = false
};
List<ExecutorBinding> teamBindings = [];
foreach (AIAgent agent in team)
{
ExecutorBinding binding = agent.BindAsExecutor(options);
teamBindings.Add(binding);
result.AddEdge(binding, orchestrator);
}
result.AddFanOutEdge(orchestrator, teamBindings)
.WithOutputFrom(orchestrator);
if (!string.IsNullOrWhiteSpace(this._name))
{
result.WithName(this._name);
}
if (!string.IsNullOrWhiteSpace(this._description))
{
result.WithDescription(this._description);
}
return result;
}
/// <inheritdoc cref="WorkflowBuilder.Build"/>
public Workflow Build() => this.ReduceToWorkflowBuilder().Build();
private TaskLimits Limits => new(
MaxRoundCount: this._maxRounds,
MaxResetCount: this._maxResets,
MaxStallCount: this._maxStalls);
private static ExecutorBinding CreateOrchestratorBinding(AIAgent managerAgent, List<AIAgent> team, TaskLimits limits, bool requirePlanSignoff)
{
ExecutorFactoryFunc factory = CreateOrchestratorAsync;
return factory.BindExecutor(nameof(MagenticOrchestrator));
ValueTask<MagenticOrchestrator> CreateOrchestratorAsync(ExecutorConfig<ExecutorOptions> options, string sessionId)
{
return new(new MagenticOrchestrator(managerAgent, team, limits, requirePlanSignoff));
}
}
}
@@ -1,9 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
namespace Microsoft.Agents.AI.Workflows;
/// <summary>
/// Notifies an AIAgent-hosting executor that it should reset its conversation state, and start a new session, if appropriate.
/// Note that for Agent Orchestrations, only Magentic makes use of this functionality.
/// </summary>
public sealed record ResetChatSignal();
@@ -24,7 +24,7 @@ internal static class TurnExtensions
=> handoffState.TurnToken.ShouldEmitStreamingEvents(agentSetting);
}
internal class AIAgentHostExecutor : ChatProtocolExecutor
internal sealed class AIAgentHostExecutor : ChatProtocolExecutor
{
private readonly AIAgent _agent;
private readonly AIAgentHostOptions _options;
@@ -40,9 +40,7 @@ internal class AIAgentHostExecutor : ChatProtocolExecutor
StringMessageChatRole = ChatRole.User
};
public static string IdFor(AIAgent agent) => agent.GetDescriptiveId();
public AIAgentHostExecutor(AIAgent agent, AIAgentHostOptions options) : base(id: IdFor(agent),
public AIAgentHostExecutor(AIAgent agent, AIAgentHostOptions options) : base(id: agent.GetDescriptiveId(),
s_defaultChatProtocolOptions,
declareCrossRunShareable: false) // Explicitly false, because we maintain turn state on the instance
{
@@ -69,14 +67,7 @@ internal class AIAgentHostExecutor : ChatProtocolExecutor
protected override ProtocolBuilder ConfigureProtocol(ProtocolBuilder protocolBuilder)
{
return this.ConfigureUserInputHandling(base.ConfigureProtocol(protocolBuilder))
.ConfigureRoutes(routeBuilder => routeBuilder.AddHandler<ResetChatSignal>(this.ResetChat));
}
internal void ResetChat(ResetChatSignal signal, IWorkflowContext context)
{
this._session = null;
this._currentTurnEmitEvents = null;
return this.ConfigureUserInputHandling(base.ConfigureProtocol(protocolBuilder));
}
private ValueTask HandleUserInputResponseAsync(
@@ -1,175 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Text;
using System.Text.Json;
using System.Text.RegularExpressions;
using Microsoft.Extensions.AI;
namespace Microsoft.Agents.AI.Workflows.Specialized.Magentic;
internal static partial class ChatMessageExtensions
{
private static void ProcessAIContents(StringBuilder resultBuilder, IEnumerable<AIContent> contents, StreamingToolCallResultPairMatcher? pairMatcher = null)
{
pairMatcher ??= new();
foreach (AIContent content in contents)
{
switch (content)
{
case TextContent textContent:
resultBuilder.AppendLine(textContent.Text);
break;
//case DataContent dataContent:
// // We really do not know how to deal with anything other than image data with descriptions, which is not
// // a well-defined concept in MEAI (as contrasted with AutoGen's ImageContent type)
// break;
case ErrorContent errorContent:
resultBuilder.AppendLine($"[ERROR{(errorContent.ErrorCode != null ? $"(Code={errorContent.ErrorCode})" : string.Empty)}]");
resultBuilder.AppendLine(errorContent.Message);
if (errorContent.Details != null)
{
resultBuilder.Append("Details:").AppendLine(errorContent.Details);
}
break;
case FunctionCallContent functionCallContent:
pairMatcher.CollectFunctionCall(functionCallContent);
break;
case FunctionResultContent functionResultContent:
pairMatcher.TryResolveFunctionCall(functionResultContent, out string? functionName);
string result = functionResultContent.Result?.ToString() ?? string.Empty;
resultBuilder.AppendLine($"[Tool Call '{functionName ?? functionResultContent.CallId}' Result]")
.AppendLine(result);
break;
case McpServerToolCallContent mstContent:
pairMatcher.CollectMcpServerToolCall(mstContent);
break;
case McpServerToolResultContent mstResultContent:
if (mstResultContent.Outputs?.Any() is true)
{
pairMatcher.TryResolveMcpServerToolCall(mstResultContent, out string? mcpServerToolName);
resultBuilder.AppendLine($"[Start MCP Server Tool Call '{mcpServerToolName ?? mstResultContent.CallId}' Results]");
ProcessAIContents(resultBuilder, mstResultContent.Outputs!);
resultBuilder.AppendLine($"[End MCP Server Tool Call '{mcpServerToolName ?? mstResultContent.CallId}']");
}
break;
case TextReasoningContent reasoningContent:
if (!string.IsNullOrWhiteSpace(reasoningContent.Text))
{
resultBuilder.Append("[Reasoning] ")
.AppendLine(reasoningContent.Text);
}
break;
case UriContent uriContent:
resultBuilder.AppendLine(uriContent.Uri.ToString());
break;
}
}
}
public static string GetText(this List<ChatMessage> messages)
{
if (messages.Count == 0)
{
return string.Empty;
}
StringBuilder builder = new();
StreamingToolCallResultPairMatcher pairMatcher = new();
foreach (ChatMessage message in messages)
{
ProcessAIContents(builder, message.Contents, pairMatcher);
}
return builder.ToString();
}
private const string FencedJsonRegexPattern = @"```(?<lang>[a-z]+)?\s*(?<json>\{[\s\S]*?\})\s*```";
#if NET
[GeneratedRegex(FencedJsonRegexPattern, RegexOptions.IgnoreCase | RegexOptions.CultureInvariant | RegexOptions.ExplicitCapture)]
public static partial Regex FencedJsonRegex();
#else
public static Regex FencedJsonRegex() => s_fencedJsonRegex;
private static readonly Regex s_fencedJsonRegex =
new(FencedJsonRegexPattern, RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.ExplicitCapture);
#endif
internal static JsonElement ExtractJson(string messageText)
{
Match match = FencedJsonRegex().Match(messageText);
if (match.Success)
{
return JsonElement.Parse(match.Groups["json"].Value);
}
int start = messageText.IndexOf('{'), scanHead = start;
int? end = null;
if (scanHead < 0)
{
throw new InvalidOperationException("No JSON object found.");
}
int depth = 0;
bool inQuotes = false, inEscape = false;
for (; scanHead < messageText.Length && end is null; scanHead++)
{
if (inEscape)
{
inEscape = false;
continue;
}
switch (messageText[scanHead])
{
case '{' when !inQuotes:
depth++;
break;
case '}' when !inQuotes:
depth--;
if (depth == 0)
{
end = scanHead;
}
break;
case '\"':
// We already handled inEscape, so we can always flip inQuotes here
inQuotes = !inQuotes;
break;
case '\\':
Debug.Assert(!inEscape);
inEscape = true;
break;
}
}
if (end is null)
{
throw new InvalidOperationException("Unbalanced JSON braces.");
}
return JsonElement.Parse(messageText.Substring(start, end.Value - start + 1));
}
public static JsonElement ExtractJson(this ChatMessage message) => ExtractJson(message.Text);
}
@@ -1,72 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
using System.Collections.Generic;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.AI;
namespace Microsoft.Agents.AI.Workflows.Specialized.Magentic;
internal sealed class ExecutorAgentHarness(AIAgent agent, AIAgentUnservicedRequestsCollector collector)
{
internal const string AgentSessionKey = nameof(AgentSession);
private AgentSession? _session;
private async ValueTask<AgentSession> EnsureSessionAsync(IWorkflowContext context, CancellationToken cancellationToken) =>
this._session ??= await agent.CreateSessionAsync(cancellationToken).ConfigureAwait(false);
public async ValueTask<AgentResponse> InvokeAgentAsync(IEnumerable<ChatMessage> messages, IWorkflowContext context, bool emitUpdateEvents, CancellationToken cancellationToken = default)
{
AgentResponse response;
if (emitUpdateEvents)
{
// Run the agent in streaming mode only when agent run update events are to be emitted.
IAsyncEnumerable<AgentResponseUpdate> agentStream = agent.RunStreamingAsync(
messages,
await this.EnsureSessionAsync(context, cancellationToken).ConfigureAwait(false),
cancellationToken: cancellationToken);
List<AgentResponseUpdate> updates = [];
await foreach (AgentResponseUpdate update in agentStream.ConfigureAwait(false))
{
await context.YieldOutputAsync(update, cancellationToken).ConfigureAwait(false);
collector.ProcessAgentResponseUpdate(update);
updates.Add(update);
}
response = updates.ToAgentResponse();
}
else
{
// Otherwise, run the agent in non-streaming mode.
response = await agent.RunAsync(messages,
await this.EnsureSessionAsync(context, cancellationToken).ConfigureAwait(false),
cancellationToken: cancellationToken)
.ConfigureAwait(false);
collector.ProcessAgentResponse(response);
}
return response;
}
public async ValueTask<JsonElement?> SerializeSessionAsync(CancellationToken cancellationToken)
=> this._session == null
? null
: await agent.SerializeSessionAsync(this._session, cancellationToken: cancellationToken).ConfigureAwait(false);
public async ValueTask DeserializeSessionAsync(JsonElement? serializedSession, CancellationToken cancellationToken)
{
this._session = serializedSession == null
? null
: await agent.DeserializeSessionAsync(serializedSession.Value, cancellationToken: cancellationToken)
.ConfigureAwait(false);
}
public void ResetSession()
{
this._session = null;
}
}
@@ -1,8 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
namespace Microsoft.Agents.AI.Workflows.Specialized.Magentic;
internal static class MagenticConstants
{
public const string MagenticTaskContextKey = nameof(MagenticTaskContextKey);
}
@@ -1,122 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
using System.Collections.Generic;
using System.Runtime.ExceptionServices;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.AI;
namespace Microsoft.Agents.AI.Workflows.Specialized.Magentic;
internal class MagenticManager(AIAgent managerAgent)
{
private static async ValueTask<ChatMessage> CheckResponseAsync(Task<AgentResponse> responseTask, IWorkflowContext context, CancellationToken cancellationToken)
{
AgentResponse response = await responseTask.ConfigureAwait(false);
if (response.Messages.Count == 0)
{
throw new InvalidOperationException("Planner Agent did not return any messages.");
}
if (response.Messages.Count > 1)
{
await context.AddEventAsync(new WorkflowWarningEvent("Planner Agent returned multiple messages; using the last one."), cancellationToken)
.ConfigureAwait(false);
}
return response.Messages[response.Messages.Count - 1];
}
private ValueTask<ChatMessage> InvokeAgentAsync(IEnumerable<ChatMessage> messages, IWorkflowContext context, CancellationToken cancellationToken, AgentSession? session = null)
=> CheckResponseAsync(managerAgent.RunAsync(messages, session, cancellationToken: cancellationToken), context, cancellationToken);
public async ValueTask<TaskLedger> UpdatePlanAsync(MagenticTaskContext taskContext, IWorkflowContext context, CancellationToken cancellationToken)
{
// If we already have a TaskLedger, we need to update the facts based on the existing factset; otherwise, we use the initial facts construction
bool isReplan = taskContext.TaskLedger != null;
AgentSession localSession = await managerAgent.CreateSessionAsync(cancellationToken).ConfigureAwait(false);
ChatMessage factsRequest = new(ChatRole.User, isReplan ? taskContext.ToTaskLedgerFactsUpdatePrompt() : taskContext.ToTaskLedgerFactsPrompt());
ChatMessage updatedFacts = await this.InvokeAgentAsync(
messages: [.. taskContext.ChatHistory, factsRequest],
context,
cancellationToken,
localSession)
.ConfigureAwait(false);
ChatMessage planRequest = new(ChatRole.User, isReplan ? taskContext.ToTaskLedgerPlanUpdatePrompt() : taskContext.ToTaskLedgerPlanPrompt());
ChatMessage updatedPlan = await this.InvokeAgentAsync(
// We rely on the AgentSession to maintain the context of the conversation, so we don't include the
// history, facts request, or updated facts in the messages list.
messages: [planRequest],
context,
cancellationToken,
localSession)
.ConfigureAwait(false);
taskContext.ChatHistory.AddRange([factsRequest, updatedFacts, planRequest, updatedPlan]);
return new(updatedFacts, updatedPlan);
}
public async ValueTask UpdateProgressLedgerAsync(MagenticTaskContext taskContext, IWorkflowContext context, CancellationToken cancellationToken)
{
ChatMessage progressRequest = new(ChatRole.User, taskContext.ToProgressLedgerPrompt());
ExceptionDispatchInfo? lastException = null;
int maxRetryCount = taskContext.TaskLimits.MaxProgressLedgerRetryCount;
for (int attempts = 0; attempts < maxRetryCount; attempts++)
{
ChatMessage progressUpdateMessage = await this.InvokeAgentAsync(
messages: [.. taskContext.ChatHistory, progressRequest],
context,
cancellationToken)
.ConfigureAwait(false);
try
{
lastException = null;
JsonElement stateUpdateJson = progressUpdateMessage.ExtractJson();
if (!taskContext.ProgressLedger.TryUpdateState(stateUpdateJson))
{
throw new InvalidOperationException("Could not answer progress ledger questions with provided JSON.");
}
break;
}
catch (Exception e)
{
lastException = ExceptionDispatchInfo.Capture(e);
string warnString = $"Progress ledger JSON parse failed (attempt {attempts}/{maxRetryCount}): {e}";
await context.AddEventAsync(new WorkflowWarningEvent(warnString), cancellationToken).ConfigureAwait(false);
if (attempts < maxRetryCount)
{
await Task.Delay(250 * attempts, cancellationToken).ConfigureAwait(false);
}
}
}
lastException?.Throw();
}
public async ValueTask<ChatMessage> PrepareFinalAnswerAsync(MagenticTaskContext taskContext, IWorkflowContext context, CancellationToken cancellationToken)
{
ChatMessage finalAnswerRequest = new(ChatRole.User, taskContext.ToFinalAnswerPrompt());
ChatMessage finalAnswer = await this.InvokeAgentAsync([.. taskContext.ChatHistory, finalAnswerRequest], context, cancellationToken)
.ConfigureAwait(false);
return new(ChatRole.Assistant, finalAnswer.Text)
{
AuthorName = finalAnswer.AuthorName ?? nameof(MagenticManager),
MessageId = finalAnswer.MessageId ?? Guid.NewGuid().ToString("N"),
CreatedAt = finalAnswer.CreatedAt ?? DateTimeOffset.UtcNow,
RawRepresentation = finalAnswer.RawRepresentation,
};
}
}
@@ -1,326 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.Json.Serialization;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.AI;
namespace Microsoft.Agents.AI.Workflows.Specialized.Magentic;
/// <summary>
/// Base type for Magentic Orchestration Events
/// </summary>
/// <param name="data"></param>
[JsonDerivedType(typeof(MagenticPlanCreatedEvent))]
[JsonDerivedType(typeof(MagenticReplannedEvent))]
[JsonDerivedType(typeof(MagenticProgressLedgerUpdatedEvent))]
public abstract class MagenticOrchestratorEvent(object? data) : WorkflowEvent(data)
{
}
/// <summary>
/// Represents the creation of the initial plan
/// </summary>
/// <param name="fullTaskLeger"></param>
public sealed class MagenticPlanCreatedEvent(ChatMessage fullTaskLeger) : MagenticOrchestratorEvent(fullTaskLeger)
{
/// <summary>
/// A <see cref="ChatMessage"/> containing the initial plan.
/// </summary>
public ChatMessage FullTaskLedger { get; } = fullTaskLeger;
}
/// <summary>
/// Represents the creation of a new plan in response to a stall.
/// </summary>
/// <param name="fullTaskLeger"></param>
public sealed class MagenticReplannedEvent(ChatMessage fullTaskLeger) : MagenticOrchestratorEvent(fullTaskLeger)
{
/// <summary>
/// A <see cref="ChatMessage"/> containing the new plan.
/// </summary>
public ChatMessage FullTaskLedger { get; } = fullTaskLeger;
}
/// <summary>
/// Represents an update to the <see cref="MagenticProgressLedger"/> when running a coordination round.
/// </summary>
/// <param name="progressLedger"></param>
public sealed class MagenticProgressLedgerUpdatedEvent(MagenticProgressLedger progressLedger) : MagenticOrchestratorEvent(progressLedger)
{
/// <summary>
/// The new state of the <see cref="MagenticProgressLedger"/>
/// </summary>
public MagenticProgressLedger ProgressLedger { get; } = progressLedger;
}
/// <summary>
/// Magentic orchestrator that defines the workflow structure.
///
/// This orchestrator manages the overall Magentic workflow in the following structure:
///
/// 1. Upon receiving the task(a list of messages), it creates the plan using the manager then runs the inner loop.
/// 2. The inner loop is distributed and implementation is decentralized. In the orchestrator, it is responsible for:
/// - Creating the progress ledger using the manager.
/// - Checking for task completion.
/// - Detecting stalling or looping and triggering replanning if needed.
/// - Sending requests to participants based on the progress ledger's next speaker.
/// - Issue requests for human intervention if enabled and needed.
/// 3. The inner loop waits for responses from the selected participant, then continues the loop.
/// 4. The orchestrator breaks out of the inner loop when the replanning or final answer conditions are met.
/// 5. The outer loop handles replanning and reenters the inner loop.
/// </summary>
/// <param name="managerAgent"></param>
/// <param name="team"></param>
/// <param name="limits"></param>
/// <param name="requirePlanSignoff"></param>
internal class MagenticOrchestrator(AIAgent managerAgent, List<AIAgent> team, TaskLimits limits, bool requirePlanSignoff)
: ChatProtocolExecutor(nameof(MagenticOrchestrator), s_options, declareCrossRunShareable: false)
{
private readonly MagenticManager _manager = new(managerAgent);
private static readonly ChatProtocolExecutorOptions s_options = new()
{
StringMessageChatRole = ChatRole.User,
AutoSendTurnToken = false
};
private MagenticTaskContext? _taskContext;
private PortBinding? _planReviewPort;
protected override ProtocolBuilder ConfigureProtocol(ProtocolBuilder protocolBuilder)
{
return base.ConfigureProtocol(protocolBuilder).ConfigureRoutes(ConfigureRoutes);
void ConfigureRoutes(RouteBuilder routeBuilder) => routeBuilder.AddPortHandler<MagenticPlanReviewRequest, MagenticPlanReviewResponse>(
"RequestPlanReview",
this.ProcessPlanReviewAsync,
out this._planReviewPort);
}
private ValueTask SubmitPlanReviewRequestAsync(MagenticTaskContext taskContext, IWorkflowContext workflowContext)
{
MagenticProgressLedger? progressLedger = taskContext.ProgressLedger;
if (progressLedger?.IsStarted is not true)
{
progressLedger = null;
}
MagenticPlanReviewRequest request = new(taskContext.TaskLedger!.CurrentPlan, progressLedger, taskContext.IsStalled);
return this._planReviewPort!.PostRequestAsync(request);
}
private async ValueTask ProcessPlanReviewAsync(MagenticPlanReviewResponse response, IWorkflowContext context, CancellationToken cancellationToken)
{
/*
Handle the human response to the plan review request.
Logic:
There are code paths which will trigger a plan review request to the human:
- Initial plan creation if `require_plan_signoff` is True.
- Potentially during the inner loop if stalling is detected (resetting and replanning).
The human can either approve the plan or request revisions with comments.
- If approved, proceed to run the outer loop, which simply adds the task ledger
to the conversation and enters the inner loop.
- If revision requested, append the review comments to the chat history,
trigger replanning via the manager, emit a REPLANNED event, then run the outer loop.
*/
if (this._taskContext == null || this._taskContext.TaskLedger == null)
{
throw new InvalidOperationException("Magentic Orchestration was not initialized correctly.");
}
if (this._taskContext.IsTerminated)
{
throw new InvalidOperationException("Magentic Orchestration has already been terminated and cannot process new messages. Please start a new session.");
}
if (response.IsApproved)
{
await this.DelegateToTeamAsync(this._taskContext, context, cancellationToken).ConfigureAwait(false);
}
else
{
this._taskContext.ChatHistory.AddRange(response.Review);
await this.UpdatePlanAndDelegateAsync(this._taskContext, context, cancellationToken).ConfigureAwait(false);
}
}
private async ValueTask UpdatePlanAndDelegateAsync(MagenticTaskContext taskContext, IWorkflowContext context, CancellationToken cancellationToken)
{
bool isReplan = taskContext.TaskLedger != null;
taskContext.TaskLedger = await this._manager.UpdatePlanAsync(taskContext, context, cancellationToken)
.ConfigureAwait(false);
this._fullTaskLedgerMessage = new(ChatRole.User, taskContext.ToTaskLedgerFullPrompt());
taskContext.ChatHistory.Add(this._fullTaskLedgerMessage);
await context.AddEventAsync(isReplan
? new MagenticReplannedEvent(this._fullTaskLedgerMessage)
: new MagenticPlanCreatedEvent(this._fullTaskLedgerMessage), cancellationToken).ConfigureAwait(false);
if (requirePlanSignoff)
{
await this.SubmitPlanReviewRequestAsync(taskContext, context).ConfigureAwait(false);
}
else
{
await this.DelegateToTeamAsync(taskContext, context, cancellationToken).ConfigureAwait(false);
}
}
protected override async ValueTask TakeTurnAsync(List<ChatMessage> messages, IWorkflowContext context, bool? emitEvents, CancellationToken cancellationToken = default)
{
// First Turn: Initialize the task context and send the initial messages to the planner agent
this._taskContext ??= new(messages, team, limits, emitEvents, []);
await this.UpdatePlanAndDelegateAsync(this._taskContext, context, cancellationToken).ConfigureAwait(false);
}
private ChatMessage? _fullTaskLedgerMessage;
private ValueTask DelegateToTeamAsync(MagenticTaskContext taskContext, IWorkflowContext context, CancellationToken cancellationToken)
{
return this.RunCoordinationRoundAsync(taskContext, context, cancellationToken);
}
private async ValueTask RunCoordinationRoundAsync(MagenticTaskContext taskContext, IWorkflowContext context, CancellationToken cancellationToken)
{
(bool hitRoundLimit, bool hitResetLimit) = taskContext.CheckLimits();
if (hitRoundLimit || hitResetLimit)
{
string limitType = hitRoundLimit ? "round" : "reset";
List<ChatMessage> messages = [new(ChatRole.Assistant, $"Task execution stopped due to hitting the maximum {limitType} count limit.")];
await context.YieldOutputAsync(messages, cancellationToken).ConfigureAwait(false);
taskContext.IsTerminated = true;
return;
}
taskContext.TaskCounters.RoundCount++;
// Update the Progress Ledger
try
{
await this._manager.UpdateProgressLedgerAsync(taskContext, context, cancellationToken).ConfigureAwait(false);
await context.AddEventAsync(new MagenticProgressLedgerUpdatedEvent(taskContext.ProgressLedger), cancellationToken)
.ConfigureAwait(false);
}
// Retry on exception to max retry count, unless it is OperationCancelledException - in that case exit the loop right away
catch (Exception ex) when (ex is not OperationCanceledException)
{
await context.AddEventAsync(new WorkflowWarningEvent($"Magentic Orchestrator: Progress ledger creation failed, triggering reset: {ex}"), cancellationToken)
.ConfigureAwait(false);
await this.ResetAndReplanAsync(taskContext, context, cancellationToken).ConfigureAwait(false);
return;
}
// Check and handle finish condition
if (taskContext.ProgressLedger.IsRequestSatisfied)
{
await this.PrepareFinalAnswerAsync(taskContext, context, cancellationToken).ConfigureAwait(false);
return;
}
// Check and handle stalls
if (taskContext.ProgressLedger.IsInLoop || !taskContext.ProgressLedger.IsProgressBeingMade)
{
taskContext.TaskCounters.StallCount++;
}
else
{
taskContext.TaskCounters.StallCount = Math.Max(0, taskContext.TaskCounters.StallCount - 1);
}
if (taskContext.IsStalled)
{
await this.ResetAndReplanAsync(taskContext, context, cancellationToken).ConfigureAwait(false);
return;
}
// Prepare to delegate to the next speaker
string nextSpeaker = taskContext.ProgressLedger.NextSpeaker;
if (string.IsNullOrEmpty(nextSpeaker))
{
await context.AddEventAsync(new WorkflowWarningEvent("Next speaker answer empty; selecting first participant as fallback"), cancellationToken)
.ConfigureAwait(false);
nextSpeaker = team.First().Name!;
}
AIAgent? nextAgent = team.FirstOrDefault(agent => agent.Name == nextSpeaker);
if (nextAgent == null)
{
await context.AddEventAsync(new WorkflowWarningEvent($"Invalid next speaker: {nextSpeaker}"), cancellationToken)
.ConfigureAwait(false);
await this.PrepareFinalAnswerAsync(taskContext, context, cancellationToken).ConfigureAwait(false);
return;
}
if (!string.IsNullOrWhiteSpace(taskContext.ProgressLedger.InstructionOrQuestion))
{
ChatMessage instruction = new(ChatRole.Assistant, taskContext.ProgressLedger.InstructionOrQuestion);
taskContext.ChatHistory.Add(instruction);
await context.SendMessageAsync(instruction, cancellationToken).ConfigureAwait(false);
}
string nextExecutorId = AIAgentHostExecutor.IdFor(nextAgent);
await context.SendMessageAsync(new TurnToken(taskContext.EmitUpdateEvents), nextExecutorId, cancellationToken).ConfigureAwait(false);
}
private async ValueTask ResetAndReplanAsync(MagenticTaskContext taskContext, IWorkflowContext context, CancellationToken cancellationToken)
{
taskContext.Reset();
await context.SendMessageAsync(new ResetChatSignal(), cancellationToken: cancellationToken).ConfigureAwait(false);
await this.UpdatePlanAndDelegateAsync(taskContext, context, cancellationToken).ConfigureAwait(false);
}
private async ValueTask PrepareFinalAnswerAsync(MagenticTaskContext taskContext, IWorkflowContext context, CancellationToken cancellationToken)
{
List<ChatMessage> messages = [await this._manager.PrepareFinalAnswerAsync(taskContext, context, cancellationToken).ConfigureAwait(false)];
await context.YieldOutputAsync(messages, cancellationToken).ConfigureAwait(false);
taskContext.IsTerminated = true;
}
private const string CurrentTurnEmitUpdateEventsKey = nameof(CurrentTurnEmitUpdateEventsKey);
protected internal override async ValueTask OnCheckpointingAsync(IWorkflowContext context, CancellationToken cancellationToken = default)
{
Task contextStateTask = this._taskContext == null
? Task.CompletedTask
: context.QueueStateUpdateAsync(MagenticConstants.MagenticTaskContextKey,
this._taskContext.ExportState(),
cancellationToken: cancellationToken)
.AsTask();
await Task.WhenAll(base.OnCheckpointingAsync(context, cancellationToken).AsTask(),
contextStateTask).ConfigureAwait(false);
}
protected internal override async ValueTask OnCheckpointRestoredAsync(IWorkflowContext context, CancellationToken cancellationToken = default)
{
await Task.WhenAll(base.OnCheckpointRestoredAsync(context, cancellationToken).AsTask(), LoadContextStateAsync())
.ConfigureAwait(false);
async Task LoadContextStateAsync()
{
MagenticTaskState? state = await context.ReadStateAsync<MagenticTaskState>(MagenticConstants.MagenticTaskContextKey, cancellationToken: cancellationToken)
.ConfigureAwait(false);
if (state != null)
{
this._taskContext = new MagenticTaskContext(state, team, limits, []);
}
}
}
}
@@ -1,95 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.Json;
using Microsoft.Extensions.AI;
namespace Microsoft.Agents.AI.Workflows.Specialized.Magentic;
internal record TaskLimits(int MaxStallCount = TaskLimits.DefaultMaxStallCount,
int? MaxRoundCount = null,
int? MaxResetCount = null,
int MaxProgressLedgerRetryCount = TaskLimits.DefaultMaxProgressLedgerRetryCount)
{
public const int DefaultMaxStallCount = 3;
public const int DefaultMaxProgressLedgerRetryCount = 3;
}
internal record TaskLedger(ChatMessage CurrentFacts, ChatMessage CurrentPlan);
internal class TaskCounters
{
public int RoundCount { get; set; }
public int StallCount { get; set; }
public int ResetCount { get; set; }
}
internal record MagenticTaskState(List<ChatMessage> TaskDefinition, List<ChatMessage> ChatHistory, TaskLedger? TaskLedger, JsonElement? ProgressLedgerState, TaskCounters Counters, bool Terminated, bool? EmitUpdateEvents)
{
}
internal class MagenticTaskContext(List<ChatMessage> taskDefinition, List<AIAgent> team, TaskLimits limits, bool? emitUpdateEvents, IEnumerable<ProgressLedgerSlot> additionalProgressQuestions)
{
internal MagenticTaskContext(MagenticTaskState state, List<AIAgent> team, TaskLimits limits, IEnumerable<ProgressLedgerSlot> additionalProgressQuestions)
: this(state.TaskDefinition, team, limits, state.EmitUpdateEvents, additionalProgressQuestions)
{
this.TaskLedger = state.TaskLedger;
this.TaskCounters = state.Counters;
this.ChatHistory = state.ChatHistory;
this.IsTerminated = state.Terminated;
if (state.ProgressLedgerState.HasValue && !this.ProgressLedger.TryUpdateState(state.ProgressLedgerState.Value))
{
throw new InvalidOperationException("Could not load progress ledger state value");
}
}
public string Task { get; } = taskDefinition.GetText();
public string TeamDescription { get; } = GetTeamDescription(team);
public List<ChatMessage> ChatHistory { get; internal set; } = new();
public TaskLedger? TaskLedger { get; internal set; }
public TaskLimits TaskLimits => limits;
public bool IsTerminated { get; internal set; }
public bool IsStalled => this.TaskCounters.StallCount >= this.TaskLimits.MaxStallCount;
public (bool HitRoundLimit, bool HitResetLimit) CheckLimits()
{
return (this.TaskLimits.MaxRoundCount.HasValue && this.TaskLimits.MaxRoundCount.Value <= this.TaskCounters.RoundCount,
this.TaskLimits.MaxResetCount.HasValue && this.TaskLimits.MaxResetCount.Value <= this.TaskCounters.ResetCount);
}
public TaskCounters TaskCounters { get; internal set; } = new();
public MagenticProgressLedger ProgressLedger { get; } = new(GetTeamNames(team), additionalProgressQuestions);
public bool? EmitUpdateEvents => emitUpdateEvents;
public static string GetTeamDescription(IEnumerable<AIAgent> team)
{
return string.Join("\n", team.Select(agent => $"- {agent.Name}: {agent.Description}"));
}
public static string GetTeamNames(IEnumerable<AIAgent> team)
{
return string.Join(", ", team.Select(agent => agent.Name));
}
public MagenticTaskState ExportState()
{
return new(taskDefinition, this.ChatHistory, this.TaskLedger, this.ProgressLedger.State, this.TaskCounters, this.IsTerminated, this.EmitUpdateEvents);
}
internal void Reset()
{
this.ChatHistory.Clear();
this.TaskCounters.ResetCount++;
this.TaskCounters.StallCount = 0;
}
}
@@ -1,151 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
using Microsoft.Extensions.AI;
namespace Microsoft.Agents.AI.Workflows.Specialized.Magentic;
internal static class PromptTemplateExtensions
{
public static string ToTaskLedgerFactsPrompt(this MagenticTaskContext taskContext)
{
return $"""
Below I will present you a request.
Before we begin addressing the request, please answer the following pre-survey to the best of your ability.
Keep in mind that you are Ken Jennings-level with trivia, and Mensa-level with puzzles, so there should be
a deep well to draw from.
Here is the request:
{taskContext.Task}
Here is the pre-survey:
1. Please list any specific facts or figures that are GIVEN in the request itself.It is possible that
there are none.
2. Please list any facts that may need to be looked up, and WHERE SPECIFICALLY they might be found.
In some cases, authoritative sources are mentioned in the request itself.
3. Please list any facts that may need to be derived(e.g., via logical deduction, simulation, or computation)
4. Please list any facts that are recalled from memory, hunches, well-reasoned guesses, etc.
When answering this survey, keep in mind that "facts" will typically be specific names, dates, statistics, etc.
Your answer should use headings:
1. GIVEN OR VERIFIED FACTS
2. FACTS TO LOOK UP
3. FACTS TO DERIVE
4. EDUCATED GUESSES
DO NOT include any other headings or sections in your response.DO NOT list next steps or plans until asked to do so.
""";
}
public static string ToTaskLedgerFactsUpdatePrompt(this MagenticTaskContext taskContext)
{
return $"""
As a reminder, we are working to solve the following task:
{taskContext.Task}
It is clear we are not making as much progress as we would like, but we may have learned something new.
Please rewrite the following fact sheet, updating it to include anything new we have learned that may be helpful.
Example edits can include (but are not limited to) adding new guesses, moving educated guesses to verified facts
if appropriate, etc. Updates may be made to any section of the fact sheet, and more than one section of the fact
sheet can be edited. This is an especially good time to update educated guesses, so please at least add or update
one educated guess or hunch, and explain your reasoning.
Here is the old fact sheet:
{taskContext.TaskLedger?.CurrentFacts ?? new(ChatRole.Assistant, string.Empty)}
""";
}
public static string ToTaskLedgerPlanPrompt(this MagenticTaskContext taskContext)
{
return $"""
Fantastic. To address this request we have assembled the following team:
{taskContext.TeamDescription}
Based on the team composition, and known and unknown facts, please devise a short bullet-point plan for addressing the
original request. Remember, there is no requirement to involve all team members. A team member's particular expertise
may not be needed for this task.
""";
}
public static string ToTaskLedgerPlanUpdatePrompt(this MagenticTaskContext taskContext)
{
return $"""
Please briefly explain what went wrong on this last run
(the root cause of the failure), and then come up with a new plan that takes steps and includes hints to overcome prior
challenges and especially avoids repeating the same mistakes. As before, the new plan should be concise, expressed in
bullet-point form, and consider the following team composition:
{taskContext.TeamDescription}
""";
}
public static string ToTaskLedgerFullPrompt(this MagenticTaskContext taskContext)
{
return $"""
We are working to address the following user request:
{taskContext.Task}
To answer this request we have assembled the following team:
{taskContext.TeamDescription}
Here is an initial fact sheet to consider:
{taskContext.TaskLedger!.CurrentFacts ?? new(ChatRole.Assistant, string.Empty)}
Here is the plan to follow as best as possible:
{taskContext.TaskLedger!.CurrentPlan}
""";
}
public static string ToProgressLedgerPrompt(this MagenticTaskContext taskContext)
{
(string questions, string schema) = taskContext.ProgressLedger.FormatQuestions();
return $"""
Recall we are working on the following request:
{taskContext.Task}
And we have assembled the following team:
{taskContext.TeamDescription}
To make progress on the request, please answer the following questions, including necessary reasoning:
{questions}
Please output an answer in pure JSON format according to the following schema. The JSON object must be parsable as-is.
DO NOT OUTPUT ANYTHING OTHER THAN JSON, AND DO NOT DEVIATE FROM THIS SCHEMA:
{schema}
""";
}
public static string ToFinalAnswerPrompt(this MagenticTaskContext taskContext)
{
return $"""
We are working on the following task:
{taskContext.Task}
We have completed the task.
The above messages contain the conversation that took place to complete the task.
Based on the information gathered, provide the final answer to the original request.
The answer should be phrased as if you were speaking to the user.
""";
}
}
@@ -1,77 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using Microsoft.Extensions.AI;
namespace Microsoft.Agents.AI.Workflows.Specialized.Magentic;
internal sealed class StreamingToolCallResultPairMatcher
{
private enum CallType
{
Function,
McpServerTool
}
private record CallSummaryKey(CallType Type, string CallId);
private struct ToolCallSummary(CallType callType, string callId, string name)
{
public CallType CallType => callType;
public string? CallId => callId;
public string Name => name;
}
private readonly Dictionary<CallSummaryKey, ToolCallSummary> _callSummaries = new();
private void Collect(CallType callType, string callId, string name, string callContentTypeName, string resultContentTypeName)
{
CallSummaryKey key = new(callType, callId);
if (this._callSummaries.ContainsKey(key))
{
throw new InvalidOperationException($"Duplicate {callContentTypeName} with CallId '{callId}' without corresponding {resultContentTypeName}.");
}
this._callSummaries[key] = new ToolCallSummary(callType, callId, name);
}
public void CollectFunctionCall(FunctionCallContent callContent)
{
const string FunctionCallContentTypeName = nameof(FunctionCallContent);
const string FunctionResultContentTypeName = nameof(FunctionResultContent);
this.Collect(CallType.Function, callContent.CallId, callContent.Name, FunctionCallContentTypeName, FunctionResultContentTypeName);
}
public void CollectMcpServerToolCall(McpServerToolCallContent callContent)
{
const string McpServerToolCallContentTypeName = nameof(McpServerToolCallContent);
const string McpServerToolResultContentTypeName = nameof(McpServerToolResultContent);
this.Collect(CallType.McpServerTool, callContent.CallId, callContent.Name, McpServerToolCallContentTypeName, McpServerToolResultContentTypeName);
}
private bool TryResolve(CallType callType, string callId, [NotNullWhen(true)] out string? name)
{
CallSummaryKey key = new(callType, callId);
bool hasMatchingCall = this._callSummaries.TryGetValue(key, out ToolCallSummary callSummary);
if (hasMatchingCall)
{
this._callSummaries.Remove(key);
}
name = hasMatchingCall ? callSummary.Name : null;
return hasMatchingCall;
}
public bool TryResolveFunctionCall(FunctionResultContent resultContent, [NotNullWhen(true)] out string? name)
=> this.TryResolve(CallType.Function, resultContent.CallId, out name);
public bool TryResolveMcpServerToolCall(McpServerToolResultContent resultContent, [NotNullWhen(true)] out string? name)
=> this.TryResolve(CallType.McpServerTool, resultContent.CallId, out name);
}
@@ -1,7 +1,6 @@
// Copyright (c) Microsoft. All rights reserved.
using System.Text.Json.Serialization;
using Microsoft.Agents.AI.Workflows.Specialized.Magentic;
namespace Microsoft.Agents.AI.Workflows;
@@ -15,8 +14,6 @@ namespace Microsoft.Agents.AI.Workflows;
[JsonDerivedType(typeof(WorkflowWarningEvent))]
[JsonDerivedType(typeof(WorkflowOutputEvent))]
[JsonDerivedType(typeof(RequestInfoEvent))]
[JsonDerivedType(typeof(MagenticOrchestratorEvent))]
public class WorkflowEvent(object? data = null)
{
/// <summary>
@@ -7,7 +7,6 @@ using System.Text.Json.Serialization;
using Microsoft.Agents.AI.Workflows.Checkpointing;
using Microsoft.Agents.AI.Workflows.Execution;
using Microsoft.Agents.AI.Workflows.Specialized;
using Microsoft.Agents.AI.Workflows.Specialized.Magentic;
using Microsoft.Extensions.AI;
namespace Microsoft.Agents.AI.Workflows;
@@ -98,10 +97,6 @@ internal static partial class WorkflowsJsonUtilities
[JsonSerializable(typeof(AIAgentHostState))]
[JsonSerializable(typeof(HandoffSharedState))]
[JsonSerializable(typeof(HandoffAgentHostState))]
[JsonSerializable(typeof(MagenticPlanReviewRequest))]
[JsonSerializable(typeof(MagenticPlanReviewResponse))]
[JsonSerializable(typeof(MagenticTaskState))]
[JsonSerializable(typeof(ResetChatSignal))]
// Event Types
//[JsonSerializable(typeof(WorkflowEvent))]
@@ -17,6 +17,9 @@ namespace AnthropicChatCompletion.IntegrationTests;
public class AnthropicChatCompletionFixture : IChatClientAgentFixture
{
// All tests for Anthropic are intended to be ran locally as the CI pipeline for Anthropic is not setup.
internal const string SkipReason = "Integrations tests for local execution only";
private readonly bool _useReasoningModel;
private readonly bool _useBeta;
@@ -102,22 +105,7 @@ public class AnthropicChatCompletionFixture : IChatClientAgentFixture
public async ValueTask InitializeAsync()
{
// Temporarily disabled: Anthropic SDK has a binary incompatibility with the current
// Microsoft.Extensions.AI version (WebSearchToolResultContent.Results method not found).
// See: https://github.com/microsoft/agent-framework/pull/5515
Assert.Skip("Anthropic integration tests temporarily disabled due to SDK incompatibility with Microsoft.Extensions.AI");
try
{
_ = TestConfiguration.GetRequiredValue(TestSettings.AnthropicApiKey);
_ = TestConfiguration.GetRequiredValue(TestSettings.AnthropicChatModelName);
_ = TestConfiguration.GetRequiredValue(TestSettings.AnthropicReasoningModelName);
}
catch (InvalidOperationException ex)
{
Assert.Skip("Anthropic configuration could not be loaded. Error:" + ex.Message);
}
Assert.SkipWhen(SkipReason is not null, SkipReason ?? string.Empty);
this._agent = await this.CreateChatClientAgentAsync();
}
@@ -1,6 +1,5 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
using System.Threading.Tasks;
using AgentConformance.IntegrationTests.Support;
using Anthropic;
@@ -18,28 +17,19 @@ namespace AnthropicChatCompletion.IntegrationTests;
/// Integration tests for Anthropic Skills functionality.
/// These tests are designed to be run locally with a valid Anthropic API key.
/// </summary>
/// <remarks>
/// Temporarily disabled due to Anthropic SDK binary incompatibility with
/// the current Microsoft.Extensions.AI version (WebSearchToolResultContent.Results).
/// </remarks>
[Trait("Category", "IntegrationDisabled")]
public sealed class AnthropicSkillsIntegrationTests
{
// All tests for Anthropic are intended to be ran locally as the CI pipeline for Anthropic is not setup.
private const string SkipReason = "Integrations tests for local execution only";
[Fact]
public async Task CreateAgentWithPptxSkillAsync()
{
AnthropicClient? anthropicClient;
string? model;
try
{
anthropicClient = new() { ApiKey = TestConfiguration.GetRequiredValue(TestSettings.AnthropicApiKey) };
model = TestConfiguration.GetRequiredValue(TestSettings.AnthropicChatModelName);
}
catch (InvalidOperationException ex)
{
Assert.Skip("Anthropic configuration could not be loaded. Error:" + ex.Message);
return;
}
Assert.SkipWhen(SkipReason is not null, SkipReason ?? string.Empty);
// Arrange
AnthropicClient anthropicClient = new() { ApiKey = TestConfiguration.GetRequiredValue(TestSettings.AnthropicApiKey) };
string model = TestConfiguration.GetRequiredValue(TestSettings.AnthropicChatModelName);
BetaSkillParams pptxSkill = new()
{
@@ -66,16 +56,10 @@ public sealed class AnthropicSkillsIntegrationTests
[Fact]
public async Task ListAnthropicManagedSkillsAsync()
{
AnthropicClient? anthropicClient;
try
{
anthropicClient = new() { ApiKey = TestConfiguration.GetRequiredValue(TestSettings.AnthropicApiKey) };
}
catch (InvalidOperationException ex)
{
Assert.Skip("Anthropic configuration could not be loaded. Error:" + ex.Message);
return;
}
Assert.SkipWhen(SkipReason is not null, SkipReason ?? string.Empty);
// Arrange
AnthropicClient anthropicClient = new() { ApiKey = TestConfiguration.GetRequiredValue(TestSettings.AnthropicApiKey) };
// Act
SkillListPage skills = await anthropicClient.Beta.Skills.List(
@@ -41,14 +41,7 @@ param(
[string] $Repository = "foundry-hosting-it",
[string] $TestContainerProject = "dotnet/tests/Foundry.Hosting.IntegrationTests.TestContainer",
# Explicit opt-in for the no-rebuild fast path. CI sets this after running the
# "Build Foundry hosted IT (and its deps)" step, which guarantees the prebuilt
# library DLLs match current source. Off by default so local invocations always
# let publish rebuild ProjectReferences and never produce an image whose tag is
# computed from current source while the contents come from a stale build.
[switch] $UsePrebuiltProjectReferences
[string] $TestContainerProject = "dotnet/tests/Foundry.Hosting.IntegrationTests.TestContainer"
)
$ErrorActionPreference = "Stop"
@@ -107,60 +100,7 @@ if (Test-Path $out) {
Remove-Item -Recurse -Force $out
}
# Conditionally tell publish to skip rebuilding ProjectReferences and consume the
# prebuilt library DLLs in place. This avoids two failure modes that arise when
# the CI workflow runs a `dotnet build` of the same library projects immediately
# before this script:
# 1) MSB3026 "file is being used by another process" when publish's MSBuild
# tries to overwrite src/<lib>/bin/Release/net10.0/<lib>.dll while the
# previous build's shared-compilation server still holds a file handle.
# 2) Publish needlessly rebuilding identical managed (RID-agnostic) library
# DLLs that prebuild already produced.
# Gated on -UsePrebuiltProjectReferences (a strict opt-in) instead of marker
# detection, because a developer machine may have a stale Release build of the
# libraries from days ago; using those would silently produce an image whose
# content is older than the source the tag is computed from.
$publishExtraArgs = @()
if ($UsePrebuiltProjectReferences) {
Write-Host "-UsePrebuiltProjectReferences: skipping ProjectReference rebuild." -ForegroundColor DarkGray
$publishExtraArgs += "-p:BuildProjectReferences=false"
} else {
# Preflight: in default (rebuild) mode, publish propagates RuntimeIdentifier=linux-musl-x64
# to library ProjectReferences and writes their intermediates to a RID-suffixed obj path
# (e.g. obj/Release/net10.0/linux-musl-x64/). DefaultItemExcludes follows the new
# IntermediateOutputPath, so any *.AssemblyInfo.cs left in obj/Release/net10.0/ from a
# prior `dotnet build` is no longer excluded and gets picked up by the **/*.cs Compile
# glob, producing CS0579 "duplicate attribute" errors. Detect that state up front and
# tell the user exactly how to recover.
$staleObjProbes = @(
"dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/obj/Release/net10.0",
"dotnet/src/Microsoft.Agents.AI.Foundry/obj/Release/net10.0",
"dotnet/src/Microsoft.Agents.AI/obj/Release/net10.0",
"dotnet/src/Microsoft.Agents.AI.Abstractions/obj/Release/net10.0"
)
$stale = @($staleObjProbes | Where-Object { Test-Path (Join-Path $_ "*.AssemblyInfo.cs") })
if ($stale.Count -gt 0) {
$msg = @(
"Detected prior Release/net10.0 build outputs in:"
($stale | ForEach-Object { " - $_" })
""
"Publish would propagate -r linux-musl-x64 to those ProjectReferences and the"
"leftover obj/Release/net10.0/*.AssemblyInfo.cs files would cause CS0579 duplicate"
"attribute errors. Pick one:"
" (a) Pass -UsePrebuiltProjectReferences (skips ProjectReference rebuild and"
" uses the existing src/<lib>/bin/Release/net10.0/*.dll outputs in place)."
" Only safe when you know those DLLs match current source - this is the path"
" CI uses immediately after its 'Build Foundry hosted IT (and its deps)' step."
" (b) Remove the stale obj/Release trees, e.g.:"
" Remove-Item -Recurse -Force dotnet/src/Microsoft.Agents.AI*/obj/Release"
" and re-run."
) -join "`n"
throw $msg
}
Write-Host "Letting publish build ProjectReferences (pass -UsePrebuiltProjectReferences in CI to skip)." -ForegroundColor DarkGray
}
dotnet publish $TestContainerProject -c Release -f net10.0 -r linux-musl-x64 --self-contained false -o $out @publishExtraArgs --tl:off | Out-Host
dotnet publish $TestContainerProject -c Release -f net10.0 -r linux-musl-x64 --self-contained false -o $out --tl:off | Out-Host
if ($LASTEXITCODE -ne 0) {
throw "dotnet publish failed with exit code $LASTEXITCODE."
}
@@ -13,6 +13,8 @@ namespace Microsoft.Agents.AI.DurableTask.IntegrationTests;
[Trait("Category", "SampleValidation")]
public sealed class ConsoleAppSamplesValidation(ITestOutputHelper outputHelper) : SamplesValidationBase(outputHelper)
{
private const string SkipFlakyTimingTest = "Flaky: timing-dependent LLM test, see https://github.com/microsoft/agent-framework/issues/4971";
private static readonly string s_samplesPath = Path.GetFullPath(
Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "..", "..", "..", "..", "..", "samples", "04-hosting", "DurableAgents", "ConsoleApps"));
@@ -67,7 +69,7 @@ public sealed class ConsoleAppSamplesValidation(ITestOutputHelper outputHelper)
});
}
[RetryFact(2, 5000)]
[Fact]
public async Task SingleAgentOrchestrationChainingSampleValidationAsync()
{
using CancellationTokenSource testTimeoutCts = this.CreateTestTimeoutCts();
@@ -103,7 +105,7 @@ public sealed class ConsoleAppSamplesValidation(ITestOutputHelper outputHelper)
});
}
[RetryFact(2, 5000)]
[Fact]
public async Task MultiAgentConcurrencySampleValidationAsync()
{
using CancellationTokenSource testTimeoutCts = this.CreateTestTimeoutCts();
@@ -158,7 +160,7 @@ public sealed class ConsoleAppSamplesValidation(ITestOutputHelper outputHelper)
});
}
[RetryFact(2, 5000)]
[Fact]
public async Task MultiAgentConditionalSampleValidationAsync()
{
using CancellationTokenSource testTimeoutCts = this.CreateTestTimeoutCts();
@@ -235,14 +237,14 @@ public sealed class ConsoleAppSamplesValidation(ITestOutputHelper outputHelper)
Assert.True(foundSuccess, "Orchestration did not complete successfully.");
}
[RetryFact(2, 5000)]
[Fact(Skip = SkipFlakyTimingTest)]
public async Task SingleAgentOrchestrationHITLSampleValidationAsync()
{
string samplePath = Path.Combine(s_samplesPath, "05_AgentOrchestration_HITL");
await this.RunSampleTestAsync(samplePath, async (process, logs) =>
{
using CancellationTokenSource testTimeoutCts = this.CreateTestTimeoutCts(TimeSpan.FromSeconds(180));
using CancellationTokenSource testTimeoutCts = this.CreateTestTimeoutCts();
// Start the HITL orchestration following the happy path from README
await this.WriteInputAsync(process, "The Future of Artificial Intelligence", testTimeoutCts.Token);
@@ -258,7 +260,7 @@ public sealed class ConsoleAppSamplesValidation(ITestOutputHelper outputHelper)
while ((line = this.ReadLogLine(logs, testTimeoutCts.Token)) != null)
{
// Look for notification that content is ready. The first time we see this, we should send a rejection.
// Subsequent times we see this, we should send approval (LLM may produce extra review cycles).
// The second time we see this, we should send approval.
if (line.Contains("Content is ready for review", StringComparison.OrdinalIgnoreCase))
{
if (!rejectionSent)
@@ -273,15 +275,20 @@ public sealed class ConsoleAppSamplesValidation(ITestOutputHelper outputHelper)
testTimeoutCts.Token);
rejectionSent = true;
}
else
else if (!approvalSent)
{
// Approve any subsequent draft (LLM non-determinism may produce extra review cycles)
// Prompt: Approve? (y/n):
await this.WriteInputAsync(process, "y", testTimeoutCts.Token);
// Prompt: Feedback (optional):
await this.WriteInputAsync(process, "Looks good!", testTimeoutCts.Token);
approvalSent = true;
}
else
{
// This should never happen
Assert.Fail("Unexpected message found.");
}
}
// Look for success message
@@ -304,14 +311,14 @@ public sealed class ConsoleAppSamplesValidation(ITestOutputHelper outputHelper)
});
}
[RetryFact(2, 5000)]
[Fact(Skip = SkipFlakyTimingTest)]
public async Task LongRunningToolsSampleValidationAsync()
{
string samplePath = Path.Combine(s_samplesPath, "06_LongRunningTools");
await this.RunSampleTestAsync(samplePath, async (process, logs) =>
{
// This test takes a bit longer to run due to the multiple agent interactions and the lengthy content generation.
using CancellationTokenSource testTimeoutCts = this.CreateTestTimeoutCts(TimeSpan.FromSeconds(180));
using CancellationTokenSource testTimeoutCts = this.CreateTestTimeoutCts(TimeSpan.FromSeconds(90));
// Test starting an agent that schedules a content generation orchestration
await this.WriteInputAsync(
@@ -328,7 +335,7 @@ public sealed class ConsoleAppSamplesValidation(ITestOutputHelper outputHelper)
while ((line = this.ReadLogLine(logs, testTimeoutCts.Token)) != null)
{
// Look for notification that content is ready. The first time we see this, we should send a rejection.
// Subsequent times we see this, we should send approval (LLM may produce extra review cycles).
// The second time we see this, we should send approval.
if (line.Contains("NOTIFICATION: Please review the following content for approval", StringComparison.OrdinalIgnoreCase))
{
// Wait for the notification to be fully written to the console
@@ -343,15 +350,20 @@ public sealed class ConsoleAppSamplesValidation(ITestOutputHelper outputHelper)
testTimeoutCts.Token);
rejectionSent = true;
}
else
else if (!approvalSent)
{
// Approve any subsequent draft (LLM non-determinism may produce extra review cycles)
// Approve the content. Note that we need to send a newline character to the console first before sending the input.
await this.WriteInputAsync(
process,
"\nApprove the content",
testTimeoutCts.Token);
approvalSent = true;
}
else
{
// This should never happen
Assert.Fail("Unexpected message found.");
}
}
// Look for success message
@@ -384,14 +396,14 @@ public sealed class ConsoleAppSamplesValidation(ITestOutputHelper outputHelper)
});
}
[RetryFact(2, 5000)]
[Fact(Skip = SkipFlakyTimingTest)]
public async Task ReliableStreamingSampleValidationAsync()
{
string samplePath = Path.Combine(s_samplesPath, "07_ReliableStreaming");
await this.RunSampleTestAsync(samplePath, async (process, logs) =>
{
// This test takes a bit longer to run due to the multiple agent interactions and the lengthy content generation.
using CancellationTokenSource testTimeoutCts = this.CreateTestTimeoutCts(TimeSpan.FromSeconds(150));
using CancellationTokenSource testTimeoutCts = this.CreateTestTimeoutCts(TimeSpan.FromSeconds(90));
// Test the agent endpoint with a simple prompt
await this.WriteInputAsync(process, "Plan a 5-day trip to Seattle. Include daily activities.", testTimeoutCts.Token);
@@ -19,9 +19,11 @@ namespace Microsoft.Agents.AI.DurableTask.IntegrationTests;
[Trait("Category", "Integration")]
public sealed class ExternalClientTests(ITestOutputHelper outputHelper) : IDisposable
{
private const string SkipFlakyTimingTest = "Flaky: timing-dependent LLM test, see https://github.com/microsoft/agent-framework/issues/4971";
private static readonly TimeSpan s_defaultTimeout = Debugger.IsAttached
? TimeSpan.FromMinutes(5)
: TimeSpan.FromSeconds(120);
: TimeSpan.FromSeconds(60);
private static readonly IConfiguration s_configuration =
new ConfigurationBuilder()
@@ -36,7 +38,7 @@ public sealed class ExternalClientTests(ITestOutputHelper outputHelper) : IDispo
public void Dispose() => this._cts.Dispose();
[RetryFact(2, 5000)]
[Fact]
public async Task SimplePromptAsync()
{
// Setup
@@ -75,7 +77,7 @@ public sealed class ExternalClientTests(ITestOutputHelper outputHelper) : IDispo
Assert.Contains(agentLogs, log => log.EventId.Name == "LogAgentResponse");
}
[RetryFact(2, 5000)]
[Fact(Skip = SkipFlakyTimingTest)]
public async Task CallFunctionToolsAsync()
{
int weatherToolInvocationCount = 0;
@@ -127,7 +129,7 @@ public sealed class ExternalClientTests(ITestOutputHelper outputHelper) : IDispo
Assert.Equal(1, packingListToolInvocationCount);
}
[RetryFact(2, 5000)]
[Fact(Skip = SkipFlakyTimingTest)]
public async Task CallLongRunningFunctionToolsAsync()
{
[Description("Starts a greeting workflow and returns the workflow instance ID")]
@@ -217,7 +217,7 @@ public abstract class SamplesValidationBase : IAsyncLifetime
/// </summary>
protected CancellationTokenSource CreateTestTimeoutCts(TimeSpan? timeout = null)
{
TimeSpan testTimeout = Debugger.IsAttached ? TimeSpan.FromMinutes(5) : timeout ?? TimeSpan.FromSeconds(120);
TimeSpan testTimeout = Debugger.IsAttached ? TimeSpan.FromMinutes(5) : timeout ?? TimeSpan.FromSeconds(60);
return new CancellationTokenSource(testTimeout);
}
@@ -22,7 +22,7 @@ public sealed class WorkflowConsoleAppSamplesValidation(ITestOutputHelper output
/// <inheritdoc />
protected override string TaskHubPrefix => "workflow";
[RetryFact(2, 5000)]
[Fact]
public async Task SequentialWorkflowSampleValidationAsync()
{
using CancellationTokenSource testTimeoutCts = this.CreateTestTimeoutCts(s_testTimeout);
@@ -71,7 +71,7 @@ public sealed class WorkflowConsoleAppSamplesValidation(ITestOutputHelper output
});
}
[RetryFact(2, 5000)]
[Fact]
public async Task ConcurrentWorkflowSampleValidationAsync()
{
using CancellationTokenSource testTimeoutCts = this.CreateTestTimeoutCts(s_testTimeout);
@@ -120,7 +120,7 @@ public sealed class WorkflowConsoleAppSamplesValidation(ITestOutputHelper output
});
}
[RetryFact(2, 5000)]
[Fact]
public async Task ConditionalEdgesWorkflowSampleValidationAsync()
{
using CancellationTokenSource testTimeoutCts = this.CreateTestTimeoutCts(s_testTimeout);
@@ -182,7 +182,7 @@ public sealed class WorkflowConsoleAppSamplesValidation(ITestOutputHelper output
}
}
[RetryFact(2, 5000)]
[Fact]
public async Task WorkflowEventsSampleValidationAsync()
{
using CancellationTokenSource testTimeoutCts = this.CreateTestTimeoutCts(s_testTimeout);
@@ -278,7 +278,7 @@ public sealed class WorkflowConsoleAppSamplesValidation(ITestOutputHelper output
});
}
[RetryFact(2, 5000)]
[Fact]
public async Task WorkflowSharedStateSampleValidationAsync()
{
using CancellationTokenSource testTimeoutCts = this.CreateTestTimeoutCts(s_testTimeout);
@@ -376,7 +376,7 @@ public sealed class WorkflowConsoleAppSamplesValidation(ITestOutputHelper output
});
}
[RetryFact(2, 5000)]
[Fact]
public async Task SubWorkflowsSampleValidationAsync()
{
using CancellationTokenSource testTimeoutCts = this.CreateTestTimeoutCts(s_testTimeout);
@@ -452,7 +452,7 @@ public sealed class WorkflowConsoleAppSamplesValidation(ITestOutputHelper output
});
}
[RetryFact(2, 5000)]
[Fact]
public async Task WorkflowHITLSampleValidationAsync()
{
using CancellationTokenSource testTimeoutCts = this.CreateTestTimeoutCts(s_testTimeout);
@@ -505,7 +505,7 @@ public sealed class WorkflowConsoleAppSamplesValidation(ITestOutputHelper output
});
}
[RetryFact(2, 5000)]
[Fact]
public async Task WorkflowAndAgentsSampleValidationAsync()
{
using CancellationTokenSource testTimeoutCts = this.CreateTestTimeoutCts(s_testTimeout);
@@ -37,7 +37,7 @@ public class AgentFrameworkResponseHandlerTelemetryTests
public async Task CreateAsync_DefaultAgent_EmitsInvokeAgentSpanAsync()
{
// Arrange
var activities = new ConcurrentActivityList();
var activities = new List<Activity>();
using var tracerProvider = Sdk.CreateTracerProviderBuilder()
.AddSource(ResponsesSourceName)
.AddInMemoryExporter(activities)
@@ -56,7 +56,7 @@ public class AgentFrameworkResponseHandlerTelemetryTests
await foreach (var _ in handler.CreateAsync(request, context, CancellationToken.None)) { }
// Assert — filter by agent name to isolate this test's span from any parallel test spans
var mySpan = Assert.Single(activities.Snapshot().Where(a => TelemetryTestAgent.AgentName.Equals(a.GetTagItem("gen_ai.agent.name"))).ToList());
var mySpan = Assert.Single(activities.Where(a => TelemetryTestAgent.AgentName.Equals(a.GetTagItem("gen_ai.agent.name"))).ToList());
Assert.Equal("invoke_agent", mySpan.GetTagItem("gen_ai.operation.name"));
Assert.NotNull(mySpan.GetTagItem("gen_ai.agent.id"));
}
@@ -65,7 +65,7 @@ public class AgentFrameworkResponseHandlerTelemetryTests
public async Task CreateAsync_KeyedAgent_EmitsInvokeAgentSpanAsync()
{
// Arrange
var activities = new ConcurrentActivityList();
var activities = new List<Activity>();
using var tracerProvider = Sdk.CreateTracerProviderBuilder()
.AddSource(ResponsesSourceName)
.AddInMemoryExporter(activities)
@@ -84,7 +84,7 @@ public class AgentFrameworkResponseHandlerTelemetryTests
await foreach (var _ in handler.CreateAsync(request, context, CancellationToken.None)) { }
// Assert — filter by agent name to isolate this test's span
var mySpan = Assert.Single(activities.Snapshot().Where(a => TelemetryTestAgent.AgentName.Equals(a.GetTagItem("gen_ai.agent.name"))).ToList());
var mySpan = Assert.Single(activities.Where(a => TelemetryTestAgent.AgentName.Equals(a.GetTagItem("gen_ai.agent.name"))).ToList());
Assert.Equal("invoke_agent", mySpan.GetTagItem("gen_ai.operation.name"));
}
@@ -95,8 +95,8 @@ public class AgentFrameworkResponseHandlerTelemetryTests
// If ApplyOpenTelemetry double-wraps, an extra span would appear on ResponsesSourceName.
// If it correctly skips wrapping, only the pre-wrap's unique source emits spans.
var preWrapSource = Guid.NewGuid().ToString();
var preWrapActivities = new ConcurrentActivityList();
var responsesActivities = new ConcurrentActivityList();
var preWrapActivities = new List<Activity>();
var responsesActivities = new List<Activity>();
using var preWrapProvider = Sdk.CreateTracerProviderBuilder()
.AddSource(preWrapSource)
@@ -125,19 +125,18 @@ public class AgentFrameworkResponseHandlerTelemetryTests
await foreach (var _ in handler.CreateAsync(request, context, CancellationToken.None)) { }
// Assert — pre-wrap source emits exactly 1 span (agent ran)
var preWrapSnapshot = preWrapActivities.Snapshot();
Assert.Single(preWrapSnapshot);
Assert.Equal("invoke_agent", preWrapSnapshot[0].GetTagItem("gen_ai.operation.name"));
Assert.Single(preWrapActivities);
Assert.Equal("invoke_agent", preWrapActivities[0].GetTagItem("gen_ai.operation.name"));
// ResponsesSourceName emits 0 spans — ApplyOpenTelemetry skipped wrapping the pre-instrumented agent
Assert.DoesNotContain(responsesActivities.Snapshot(), a => TelemetryTestAgent.AgentName.Equals(a.GetTagItem("gen_ai.agent.name")));
Assert.DoesNotContain(responsesActivities, a => TelemetryTestAgent.AgentName.Equals(a.GetTagItem("gen_ai.agent.name")));
}
[Fact]
public async Task CreateAsync_DefaultAgent_SpanDisplayNameContainsAgentNameAsync()
{
// Arrange
var activities = new ConcurrentActivityList();
var activities = new List<Activity>();
using var tracerProvider = Sdk.CreateTracerProviderBuilder()
.AddSource(ResponsesSourceName)
.AddInMemoryExporter(activities)
@@ -156,7 +155,7 @@ public class AgentFrameworkResponseHandlerTelemetryTests
await foreach (var _ in handler.CreateAsync(request, context, CancellationToken.None)) { }
// Assert — display name follows "invoke_agent {Name}({Id})" convention; filter by agent name to isolate
var mySpan = Assert.Single(activities.Snapshot().Where(a => TelemetryTestAgent.AgentName.Equals(a.GetTagItem("gen_ai.agent.name"))).ToList());
var mySpan = Assert.Single(activities.Where(a => TelemetryTestAgent.AgentName.Equals(a.GetTagItem("gen_ai.agent.name"))).ToList());
Assert.Contains("invoke_agent", mySpan.DisplayName, StringComparison.Ordinal);
Assert.Contains(TelemetryTestAgent.AgentName, mySpan.DisplayName, StringComparison.Ordinal);
}
@@ -232,35 +231,4 @@ public class AgentFrameworkResponseHandlerTelemetryTests
}
private sealed class TelemetryAgentSession : AgentSession;
/// <summary>
/// Thread-safe <see cref="ICollection{Activity}"/> used by OTel's InMemoryExporter to capture
/// activities emitted on globally-listened sources. Required because the exporter writes into
/// the supplied collection from background Activity completion callbacks while the test thread
/// may be enumerating it for assertions, and other tests in the same assembly may emit on the
/// same source concurrently. A plain <see cref="List{Activity}"/> trips
/// "Collection was modified; enumeration operation may not execute." in that scenario.
/// </summary>
private sealed class ConcurrentActivityList : ICollection<Activity>
{
private readonly List<Activity> _items = new();
private readonly object _gate = new();
public int Count { get { lock (this._gate) { return this._items.Count; } } }
public bool IsReadOnly => false;
public void Add(Activity item) { lock (this._gate) { this._items.Add(item); } }
public void Clear() { lock (this._gate) { this._items.Clear(); } }
public bool Contains(Activity item) { lock (this._gate) { return this._items.Contains(item); } }
public void CopyTo(Activity[] array, int arrayIndex) { lock (this._gate) { this._items.CopyTo(array, arrayIndex); } }
public bool Remove(Activity item) { lock (this._gate) { return this._items.Remove(item); } }
public Activity[] Snapshot()
{
lock (this._gate) { return this._items.ToArray(); }
}
public IEnumerator<Activity> GetEnumerator() => ((IEnumerable<Activity>)this.Snapshot()).GetEnumerator();
System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() => this.GetEnumerator();
}
}
@@ -207,10 +207,9 @@ public class InputConverterTests
[Fact]
public void ConvertOutputItemsToMessages_FunctionToolCallOutput_ReturnsToolMessage()
{
// Spec-compliant payload: a JSON string literal.
var funcOutput = new OutputItemFunctionToolCallOutput(
callId: "call_def",
output: BinaryData.FromString("\"result data\""));
output: BinaryData.FromString("result data"));
var messages = InputConverter.ConvertOutputItemsToMessages([funcOutput]);
@@ -219,52 +218,6 @@ public class InputConverterTests
var result = messages[0].Contents.OfType<FunctionResultContent>().FirstOrDefault();
Assert.NotNull(result);
Assert.Equal("call_def", result.CallId);
// Round-trip: the JSON-string wire payload is unwrapped to the original tool result text.
Assert.Equal("result data", result.Result as string);
}
[Fact]
public void ConvertOutputItemsToMessages_FunctionToolCallOutput_LegacyRawJsonArray_PassesThrough()
{
// Legacy/non-conforming producers that emitted a raw JSON value (array/object) in
// `output` are tolerated: the raw text is forwarded as the FunctionResultContent.Result
// so the model still sees the original tool-output shape on replay.
var funcOutput = new OutputItemFunctionToolCallOutput(
callId: "call_legacy",
output: BinaryData.FromString("[{\"id\":1}]"));
var messages = InputConverter.ConvertOutputItemsToMessages([funcOutput]);
var result = messages[0].Contents.OfType<FunctionResultContent>().FirstOrDefault();
Assert.NotNull(result);
Assert.Equal("[{\"id\":1}]", result.Result as string);
}
[Fact]
public void ConvertInputToMessages_FunctionCallOutput_JsonStringPayload_Unwraps()
{
// Spec-compliant inbound payload — a JSON string literal — must be unwrapped so
// FunctionResultContent.Result is the original tool result text, not the JSON-encoded form.
var input = new[]
{
new
{
type = "function_call_output",
id = "fc_out_002",
call_id = "call_456",
output = "sunny"
}
};
var request = new CreateResponse();
request.Input = BinaryData.FromObjectAsJson(input);
var messages = InputConverter.ConvertInputToMessages(request);
Assert.Single(messages);
var funcResult = messages[0].Contents.OfType<FunctionResultContent>().FirstOrDefault();
Assert.NotNull(funcResult);
Assert.Equal("sunny", funcResult.Result as string);
}
[Fact]
@@ -616,10 +616,9 @@ public class OutputConverterTests
Assert.IsType<ResponseCompletedEvent>(events[^1]);
}
// K-06: FRC payloads are wrapped as JSON string literals on the wire so the field is
// always a spec-compliant OpenAI Responses `function_call_output.output` string value.
// K-06: FRC string results are emitted as raw text on the wire (not JSON-quoted).
[Fact]
public async Task ConvertUpdatesToEventsAsync_FunctionResultStringPayload_EmittedAsJsonStringAsync()
public async Task ConvertUpdatesToEventsAsync_FunctionResultStringPayload_EmittedAsRawTextAsync()
{
var (stream, _) = CreateTestStream();
var update = new AgentResponseUpdate { Contents = [new FunctionResultContent("call_1", "sunny")] };
@@ -632,76 +631,8 @@ public class OutputConverterTests
var added = Assert.Single(events.OfType<ResponseOutputItemAddedEvent>());
var output = Assert.IsType<OutputItemFunctionToolCallOutput>(added.Item);
// The wire payload is a JSON string literal — `"sunny"`, not the bare bytes `sunny`.
Assert.Equal("\"sunny\"", output.Output.ToString());
}
// K-06b: List/object FRC payloads must be JSON-stringified into a JSON string value
// so the OpenAI .NET client (FunctionCallOutputResponseItem.Output: string) can parse them.
[Fact]
public async Task ConvertUpdatesToEventsAsync_FunctionResultObjectPayload_EmittedAsJsonStringAsync()
{
var (stream, _) = CreateTestStream();
var todoList = new[] { new { id = 1, text = "Buy milk" } };
var update = new AgentResponseUpdate { Contents = [new FunctionResultContent("call_1", todoList)] };
var events = new List<ResponseStreamEvent>();
await foreach (var evt in OutputConverter.ConvertUpdatesToEventsAsync(ToAsync(new[] { update }), stream))
{
events.Add(evt);
}
var added = Assert.Single(events.OfType<ResponseOutputItemAddedEvent>());
var output = Assert.IsType<OutputItemFunctionToolCallOutput>(added.Item);
// The wire payload must be a quoted JSON string containing the JSON-serialized object.
var raw = output.Output.ToString();
Assert.StartsWith("\"", raw);
Assert.EndsWith("\"", raw);
// The unwrapped value must round-trip back to the original JSON.
var inner = System.Text.Json.JsonSerializer.Deserialize<string>(raw);
Assert.Equal("[{\"id\":1,\"text\":\"Buy milk\"}]", inner);
}
// K-06c: A JsonElement of kind String must not be double-encoded.
[Fact]
public async Task ConvertUpdatesToEventsAsync_FunctionResultJsonElementStringPayload_NotDoubleEncodedAsync()
{
var (stream, _) = CreateTestStream();
using var doc = System.Text.Json.JsonDocument.Parse("\"sunny\"");
var update = new AgentResponseUpdate { Contents = [new FunctionResultContent("call_1", doc.RootElement.Clone())] };
var events = new List<ResponseStreamEvent>();
await foreach (var evt in OutputConverter.ConvertUpdatesToEventsAsync(ToAsync(new[] { update }), stream))
{
events.Add(evt);
}
var added = Assert.Single(events.OfType<ResponseOutputItemAddedEvent>());
var output = Assert.IsType<OutputItemFunctionToolCallOutput>(added.Item);
// Must be `"sunny"`, not `"\"sunny\""`.
Assert.Equal("\"sunny\"", output.Output.ToString());
}
// K-06d: A JsonElement of non-string kind (e.g. array) must be JSON-stringified, not
// emitted as a raw JSON array on the wire.
[Fact]
public async Task ConvertUpdatesToEventsAsync_FunctionResultJsonElementArrayPayload_EmittedAsJsonStringAsync()
{
var (stream, _) = CreateTestStream();
using var doc = System.Text.Json.JsonDocument.Parse("[{\"id\":1}]");
var update = new AgentResponseUpdate { Contents = [new FunctionResultContent("call_1", doc.RootElement.Clone())] };
var events = new List<ResponseStreamEvent>();
await foreach (var evt in OutputConverter.ConvertUpdatesToEventsAsync(ToAsync(new[] { update }), stream))
{
events.Add(evt);
}
var added = Assert.Single(events.OfType<ResponseOutputItemAddedEvent>());
var output = Assert.IsType<OutputItemFunctionToolCallOutput>(added.Item);
var raw = output.Output.ToString();
var inner = System.Text.Json.JsonSerializer.Deserialize<string>(raw);
Assert.Equal("[{\"id\":1}]", inner);
// String FRC payloads must not be double-encoded — `sunny`, not `"sunny"`.
Assert.Equal("sunny", output.Output.ToString());
}
// L-01
@@ -15,6 +15,8 @@ namespace Microsoft.Agents.AI.Hosting.AzureFunctions.IntegrationTests;
[Trait("Category", "SampleValidation")]
public sealed class SamplesValidation(ITestOutputHelper outputHelper) : IAsyncLifetime
{
private const string SkipFlakyTimingTest = "Flaky: timing-dependent LLM test, see https://github.com/microsoft/agent-framework/issues/4971";
private const string AzureFunctionsPort = "7071";
private const string AzuritePort = "10000";
private const string DtsPort = "8080";
@@ -35,7 +37,7 @@ public sealed class SamplesValidation(ITestOutputHelper outputHelper) : IAsyncLi
.Build();
private static bool s_infrastructureStarted;
private static readonly TimeSpan s_orchestrationTimeout = TimeSpan.FromMinutes(3);
private static readonly TimeSpan s_orchestrationTimeout = TimeSpan.FromMinutes(2);
// In CI, `dotnet run` builds the Functions project from scratch before the host starts, so 60s is not enough.
private static readonly TimeSpan s_functionsReadyTimeout = TimeSpan.FromSeconds(180);
@@ -60,7 +62,7 @@ public sealed class SamplesValidation(ITestOutputHelper outputHelper) : IAsyncLi
await Task.CompletedTask;
}
[RetryFact(2, 5000)]
[Fact]
public async Task SingleAgentSampleValidationAsync()
{
string samplePath = Path.Combine(s_samplesPath, "01_SingleAgent");
@@ -105,7 +107,7 @@ public sealed class SamplesValidation(ITestOutputHelper outputHelper) : IAsyncLi
});
}
[Fact(Skip = "Flaky: LLM non-determinism can produce null orchestration results")]
[Fact]
public async Task SingleAgentOrchestrationChainingSampleValidationAsync()
{
string samplePath = Path.Combine(s_samplesPath, "02_AgentOrchestration_Chaining");
@@ -148,7 +150,7 @@ public sealed class SamplesValidation(ITestOutputHelper outputHelper) : IAsyncLi
});
}
[RetryFact(2, 5000)]
[Fact]
public async Task MultiAgentOrchestrationConcurrentSampleValidationAsync()
{
string samplePath = Path.Combine(s_samplesPath, "03_AgentOrchestration_Concurrency");
@@ -198,7 +200,7 @@ public sealed class SamplesValidation(ITestOutputHelper outputHelper) : IAsyncLi
});
}
[RetryFact(2, 5000)]
[Fact]
public async Task MultiAgentOrchestrationConditionalsSampleValidationAsync()
{
string samplePath = Path.Combine(s_samplesPath, "04_AgentOrchestration_Conditionals");
@@ -216,7 +218,7 @@ public sealed class SamplesValidation(ITestOutputHelper outputHelper) : IAsyncLi
});
}
[RetryFact(2, 5000)]
[Fact]
public async Task SingleAgentOrchestrationHITLSampleValidationAsync()
{
string samplePath = Path.Combine(s_samplesPath, "05_AgentOrchestration_HITL");
@@ -272,7 +274,7 @@ public sealed class SamplesValidation(ITestOutputHelper outputHelper) : IAsyncLi
});
}
[RetryFact(2, 5000)]
[Fact(Skip = SkipFlakyTimingTest)]
public async Task LongRunningToolsSampleValidationAsync()
{
string samplePath = Path.Combine(s_samplesPath, "06_LongRunningTools");
@@ -314,7 +316,7 @@ public sealed class SamplesValidation(ITestOutputHelper outputHelper) : IAsyncLi
}
},
message: "Orchestration is requesting human feedback",
timeout: TimeSpan.FromSeconds(180));
timeout: TimeSpan.FromSeconds(60));
// Approve the content
Uri approvalUri = new($"{runAgentUri}?thread_id={sessionId}");
@@ -334,7 +336,7 @@ public sealed class SamplesValidation(ITestOutputHelper outputHelper) : IAsyncLi
}
},
message: "Content published notification is logged",
timeout: TimeSpan.FromSeconds(180));
timeout: TimeSpan.FromSeconds(60));
// Verify the final orchestration status by asking the agent for the status
Uri statusUri = new($"{runAgentUri}?thread_id={sessionId}");
@@ -358,11 +360,11 @@ public sealed class SamplesValidation(ITestOutputHelper outputHelper) : IAsyncLi
return isCompleted && hasContent;
},
message: "Orchestration is completed",
timeout: TimeSpan.FromSeconds(180));
timeout: TimeSpan.FromSeconds(60));
});
}
[RetryFact(2, 5000)]
[Fact]
public async Task AgentAsMcpToolAsync()
{
string samplePath = Path.Combine(s_samplesPath, "07_AgentAsMcpTool");
@@ -402,7 +404,7 @@ public sealed class SamplesValidation(ITestOutputHelper outputHelper) : IAsyncLi
});
}
[RetryFact(2, 5000)]
[Fact(Skip = SkipFlakyTimingTest)]
public async Task ReliableStreamingSampleValidationAsync()
{
string samplePath = Path.Combine(s_samplesPath, "08_ReliableStreaming");
@@ -1,153 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
//using System.Collections.Generic;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using FluentAssertions;
using Microsoft.Agents.AI.Workflows.Specialized.Magentic;
using Microsoft.Extensions.AI;
namespace Microsoft.Agents.AI.Workflows.UnitTests;
public class MagenticManagerTests
{
private static void CheckMessage(ChatMessage message, string expectedText, bool runPropertySmokeTest = false, bool skipCreatedAt = true)
{
message.Text.Should().Be(expectedText);
if (runPropertySmokeTest)
{
message.AuthorName.Should().Be(nameof(MagenticOrchestrator));
if (!skipCreatedAt)
{
message.CreatedAt.Should().NotBeNull().And.NotBeBefore(DateTimeOffset.UtcNow.AddDays(-1));
}
message.Role.Should().Be(ChatRole.Assistant);
message.MessageId.Should().NotBeNull();
}
}
[Theory]
[InlineData(false)]
[InlineData(true)]
public async Task Test_MagenticManager_UpdatePlanAsync(bool hasExistingPlan)
{
TestReplayAgent testAgent = new(name: nameof(MagenticOrchestrator),
messages:
[
[new(ChatRole.Assistant, "Facts")],
[new(ChatRole.Assistant, "Plan")],
]);
TestEchoAgent participant = new(name: "Echo");
MagenticManager manager = new(testAgent);
MagenticTaskContext taskContext = new([new(ChatRole.User, "Task")], [participant], new TaskLimits(), null, []);
if (hasExistingPlan)
{
taskContext.TaskLedger = new(new(ChatRole.Assistant, "OldFacts"), new(ChatRole.Assistant, "OldPlan"));
}
TestRunContext runContext = new();
IWorkflowContext workflowContext = runContext.BindWorkflowContext(nameof(MagenticOrchestrator));
TaskLedger newPlan = await manager.UpdatePlanAsync(taskContext, workflowContext, CancellationToken.None);
CheckMessage(newPlan.CurrentFacts, "Facts");
CheckMessage(newPlan.CurrentPlan, "Plan");
taskContext.ChatHistory.Should().HaveCount(4);
if (hasExistingPlan)
{
ChatMessage factsRequest = taskContext.ChatHistory[0];
factsRequest.Text.Should().Contain("OldFacts");
}
ChatMessage facts = taskContext.ChatHistory[1];
facts.Should().Be(newPlan.CurrentFacts);
ChatMessage plan = taskContext.ChatHistory[3];
plan.Should().Be(newPlan.CurrentPlan);
}
[Theory]
[InlineData(0)]
[InlineData(1)]
[InlineData(2)]
[InlineData(3)]
[InlineData(4)]
public async Task Test_MagenticManager_UpdateProgressLedgerAsync(int failures)
{
List<List<ChatMessage>> turns =
TestProgressLedgerState.MissingRequired.Take(failures)
.Select<TestProgressLedgerState, List<ChatMessage>>(
state => [new ChatMessage(ChatRole.Assistant, state.ToJsonString())])
.ToList();
turns.Should().HaveCount(failures);
turns.Add([new ChatMessage(ChatRole.Assistant, TestProgressLedgerState.Default.ToJsonString())]);
TestReplayAgent testAgent = new(name: nameof(MagenticOrchestrator),
messages: turns);
TestEchoAgent participant = new(name: "Echo");
MagenticManager manager = new(testAgent);
MagenticTaskContext taskContext = new([new(ChatRole.User, "Task")], [participant], new TaskLimits(), null, []);
taskContext.TaskLedger = new(new(ChatRole.Assistant, "OldFacts"), new(ChatRole.Assistant, "OldPlan"));
TestRunContext runContext = new();
IWorkflowContext workflowContext = runContext.BindWorkflowContext(nameof(MagenticOrchestrator));
// Precondition check: ProgressLedger should be not "started"
taskContext.ProgressLedger.IsStarted.Should().BeFalse();
Func<Task> action = () => manager.UpdateProgressLedgerAsync(taskContext, workflowContext, CancellationToken.None).AsTask();
if (failures >= taskContext.TaskLimits.MaxProgressLedgerRetryCount)
{
// We expect to see an exception if the number of failures exceeds the maximum retry count
await action.Should().ThrowAsync();
taskContext.ProgressLedger.IsStarted.Should().BeFalse();
}
else
{
await action.Should().NotThrowAsync();
taskContext.ProgressLedger.IsStarted.Should().BeTrue();
TestProgressLedgerState.Default.Validate(taskContext.ProgressLedger);
}
int expectedWarnings = Math.Min(failures, 3);
runContext.Events.Should().HaveCount(expectedWarnings).And.AllBeOfType<WorkflowWarningEvent>();
}
[Fact]
public async Task Test_MagenticManager_PrepareFinalAnswerAsync()
{
TestReplayAgent testAgent = new(name: nameof(MagenticOrchestrator),
messages:
[
[
new(ChatRole.Assistant, "FinalAnswer")
],
]);
TestEchoAgent participant = new(name: "Echo");
MagenticManager manager = new(testAgent);
MagenticTaskContext taskContext = new([new(ChatRole.User, "Task")], [participant], new TaskLimits(), null, []);
TestRunContext runContext = new();
IWorkflowContext workflowContext = runContext.BindWorkflowContext(nameof(MagenticOrchestrator));
ChatMessage answer = await manager.PrepareFinalAnswerAsync(taskContext, workflowContext, CancellationToken.None);
CheckMessage(answer, "FinalAnswer", true, false);
}
}
@@ -1,214 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
using System.Text.Json;
using FluentAssertions;
using Microsoft.Agents.AI.Workflows.Specialized.Magentic;
using Microsoft.Extensions.AI;
namespace Microsoft.Agents.AI.Workflows.UnitTests;
public class MagenticProgressLedgerTests
{
public record KVPair(string key);
public record AnswerReasonPair(bool answer, string reason);
[Theory]
[InlineData(false)]
[InlineData(true)]
public void Test_ExtractJson_SucceedsWhenInBlockQuote(bool isTagged)
{
// Arrange
string json = isTagged
? "```json\n{\"key\": \"value\"}\n```"
: "```{\"key\": \"value\"}```";
string embedded = $"Some text before the JSON block.\n{json}\nSome text after the JSON block.";
ChatMessage message = new(ChatRole.Assistant, embedded);
// Act
JsonElement element = message.ExtractJson();
// Assert
KVPair? result = element.Deserialize<KVPair>();
result.Should().NotBeNull();
result.key.Should().Be("value");
}
[Fact]
public void Test_ExtractJson_SucceedsWhenScanning()
{
// Arrange
ChatMessage message = new(ChatRole.Assistant,
"""
Some text before the JSON embed.
{"key": "value"}
Some text after the JSON embed.
""");
// Act
JsonElement element = message.ExtractJson();
// Assert
KVPair? result = element.Deserialize<KVPair>();
result.Should().NotBeNull();
result.key.Should().Be("value");
}
[Fact]
public void Test_ExtractJson_FailsWhenUnbalanced()
{
// Arrange
ChatMessage message = new(ChatRole.Assistant,
"""
Some text before the JSON embed.
{"key": { "key2": "value" }
Some text after the JSON embed.
""");
// Act
Func<JsonElement> action = () => message.ExtractJson();
// Assert
action.Should().Throw();
}
[Fact]
public void Test_ExtractJson_FailsWhenNoJson()
{
// Arrange
ChatMessage message = new(ChatRole.Assistant,
"""
Some text, without JSON
""");
// Act
Func<JsonElement> action = () => message.ExtractJson();
// Assert
action.Should().Throw();
}
[Fact]
public void Test_ExtractJson_SuceedsWithQuotesBrackets()
{
// Arrange
ChatMessage message = new(ChatRole.Assistant,
"""
{"reason":"the output contained }", "answer": false}
""");
// Act
JsonElement element = message.ExtractJson();
// Assert
AnswerReasonPair? result = element.Deserialize<AnswerReasonPair>();
result.Should().NotBeNull();
result.reason.Should().Be("the output contained }");
result.answer.Should().BeFalse();
}
public static readonly string TestTeamNames = string.Join(", ", ["CodingAgent", "CodeExecutor", "WebSurferAgent", "FileSurferAgent"]);
[Fact]
public void Test_ProgressLedgerState_IsEmptyWhenStarted()
{
// Arrange/Act
MagenticProgressLedger ledger = new(TestTeamNames, []);
// Assert
ledger.State.Should().BeNull();
ledger.IsStarted.Should().BeFalse();
ledger.TryGetCurrentSlotValue(TestProgressLedgerState.CustomSlot1, out _).Should().BeFalse();
ledger.TryGetCurrentSlotValue(TestProgressLedgerState.CustomSlot2, out _).Should().BeFalse();
}
[Theory]
[InlineData(0, "RequiredOnly")]
[InlineData(1, "IncludeCustom")]
public void Test_ProgressLedgerState_IsNotEmptyWhenRestored(int caseIndex, string _)
{
// Arrange
TestProgressLedgerState state = TestProgressLedgerState.Working[caseIndex];
JsonElement element = state.ToJson();
// Act
MagenticProgressLedger ledger = new(TestTeamNames, [], element);
// Assert
ledger.State.Should().Be(element);
state.Validate(ledger);
}
[Theory]
[InlineData(0, "RequiredOnly")]
[InlineData(1, "IncludeCustom")]
public void Test_ProgressLedgerState_SwitchesToStartedWhenStateUpdates(int caseIndex, string _)
{
// Arrange
MagenticProgressLedger ledger = new(TestTeamNames, []);
TestProgressLedgerState targetState = TestProgressLedgerState.Working[caseIndex];
JsonElement element = targetState.ToJson();
ledger.State.Should().BeNull();
// Act
ledger.TryUpdateState(element).Should().BeTrue();
// Assert
ledger.State.Should().Be(element);
targetState.Validate(ledger);
}
[Theory]
[InlineData(0, "is_request_satisfied")]
[InlineData(1, "is_in_loop")]
[InlineData(2, "is_progress_being_made")]
[InlineData(3, "instruction_or_question")]
[InlineData(4, "next_speaker")]
public void Test_ProgressLedgerState_FailsToUpdateWhenRequiredAnswersMissing(int caseIndex, string _)
{
// Arrange
MagenticProgressLedger ledger = new(TestTeamNames, []);
TestProgressLedgerState targetState = TestProgressLedgerState.MissingRequired[caseIndex];
JsonElement element = targetState.ToJson();
ledger.State.Should().BeNull();
// Act
ledger.TryUpdateState(element).Should().BeFalse();
ledger.State.Should().BeNull();
}
[Theory]
[InlineData(true)]
[InlineData(false)]
public void Test_ProgressLedgerState_GeneratesCorrectSchema(bool includeCustom)
{
// Arrange
MagenticProgressLedger ledger = new(TestTeamNames, includeCustom
? [TestProgressLedgerState.CustomSlot1, TestProgressLedgerState.CustomSlot2]
: []);
// Act
(string questionBlock, string answerSchema) = ledger.FormatQuestions();
foreach (ProgressLedgerSlot slot in ledger.Slots)
{
// Best-efforts validation: I do not want to make it super-brittle and check for 1:1: with the template
// since that is effectively checking that string formatting works right to some extent.
questionBlock.Should().Contain(slot.Question);
answerSchema.Should().Contain(slot.Key);
answerSchema.Should().Contain(slot.SchemaType);
if (!string.IsNullOrWhiteSpace(slot.SchemaTypeSuffix))
{
answerSchema.Should().Contain(slot.SchemaTypeSuffix);
}
}
}
}
@@ -1,88 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
using System.Text.Json;
using System.Text.Json.Serialization;
using FluentAssertions;
namespace Microsoft.Agents.AI.Workflows.UnitTests;
public sealed record Slot<T>(T? answer, string? reason);
public record TestProgressLedgerState(Slot<bool?>? is_request_satisfied = null,
Slot<bool?>? is_in_loop = null,
Slot<bool?>? is_progress_being_made = null,
Slot<string>? instruction_or_question = null,
Slot<string>? next_speaker = null,
Slot<bool?>? custom1 = null,
Slot<string>? custom2 = null)
{
public TestProgressLedgerState() : this(new Slot<bool?>(false, "is_request_satisfied_reason"),
new Slot<bool?>(false, "is_in_loop_reason"),
new Slot<bool?>(false, "is_progress_being_made_reason"),
new Slot<string>("Answer", "instruction_or_question_reason"),
new Slot<string>("Lorem Ipsum", "next_speaker_reason"),
new Slot<bool?>(false, "custom1_reason"),
new Slot<string>("Custom2", "custom2_reason"))
{ }
public string ToJsonString() => this.ToJson().ToString();
private static readonly JsonSerializerOptions s_options = new()
{
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull
};
public JsonElement ToJson() => JsonSerializer.SerializeToElement(this, s_options);
internal static BooleanProgressLedgerSlot CustomSlot1 = new("custom1", "Custom Slot 1");
internal static StringProgressLedgerSlot CustomSlot2 = new("custom2", "Custom Slot 2");
public static bool TryGetCustom1(MagenticProgressLedger state, out bool result)
=> state.TryGetCurrentSlotValue(CustomSlot1, out result);
public static bool TryGetCustom2(MagenticProgressLedger state, out string? result)
=> state.TryGetCurrentSlotValue(CustomSlot2, out result);
public void Validate(MagenticProgressLedger state)
{
state.IsRequestSatisfied.Should().Be(this.is_request_satisfied!.answer!.Value);
state.IsInLoop.Should().Be(this.is_in_loop!.answer!.Value);
state.IsProgressBeingMade.Should().Be(this.is_progress_being_made!.answer!.Value);
state.InstructionOrQuestion.Should().Be(this.instruction_or_question!.answer);
state.NextSpeaker.Should().Be(this.next_speaker!.answer);
if (this.custom1 != null)
{
TryGetCustom1(state, out bool custom1Value).Should().BeTrue();
custom1Value.Should().Be(this.custom1.answer!.Value);
}
else
{
TryGetCustom1(state, out _).Should().BeFalse();
}
if (this.custom2 != null)
{
TryGetCustom2(state, out string? custom2Value).Should().BeTrue();
custom2Value.Should().Be(this.custom2.answer);
}
else
{
TryGetCustom2(state, out _).Should().BeFalse();
}
}
public static readonly TestProgressLedgerState Default = new();
public static readonly TestProgressLedgerState RequiredOnly = Default with { custom1 = null, custom2 = null };
public static readonly TestProgressLedgerState[] Working = [RequiredOnly, Default];
public static readonly TestProgressLedgerState[] MissingRequired =
[
Default with { is_request_satisfied = null },
Default with { is_in_loop = null},
Default with { is_progress_being_made = null},
Default with { instruction_or_question = null},
Default with { next_speaker = null},
];
}
@@ -11,14 +11,8 @@ using Microsoft.Extensions.AI;
namespace Microsoft.Agents.AI.Workflows.UnitTests;
public class TestReplayAgent(List<List<ChatMessage>> messages, string? id = null, string? name = null) : AIAgent
public class TestReplayAgent(List<ChatMessage>? messages = null, string? id = null, string? name = null) : AIAgent
{
public TestReplayAgent(List<ChatMessage> messages, string? id = null, string? name = null) : this([messages ?? []], id, name)
{ }
public TestReplayAgent(string? id = null, string? name = null) : this([[]], id, name)
{ }
protected override string? IdCore => id;
public override string? Name => name;
@@ -63,55 +57,46 @@ public class TestReplayAgent(List<List<ChatMessage>> messages, string? id = null
public static TestReplayAgent FromStrings(params string[] messages) =>
new(ToChatMessages(messages));
public List<List<ChatMessage>> Messages { get; } = Validate(messages) ?? [];
public List<ChatMessage> Messages { get; } = Validate(messages) ?? [];
protected override Task<AgentResponse> RunCoreAsync(IEnumerable<ChatMessage> messages, AgentSession? session = null, AgentRunOptions? options = null, CancellationToken cancellationToken = default)
=> this.RunStreamingAsync(messages, session, options, cancellationToken).ToAgentResponseAsync(cancellationToken);
public int Turn { get; set; }
protected override async IAsyncEnumerable<AgentResponseUpdate> RunCoreStreamingAsync(IEnumerable<ChatMessage> messages, AgentSession? session = null, AgentRunOptions? options = null, [EnumeratorCancellation] CancellationToken cancellationToken = default)
{
string responseId = Guid.NewGuid().ToString("N");
if (this.Turn < this.Messages.Count)
foreach (ChatMessage message in this.Messages)
{
foreach (ChatMessage message in this.Messages[this.Turn++])
foreach (AIContent content in message.Contents)
{
foreach (AIContent content in message.Contents)
yield return new AgentResponseUpdate()
{
yield return new AgentResponseUpdate()
{
AgentId = this.Id,
AuthorName = this.Name,
MessageId = message.MessageId,
ResponseId = responseId,
Contents = [content],
Role = message.Role,
};
}
AgentId = this.Id,
AuthorName = this.Name,
MessageId = message.MessageId,
ResponseId = responseId,
Contents = [content],
Role = message.Role,
};
}
}
}
private static List<List<ChatMessage>>? Validate(List<List<ChatMessage>>? candidateMessages)
private static List<ChatMessage>? Validate(List<ChatMessage>? candidateMessages)
{
string? lastMessageId = null;
string? currentMessageId = null;
if (candidateMessages != null)
if (candidateMessages is not null)
{
foreach (List<ChatMessage> candidateMessagesTurn in candidateMessages)
foreach (ChatMessage message in candidateMessages)
{
foreach (ChatMessage message in candidateMessagesTurn)
if (currentMessageId is null)
{
if (lastMessageId is null || lastMessageId != message.MessageId)
{
lastMessageId = message.MessageId;
}
else if (lastMessageId == message.MessageId)
{
throw new ArgumentException("Duplicate consecutive message ids");
}
currentMessageId = message.MessageId;
}
else if (currentMessageId == message.MessageId)
{
throw new ArgumentException("Duplicate consecutive message ids");
}
}
}
@@ -216,12 +216,10 @@ class AnthropicSettings(TypedDict, total=False):
Keys:
api_key: The Anthropic API key.
chat_model: The Anthropic chat model.
base_url: Optional base URL for the Anthropic API endpoint.
"""
api_key: SecretString | None
chat_model: str | None
base_url: str | None
class RawAnthropicClient(
@@ -250,7 +248,6 @@ class RawAnthropicClient(
*,
api_key: str | None = None,
model: str | None = None,
base_url: str | None = None,
anthropic_client: AnthropicAsyncClient | None = None,
additional_beta_flags: list[str] | None = None,
additional_properties: dict[str, Any] | None = None,
@@ -262,8 +259,6 @@ class RawAnthropicClient(
Keyword Args:
api_key: The Anthropic API key to use for authentication.
model: The model to use.
base_url: Optional base URL for the Anthropic API endpoint. Useful for Foundry or
other compatible deployments. Falls back to ``ANTHROPIC_BASE_URL`` env variable.
anthropic_client: An existing Anthropic client to use. If not provided, one will be created.
This can be used to further configure the client before passing it in.
For instance if you need to set a different base_url for testing or private deployments.
@@ -289,13 +284,6 @@ class RawAnthropicClient(
api_key="your_anthropic_api_key",
)
# Or with a custom base URL (e.g. for Foundry-compatible endpoints)
client = RawAnthropicClient(
model="claude-sonnet-4-5-20250929",
api_key="your_anthropic_api_key",
base_url="https://custom-anthropic-endpoint.com",
)
# Or loading from a .env file
client = RawAnthropicClient(env_file_path="path/to/.env")
@@ -328,14 +316,12 @@ class RawAnthropicClient(
env_prefix="ANTHROPIC_",
api_key=api_key,
chat_model=model,
base_url=base_url,
env_file_path=env_file_path,
env_file_encoding=env_file_encoding,
)
api_key_secret = anthropic_settings.get("api_key")
model_setting = anthropic_settings.get("chat_model")
base_url_setting = anthropic_settings.get("base_url")
if anthropic_client is None:
if api_key_secret is None:
@@ -346,7 +332,6 @@ class RawAnthropicClient(
anthropic_client = AsyncAnthropic(
api_key=api_key_secret.get_secret_value(),
base_url=base_url_setting,
default_headers={"User-Agent": get_user_agent()},
)
@@ -1424,7 +1409,6 @@ class AnthropicClient(
*,
api_key: str | None = None,
model: str | None = None,
base_url: str | None = None,
anthropic_client: AnthropicAsyncClient | None = None,
additional_beta_flags: list[str] | None = None,
additional_properties: dict[str, Any] | None = None,
@@ -1438,8 +1422,6 @@ class AnthropicClient(
Keyword Args:
api_key: The Anthropic API key to use for authentication.
model: The model to use.
base_url: Optional base URL for the Anthropic API endpoint. Useful for Foundry or
other compatible deployments. Falls back to ``ANTHROPIC_BASE_URL`` env variable.
anthropic_client: An existing Anthropic client to use. If not provided, one will be created.
This can be used to further configure the client before passing it in.
For instance if you need to set a different base_url for testing or private deployments.
@@ -1466,13 +1448,6 @@ class AnthropicClient(
api_key="your_anthropic_api_key",
)
# Or with a custom base URL (e.g. for Foundry-compatible endpoints)
client = AnthropicClient(
model="claude-sonnet-4-5-20250929",
api_key="your_anthropic_api_key",
base_url="https://custom-anthropic-endpoint.com",
)
# Or loading from a .env file
client = AnthropicClient(env_file_path="path/to/.env")
@@ -1502,7 +1477,6 @@ class AnthropicClient(
super().__init__(
api_key=api_key,
model=model,
base_url=base_url,
anthropic_client=anthropic_client,
additional_beta_flags=additional_beta_flags,
additional_properties=additional_properties,
@@ -149,108 +149,6 @@ def test_anthropic_client_init_auto_create_client(
assert client.model == anthropic_unit_test_env["ANTHROPIC_CHAT_MODEL"]
def test_anthropic_client_init_with_base_url(
anthropic_unit_test_env: dict[str, str],
) -> None:
"""Test AnthropicClient accepts a base_url and passes it to the underlying AsyncAnthropic client."""
custom_url = "https://custom-anthropic-endpoint.com"
client = AnthropicClient(
api_key=anthropic_unit_test_env["ANTHROPIC_API_KEY"],
model=anthropic_unit_test_env["ANTHROPIC_CHAT_MODEL"],
base_url=custom_url,
)
assert custom_url in str(client.anthropic_client.base_url)
def test_raw_anthropic_client_init_with_base_url(
anthropic_unit_test_env: dict[str, str],
) -> None:
"""Test RawAnthropicClient accepts a base_url and passes it to the underlying AsyncAnthropic client."""
custom_url = "https://custom-anthropic-endpoint.com"
client = RawAnthropicClient(
api_key=anthropic_unit_test_env["ANTHROPIC_API_KEY"],
model=anthropic_unit_test_env["ANTHROPIC_CHAT_MODEL"],
base_url=custom_url,
)
assert custom_url in str(client.anthropic_client.base_url)
@pytest.mark.parametrize(
"override_env_param_dict",
[{"ANTHROPIC_BASE_URL": "https://env-base-url.example.com"}],
indirect=True,
)
def test_anthropic_client_init_base_url_from_env(
anthropic_unit_test_env: dict[str, str],
) -> None:
"""Test AnthropicClient picks up base_url from ANTHROPIC_BASE_URL env variable when not passed explicitly."""
client = AnthropicClient(
api_key=anthropic_unit_test_env["ANTHROPIC_API_KEY"],
model=anthropic_unit_test_env["ANTHROPIC_CHAT_MODEL"],
)
assert anthropic_unit_test_env["ANTHROPIC_BASE_URL"] in str(client.anthropic_client.base_url)
@pytest.mark.parametrize(
"override_env_param_dict",
[{"ANTHROPIC_BASE_URL": "https://env-base-url.example.com"}],
indirect=True,
)
def test_raw_anthropic_client_init_base_url_from_env(
anthropic_unit_test_env: dict[str, str],
) -> None:
"""Test RawAnthropicClient picks up base_url from ANTHROPIC_BASE_URL env variable when not passed explicitly."""
client = RawAnthropicClient(
api_key=anthropic_unit_test_env["ANTHROPIC_API_KEY"],
model=anthropic_unit_test_env["ANTHROPIC_CHAT_MODEL"],
)
assert anthropic_unit_test_env["ANTHROPIC_BASE_URL"] in str(client.anthropic_client.base_url)
@pytest.mark.parametrize(
"override_env_param_dict",
[{"ANTHROPIC_BASE_URL": "https://env-base-url.example.com"}],
indirect=True,
)
def test_anthropic_client_init_explicit_base_url_wins_over_env(
anthropic_unit_test_env: dict[str, str],
) -> None:
"""Test that an explicit base_url kwarg takes priority over ANTHROPIC_BASE_URL env variable."""
explicit_url = "https://explicit-endpoint.example.com"
client = AnthropicClient(
api_key=anthropic_unit_test_env["ANTHROPIC_API_KEY"],
model=anthropic_unit_test_env["ANTHROPIC_CHAT_MODEL"],
base_url=explicit_url,
)
assert explicit_url in str(client.anthropic_client.base_url)
assert anthropic_unit_test_env["ANTHROPIC_BASE_URL"] not in str(client.anthropic_client.base_url)
@pytest.mark.parametrize(
"override_env_param_dict",
[{"ANTHROPIC_BASE_URL": "https://env-base-url.example.com"}],
indirect=True,
)
def test_raw_anthropic_client_init_explicit_base_url_wins_over_env(
anthropic_unit_test_env: dict[str, str],
) -> None:
"""Test that an explicit base_url kwarg takes priority over ANTHROPIC_BASE_URL env variable."""
explicit_url = "https://explicit-endpoint.example.com"
client = RawAnthropicClient(
api_key=anthropic_unit_test_env["ANTHROPIC_API_KEY"],
model=anthropic_unit_test_env["ANTHROPIC_CHAT_MODEL"],
base_url=explicit_url,
)
assert explicit_url in str(client.anthropic_client.base_url)
assert anthropic_unit_test_env["ANTHROPIC_BASE_URL"] not in str(client.anthropic_client.base_url)
def test_anthropic_client_init_missing_api_key() -> None:
"""Test AnthropicClient initialization when API key is missing."""
with patch("agent_framework_anthropic._chat_client.load_settings") as mock_load:
@@ -135,7 +135,6 @@ from ._sessions import (
from ._settings import SecretString, load_settings
from ._skills import (
AggregatingSkillsSource,
ClassSkill,
DeduplicatingSkillsSource,
DelegatingSkillsSource,
FileSkill,
@@ -346,7 +345,6 @@ __all__ = [
"ChatResponseUpdate",
"CheckResult",
"CheckpointStorage",
"ClassSkill",
"CompactionProvider",
"CompactionStrategy",
"Content",
@@ -354,8 +352,8 @@ __all__ = [
"ContinuationToken",
"ConversationSplit",
"ConversationSplitter",
"DeduplicatingSkillsSource",
"Default",
"DeduplicatingSkillsSource",
"DelegatingSkillsSource",
"Edge",
"EdgeCondition",
+10 -53
View File
@@ -158,22 +158,6 @@ def streamable_http_client(*args: Any, **kwargs: Any) -> _AsyncGeneratorContextM
return _streamable_http_client(*args, **kwargs) # type: ignore[return-value]
def _should_propagate_cancelled_error(ex: BaseException) -> bool:
"""Return True if *ex* is a genuine task-cancellation that should propagate unchanged.
On Python >= 3.11, ``task.cancelling() > 0`` distinguishes a real caller-driven
cancellation from a CancelledError raised internally by a library (e.g. via an
anyio cancel scope). On older Python versions the API is unavailable, so we
always return False and let callers wrap the error in ToolException instead.
"""
if not isinstance(ex, asyncio.CancelledError):
return False
if sys.version_info < (3, 11):
return False
task = asyncio.current_task()
return task is not None and task.cancelling() > 0
# region: MCP Plugin
@@ -643,17 +627,6 @@ class MCPTool:
except asyncio.CancelledError:
logger.warning("Could not cleanly close MCP exit stack because the lifecycle owner task was cancelled.")
async def _close_and_check_cancelled(self, ex: BaseException) -> bool:
"""Close the exit stack and return True if *ex* is a genuine task cancellation.
Callers should immediately re-raise when this returns True::
if await self._close_and_check_cancelled(ex):
raise
"""
await self._safe_close_exit_stack()
return _should_propagate_cancelled_error(ex)
async def connect(self, *, reset: bool = False) -> None:
if self._is_lifecycle_owner_task():
await self._connect_on_owner(reset=reset)
@@ -682,23 +655,14 @@ class MCPTool:
if not self.session:
try:
transport = await self._exit_stack.enter_async_context(self.get_mcp_client())
except (Exception, asyncio.CancelledError) as ex:
# On Python >= 3.11, re-raise genuine task cancellation (task.cancelling() > 0)
# instead of wrapping it in ToolException. On Python < 3.11, task.cancelling()
# is unavailable so MCP-internal CancelledErrors cannot be distinguished from
# caller-driven cancellation; they are wrapped as ToolException in that case.
if await self._close_and_check_cancelled(ex):
raise
except Exception as ex:
await self._safe_close_exit_stack()
command = getattr(self, "command", None)
if command:
error_msg = f"Failed to start MCP server '{command}': {ex}"
else:
error_msg = f"Failed to connect to MCP server: {ex}"
# CancelledError is a BaseException (not Exception) on Python >= 3.8, so
# inner_exception=None and ToolException.__init__ won't log exc_info.
if isinstance(ex, asyncio.CancelledError):
logger.debug(error_msg, exc_info=True)
raise ToolException(error_msg, inner_exception=ex if isinstance(ex, Exception) else None) from ex
raise ToolException(error_msg, inner_exception=ex) from ex
try:
try:
from mcp import types
@@ -728,21 +692,16 @@ class MCPTool:
sampling_capabilities=sampling_capabilities,
)
)
except (Exception, asyncio.CancelledError) as ex:
if await self._close_and_check_cancelled(ex):
raise
session_error_msg = f"Failed to create MCP session: {ex}"
if isinstance(ex, asyncio.CancelledError):
logger.debug(session_error_msg, exc_info=True)
except Exception as ex:
await self._safe_close_exit_stack()
raise ToolException(
message=session_error_msg,
inner_exception=ex if isinstance(ex, Exception) else None,
message="Failed to create MCP session. Please check your configuration.",
inner_exception=ex,
) from ex
try:
await session.initialize()
except (Exception, asyncio.CancelledError) as ex:
if await self._close_and_check_cancelled(ex):
raise
except Exception as ex:
await self._safe_close_exit_stack()
# Provide context about initialization failure
command = getattr(self, "command", None)
if command:
@@ -751,9 +710,7 @@ class MCPTool:
error_msg = f"MCP server '{full_command}' failed to initialize: {ex}"
else:
error_msg = f"MCP server failed to initialize: {ex}"
if isinstance(ex, asyncio.CancelledError):
logger.debug(error_msg, exc_info=True)
raise ToolException(error_msg, inner_exception=ex if isinstance(ex, Exception) else None) from ex
raise ToolException(error_msg, inner_exception=ex) from ex
self.session = session
elif self.session._request_id == 0: # type: ignore[attr-defined]
# If the session is not initialized, we need to reinitialize it
+53 -488
View File
@@ -5,7 +5,7 @@
Defines the core data model classes for the agent skills system:
- **Skills:** :class:`Skill` (abstract base), :class:`InlineSkill` (code-defined),
:class:`ClassSkill` (class-based), and :class:`FileSkill` (filesystem-backed).
and :class:`FileSkill` (filesystem-backed).
- **Resources:** :class:`SkillResource` (abstract base), :class:`InlineSkillResource`
(static content or callable).
- **Scripts:** :class:`SkillScript` (abstract base), :class:`InlineSkillScript`
@@ -27,9 +27,6 @@ Skills can come from different sources:
Represented as :class:`FileSkill` instances.
- **Code-defined** — created as :class:`InlineSkill` instances in Python code,
with optional callable resources attached via the ``@skill.resource`` decorator.
- **Class-based** — created by subclassing :class:`ClassSkill` to define
self-contained, reusable skill types with ``create_resource()`` and
``create_script()`` factory methods.
- **Custom sources** — any :class:`SkillsSource` implementation that provides
skills from arbitrary origins (REST APIs, databases, etc.).
@@ -449,10 +446,14 @@ class FileSkillScript(SkillScript):
"""
if not isinstance(skill, FileSkill):
raise TypeError(
f"File-based script '{self.name}' requires a FileSkill but received '{type(skill).__name__}'."
f"File-based script '{self.name}' requires a FileSkill "
f"but received '{type(skill).__name__}'."
)
if self._runner is None:
raise ValueError(f"Script '{self.name}' requires a runner. Provide a script_runner for file-based scripts.")
raise ValueError(
f"Script '{self.name}' requires a runner. "
"Provide a script_runner for file-based scripts."
)
result = self._runner(skill, self, args)
if inspect.isawaitable(result):
return await result
@@ -569,69 +570,11 @@ def _validate_skill_description(name: str, description: str) -> None:
raise ValueError("Skill description cannot be empty.")
if len(description) > MAX_DESCRIPTION_LENGTH:
raise ValueError(
f"Skill '{name}' has an invalid description: Must be {MAX_DESCRIPTION_LENGTH} characters or fewer."
f"Skill '{name}' has an invalid description: "
f"Must be {MAX_DESCRIPTION_LENGTH} characters or fewer."
)
def _build_skill_content(
name: str,
description: str,
instructions: str,
resources: Sequence[SkillResource] | None = None,
scripts: Sequence[SkillScript] | None = None,
) -> str:
"""Build XML-structured content for code-defined and class-based skills.
Produces an XML document containing name, description, instructions,
resources, and scripts elements. Used by both :class:`InlineSkill`
and :class:`ClassSkill` to generate their ``content`` property.
Args:
name: The skill name.
description: The skill description.
instructions: The raw instructions text.
resources: Optional resources associated with the skill.
scripts: Optional scripts associated with the skill.
Returns:
An XML-structured content string.
"""
result = (
f"<name>{xml_escape(name)}</name>\n"
f"<description>{xml_escape(description)}</description>\n"
"\n"
"<instructions>\n"
f"{instructions}\n"
"</instructions>"
)
if resources:
resource_lines = "\n".join(_create_resource_element(r) for r in resources)
result += f"\n\n<resources>\n{resource_lines}\n</resources>"
if scripts:
script_lines = "\n".join(_create_script_element(s) for s in scripts)
result += f"\n\n<scripts>\n{script_lines}\n</scripts>"
return result
def _create_resource_element(resource: SkillResource) -> str:
"""Create a self-closing ``<resource …/>`` XML element from a :class:`SkillResource`.
Args:
resource: The resource to create the element from.
Returns:
A single indented XML element string with ``name`` and optional
``description`` attributes.
"""
attrs = f'name="{xml_escape(resource.name, quote=True)}"'
if resource.description:
attrs += f' description="{xml_escape(resource.description, quote=True)}"'
return f" <resource {attrs}/>"
@experimental(feature_id=ExperimentalFeature.SKILLS)
class InlineSkill(Skill):
"""A skill defined entirely in code with resources and scripts.
@@ -696,10 +639,25 @@ class InlineSkill(Skill):
if self._cached_content is not None:
return self._cached_content
self._cached_content = _build_skill_content(
self.name, self.description, self.instructions, self._resources, self._scripts
result = (
f"<name>{xml_escape(self.name)}</name>\n"
f"<description>{xml_escape(self.description)}</description>\n"
"\n"
"<instructions>\n"
f"{self.instructions}\n"
"</instructions>"
)
return self._cached_content
if self._resources:
resource_lines = "\n".join(self._create_resource_element(r) for r in self._resources)
result += f"\n\n<resources>\n{resource_lines}\n</resources>"
if self._scripts:
script_lines = "\n".join(_create_script_element(s) for s in self._scripts)
result += f"\n\n<scripts>\n{script_lines}\n</scripts>"
self._cached_content = result
return result
@property
def resources(self) -> list[SkillResource]:
@@ -711,6 +669,22 @@ class InlineSkill(Skill):
"""Mutable list of :class:`SkillScript` instances."""
return self._scripts
@staticmethod
def _create_resource_element(resource: SkillResource) -> str:
"""Create a self-closing ``<resource …/>`` XML element from an :class:`SkillResource`.
Args:
resource: The resource to create the element from.
Returns:
A single indented XML element string with ``name`` and optional
``description`` attributes.
"""
attrs = f'name="{xml_escape(resource.name, quote=True)}"'
if resource.description:
attrs += f' description="{xml_escape(resource.description, quote=True)}"'
return f" <resource {attrs}/>"
def resource(
self,
func: Callable[..., Any] | None = None,
@@ -731,7 +705,8 @@ class InlineSkill(Skill):
Keyword Args:
name: Resource name override. Defaults to ``func.__name__``.
description: Resource description override. Defaults to ``None``.
description: Resource description override. Defaults to the
function's docstring (via :func:`inspect.getdoc`).
Returns:
The original function unchanged, or a secondary decorator when
@@ -757,7 +732,7 @@ class InlineSkill(Skill):
def decorator(f: Callable[..., Any]) -> Callable[..., Any]:
resource_name = name or f.__name__
resource_description = description
resource_description = description or (inspect.getdoc(f) or None)
self._resources.append(
InlineSkillResource(
name=resource_name,
@@ -791,7 +766,8 @@ class InlineSkill(Skill):
Keyword Args:
name: Script name override. Defaults to ``func.__name__``.
description: Script description override. Defaults to ``None``.
description: Script description override. Defaults to the
function's docstring (via :func:`inspect.getdoc`).
Returns:
The original function unchanged, or a secondary decorator when
@@ -818,7 +794,7 @@ class InlineSkill(Skill):
def decorator(f: Callable[..., Any]) -> Callable[..., Any]:
script_name = name or f.__name__
script_description = description
script_description = description or (inspect.getdoc(f) or None)
self._scripts.append(
InlineSkillScript(
name=script_name,
@@ -833,420 +809,6 @@ class InlineSkill(Skill):
return decorator(func)
def _make_method_name(method_name: str) -> str:
"""Convert a Python method name to a skill resource/script name.
Replaces underscores with hyphens to match the skill naming convention.
Args:
method_name: The Python method name (e.g. ``"conversion_table"``).
Returns:
The converted name (e.g. ``"conversion-table"``).
"""
return method_name.replace("_", "-").strip("-")
def _validate_member_name(name: str, kind: str) -> None:
"""Validate a resource or script name at decoration time.
Args:
name: The name to validate.
kind: ``"resource"`` or ``"script"`` — used in error messages.
Raises:
ValueError: If the name is empty, too long, or contains invalid characters.
"""
if not name or not name.strip():
raise ValueError(f"@ClassSkill.{kind} name cannot be empty.")
if len(name) > MAX_NAME_LENGTH or not VALID_NAME_RE.match(name):
raise ValueError(
f"Invalid @ClassSkill.{kind} name '{name}': Must be {MAX_NAME_LENGTH} characters or fewer, "
"using only lowercase letters, numbers, and hyphens, and must not start or end with a hyphen "
"or contain consecutive hyphens."
)
def _discover_marked_members(cls: type, marker_attr: str) -> list[tuple[str, dict[str, Any]]]:
"""Scan a class for methods or properties stamped with a marker attribute.
Checks both regular callable attributes (via ``dir``) and ``property``
descriptors (via ``cls.__dict__``) whose ``fget`` carries the marker.
Args:
cls: The class to scan.
marker_attr: The marker attribute name to look for (e.g.
``"_skill_resource_marker"``).
Returns:
A list of ``(member_name, marker_dict)`` tuples.
"""
results: list[tuple[str, dict[str, Any]]] = []
seen: set[str] = set()
# Walk the MRO so that property-resources defined on a parent class
# are also discovered. ``cls.__dict__`` only sees the leaf class.
for klass in cls.__mro__:
for attr_name, attr_value in klass.__dict__.items():
if attr_name in seen:
continue
if (
isinstance(attr_value, property)
and attr_value.fget is not None
and hasattr(attr_value.fget, marker_attr)
):
results.append((attr_name, getattr(attr_value.fget, marker_attr)))
seen.add(attr_name)
# Check regular callable attributes.
for attr_name in dir(cls):
if attr_name in seen:
continue
try:
attr = getattr(cls, attr_name, None)
except Exception:
# Some descriptors (e.g. abstract properties) may raise on access.
logger.warning("Skipping '%s' during skill discovery: descriptor raised on access", attr_name)
attr = None
if attr is not None and callable(attr) and hasattr(attr, marker_attr):
results.append((attr_name, getattr(attr, marker_attr)))
return results
@experimental(feature_id=ExperimentalFeature.SKILLS)
class ClassSkill(Skill, ABC):
"""Abstract base class for defining skills as reusable Python classes.
Inherit from this class to create a self-contained skill definition.
Override :attr:`instructions` to provide the skill body.
Resources and scripts can be defined in two ways:
- **Decorator-based (recommended):** Mark methods with
:meth:`ClassSkill.resource` and :meth:`ClassSkill.script` decorators
for automatic discovery.
- **Explicit override:** Override the :attr:`resources` and
:attr:`scripts` properties, constructing :class:`InlineSkillResource`
and :class:`InlineSkillScript` instances directly.
Class-based skills can be distributed via shared libraries or PyPI
packages, making them easy to reuse across projects.
Attributes:
name: Skill name (lowercase letters, numbers, hyphens only).
description: Human-readable description of the skill.
Examples:
Decorator-based (recommended):
.. code-block:: python
class UnitConverterSkill(ClassSkill):
def __init__(self) -> None:
super().__init__(
name="unit-converter",
description="Convert between common units.",
)
@property
def instructions(self) -> str:
return "Use this skill to convert units..."
@ClassSkill.resource(name="table")
def conversion_table(self) -> str:
return "| From | To | Factor |..."
@ClassSkill.script(name="convert")
def convert(self, value: float, factor: float) -> str:
return json.dumps({"result": round(value * factor, 4)})
Explicit override:
.. code-block:: python
class UnitConverterSkill(ClassSkill):
def __init__(self) -> None:
super().__init__(
name="unit-converter",
description="Convert between common units.",
)
@property
def instructions(self) -> str:
return "Use this skill to convert units..."
@property
def resources(self) -> list[SkillResource]:
return [
InlineSkillResource(name="table", content="| From | To | Factor |..."),
]
@property
def scripts(self) -> list[SkillScript]:
return [InlineSkillScript(name="convert", function=convert_fn)]
"""
def __init__(
self,
*,
name: str,
description: str,
) -> None:
"""Initialize a ClassSkill.
Args:
name: Skill name (lowercase letters, numbers, hyphens only;
max 64 characters).
description: Human-readable description of the skill
(≤1024 characters).
"""
super().__init__(name=name, description=description)
self._cached_content: str | None = None
self._cached_resources: list[SkillResource] | None = None
self._cached_scripts: list[SkillScript] | None = None
@staticmethod
def resource(
func: Callable[..., Any] | None = None,
*,
name: str | None = None,
description: str | None = None,
) -> Any:
"""Decorator that marks a method or property as a skill resource for auto-discovery.
When applied to a method or property on a :class:`ClassSkill` subclass,
it is automatically discovered and registered as an
:class:`InlineSkillResource`. Methods are invoked each time the
resource is read. Properties are evaluated via their getter.
Can be applied to a method directly, or stacked with ``@property``
(place ``@property`` first, ``@ClassSkill.resource`` second).
Supports bare usage (``@ClassSkill.resource``) and parameterized usage
(``@ClassSkill.resource(name="custom", description="...")``).
Args:
func: The function being decorated. Populated automatically when
the decorator is applied without parentheses.
Keyword Args:
name: Resource name override. Defaults to the method name with
underscores replaced by hyphens.
description: Resource description. Defaults to ``None``.
Examples:
On a method:
.. code-block:: python
@ClassSkill.resource(name="conversion-table")
def get_table(self) -> str:
return "..."
On a property:
.. code-block:: python
@property
@ClassSkill.resource
def conversion_table(self) -> str:
return "..."
"""
def decorator(f: Callable[..., Any]) -> Callable[..., Any]:
if isinstance(f, (property, classmethod, staticmethod)):
raise TypeError(
"@ClassSkill.resource must be applied before @property, @classmethod, or @staticmethod. "
"Place @property first, then @ClassSkill.resource."
)
if name is not None:
_validate_member_name(name, "resource")
f._skill_resource_marker = { # type: ignore[attr-defined]
"name": name,
"description": description,
}
return f
if func is None:
return decorator
return decorator(func)
@staticmethod
def script(
func: Callable[..., Any] | None = None,
*,
name: str | None = None,
description: str | None = None,
) -> Any:
"""Decorator that marks a method as a skill script for auto-discovery.
When applied to a method on a :class:`ClassSkill` subclass, the method is
automatically discovered and registered as an :class:`InlineSkillScript`.
The method's parameters (excluding ``self``) are used to generate a JSON
schema, and the method is invoked in-process when the script is run.
Supports bare usage (``@ClassSkill.script``) and parameterized usage
(``@ClassSkill.script(name="custom", description="...")``).
Args:
func: The function being decorated. Populated automatically when
the decorator is applied without parentheses.
Keyword Args:
name: Script name override. Defaults to the method name with
underscores replaced by hyphens.
description: Script description. Defaults to ``None``.
Examples:
.. code-block:: python
@ClassSkill.script(name="convert")
def convert(self, value: float, factor: float) -> str:
return json.dumps({"result": round(value * factor, 4)})
"""
def decorator(f: Callable[..., Any]) -> Callable[..., Any]:
if isinstance(f, (property, classmethod, staticmethod)):
raise TypeError(
"@ClassSkill.script must be applied before @property, @classmethod, or @staticmethod."
)
if name is not None:
_validate_member_name(name, "script")
f._skill_script_marker = { # type: ignore[attr-defined]
"name": name,
"description": description,
}
return f
if func is None:
return decorator
return decorator(func)
@property
@abstractmethod
def instructions(self) -> str:
"""The raw instructions text for this skill.
Subclasses must override this property to provide the skill body.
"""
...
@property
def resources(self) -> list[SkillResource]:
"""Resources discovered from :meth:`ClassSkill.resource`-decorated methods.
On first access, scans the class for methods marked with the
:meth:`ClassSkill.resource` decorator and instantiates
:class:`InlineSkillResource` instances from them.
The result is cached after the first access.
Override this property to provide resources explicitly instead of
using decorator-based discovery.
"""
if self._cached_resources is not None:
return list(self._cached_resources)
resources: list[SkillResource] = []
seen_names: set[str] = set()
for attr_name, attr in _discover_marked_members(type(self), "_skill_resource_marker"):
marker: dict[str, Any] = attr
resource_name = marker.get("name") or _make_method_name(attr_name)
if resource_name in seen_names:
raise ValueError(
f"Skill '{self.name}' already has a resource named '{resource_name}'. "
"Ensure each @ClassSkill.resource has a unique name."
)
seen_names.add(resource_name)
# Use inspect.getattr_static to check the descriptor type without
# triggering it, and walk the MRO so inherited properties are found.
static_attr = inspect.getattr_static(self, attr_name, None)
is_property = isinstance(static_attr, property)
resource_description = marker.get("description")
if is_property:
# Property — use a lambda that reads the property value each time.
# We capture attr_name to avoid late-binding issues.
# Do NOT call getattr here to avoid triggering the getter during discovery.
resource_func = (lambda name: lambda: getattr(self, name))(attr_name)
resources.append(
InlineSkillResource(
name=resource_name,
function=resource_func,
description=resource_description,
)
)
else:
# Regular method — use the bound method directly.
bound_method = getattr(self, attr_name)
resources.append(
InlineSkillResource(
name=resource_name,
function=bound_method,
description=resource_description,
)
)
self._cached_resources = resources
return list(self._cached_resources)
@property
def scripts(self) -> list[SkillScript]:
"""Scripts discovered from :meth:`ClassSkill.script`-decorated methods.
On first access, scans the class for methods marked with the
:meth:`ClassSkill.script` decorator and instantiates
:class:`InlineSkillScript` instances from them.
The result is cached after the first access.
Override this property to provide scripts explicitly instead of
using decorator-based discovery.
"""
if self._cached_scripts is not None:
return list(self._cached_scripts)
scripts: list[SkillScript] = []
seen_names: set[str] = set()
for attr_name, attr in _discover_marked_members(type(self), "_skill_script_marker"):
marker: dict[str, Any] = attr
script_name = marker.get("name") or _make_method_name(attr_name)
if script_name in seen_names:
raise ValueError(
f"Skill '{self.name}' already has a script named '{script_name}'. "
"Ensure each @ClassSkill.script has a unique name."
)
seen_names.add(script_name)
bound_method = getattr(self, attr_name)
script_description = marker.get("description")
scripts.append(
InlineSkillScript(
name=script_name,
function=bound_method,
description=script_description,
)
)
self._cached_scripts = scripts
return list(self._cached_scripts)
@property
def content(self) -> str:
"""Synthesized XML content containing name, description, instructions, resources, and scripts.
The result is cached after the first access.
"""
if self._cached_content is not None:
return self._cached_content
self._cached_content = _build_skill_content(
self.name, self.description, self.instructions, self.resources, self.scripts
)
return self._cached_content
@experimental(feature_id=ExperimentalFeature.SKILLS)
class FileSkill(Skill):
"""A :class:`Skill` discovered from a filesystem directory backed by a SKILL.md file.
@@ -2431,7 +1993,10 @@ class FileSkillsSource(SkillsSource):
raise ValueError(f"Resource file '{resource_name}' not found in skill directory '{skill_dir}'.")
if FileSkillsSource._has_symlink_in_path(resource_full_path, root_directory_path):
raise ValueError(f"Resource file '{resource_name}' has a symlink in its path; symlinks are not allowed.")
raise ValueError(
f"Resource file '{resource_name}' "
"has a symlink in its path; symlinks are not allowed."
)
return resource_full_path
-280
View File
@@ -1,10 +1,8 @@
# Copyright (c) Microsoft. All rights reserved.
# type: ignore[reportPrivateUsage]
import asyncio
import json
import logging
import os
import sys
from contextlib import _AsyncGeneratorContextManager # type: ignore
from typing import Any
from unittest.mock import AsyncMock, Mock, patch
@@ -29,7 +27,6 @@ from agent_framework._mcp import (
_build_prefixed_mcp_name,
_get_input_model_from_mcp_prompt,
_normalize_mcp_name,
_should_propagate_cancelled_error,
logger,
)
from agent_framework._middleware import FunctionMiddlewarePipeline
@@ -2179,7 +2176,6 @@ async def test_connect_session_creation_failure():
await tool.connect()
assert "Failed to create MCP session" in str(exc_info.value)
assert "Session creation failed" in str(exc_info.value) # exception text is now part of the message
assert "Session creation failed" in str(exc_info.value.__cause__)
@@ -2268,282 +2264,6 @@ async def test_connect_cleanup_on_initialization_failure():
tool._exit_stack.aclose.assert_called_once()
async def test_connect_cancelled_error_during_transport_creation_raises_tool_exception():
"""Test that CancelledError from transport creation is wrapped in ToolException."""
tool = MCPStreamableHTTPTool(name="test", url="http://example.com")
tool._exit_stack.aclose = AsyncMock()
tool.get_mcp_client = Mock(side_effect=asyncio.CancelledError("cancel scope"))
with pytest.raises(ToolException, match="Failed to connect to MCP server"):
await tool.connect()
tool._exit_stack.aclose.assert_called_once()
async def test_connect_cancelled_error_during_transport_creation_stdio_raises_tool_exception():
"""Test that CancelledError from transport creation uses the command-specific message for MCPStdioTool."""
tool = MCPStdioTool(name="test", command="my-server")
tool._exit_stack.aclose = AsyncMock()
tool.get_mcp_client = Mock(side_effect=asyncio.CancelledError("cancel scope"))
with pytest.raises(ToolException, match="Failed to start MCP server 'my-server'"):
await tool.connect()
tool._exit_stack.aclose.assert_called_once()
async def test_connect_cancelled_error_during_session_creation_raises_tool_exception():
"""Test that CancelledError from session creation is wrapped in ToolException."""
tool = MCPStreamableHTTPTool(name="test", url="http://example.com")
mock_transport = (Mock(), Mock())
mock_context_manager = Mock()
mock_context_manager.__aenter__ = AsyncMock(return_value=mock_transport)
mock_context_manager.__aexit__ = AsyncMock(return_value=None)
tool.get_mcp_client = Mock(return_value=mock_context_manager)
with patch("mcp.client.session.ClientSession") as mock_session_class:
mock_session_class.return_value.__aenter__ = AsyncMock(side_effect=asyncio.CancelledError("cancel scope"))
mock_session_class.return_value.__aexit__ = AsyncMock(return_value=None)
with pytest.raises(ToolException, match="Failed to create MCP session"):
await tool.connect()
async def test_connect_cancelled_error_during_initialize_raises_tool_exception():
"""Test that CancelledError from session.initialize() is wrapped in ToolException.
This is the primary regression test for the bug: when an MCP server is unreachable,
the MCP library raises asyncio.CancelledError internally, which previously escaped
all except Exception handlers and could not be caught by user code.
"""
tool = MCPStreamableHTTPTool(name="test", url="http://example.com")
mock_transport = (Mock(), Mock())
mock_context_manager = Mock()
mock_context_manager.__aenter__ = AsyncMock(return_value=mock_transport)
mock_context_manager.__aexit__ = AsyncMock(return_value=None)
tool.get_mcp_client = Mock(return_value=mock_context_manager)
mock_session = Mock()
mock_session.initialize = AsyncMock(side_effect=asyncio.CancelledError("Cancelled via cancel scope"))
with patch("mcp.client.session.ClientSession") as mock_session_class:
mock_session_class.return_value.__aenter__ = AsyncMock(return_value=mock_session)
mock_session_class.return_value.__aexit__ = AsyncMock(return_value=None)
with pytest.raises(ToolException, match="MCP server failed to initialize"):
await tool.connect()
async def test_connect_cancelled_error_during_initialize_stdio_raises_tool_exception():
"""Test that CancelledError from session.initialize() uses the command-specific message for MCPStdioTool."""
tool = MCPStdioTool(name="test", command="my-server", args=["--port", "8080"])
mock_transport = (Mock(), Mock())
mock_context_manager = Mock()
mock_context_manager.__aenter__ = AsyncMock(return_value=mock_transport)
mock_context_manager.__aexit__ = AsyncMock(return_value=None)
tool.get_mcp_client = Mock(return_value=mock_context_manager)
mock_session = Mock()
mock_session.initialize = AsyncMock(side_effect=asyncio.CancelledError("Cancelled via cancel scope"))
with patch("mcp.client.session.ClientSession") as mock_session_class:
mock_session_class.return_value.__aenter__ = AsyncMock(return_value=mock_session)
mock_session_class.return_value.__aexit__ = AsyncMock(return_value=None)
with pytest.raises(ToolException, match="MCP server 'my-server --port 8080' failed to initialize"):
await tool.connect()
@pytest.mark.skipif(sys.version_info < (3, 11), reason="task.cancelling() requires Python >= 3.11")
async def test_connect_genuine_cancellation_during_transport_creation_propagates():
"""Test that genuine task cancellation (task.cancelling() > 0) propagates as CancelledError."""
tool = MCPStreamableHTTPTool(name="test", url="http://example.com")
tool._exit_stack.aclose = AsyncMock()
mock_cancelled_task = Mock()
mock_cancelled_task.cancelling.return_value = 1
with patch("asyncio.current_task", return_value=mock_cancelled_task):
tool.get_mcp_client = Mock(side_effect=asyncio.CancelledError("task cancelled"))
with pytest.raises(asyncio.CancelledError):
await tool.connect()
tool._exit_stack.aclose.assert_called_once()
@pytest.mark.skipif(sys.version_info < (3, 11), reason="task.cancelling() requires Python >= 3.11")
async def test_connect_genuine_cancellation_during_initialize_propagates():
"""Test that genuine task cancellation during initialize() propagates as CancelledError."""
tool = MCPStreamableHTTPTool(name="test", url="http://example.com")
tool._exit_stack.aclose = AsyncMock()
mock_transport = (Mock(), Mock())
mock_context_manager = Mock()
mock_context_manager.__aenter__ = AsyncMock(return_value=mock_transport)
mock_context_manager.__aexit__ = AsyncMock(return_value=None)
tool.get_mcp_client = Mock(return_value=mock_context_manager)
mock_session = Mock()
mock_session.initialize = AsyncMock(side_effect=asyncio.CancelledError("task cancelled"))
mock_cancelled_task = Mock()
mock_cancelled_task.cancelling.return_value = 1
with (
patch("asyncio.current_task", return_value=mock_cancelled_task),
patch("mcp.client.session.ClientSession") as mock_session_class,
):
mock_session_class.return_value.__aenter__ = AsyncMock(return_value=mock_session)
mock_session_class.return_value.__aexit__ = AsyncMock(return_value=None)
with pytest.raises(asyncio.CancelledError):
await tool.connect()
tool._exit_stack.aclose.assert_called_once()
@pytest.mark.skipif(sys.version_info < (3, 11), reason="task.cancelling() requires Python >= 3.11")
async def test_connect_genuine_cancellation_during_session_creation_propagates():
"""Test that genuine task cancellation during session creation propagates as CancelledError."""
tool = MCPStreamableHTTPTool(name="test", url="http://example.com")
tool._exit_stack.aclose = AsyncMock()
mock_transport = (Mock(), Mock())
mock_context_manager = Mock()
mock_context_manager.__aenter__ = AsyncMock(return_value=mock_transport)
mock_context_manager.__aexit__ = AsyncMock(return_value=None)
tool.get_mcp_client = Mock(return_value=mock_context_manager)
mock_cancelled_task = Mock()
mock_cancelled_task.cancelling.return_value = 1
with (
patch("asyncio.current_task", return_value=mock_cancelled_task),
patch("mcp.client.session.ClientSession") as mock_session_class,
):
mock_session_class.return_value.__aenter__ = AsyncMock(side_effect=asyncio.CancelledError("task cancelled"))
mock_session_class.return_value.__aexit__ = AsyncMock(return_value=None)
with pytest.raises(asyncio.CancelledError):
await tool.connect()
tool._exit_stack.aclose.assert_called_once()
async def test_aenter_cancelled_error_during_connect_is_catchable_as_exception():
"""Test that CancelledError during __aenter__ is catchable as Exception.
Verifies the end-to-end fix: async with MCPStreamableHTTPTool(...) raises an
exception that can be caught by a normal `except Exception` block.
"""
tool = MCPStreamableHTTPTool(name="test", url="http://example.com")
mock_session = Mock()
mock_session.initialize = AsyncMock(side_effect=asyncio.CancelledError("Cancelled via cancel scope"))
mock_transport = (Mock(), Mock())
mock_context_manager = Mock()
mock_context_manager.__aenter__ = AsyncMock(return_value=mock_transport)
mock_context_manager.__aexit__ = AsyncMock(return_value=None)
tool.get_mcp_client = Mock(return_value=mock_context_manager)
with patch("mcp.client.session.ClientSession") as mock_session_class:
mock_session_class.return_value.__aenter__ = AsyncMock(return_value=mock_session)
mock_session_class.return_value.__aexit__ = AsyncMock(return_value=None)
caught = None
try:
async with tool:
pass
except Exception as e:
caught = e
assert caught is not None, "Expected an exception to be caught by except Exception"
assert isinstance(caught, ToolException)
# Tests for _should_propagate_cancelled_error helper
def test_should_propagate_cancelled_error_returns_false_for_non_cancelled_error():
assert _should_propagate_cancelled_error(RuntimeError("boom")) is False
def test_should_propagate_cancelled_error_returns_false_when_no_current_task():
with patch("asyncio.current_task", return_value=None):
assert _should_propagate_cancelled_error(asyncio.CancelledError()) is False
@pytest.mark.skipif(sys.version_info < (3, 11), reason="task.cancelling() requires Python >= 3.11")
def test_should_propagate_cancelled_error_returns_true_when_task_is_cancelling():
mock_task = Mock()
mock_task.cancelling.return_value = 1
with patch("asyncio.current_task", return_value=mock_task):
assert _should_propagate_cancelled_error(asyncio.CancelledError()) is True
@pytest.mark.skipif(sys.version_info < (3, 11), reason="task.cancelling() requires Python >= 3.11")
def test_should_propagate_cancelled_error_returns_false_when_task_not_cancelling():
mock_task = Mock()
mock_task.cancelling.return_value = 0
with patch("asyncio.current_task", return_value=mock_task):
assert _should_propagate_cancelled_error(asyncio.CancelledError()) is False
async def test_connect_cancelled_error_during_session_creation_includes_exception_in_message():
"""Test that CancelledError from session creation includes exception details in ToolException message."""
tool = MCPStreamableHTTPTool(name="test", url="http://example.com")
mock_transport = (Mock(), Mock())
mock_context_manager = Mock()
mock_context_manager.__aenter__ = AsyncMock(return_value=mock_transport)
mock_context_manager.__aexit__ = AsyncMock(return_value=None)
tool.get_mcp_client = Mock(return_value=mock_context_manager)
with patch("mcp.client.session.ClientSession") as mock_session_class:
mock_session_class.return_value.__aenter__ = AsyncMock(
side_effect=asyncio.CancelledError("cancel scope detail")
)
mock_session_class.return_value.__aexit__ = AsyncMock(return_value=None)
with pytest.raises(ToolException) as exc_info:
await tool.connect()
assert "Failed to create MCP session" in str(exc_info.value)
assert "cancel scope detail" in str(exc_info.value)
async def test_connect_cancelled_error_during_session_creation_logs_with_exc_info():
"""Test that CancelledError from session creation is logged with exc_info=True."""
tool = MCPStreamableHTTPTool(name="test", url="http://example.com")
mock_transport = (Mock(), Mock())
mock_context_manager = Mock()
mock_context_manager.__aenter__ = AsyncMock(return_value=mock_transport)
mock_context_manager.__aexit__ = AsyncMock(return_value=None)
tool.get_mcp_client = Mock(return_value=mock_context_manager)
with patch("mcp.client.session.ClientSession") as mock_session_class:
mock_session_class.return_value.__aenter__ = AsyncMock(side_effect=asyncio.CancelledError("cancel scope"))
mock_session_class.return_value.__aexit__ = AsyncMock(return_value=None)
from agent_framework._mcp import logger as mcp_logger
with patch.object(mcp_logger, "debug") as mock_debug:
with pytest.raises(ToolException):
await tool.connect()
# Verify logger.debug was called with exc_info=True (not an exception instance)
debug_calls = mock_debug.call_args_list
cancel_calls = [c for c in debug_calls if "Failed to create MCP session" in str(c)]
assert cancel_calls, "Expected a debug log for the cancelled session creation"
_, kwargs = cancel_calls[0]
assert kwargs.get("exc_info") is True
def test_mcp_stdio_tool_get_mcp_client_with_env_and_kwargs():
"""Test MCPStdioTool.get_mcp_client() with environment variables and client kwargs."""
env_vars = {"PATH": "/usr/bin", "DEBUG": "1"}
+17 -777
View File
@@ -5,7 +5,6 @@
from __future__ import annotations
import os
from abc import ABC
from collections.abc import Sequence
from pathlib import Path
from typing import Any
@@ -15,7 +14,6 @@ import pytest
from agent_framework import (
AggregatingSkillsSource,
ClassSkill,
DeduplicatingSkillsSource,
FileSkill,
FileSkillScript,
@@ -34,7 +32,6 @@ from agent_framework._skills import (
DEFAULT_SCRIPT_EXTENSIONS,
InlineSkillResource,
InlineSkillScript,
_create_resource_element,
_create_script_element,
_FileSkillResource,
)
@@ -1007,7 +1004,7 @@ class TestInlineSkill:
assert len(skill.resources) == 1
assert skill.resources[0].name == "get_schema"
assert skill.resources[0].description is None
assert skill.resources[0].description == "Get the database schema."
assert isinstance(skill.resources[0], InlineSkillResource)
assert skill.resources[0].function is get_schema
@@ -1193,9 +1190,7 @@ class TestSkillsProviderCodeSkill:
provider = SkillsProvider([skill])
await _init_provider(provider)
result = await provider._read_skill_resource(
_raw_skills(provider), "prog-skill", "get_user_data", auth_token="abc"
)
result = await provider._read_skill_resource(_raw_skills(provider), "prog-skill", "get_user_data", auth_token="abc")
assert result == "data with token=abc"
async def test_read_callable_resource_without_kwargs_ignores_extra_args(self) -> None:
@@ -1680,22 +1675,22 @@ class TestCreateResourceElement:
def test_name_only(self) -> None:
r = InlineSkillResource(name="my-ref", content="data")
elem = _create_resource_element(r)
elem = InlineSkill._create_resource_element(r)
assert elem == ' <resource name="my-ref"/>'
def test_with_description(self) -> None:
r = InlineSkillResource(name="my-ref", description="A reference.", content="data")
elem = _create_resource_element(r)
elem = InlineSkill._create_resource_element(r)
assert elem == ' <resource name="my-ref" description="A reference."/>'
def test_xml_escapes_name(self) -> None:
r = InlineSkillResource(name='ref"special', content="data")
elem = _create_resource_element(r)
elem = InlineSkill._create_resource_element(r)
assert "&quot;" in elem
def test_xml_escapes_description(self) -> None:
r = InlineSkillResource(name="ref", description='Uses <tags> & "quotes"', content="data")
elem = _create_resource_element(r)
elem = InlineSkill._create_resource_element(r)
assert "&lt;tags&gt;" in elem
assert "&amp;" in elem
assert "&quot;" in elem
@@ -2064,7 +2059,6 @@ class TestSkillResourceRead:
async def test_read_async_function(self) -> None:
"""read() awaits an async function and returns its result."""
async def get_data() -> str:
return "async result"
@@ -2074,7 +2068,6 @@ class TestSkillResourceRead:
async def test_read_function_with_kwargs(self) -> None:
"""read() forwards kwargs to functions that accept them."""
def get_config(**kwargs: Any) -> str:
return f"user={kwargs.get('user_id')}"
@@ -2084,7 +2077,6 @@ class TestSkillResourceRead:
async def test_read_async_function_with_kwargs(self) -> None:
"""read() forwards kwargs to async functions that accept them."""
async def get_config(**kwargs: Any) -> str:
return f"user={kwargs.get('user_id')}"
@@ -2094,7 +2086,6 @@ class TestSkillResourceRead:
async def test_read_function_without_kwargs_ignores_extra(self) -> None:
"""read() does not pass kwargs to functions that don't accept them."""
def simple() -> str:
return "fixed"
@@ -2104,7 +2095,6 @@ class TestSkillResourceRead:
async def test_read_function_raises_propagates(self) -> None:
"""read() propagates exceptions from the function."""
def failing() -> str:
raise RuntimeError("boom")
@@ -2139,8 +2129,8 @@ class TestSkillResourceDecoratorEdgeCases:
return "data"
assert skill.resources[0].name == "custom-name"
# description is None when not explicitly provided
assert skill.resources[0].description is None
# description falls back to docstring
assert skill.resources[0].description == "Some docs."
def test_decorator_with_description_only(self) -> None:
skill = InlineSkill(name="my-skill", description="A skill.", instructions="Body")
@@ -2323,7 +2313,7 @@ class TestSkillScriptDecorator:
assert len(skill.scripts) == 1
assert skill.scripts[0].name == "analyze"
assert skill.scripts[0].description is None
assert skill.scripts[0].description == "Run analysis."
assert isinstance(skill.scripts[0], InlineSkillScript)
assert skill.scripts[0].function is analyze
@@ -2757,7 +2747,6 @@ class TestSkillsProviderFactories:
async def test_code_script_returns_object(self) -> None:
"""Code-defined scripts can return non-string objects."""
def returns_dict() -> dict:
return {"status": "ok", "value": 42}
@@ -2866,8 +2855,8 @@ class TestSkillsProviderFactories:
provider = SkillsProvider([skill])
await _init_provider(provider)
result = await provider._run_skill_script(
_raw_skills(provider), "my-skill", "process", args={"mode": "llm-value"}, mode="runtime-value"
result = await provider._run_skill_script(_raw_skills(provider),
"my-skill", "process", args={"mode": "llm-value"}, mode="runtime-value"
)
assert "Error" in result
@@ -2957,7 +2946,6 @@ class TestSkillsProviderFactories:
async def test_code_script_exception_returns_error(self) -> None:
"""A code script function that raises should return an error string."""
def failing_script() -> str:
raise RuntimeError("Something went wrong")
@@ -3180,760 +3168,8 @@ class TestLoadSkillWithScripts:
result = provider._load_skill(_raw_skills(provider), "my-skill")
assert "<scripts>" not in result
# ---------------------------------------------------------------------------
# Tests: ClassSkill
# ---------------------------------------------------------------------------
class _MinimalClassSkill(ClassSkill):
"""A minimal class-based skill with no resources or scripts."""
def __init__(self) -> None:
super().__init__(name="minimal-skill", description="A minimal skill.")
@property
def instructions(self) -> str:
return "Do minimal things."
class _FullClassSkill(ClassSkill):
"""A class-based skill with resources and scripts."""
def __init__(self) -> None:
super().__init__(name="full-skill", description="A full skill.")
self._resources: list[SkillResource] | None = None
self._scripts: list[SkillScript] | None = None
@property
def instructions(self) -> str:
return "Use this skill for full tasks."
@property
def resources(self) -> list[SkillResource]:
if self._resources is None:
self._resources = [
InlineSkillResource(name="test-resource", content="Static resource content."),
]
return self._resources
@property
def scripts(self) -> list[SkillScript]:
if self._scripts is None:
self._scripts = [
InlineSkillScript(name="test-script", function=_class_skill_test_fn),
]
return self._scripts
def _class_skill_test_fn(value: float, factor: float) -> str:
"""Multiply value by factor."""
import json as _json
return _json.dumps({"result": round(value * factor, 4)})
class TestClassSkill:
"""Tests for ClassSkill abstract base class."""
def test_minimal_skill_has_no_resources(self) -> None:
skill = _MinimalClassSkill()
assert skill.resources == []
def test_minimal_skill_has_no_scripts(self) -> None:
skill = _MinimalClassSkill()
assert skill.scripts == []
def test_minimal_skill_content_contains_name(self) -> None:
skill = _MinimalClassSkill()
assert "<name>minimal-skill</name>" in skill.content
def test_minimal_skill_content_contains_description(self) -> None:
skill = _MinimalClassSkill()
assert "<description>A minimal skill.</description>" in skill.content
def test_minimal_skill_content_contains_instructions(self) -> None:
skill = _MinimalClassSkill()
assert "Do minimal things." in skill.content
def test_minimal_skill_content_no_resources_element(self) -> None:
skill = _MinimalClassSkill()
assert "<resources>" not in skill.content
def test_minimal_skill_content_no_scripts_element(self) -> None:
skill = _MinimalClassSkill()
assert "<scripts>" not in skill.content
def test_full_skill_has_resources(self) -> None:
skill = _FullClassSkill()
assert len(skill.resources) == 1
assert skill.resources[0].name == "test-resource"
def test_full_skill_has_scripts(self) -> None:
skill = _FullClassSkill()
assert len(skill.scripts) == 1
assert skill.scripts[0].name == "test-script"
def test_full_skill_content_contains_resources(self) -> None:
skill = _FullClassSkill()
assert "<resources>" in skill.content
assert 'name="test-resource"' in skill.content
def test_full_skill_content_contains_scripts(self) -> None:
skill = _FullClassSkill()
assert "<scripts>" in skill.content
assert 'name="test-script"' in skill.content
def test_content_is_cached(self) -> None:
skill = _MinimalClassSkill()
content1 = skill.content
content2 = skill.content
assert content1 is content2
def test_resources_are_lazy_cached(self) -> None:
skill = _FullClassSkill()
resources1 = skill.resources
resources2 = skill.resources
assert resources1 is resources2
def test_scripts_are_lazy_cached(self) -> None:
skill = _FullClassSkill()
scripts1 = skill.scripts
scripts2 = skill.scripts
assert scripts1 is scripts2
def test_script_has_parameters_schema(self) -> None:
skill = _FullClassSkill()
script = skill.scripts[0]
assert isinstance(script, InlineSkillScript)
schema = script.parameters_schema
assert schema is not None
assert "value" in schema.get("properties", {})
assert "factor" in schema.get("properties", {})
async def test_provider_with_class_skill(self) -> None:
skill = _FullClassSkill()
provider = SkillsProvider([skill])
await _init_provider(provider)
skills = _raw_skills(provider)
assert len(skills) == 1
assert skills[0].name == "full-skill"
async def test_provider_loads_class_skill_content(self) -> None:
skill = _FullClassSkill()
provider = SkillsProvider([skill])
await _init_provider(provider)
result = provider._load_skill(_raw_skills(provider), "full-skill")
assert "Use this skill for full tasks." in result
assert "<resources>" in result
assert "<scripts>" in result
async def test_in_memory_source_with_class_skill(self) -> None:
skill = _MinimalClassSkill()
source = InMemorySkillsSource([skill])
skills = await source.get_skills()
assert len(skills) == 1
assert skills[0].name == "minimal-skill"
async def test_mixed_inline_and_class_skills(self) -> None:
inline = InlineSkill(name="inline-skill", description="Inline", instructions="inline body")
class_skill = _MinimalClassSkill()
provider = SkillsProvider([inline, class_skill])
await _init_provider(provider)
skills = _raw_skills(provider)
names = {s.name for s in skills}
assert names == {"inline-skill", "minimal-skill"}
async def test_class_skill_script_runs(self) -> None:
skill = _FullClassSkill()
script = skill.scripts[0]
result = await script.run(skill, {"value": 10.0, "factor": 2.5})
import json as _json
parsed = _json.loads(result)
assert parsed["result"] == 25.0
async def test_class_skill_resource_reads(self) -> None:
skill = _FullClassSkill()
resource = skill.resources[0]
content = await resource.read()
assert content == "Static resource content."
# ---------------------------------------------------------------------------
# Tests: ClassSkill with decorator-based discovery
# ---------------------------------------------------------------------------
class _DecoratorClassSkill(ClassSkill):
"""A class-based skill using @ClassSkill.resource and @ClassSkill.script decorators."""
def __init__(self) -> None:
super().__init__(name="decorator-skill", description="A decorator-discovered skill.")
@property
def instructions(self) -> str:
return "Use this skill for decorator tests."
@ClassSkill.resource(name="lookup-table")
def get_table(self) -> str:
"""Conversion lookup table."""
return "| From | To | Factor |"
@ClassSkill.script(name="convert")
def run_convert(self, value: float, factor: float) -> str:
"""Convert a value."""
import json as _json
return _json.dumps({"result": round(value * factor, 4)})
class _BareDecoratorSkill(ClassSkill):
"""Skill using bare decorators (no arguments) — name/description from method."""
def __init__(self) -> None:
super().__init__(name="bare-skill", description="Bare decorator skill.")
@property
def instructions(self) -> str:
return "Bare instructions."
@ClassSkill.resource
def my_table(self) -> str:
"""The table docs."""
return "table content"
@ClassSkill.script
def my_script(self, x: int) -> int:
"""Double x."""
return x * 2
class _DuplicateResourceSkill(ClassSkill):
"""Skill with duplicate resource names — should raise."""
def __init__(self) -> None:
super().__init__(name="dup-skill", description="Dup.")
@property
def instructions(self) -> str:
return "x"
@ClassSkill.resource(name="same-name")
def res_a(self) -> str:
return "a"
@ClassSkill.resource(name="same-name")
def res_b(self) -> str:
return "b"
class _DuplicateScriptSkill(ClassSkill):
"""Skill with duplicate script names — should raise."""
def __init__(self) -> None:
super().__init__(name="dup-script-skill", description="Dup.")
@property
def instructions(self) -> str:
return "x"
@ClassSkill.script(name="same-name")
def script_a(self, x: int) -> int:
return x
@ClassSkill.script(name="same-name")
def script_b(self, x: int) -> int:
return x
class _SelfAccessSkill(ClassSkill):
"""Skill where resource/script access instance state via self."""
def __init__(self, multiplier: int = 10) -> None:
super().__init__(name="self-access", description="Self access skill.")
self.multiplier = multiplier
@property
def instructions(self) -> str:
return "Use multiplier."
@ClassSkill.resource(name="config")
def get_config(self) -> str:
return f"multiplier={self.multiplier}"
@ClassSkill.script(name="multiply")
def multiply(self, value: int) -> int:
return value * self.multiplier
class TestClassSkillDecoratorDiscovery:
"""Tests for decorator-based resource/script discovery on ClassSkill."""
def test_discovers_resources(self) -> None:
skill = _DecoratorClassSkill()
assert len(skill.resources) == 1
assert skill.resources[0].name == "lookup-table"
def test_discovers_scripts(self) -> None:
skill = _DecoratorClassSkill()
assert len(skill.scripts) == 1
assert skill.scripts[0].name == "convert"
def test_resource_description_from_decorator(self) -> None:
skill = _DecoratorClassSkill()
assert skill.resources[0].description is None
def test_script_description_from_decorator(self) -> None:
skill = _DecoratorClassSkill()
assert skill.scripts[0].description is None
def test_bare_decorator_name_from_method(self) -> None:
skill = _BareDecoratorSkill()
assert skill.resources[0].name == "my-table"
assert skill.scripts[0].name == "my-script"
def test_bare_decorator_description_is_none(self) -> None:
skill = _BareDecoratorSkill()
assert skill.resources[0].description is None
assert skill.scripts[0].description is None
async def test_resource_reads(self) -> None:
skill = _DecoratorClassSkill()
content = await skill.resources[0].read()
assert content == "| From | To | Factor |"
async def test_script_runs(self) -> None:
skill = _DecoratorClassSkill()
import json as _json
result = await skill.scripts[0].run(skill, {"value": 10.0, "factor": 2.5})
parsed = _json.loads(result)
assert parsed["result"] == 25.0
def test_script_schema_excludes_self(self) -> None:
skill = _DecoratorClassSkill()
script = skill.scripts[0]
assert isinstance(script, InlineSkillScript)
schema = script.parameters_schema
assert schema is not None
props = schema.get("properties", {})
assert "self" not in props
assert "value" in props
assert "factor" in props
def test_resources_cached(self) -> None:
skill = _DecoratorClassSkill()
r1 = skill.resources
r2 = skill.resources
assert r1 == r2
assert r1 is not r2 # defensive copy
def test_scripts_cached(self) -> None:
skill = _DecoratorClassSkill()
s1 = skill.scripts
s2 = skill.scripts
assert s1 == s2
assert s1 is not s2 # defensive copy
def test_content_includes_discovered_resources(self) -> None:
skill = _DecoratorClassSkill()
assert "<resources>" in skill.content
assert 'name="lookup-table"' in skill.content
def test_content_includes_discovered_scripts(self) -> None:
skill = _DecoratorClassSkill()
assert "<scripts>" in skill.content
assert 'name="convert"' in skill.content
def test_duplicate_resource_name_raises(self) -> None:
skill = _DuplicateResourceSkill()
with pytest.raises(ValueError, match="already has a resource named"):
_ = skill.resources
def test_duplicate_script_name_raises(self) -> None:
skill = _DuplicateScriptSkill()
with pytest.raises(ValueError, match="already has a script named"):
_ = skill.scripts
async def test_self_access_resource(self) -> None:
skill = _SelfAccessSkill(multiplier=42)
content = await skill.resources[0].read()
assert content == "multiplier=42"
async def test_self_access_script(self) -> None:
skill = _SelfAccessSkill(multiplier=3)
result = await skill.scripts[0].run(skill, {"value": 7})
assert result == 21
def test_no_decorators_yields_empty(self) -> None:
skill = _MinimalClassSkill()
assert skill.resources == []
assert skill.scripts == []
async def test_provider_with_decorator_skill(self) -> None:
skill = _DecoratorClassSkill()
provider = SkillsProvider([skill])
await _init_provider(provider)
skills = _raw_skills(provider)
assert len(skills) == 1
assert skills[0].name == "decorator-skill"
def test_manual_override_wins(self) -> None:
"""A subclass that overrides resources/scripts bypasses decorator discovery."""
skill = _FullClassSkill()
assert len(skill.resources) == 1
assert skill.resources[0].name == "test-resource"
async def test_property_resource_reads(self) -> None:
"""@ClassSkill.resource on a @property works correctly."""
skill = _PropertyResourceSkill()
assert len(skill.resources) == 1
assert skill.resources[0].name == "static-table"
content = await skill.resources[0].read()
assert "miles" in content
def test_property_resource_description_is_none_without_explicit(self) -> None:
skill = _PropertyResourceSkill()
assert skill.resources[0].description is None
def test_property_resource_in_content(self) -> None:
skill = _PropertyResourceSkill()
assert 'name="static-table"' in skill.content
async def test_mixed_property_and_method_resources(self) -> None:
"""Property and method resources can coexist."""
skill = _MixedPropertyMethodSkill()
names = {r.name for r in skill.resources}
assert names == {"prop-data", "method-data"}
for r in skill.resources:
content = await r.read()
assert "content" in content.lower()
def test_explicit_resource_description_in_object(self) -> None:
"""Explicit description= on @ClassSkill.resource is stored on the object."""
skill = _ExplicitDescriptionSkill()
res = next(r for r in skill.resources if r.name == "described-res")
assert res.description == "A described resource."
def test_explicit_script_description_in_object(self) -> None:
"""Explicit description= on @ClassSkill.script is stored on the object."""
skill = _ExplicitDescriptionSkill()
scr = next(s for s in skill.scripts if s.name == "described-scr")
assert scr.description == "A described script."
def test_explicit_description_in_content_xml(self) -> None:
"""Explicit descriptions appear in the skill content XML."""
skill = _ExplicitDescriptionSkill()
assert 'description="A described resource."' in skill.content
assert 'description="A described script."' in skill.content
def test_property_getter_not_called_during_discovery(self) -> None:
"""Property getter must NOT be evaluated when resources are discovered."""
skill = _PropertyCallCountSkill()
assert skill.getter_call_count == 0
_ = skill.resources # discovery should NOT call the getter
assert skill.getter_call_count == 0
async def test_property_getter_called_on_read(self) -> None:
"""Property getter IS evaluated when the resource is read."""
skill = _PropertyCallCountSkill()
_ = skill.resources
assert skill.getter_call_count == 0
await skill.resources[0].read()
assert skill.getter_call_count == 1
def test_make_method_name_strips_leading_trailing_hyphens(self) -> None:
"""_make_method_name strips leading/trailing underscores turned to hyphens."""
from agent_framework._skills import _make_method_name
assert _make_method_name("my_method") == "my-method"
assert _make_method_name("_private_method_") == "private-method"
assert _make_method_name("__dunder__") == "dunder"
assert _make_method_name("already_good") == "already-good"
def test_inherited_decorated_resources_are_discovered(self) -> None:
"""Decorated resources from a parent class are discovered on subclass."""
skill = _ChildSkill()
names = {r.name for r in skill.resources}
assert "parent-data" in names
def test_inherited_decorated_scripts_are_discovered(self) -> None:
"""Decorated scripts from a parent class are discovered on subclass."""
skill = _ChildSkill()
names = {s.name for s in skill.scripts}
assert "parent-action" in names
def test_child_can_add_own_resources(self) -> None:
"""A child class can add resources alongside inherited ones."""
skill = _ChildSkill()
names = {r.name for r in skill.resources}
assert "parent-data" in names
assert "child-data" in names
async def test_script_receives_kwargs(self) -> None:
"""ClassSkill scripts receive **kwargs forwarded from the runtime."""
skill = _KwargsSkill()
script = skill.scripts[0]
result = await script.run(skill, {"x": 5}, custom_key="hello")
assert result == "5-hello"
def test_wrong_decorator_order_resource_raises(self) -> None:
"""@ClassSkill.resource above @property raises TypeError at class definition."""
with pytest.raises(TypeError, match="must be applied before @property"):
class _BadOrder(ClassSkill):
def __init__(self) -> None:
super().__init__(name="bad", description="bad")
@property
def instructions(self) -> str:
return "x"
@ClassSkill.resource(name="oops") # wrong: should be below @property
@property
def bad_prop(self) -> str:
return "x"
def test_wrong_decorator_order_script_raises(self) -> None:
"""@ClassSkill.script on a property raises TypeError."""
with pytest.raises(TypeError, match="must be applied before"):
class _BadOrder(ClassSkill):
def __init__(self) -> None:
super().__init__(name="bad", description="bad")
@property
def instructions(self) -> str:
return "x"
@ClassSkill.script(name="oops")
@property
def bad_prop(self) -> str:
return "x"
def test_invalid_explicit_resource_name_raises(self) -> None:
"""Invalid name= on @ClassSkill.resource raises ValueError at decoration."""
with pytest.raises(ValueError, match="Invalid @ClassSkill.resource name"):
class _BadName(ClassSkill):
def __init__(self) -> None:
super().__init__(name="bad", description="bad")
@property
def instructions(self) -> str:
return "x"
@ClassSkill.resource(name="UPPER CASE!")
def res(self) -> str:
return "x"
def test_invalid_explicit_script_name_raises(self) -> None:
"""Invalid name= on @ClassSkill.script raises ValueError at decoration."""
with pytest.raises(ValueError, match="Invalid @ClassSkill.script name"):
class _BadName(ClassSkill):
def __init__(self) -> None:
super().__init__(name="bad", description="bad")
@property
def instructions(self) -> str:
return "x"
@ClassSkill.script(name="has spaces")
def scr(self, x: int) -> int:
return x
def test_empty_explicit_name_raises(self) -> None:
"""Empty name= on @ClassSkill.resource raises ValueError."""
with pytest.raises(ValueError, match="name cannot be empty"):
class _EmptyName(ClassSkill):
def __init__(self) -> None:
super().__init__(name="bad", description="bad")
@property
def instructions(self) -> str:
return "x"
@ClassSkill.resource(name="")
def res(self) -> str:
return "x"
def test_resources_copy_prevents_cache_mutation(self) -> None:
"""Mutating the returned resources list does not affect the cache."""
skill = _DecoratorClassSkill()
r1 = skill.resources
r1.clear()
r2 = skill.resources
assert len(r2) == 1 # original cached list is intact
def test_scripts_copy_prevents_cache_mutation(self) -> None:
"""Mutating the returned scripts list does not affect the cache."""
skill = _DecoratorClassSkill()
s1 = skill.scripts
s1.clear()
s2 = skill.scripts
assert len(s2) == 1 # original cached list is intact
async def test_inherited_property_resource_discovered(self) -> None:
"""A @property @ClassSkill.resource on a parent class is discovered on child."""
skill = _ChildWithInheritedPropertySkill()
names = {r.name for r in skill.resources}
assert "parent-prop" in names
content = await next(r for r in skill.resources if r.name == "parent-prop").read()
assert content == "parent property content"
# ---------------------------------------------------------------------------
# Helper skills for additional tests
# ---------------------------------------------------------------------------
class _ExplicitDescriptionSkill(ClassSkill):
"""Skill with explicit descriptions on decorator."""
def __init__(self) -> None:
super().__init__(name="desc-skill", description="Explicit desc.")
@property
def instructions(self) -> str:
return "x"
@ClassSkill.resource(name="described-res", description="A described resource.")
def res(self) -> str:
return "data"
@ClassSkill.script(name="described-scr", description="A described script.")
def scr(self, x: int) -> int:
return x
class _PropertyCallCountSkill(ClassSkill):
"""Tracks how many times the property getter is called."""
def __init__(self) -> None:
super().__init__(name="callcount-skill", description="Tracks calls.")
self.getter_call_count = 0
@property
def instructions(self) -> str:
return "x"
@property
@ClassSkill.resource(name="counted")
def counted_resource(self) -> str:
self.getter_call_count += 1
return "counted"
class _ParentSkill(ClassSkill, ABC):
"""Parent with decorated resources/scripts."""
@ClassSkill.resource(name="parent-data")
def parent_resource(self) -> str:
return "parent"
@ClassSkill.script(name="parent-action")
def parent_script(self, x: int) -> int:
return x
class _ChildSkill(_ParentSkill):
"""Child inheriting parent resources and adding its own."""
def __init__(self) -> None:
super().__init__(name="child-skill", description="Child.")
@property
def instructions(self) -> str:
return "child"
@ClassSkill.resource(name="child-data")
def child_resource(self) -> str:
return "child"
class _KwargsSkill(ClassSkill):
"""Skill that uses **kwargs from runtime."""
def __init__(self) -> None:
super().__init__(name="kwargs-skill", description="Kwargs.")
@property
def instructions(self) -> str:
return "x"
@ClassSkill.script(name="echo")
def echo(self, x: int, **kwargs: Any) -> str:
return f"{x}-{kwargs.get('custom_key', 'none')}"
class _ParentWithPropertyResource(ClassSkill, ABC):
"""Parent with a property-based resource."""
@property
@ClassSkill.resource(name="parent-prop")
def parent_property(self) -> str:
return "parent property content"
class _ChildWithInheritedPropertySkill(_ParentWithPropertyResource):
"""Child that should discover inherited property resource."""
def __init__(self) -> None:
super().__init__(name="child-prop-skill", description="Child prop.")
@property
def instructions(self) -> str:
return "x"
class _PropertyResourceSkill(ClassSkill):
"""Skill with a property-based resource."""
def __init__(self) -> None:
super().__init__(name="prop-skill", description="Property skill.")
@property
def instructions(self) -> str:
return "Use this skill."
@property
@ClassSkill.resource(name="static-table")
def conversion_table(self) -> str:
"""Static conversion table."""
return "| miles | km | 1.60934 |"
class _MixedPropertyMethodSkill(ClassSkill):
"""Skill with both property and method resources."""
def __init__(self) -> None:
super().__init__(name="mixed-prop", description="Mixed.")
@property
def instructions(self) -> str:
return "x"
@property
@ClassSkill.resource(name="prop-data")
def static_data(self) -> str:
"""Static content."""
return "Property Content"
@ClassSkill.resource(name="method-data")
def dynamic_data(self) -> str:
"""Dynamic content."""
return "Method Content"
async def test_code_skill_scripts_element_contains_parameters(self) -> None:
"""Scripts XML includes parameters schema when the function has typed parameters."""
def analyze(query: str, limit: int = 10) -> str:
return "result"
@@ -4519,7 +3755,9 @@ class TestSourceComposition:
)
(skill_dir / "run.py").write_text("print('hi')", encoding="utf-8")
source = DeduplicatingSkillsSource(FileSkillsSource(str(tmp_path), script_runner=_noop_script_runner))
source = DeduplicatingSkillsSource(
FileSkillsSource(str(tmp_path), script_runner=_noop_script_runner)
)
provider = SkillsProvider(source)
await _init_provider(provider)
assert "my-skill" in _ctx(provider)[0]
@@ -4560,7 +3798,9 @@ class TestSourceComposition:
call_log.append("source")
return "source"
source = DeduplicatingSkillsSource(FileSkillsSource(str(tmp_path), script_runner=source_runner))
source = DeduplicatingSkillsSource(
FileSkillsSource(str(tmp_path), script_runner=source_runner)
)
provider = SkillsProvider(source)
await _init_provider(provider)
+10 -12
View File
@@ -10,8 +10,7 @@ Start with file-based or code-defined skills, then explore combining them and ad
|--------|-------------|
| [**file_based_skill**](file_based_skill/) | Define skills as `SKILL.md` files on disk with reference documents and executable scripts. Uses the unit-converter skill. |
| [**code_defined_skill**](code_defined_skill/) | Define skills entirely in Python code using `Skill`, `@skill.resource`, and `@skill.script` decorators. Uses a code-defined unit-converter skill. |
| [**class_based_skill**](class_based_skill/) | Define skills as Python classes using `ClassSkill` with `@ClassSkill.resource` and `@ClassSkill.script` decorators for auto-discovery. Uses a class-based unit-converter skill. |
| [**mixed_skills**](mixed_skills/) | Combine code-defined, class-based, and file-based skills in a single agent. Uses a code-defined volume-converter, a class-based temperature-converter, and a file-based unit-converter. |
| [**mixed_skills**](mixed_skills/) | Combine code-defined and file-based skills in a single agent. Uses a code-defined volume-converter and a file-based unit-converter. |
| [**script_approval**](script_approval/) | Require human-in-the-loop approval before executing skill scripts |
## Key Concepts
@@ -24,18 +23,17 @@ Skills use a three-step interaction model to minimize token usage:
2. **Load** — Full instructions are loaded on-demand via the `load_skill` tool
3. **Access** — Resources are read via `read_skill_resource`; scripts are executed via `run_skill_script`
### File-Based vs Code-Defined vs Class-Based Skills
### File-Based vs Code-Defined Skills
| Aspect | File-Based | Code-Defined | Class-Based |
|--------|-----------|--------------|-------------|
| Definition | `SKILL.md` files on disk | `Skill` instances in Python | Classes extending `ClassSkill` |
| Resources | Static files in `references/` and `assets/` directories | Callable functions via `@skill.resource` decorator | `@ClassSkill.resource` decorator (auto-discovered) |
| Scripts | Python files in `scripts/` directory (executed via subprocess) | Callable functions via `@skill.script` decorator (executed in-process) | `@ClassSkill.script` decorator (executed in-process) |
| Discovery | Automatic via `skill_paths` parameter | Explicit via `skills` parameter | Explicit via `skills` parameter |
| Dynamic content | No (static files only) | Yes (functions can generate content at runtime) | Yes (functions can generate content at runtime) |
| Sharing pattern | Copy skill directory | Inline or shared instances | Package in shared libraries/PyPI |
| Aspect | File-Based | Code-Defined |
|--------|-----------|--------------|
| Definition | `SKILL.md` files on disk | `Skill` instances in Python |
| Resources | Static files in `references/` and `assets/` directories | Callable functions via `@skill.resource` decorator |
| Scripts | Python files in `scripts/` directory (executed via subprocess) | Callable functions via `@skill.script` decorator (executed in-process) |
| Discovery | Automatic via `skill_paths` parameter | Explicit via `skills` parameter |
| Dynamic content | No (static files only) | Yes (functions can generate content at runtime) |
All three types can be combined in a single `SkillsProvider` — see the [mixed_skills](mixed_skills/) sample.
Both types can be combined in a single `SkillsProvider` — see the [mixed_skills](mixed_skills/) sample.
### Script Execution
@@ -1,71 +0,0 @@
# Class-Based Agent Skills
This sample demonstrates how to define **Agent Skills as Python classes** using `ClassSkill`.
## What's Demonstrated
- Creating skills as classes that extend `ClassSkill`
- Bundling name, description, instructions, resources, and scripts into a single class
- Using `@ClassSkill.resource` decorator for automatic resource discovery
- Using `@ClassSkill.script` decorator for automatic script discovery
- Lazy-loading and caching of resources and scripts
- Registering class-based skills with `SkillsProvider`
## Skills Included
### unit-converter (class-based)
A `UnitConverterSkill` class that converts between common units. Defined in `class_based_skill.py`:
- `conversion-table` — Static resource with factor table
- `convert` — Script that performs `value × factor` conversion
## Project Structure
```
class_based_skill/
├── class_based_skill.py
└── README.md
```
## Running the Sample
### Prerequisites
- An [Azure AI Foundry](https://ai.azure.com/) project with a deployed model (e.g. `gpt-4o-mini`)
### Environment Variables
Set the required environment variables in a `.env` file (see `python/.env.example`):
- `FOUNDRY_PROJECT_ENDPOINT`: Your Azure AI Foundry project endpoint
- `FOUNDRY_MODEL`: The name of your model deployment (defaults to `gpt-4o-mini`)
### Authentication
This sample uses `AzureCliCredential` for authentication. Run `az login` in your terminal before running the sample.
### Run
```bash
cd python
uv run samples/02-agents/skills/class_based_skill/class_based_skill.py
```
### Expected Output
```
Converting units with class-based skills
------------------------------------------------------------
Agent: Here are your conversions:
1. **26.2 miles → 42.16 km** (a marathon distance)
2. **75 kg → 165.35 lbs**
```
## Learn More
- [Agent Skills Specification](https://agentskills.io/)
- [Code-Defined Skills Sample](../code_defined_skill/)
- [Mixed Skills Sample](../mixed_skills/)
- [Microsoft Agent Framework Documentation](../../../../../docs/)
@@ -1,145 +0,0 @@
# Copyright (c) Microsoft. All rights reserved.
import asyncio
import json
import os
# Uncomment this filter to suppress the experimental Skills warning before
# using the sample's Skills APIs.
# import warnings # isort: skip
# warnings.filterwarnings("ignore", message=r"\[SKILLS\].*", category=FutureWarning)
from textwrap import dedent
from agent_framework import Agent, ClassSkill, SkillsProvider
from agent_framework.foundry import FoundryChatClient
from azure.identity import AzureCliCredential
from dotenv import load_dotenv
"""
Class-Based Agent Skills Define skills as Python classes
This sample demonstrates how to define Agent Skills as reusable Python classes
by subclassing ``ClassSkill``. Class-based skills bundle all components (name,
description, instructions, resources, scripts) into a single class, making
them easy to package and distribute via shared libraries or PyPI.
Key concepts shown:
- Subclassing ``ClassSkill`` to create a self-contained skill
- Using ``@property`` + ``@ClassSkill.resource`` (bare) name defaults to method name
- Using ``@ClassSkill.script(name=..., description=...)`` explicit name and description
- Lazy-loading and caching of resources and scripts
"""
# Load environment variables from .env file
load_dotenv()
# ---------------------------------------------------------------------------
# Class-Based Skill: UnitConverterSkill
# ---------------------------------------------------------------------------
class UnitConverterSkill(ClassSkill):
"""A unit-converter skill defined as a Python class.
Converts between common units (mileskm, poundskg) using a
conversion factor. Resources and scripts are discovered automatically
via decorators.
"""
def __init__(self) -> None:
super().__init__(
name="unit-converter",
description=(
"Convert between common units using a multiplication factor. "
"Use when asked to convert miles, kilometers, pounds, or kilograms."
),
)
@property
def instructions(self) -> str:
return dedent("""\
Use this skill when the user asks to convert between units.
1. Review the conversion-table resource to find the factor for the requested conversion.
2. Use the convert script, passing the value and factor from the table.
3. Present the result clearly with both units.
""")
# 1. Property with bare decorator — name defaults to the method name
# ("conversion_table" → "conversion-table"), no description.
# Place @property first, then @ClassSkill.resource.
@property
@ClassSkill.resource
def conversion_table(self) -> str:
"""Lookup table of multiplication factors for common unit conversions."""
return dedent("""\
# Conversion Tables
Formula: **result = value × factor**
| From | To | Factor |
|-------------|-------------|----------|
| miles | kilometers | 1.60934 |
| kilometers | miles | 0.621371 |
| pounds | kilograms | 0.453592 |
| kilograms | pounds | 2.20462 |
""")
# 2. Explicit name — overrides the method name
# 3. Explicit description — provides a description for the script
@ClassSkill.script(name="convert", description="Multiplies a value by a conversion factor.")
def convert_units(self, value: float, factor: float) -> str:
"""Convert a value using a multiplication factor: result = value × factor.
Args:
value: The numeric value to convert.
factor: Conversion factor from the conversion table.
Returns:
JSON string with the inputs and converted result.
"""
result = round(value * factor, 4)
return json.dumps({"value": value, "factor": factor, "result": result})
async def main() -> None:
"""Run the class-based skills demo."""
endpoint = os.environ["FOUNDRY_PROJECT_ENDPOINT"]
deployment = os.environ.get("FOUNDRY_MODEL", "gpt-4o-mini")
client = FoundryChatClient(
project_endpoint=endpoint,
model=deployment,
credential=AzureCliCredential(),
)
# Instantiate the class-based skill and pass it to the provider
unit_converter = UnitConverterSkill()
async with Agent(
client=client,
instructions="You are a helpful assistant that can convert units.",
context_providers=[SkillsProvider(unit_converter)],
) as agent:
print("Converting units with class-based skills")
print("-" * 60)
response = await agent.run(
"How many kilometers is a marathon (26.2 miles)? And how many pounds is 75 kilograms?"
)
print(f"Agent: {response}\n")
if __name__ == "__main__":
asyncio.run(main())
"""
Sample output:
Converting units with class-based skills
------------------------------------------------------------
Agent: Here are your conversions:
1. **26.2 miles 42.16 km** (a marathon distance)
2. **75 kg 165.35 lbs**
"""
@@ -1,18 +1,17 @@
# Mixed Skills — Code, Class, and File Skills
# Mixed Skills — Code Skills and File Skills
This sample demonstrates how to combine **code-defined skills**,
**class-based skills**, and **file-based skills** in a single agent using
`SkillsProvider`.
This sample demonstrates how to combine **code-defined skills** and
**file-based skills** in a single agent using a `SkillScriptRunner` callable
and `SkillsProvider`.
## Concepts
| Concept | Description |
|---------|-------------|
| **Code skill** | A `Skill` created in Python with `@skill.script` decorators for in-process callable functions and `@skill.resource` for dynamic content |
| **Class skill** | A self-contained skill class extending `ClassSkill`, bundling instructions, resources, and scripts |
| **File skill** | A skill discovered from a `SKILL.md` file on disk, with reference documents and executable script files |
| **`script_runner`** | A callable (sync or async) satisfying the `SkillScriptRunner` protocol — required when file skills have scripts |
| **`SkillsProvider`** | Registers code-defined, class-based, and file-based skills in a single provider |
| **`SkillsProvider`** | Registers both code-defined and file-based skills in a single provider |
## Skills in This Sample
@@ -25,15 +24,6 @@ Defined entirely in Python code using decorators:
Code scripts run **in-process** — no subprocess or external runner needed.
### temperature-converter (class skill)
Defined as a `TemperatureConverterSkill` class extending `ClassSkill`:
- **`@ClassSkill.resource`** — `temperature-conversion-formulas`: °F↔°C↔K formulas
- **`@ClassSkill.script`** — `convert-temperature`: converts between temperature scales
Class-based scripts run **in-process** — no subprocess or external runner needed.
### unit-converter (file skill)
Discovered from `skills/unit-converter/SKILL.md`:
@@ -53,10 +43,7 @@ File scripts are executed as **local Python subprocesses** via the
│ AggregatingSkillsSource([ │
│ FileSkillsSource("./skills", # file skills │
│ script_runner=runner), │
│ InMemorySkillsSource([
│ volume_skill, # code skill │
│ temp_converter, # class skill │
│ ]), │
│ InMemorySkillsSource([skill]), # code skills
│ ]) │
│ ) │
│ ) │
@@ -67,7 +54,6 @@ File scripts are executed as **local Python subprocesses** via the
│ script_runner(skill, script, args) │
│ │
│ • Code scripts (@skill.script) → in-process call │
│ • Class scripts (@ClassSkill.script) → in-process call │
│ • File scripts (scripts/*.py) → subprocess via │
│ the callback function │
└─────────────────────────────────────────────────────────────┘
@@ -16,7 +16,6 @@ from typing import Any
from agent_framework import (
Agent,
AggregatingSkillsSource,
ClassSkill,
DeduplicatingSkillsSource,
FileSkillsSource,
InlineSkill,
@@ -35,32 +34,28 @@ if _SKILLS_ROOT not in sys.path:
from subprocess_script_runner import subprocess_script_runner # noqa: E402
"""
Mixed Skills Code, class, and file skills in a single agent
Mixed Skills Code skills and file skills in a single agent
This sample demonstrates how to combine **code-defined skills** (with
``@skill.script`` and ``@skill.resource`` decorators), **class-based skills**
(subclassing ``ClassSkill``), and **file-based skills** (discovered from
``SKILL.md`` files on disk) in a single agent using ``SkillsProvider`` and
a ``SkillScriptRunner`` callable.
``@skill.script`` and ``@skill.resource`` decorators) and **file-based skills**
(discovered from ``SKILL.md`` files on disk) in a single agent using
``SkillsProvider`` and a ``SkillScriptRunner`` callable.
Key concepts shown:
- Code skills with ``@skill.script``: executable Python functions the agent
can invoke directly in-process.
- Code skills with ``@skill.resource``: dynamic content the agent can read
on demand.
- Class skills: self-contained skill classes extending ``ClassSkill``.
- File skills from disk: ``SKILL.md`` files with reference documents and
executable script files.
- ``script_runner``: routes **file-based** script execution
through a callback, enabling custom handling (e.g. subprocess calls).
Code-defined and class-based scripts run in-process automatically.
Code-defined scripts (``@skill.script``) run in-process automatically.
The sample registers three skills:
The sample registers two skills:
1. **volume-converter** (code skill) converts between gallons and liters using
``@skill.script`` for conversion and ``@skill.resource`` for the factor table.
2. **temperature-converter** (class skill) converts between temperature scales
(°F°CK) using a ``ClassSkill`` subclass.
3. **unit-converter** (file skill) converts between common units (mileskm,
2. **unit-converter** (file skill) converts between common units (mileskm,
poundskg) via a subprocess-executed Python script discovered from
``skills/unit-converter/SKILL.md``.
"""
@@ -115,68 +110,9 @@ def convert_volume(value: float, factor: float) -> str:
# ---------------------------------------------------------------------------
# 2. Define a class-based skill for temperature conversion
# 2. Wire everything together and run the agent
# ---------------------------------------------------------------------------
class TemperatureConverterSkill(ClassSkill):
"""A temperature-converter skill defined as a Python class.
Converts between temperature scales (Fahrenheit, Celsius, Kelvin).
Resources and scripts are discovered automatically via decorators.
"""
def __init__(self) -> None:
super().__init__(
name="temperature-converter",
description="Convert between temperature scales (Fahrenheit, Celsius, Kelvin).",
)
@property
def instructions(self) -> str:
return dedent("""\
Use this skill when the user asks to convert temperatures.
1. Read the temperature-conversion-formulas resource to find the factor and offset
for the requested conversion.
2. Use the convert-temperature script, passing value, factor, and offset.
3. Present the result clearly with both temperature scales.
""")
@ClassSkill.resource(name="temperature-conversion-formulas")
def formulas(self) -> str:
"""Temperature conversion formulas reference table."""
return dedent("""\
# Temperature Conversion Formulas
Formula: **result = value × factor + offset**
| From | To | Factor | Offset |
|-------------|-------------|----------|-----------|
| Fahrenheit | Celsius | 0.555556 | -17.7778 |
| Celsius | Fahrenheit | 1.8 | 32 |
| Celsius | Kelvin | 1 | 273.15 |
| Kelvin | Celsius | 1 | -273.15 |
""")
@ClassSkill.script(name="convert-temperature")
def convert_temperature(self, value: float, factor: float, offset: float = 0) -> str:
"""Convert a temperature value using factor and offset from the formulas resource.
Args:
value: The numeric temperature value to convert.
factor: Conversion factor from the formulas resource.
offset: Offset to add after multiplying (default 0).
Returns:
JSON string with the conversion result.
"""
result = round(value * factor + offset, 4)
return json.dumps({"value": value, "factor": factor, "offset": offset, "result": result})
# ---------------------------------------------------------------------------
# 3. Wire everything together and run the agent
# ---------------------------------------------------------------------------
async def main() -> None:
"""Run the combined skills demo."""
@@ -190,11 +126,9 @@ async def main() -> None:
credential=AzureCliCredential(),
)
# Create the SkillsProvider with code, class, and file skills.
# The script_runner handles file-based scripts; code-defined and
# class-based scripts run in-process automatically.
temperature_converter = TemperatureConverterSkill()
# Create the SkillsProvider with both code and file skills.
# The script_runner handles file-based scripts; code-defined scripts
# (@skill.script) run in-process automatically.
skills_provider = SkillsProvider(
DeduplicatingSkillsSource(
AggregatingSkillsSource([
@@ -202,7 +136,7 @@ async def main() -> None:
str(Path(__file__).parent / "skills"),
script_runner=subprocess_script_runner,
),
InMemorySkillsSource([volume_converter_skill, temperature_converter]),
InMemorySkillsSource([volume_converter_skill]),
])
)
)
@@ -210,17 +144,14 @@ async def main() -> None:
# Run the agent
async with Agent(
client=client,
instructions="You are a helpful assistant that can convert units, volumes, and temperatures.",
instructions="You are a helpful assistant that can convert units.",
context_providers=[skills_provider],
) as agent:
# Ask the agent to use all three skills
print("Converting with mixed skills (file + code + class)")
# Ask the agent to use both skills
print("Converting units")
print("-" * 60)
response = await agent.run(
"I need three conversions: "
"1) How many kilometers is a marathon (26.2 miles)? "
"2) How many liters is a 5-gallon bucket? "
"3) What is 98.6°F in Celsius?"
"How many kilometers is a marathon (26.2 miles)? And how many liters is a 5-gallon bucket?"
)
print(f"Agent: {response}\n")
@@ -231,11 +162,12 @@ if __name__ == "__main__":
"""
Sample output:
Converting with mixed skills (file + code + class)
Converting units
------------------------------------------------------------
Agent: Here are your conversions:
1. **26.2 miles 42.16 km** (a marathon distance)
2. **5 gallons 18.93 liters**
3. **98.6°F 37.0°C**
I used the conversion factors from each skill's reference table.
"""
@@ -2,18 +2,16 @@
"""Aggregate per-provider JUnit XML test results and generate a trend report.
Parses JUnit XML files produced by CI jobs both ``pytest.xml`` (Python) and
xunit v3 ``*.junit`` (dotnet) merges them into a single run, combines
with historical data, and generates a markdown trend table.
Parses ``pytest.xml`` (JUnit XML) files produced by each CI job, merges them
into a single run, combines with historical data, and generates a markdown
trend table the same pattern used by ``scripts/sample_validation/aggregate.py``.
Usage (from CI):
python aggregate.py <reports-dir> <history-file> <output-file>
The reports directory is expected to contain artifact subdirectories. Two
layouts are supported:
- **Python (pytest):** ``test-results-<provider>/pytest.xml``
- **Dotnet (xunit):** ``dotnet-test-results-<tfm>-<os>/*.junit``
The reports directory is expected to contain subdirectories named
``test-results-<provider>/`` each containing a ``pytest.xml`` file
(created by ``actions/download-artifact``).
"""
from __future__ import annotations
@@ -48,21 +46,9 @@ def _format_run_label(timestamp: str) -> str:
def _derive_provider(directory_name: str) -> str:
"""Derive a provider label from a report directory name.
Handles both Python and dotnet naming conventions:
- ``test-results-openai`` ``OpenAI``
- ``test-results-azure-openai`` ``Azure OpenAI``
- ``dotnet-test-results-net10.0-ubuntu-latest`` ``net10.0 (ubuntu)``
``test-results-openai`` ``OpenAI``
``test-results-azure-openai`` ``Azure OpenAI``
"""
# Dotnet convention: dotnet-test-results-<framework>-<os>
if directory_name.startswith("dotnet-test-results-"):
raw = directory_name.replace("dotnet-test-results-", "")
# e.g. "net10.0-ubuntu-latest" → framework="net10.0", os="ubuntu-latest"
parts = raw.split("-", 1)
framework = parts[0]
os_label = parts[1].split("-")[0] if len(parts) > 1 else ""
return f"{framework} ({os_label})" if os_label else framework
# Python convention: test-results-<provider>
raw = directory_name.replace("test-results-", "")
known = {
"openai": "OpenAI",
@@ -116,21 +102,11 @@ def _parse_junit_xml(xml_path: Path) -> list[dict[str, str]]:
# it appends the class name, e.g.:
# "packages.foundry.tests.foundry.test_foundry_embedding_client.TestFoundryEmbeddingIntegration"
# We want the file-level module: "test_foundry_embedding_client"
#
# xunit (dotnet) writes classname as the full C# type, e.g.:
# "OpenAIChatCompletion.IntegrationTests.ChatCompletionTests"
# We want the project prefix: "OpenAIChatCompletion"
if classname:
parts = classname.rsplit(".", 2)
# If the last segment starts with uppercase it's a class name — take the one before it
if len(parts) >= 2 and parts[-1][0:1].isupper():
# For dotnet: if the penultimate part is "IntegrationTests" or "UnitTests",
# use the part before that (the project name) instead
if parts[-2] in ("IntegrationTests", "UnitTests") and len(parts) >= 3:
# parts[0] may contain dots — take the last segment of it
module = parts[0].rsplit(".", 1)[-1]
else:
module = parts[-2]
module = parts[-2]
else:
module = parts[-1]
else:
@@ -172,61 +148,28 @@ def _parse_junit_xml(xml_path: Path) -> list[dict[str, str]]:
# ---------------------------------------------------------------------------
def _discover_xml_files(reports_dir: Path) -> list[tuple[str, Path]]:
"""Discover JUnit XML test result files in artifact subdirectories.
Handles two directory layouts:
- **Python (pytest):** ``test-results-<provider>/pytest.xml``
- **Dotnet (xunit):** ``dotnet-test-results-<tfm>-<os>/*.junit``
Returns:
List of ``(directory_name, xml_path)`` tuples.
"""
xml_files: list[tuple[str, Path]] = []
if not reports_dir.is_dir():
return xml_files
for subdir in sorted(reports_dir.iterdir()):
if not subdir.is_dir():
continue
# Python layout: single pytest.xml per artifact
pytest_xml = subdir / "pytest.xml"
if pytest_xml.exists():
xml_files.append((subdir.name, pytest_xml))
continue
# Dotnet layout: multiple *.junit files per artifact
junit_files = sorted(subdir.rglob("*.junit"))
for jf in junit_files:
xml_files.append((subdir.name, jf))
# Fallback: any .xml file that looks like JUnit (not .trx, not cobertura)
if not junit_files:
for xf in sorted(subdir.rglob("*.xml")):
if xf.suffix == ".xml" and not xf.name.endswith(".cobertura.xml"):
xml_files.append((subdir.name, xf))
return xml_files
def load_current_run(reports_dir: Path) -> dict[str, Any]:
"""Load per-provider JUnit XML reports from the current CI run and merge.
Supports both pytest (Python) and xunit v3 (dotnet) JUnit XML formats.
Args:
reports_dir: Directory containing artifact subdirectories with XML reports.
reports_dir: Directory containing ``test-results-<provider>/`` subdirs.
Returns:
Merged run dict with ``timestamp``, ``summary``, ``results``.
"""
combined_results: dict[str, dict[str, str]] = {} # nodeid → {status, provider}
xml_files = _discover_xml_files(reports_dir)
# actions/download-artifact creates: reports_dir/test-results-openai/pytest.xml
xml_files: list[tuple[str, Path]] = []
if reports_dir.is_dir():
for subdir in sorted(reports_dir.iterdir()):
if subdir.is_dir():
xml_file = subdir / "pytest.xml"
if xml_file.exists():
xml_files.append((subdir.name, xml_file))
if not xml_files:
print(f"Warning: No JUnit XML files found in {reports_dir}")
print(f"Warning: No pytest.xml files found in {reports_dir}")
return {
"timestamp": datetime.now(timezone.utc).isoformat(),
"summary": {
@@ -238,42 +181,19 @@ def load_current_run(reports_dir: Path) -> dict[str, Any]:
"results": {},
}
# Dotnet tests always run under multiple frameworks, so we always
# qualify their keys with the provider to ensure deterministic,
# stable keys across runs regardless of file parse order.
is_dotnet = any(d.startswith("dotnet-test-results-") for d, _ in xml_files)
for dir_name, xml_file in xml_files:
print(f" Loading: {xml_file}")
provider = _derive_provider(dir_name)
tests = _parse_junit_xml(xml_file)
for test in tests:
raw_id = test["nodeid"]
key = f"{provider}::{raw_id}" if is_dotnet else raw_id
combined_results[key] = {
combined_results[test["nodeid"]] = {
"status": test["status"],
"provider": provider,
"module": test.get("module", ""),
}
# Build per-provider summary counts so the report can show one row per
# framework (dotnet) or per provider (Python).
provider_counts: dict[str, dict[str, int]] = {}
for r in combined_results.values():
prov = r.get("provider", "Unknown")
if prov not in provider_counts:
provider_counts[prov] = {"total": 0, "passed": 0, "failed": 0, "skipped": 0}
provider_counts[prov]["total"] += 1
st = r["status"]
if st == "passed":
provider_counts[prov]["passed"] += 1
elif st in ("failed", "error"):
provider_counts[prov]["failed"] += 1
elif st == "skipped":
provider_counts[prov]["skipped"] += 1
# Overall summary (sum across all providers).
# Build summary counts using mutually exclusive status buckets.
# Errors are folded into the failed count for display purposes.
statuses = [r["status"] for r in combined_results.values()]
summary = {
"total": len(statuses),
@@ -285,7 +205,6 @@ def load_current_run(reports_dir: Path) -> dict[str, Any]:
return {
"timestamp": datetime.now(timezone.utc).isoformat(),
"summary": summary,
"provider_summaries": provider_counts,
"results": combined_results,
}
@@ -334,29 +253,7 @@ def generate_trend_report(runs: list[dict[str, Any]]) -> str:
"",
]
# Detect whether this is a dotnet report (provider-qualified keys).
is_dotnet = False
for run in runs:
provider_sums = run.get("provider_summaries", {})
if any(p.startswith("net") for p in provider_sums):
is_dotnet = True
break
if is_dotnet:
_generate_dotnet_report(lines, runs)
else:
_generate_python_report(lines, runs)
lines.append("")
lines.append("**Legend:** ✅ Passed · ❌ Failed · ⏭️ Skipped · ⚠️ Expected Failure (xfail) · N/A Not available")
lines.append("")
return "\n".join(lines)
def _generate_python_report(lines: list[str], runs: list[dict[str, Any]]) -> None:
"""Generate the original single-table Python report format."""
# --- Overall status table ---
# --- Overall status table (most recent first) ---
lines.append("## Overall Status (Last 5 Runs)")
lines.append("")
lines.append("| Run | Total | ✅ Passed | ❌ Failed | ⏭️ Skipped |")
@@ -379,91 +276,27 @@ def _generate_python_report(lines: list[str], runs: list[dict[str, Any]]) -> Non
lines.append("")
# --- Single per-test results table ---
_generate_per_test_table(lines, runs, "## Per-Test Results")
# --- Per-test results table ---
lines.append("## Per-Test Results")
lines.append("")
def _generate_dotnet_report(lines: list[str], runs: list[dict[str, Any]]) -> None:
"""Generate per-framework tables for dotnet (net10.0, net472, etc.)."""
# Collect all providers seen across all runs, sorted for stable ordering
all_providers: set[str] = set()
for run in runs:
all_providers.update(run.get("provider_summaries", {}).keys())
providers = sorted(all_providers)
for provider in providers:
lines.append(f"## {provider}")
lines.append("")
# --- Per-provider summary table ---
lines.append("| Run | Total | ✅ Passed | ❌ Failed | ⏭️ Skipped |")
lines.append("|-----|-------|-----------|-----------|------------|")
for run in reversed(runs):
ps = run.get("provider_summaries", {}).get(provider, {})
total = ps.get("total", 0)
label = _format_run_label(run["timestamp"])
if total == 0:
lines.append(f"| {label} | N/A | N/A | N/A | N/A |")
else:
lines.append(
f"| {label} "
f"| {total} "
f"| {ps.get('passed', 0)}/{total} "
f"| {ps.get('failed', 0)}/{total} "
f"| {ps.get('skipped', 0)}/{total} |"
)
for _ in range(MAX_HISTORY - len(runs)):
lines.append("| N/A | N/A | N/A | N/A | N/A |")
lines.append("")
# --- Per-test table filtered to this provider ---
_generate_per_test_table(
lines, runs,
heading=None,
provider_filter=provider,
)
def _generate_per_test_table(
lines: list[str],
runs: list[dict[str, Any]],
heading: str | None = None,
provider_filter: str | None = None,
) -> None:
"""Emit a per-test trend table, optionally filtered to a single provider."""
if heading:
lines.append(heading)
lines.append("")
# Collect all test nodeids (and metadata) across all runs
all_tests: dict[str, str] = {} # nodeid → provider
all_modules: dict[str, str] = {} # nodeid → module
# Collect all test nodeids, providers, and modules across all runs
all_tests: dict[str, str] = {} # nodeid → provider (from most recent run)
all_modules: dict[str, str] = {} # nodeid → module (from most recent run)
for run in runs:
for nodeid, info in run.get("results", {}).items():
if not isinstance(info, dict):
continue
prov = info.get("provider", "Unknown")
if provider_filter and prov != provider_filter:
continue
module = info.get("module", "")
all_tests[nodeid] = prov
provider = info.get("provider", "Unknown") if isinstance(info, dict) else "Unknown"
module = info.get("module", "") if isinstance(info, dict) else ""
all_tests[nodeid] = provider
all_modules[nodeid] = module
if not all_tests:
lines.append("*No test results available.*")
lines.append("")
return
return "\n".join(lines)
# Build header
if provider_filter:
header = "| Test | File |"
separator = "|------|------|"
else:
header = "| Test | File | Provider |"
separator = "|------|------|----------|"
# Build header (most recent run first)
header = "| Test | File | Provider |"
separator = "|------|------|----------|"
for run in reversed(runs):
label = _format_run_label(run["timestamp"])
header += f" {label} |"
@@ -475,15 +308,12 @@ def _generate_per_test_table(
lines.append(header)
lines.append(separator)
# Sort by module then test name
for nodeid in sorted(all_tests, key=lambda n: (all_modules.get(n, ""), n)):
# Sort by provider then test name
for nodeid in sorted(all_tests, key=lambda n: (all_tests[n], n)):
provider = all_tests[nodeid]
module = all_modules.get(nodeid, "")
short = _short_name(nodeid)
if provider_filter:
row = f"| `{short}` | `{module}` |"
else:
provider = all_tests[nodeid]
row = f"| `{short}` | `{module}` | {provider} |"
row = f"| `{short}` | `{module}` | {provider} |"
for run in reversed(runs):
result = run.get("results", {}).get(nodeid)
@@ -500,6 +330,10 @@ def _generate_per_test_table(
lines.append(row)
lines.append("")
lines.append("**Legend:** ✅ Passed · ❌ Failed · ⏭️ Skipped · ⚠️ Expected Failure (xfail) · N/A Not available")
lines.append("")
return "\n".join(lines)
# ---------------------------------------------------------------------------