Merge remote-tracking branch 'origin/main' into hosted-output-formating

This commit is contained in:
alliscode
2026-05-07 14:54:47 -07:00
Unverified
43 changed files with 4108 additions and 242 deletions
+84 -1
View File
@@ -273,6 +273,8 @@ jobs:
-c ${{ matrix.configuration }} `
--no-build -v Normal `
--report-xunit-trx `
--report-junit `
--results-directory ../IntegrationTestResults/ `
--ignore-exit-code 8 `
--filter-not-trait "Category=IntegrationDisabled" `
--filter-not-trait "Category=FoundryHostedAgents" `
@@ -294,6 +296,10 @@ jobs:
AZURE_AI_PROJECT_ENDPOINT: ${{ vars.AZURE_AI_PROJECT_ENDPOINT }}
AZURE_AI_MODEL_DEPLOYMENT_NAME: ${{ vars.AZURE_AI_MODEL_DEPLOYMENT_NAME }}
AZURE_AI_BING_CONNECTION_ID: ${{ vars.AZURE_AI_BING_CONNECTION_ID }}
# Anthropic Models
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
ANTHROPIC_CHAT_MODEL_NAME: ${{ vars.ANTHROPIC_CHAT_MODEL_NAME }}
ANTHROPIC_REASONING_MODEL_NAME: ${{ vars.ANTHROPIC_REASONING_MODEL_NAME }}
# Generate test reports and check coverage
- name: Generate test reports
@@ -316,6 +322,14 @@ jobs:
shell: pwsh
run: ./dotnet/eng/scripts/dotnet-check-coverage.ps1 -JsonReportPath "TestResults/Reports/Summary.json" -CoverageThreshold $env:COVERAGE_THRESHOLD
- name: Upload integration test results
if: always() && github.event_name != 'pull_request' && matrix.integration-tests
uses: actions/upload-artifact@v7
with:
name: dotnet-test-results-${{ matrix.targetFramework }}-${{ matrix.os }}
path: IntegrationTestResults/**/*.junit
if-no-files-found: ignore
# The Foundry hosted-agent IT is costly (it builds a container, pushes to ACR, and provisions
# live agents on a separate Foundry project). Running it in its own job keeps the overall
# workflow time roughly flat: it executes in parallel to dotnet-build and dotnet-test and is
@@ -379,6 +393,14 @@ jobs:
# We rebuild and push the test container image on every IT run so framework code changes
# are picked up; the image tag is content-hashed across the test container source AND its
# framework project references, so identical content is a no-op push.
#
# `-UsePrebuiltProjectReferences` opts into the no-rebuild fast path: publish skips
# rebuilding ProjectReferences and consumes the DLLs the prior "Build Foundry hosted IT
# (and its deps)" step already produced. This avoids MSB3026 ("file is being used by
# another process") collisions caused by the previous build's shared-compilation server
# still holding file handles to those DLLs. Safe in CI because the prebuild step ran in
# the same job against the same source. Do not remove the prebuild step (the subsequent
# `dotnet test --no-build` step depends on it too).
- name: Build and push Foundry Hosted Agents test container
id: build-foundry-hosted-image
shell: pwsh
@@ -388,7 +410,7 @@ jobs:
if ([string]::IsNullOrWhiteSpace($registry)) {
throw "IT_HOSTED_AGENT_REGISTRY not set in the integration environment."
}
& "${{ github.workspace }}/dotnet/tests/Foundry.Hosting.IntegrationTests/scripts/it-build-image.ps1" -Registry $registry | Tee-Object -FilePath $env:GITHUB_ENV -Append
& "${{ github.workspace }}/dotnet/tests/Foundry.Hosting.IntegrationTests/scripts/it-build-image.ps1" -Registry $registry -UsePrebuiltProjectReferences | Tee-Object -FilePath $env:GITHUB_ENV -Append
- name: Run Foundry Hosted Agents Integration Tests
shell: pwsh
@@ -448,3 +470,64 @@ jobs:
uses: actions/github-script@v8
with:
script: core.setFailed('Integration Tests Cancelled!')
# Integration test trend report (aggregates JUnit XML results from dotnet test jobs)
dotnet-integration-test-report:
name: Integration Test Report
if: >
always() &&
github.event_name != 'pull_request' &&
(contains(join(needs.*.result, ','), 'success') ||
contains(join(needs.*.result, ','), 'failure'))
needs: [dotnet-test]
runs-on: ubuntu-latest
defaults:
run:
working-directory: python
steps:
- uses: actions/checkout@v6
with:
persist-credentials: false
sparse-checkout: |
.github/actions/python-setup
python
- name: Set up python and install the project
uses: ./.github/actions/python-setup
with:
python-version: "3.13"
os: ${{ runner.os }}
- name: Download all test results from current run
uses: actions/download-artifact@v4
with:
pattern: dotnet-test-results-*
path: dotnet-test-results/
- name: Restore report history cache
uses: actions/cache/restore@v4
with:
path: python/dotnet-integration-report-history.json
key: dotnet-integration-report-history-${{ github.run_id }}
restore-keys: |
dotnet-integration-report-history-
- name: Generate trend report
run: >
uv run python scripts/integration_test_report/aggregate.py
../dotnet-test-results/
dotnet-integration-report-history.json
dotnet-integration-test-report.md
- name: Post to Job Summary
if: always()
run: cat dotnet-integration-test-report.md >> $GITHUB_STEP_SUMMARY
- name: Save report history cache
if: always()
uses: actions/cache/save@v4
with:
path: python/dotnet-integration-report-history.json
key: dotnet-integration-report-history-${{ github.run_id }}
- name: Upload trend report
if: always()
uses: actions/upload-artifact@v7
with:
name: dotnet-integration-test-report
path: |
python/dotnet-integration-test-report.md
python/dotnet-integration-report-history.json
+1
View File
@@ -33,3 +33,4 @@ Console.WriteLine(await agent.RunAsync("Write a haiku about Microsoft Agent Fram
- [Design Documents](../docs/design)
- [Architectural Decision Records](../docs/decisions)
- [MSFT Learn Docs](https://learn.microsoft.com/agent-framework/overview/agent-framework-overview)
@@ -50,12 +50,12 @@ Console.WriteLine(await agent.RunAsync("My name is Ruaidhrí", session));
Console.WriteLine(await agent.RunAsync("I am 20 years old", session));
// We can serialize the session. The serialized state will include the state of the memory component.
JsonElement sesionElement = await agent.SerializeSessionAsync(session);
JsonElement sessionElement = await agent.SerializeSessionAsync(session);
Console.WriteLine("\n>> Use deserialized session with previously created memories\n");
// Later we can deserialize the session and continue the conversation with the previous memory component state.
var deserializedSession = await agent.DeserializeSessionAsync(sesionElement);
var deserializedSession = await agent.DeserializeSessionAsync(sessionElement);
Console.WriteLine(await agent.RunAsync("What is my name and age?", deserializedSession));
Console.WriteLine("\n>> Read memories using memory component\n");
@@ -13,5 +13,5 @@ internal sealed class SequenceNumber
/// Gets the next sequence number.
/// </summary>
/// <returns>The next sequence number.</returns>
public int Increment() => this._sequenceNumber++;
public int Increment() => System.Threading.Interlocked.Increment(ref this._sequenceNumber) - 1;
}
@@ -1,6 +1,7 @@
// Copyright (c) Microsoft. All rights reserved.
using System.Text.Json.Serialization;
using Microsoft.Agents.AI.Workflows.Specialized.Magentic;
namespace Microsoft.Agents.AI.Workflows;
@@ -10,6 +11,7 @@ namespace Microsoft.Agents.AI.Workflows;
[JsonDerivedType(typeof(ExecutorInvokedEvent))]
[JsonDerivedType(typeof(ExecutorCompletedEvent))]
[JsonDerivedType(typeof(ExecutorFailedEvent))]
[JsonDerivedType(typeof(MagenticOrchestratorEvent))]
public class ExecutorEvent(string executorId, object? data) : WorkflowEvent(data)
{
/// <summary>
@@ -0,0 +1,44 @@
// Copyright (c) Microsoft. All rights reserved.
using System.Collections.Generic;
using System.Linq;
using Microsoft.Extensions.AI;
namespace Microsoft.Agents.AI.Workflows;
/// <summary>
/// Request for human review of a proposed plan.
/// </summary>
/// <param name="Plan">The proposed plan.</param>
/// <param name="CurrentProgress">The current progress ledger, if available. During the initial plan review,
/// this will be <see langword="null"/>. In subsequent reviews after replanning (due to stalls), this will
/// contain the latest progress ledger that determined that no progress has been made or the workflow was in
/// a loop.</param>
/// <param name="IsStalled">Whether the workflow is currently stalled.</param>
public record MagenticPlanReviewRequest(ChatMessage Plan, MagenticProgressLedger? CurrentProgress, bool IsStalled)
{
/// <summary>
/// Create an approving <see cref="MagenticPlanReviewResponse"/>.
/// </summary>
/// <returns></returns>
public MagenticPlanReviewResponse Approve() => new([]);
/// <summary>
/// Create a <see cref="MagenticPlanReviewResponse"/> with revisions.
/// </summary>
/// <returns></returns>
public MagenticPlanReviewResponse Revise(string message) => new([new(ChatRole.User, message)]);
/// <summary>
/// Create a <see cref="MagenticPlanReviewResponse"/> with revisions.
/// </summary>
/// <returns></returns>
public MagenticPlanReviewResponse Revise(ChatMessage message) => new([message]);
/// <summary>
/// Create a <see cref="MagenticPlanReviewResponse"/> with revisions.
/// </summary>
/// <returns></returns>
public MagenticPlanReviewResponse Revise(IEnumerable<ChatMessage> messages)
=> new(messages is List<ChatMessage> messageList ? messageList : messages.ToList());
}
@@ -0,0 +1,18 @@
// Copyright (c) Microsoft. All rights reserved.
using System.Collections.Generic;
using Microsoft.Extensions.AI;
namespace Microsoft.Agents.AI.Workflows;
/// <summary>
/// Review feedback for a proposed plan, including any revisions if the plan is not approved as-is. An
/// empty list of review messages indicates approval of the proposed plan without any revisions.
/// </summary>
/// <param name="Review">
/// Review feedback for a generated plan. Empty if the plan is approved as-is and changes are requested.
/// </param>
public record MagenticPlanReviewResponse(List<ChatMessage> Review)
{
internal bool IsApproved => this.Review.Count == 0;
}
@@ -0,0 +1,269 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using System.Linq;
using System.Text;
using System.Text.Json;
using System.Text.Json.Serialization;
using System.Text.Json.Serialization.Metadata;
namespace Microsoft.Agents.AI.Workflows;
/// <summary>
/// Maintains a ledger of progress made by the Magentic workflow.
/// </summary>
public class MagenticProgressLedger
{
internal static readonly BooleanProgressLedgerSlot IsRequestSatisfiedSlot = new("is_request_satisfied",
"Is the request fully satisfied? (True if complete, or False if the original request has yet to be SUCCESSFULLY and FULLY addressed)");
internal static readonly BooleanProgressLedgerSlot IsInLoopSlot = new("is_in_loop",
"Are we in a loop where we are repeating the same requests and or getting the same responses as before? " +
"Loops can span multiple turns, and can include repeated actions like scrolling up or down more than a handful of times.");
internal static readonly BooleanProgressLedgerSlot IsProgressBeingMadeSlot = new("is_progress_being_made",
"Are we making forward progress? (True if just starting, or recent messages are adding value. False if recent " +
"messages show evidence of being stuck in a loop or if there is evidence of significant barriers to success " +
"such as the inability to read from a required file)");
internal readonly StringProgressLedgerSlot NextSpeakerSlot;
internal static readonly StringProgressLedgerSlot InstructionOrQuestionSlot = new("instruction_or_question",
"What instruction or question would you give this team member? (Phrase as if speaking directly to them, and " +
"include any specific information they may need)");
internal MagenticProgressLedger(string teamNames, IEnumerable<ProgressLedgerSlot> additionalQuestions, JsonElement? state = null)
{
this.NextSpeakerSlot = new("next_speaker", $"Who should speak next? (select from: {teamNames})");
this.AdditionalQuestions = additionalQuestions as ProgressLedgerSlot[] ?? additionalQuestions.ToArray();
if (state != null)
{
this.TryUpdateState(state.Value);
}
}
internal ProgressLedgerSlot[] AdditionalQuestions { get; }
internal bool TryUpdateState(JsonElement element)
{
// In principle all of these should be inlineable, but the CodeAnalysis fails to properly chain through the and-chain to realize that
// all must be true for `requiredQuestionsAnswered` to be true, meaning all of the out parameters would be initialized properly.
bool isInLoop = false;
bool isProgressBeingMade = false;
string? nextSpeaker = string.Empty;
string? instructionOrQuestion = string.Empty;
bool requiredQuestionsAnswered =
IsRequestSatisfiedSlot.TryGetValueFrom(element, out bool isRequestSatisfied) &&
IsInLoopSlot.TryGetValueFrom(element, out isInLoop) &&
IsProgressBeingMadeSlot.TryGetValueFrom(element, out isProgressBeingMade) &&
this.NextSpeakerSlot.TryGetValueFrom(element, out nextSpeaker) &&
InstructionOrQuestionSlot.TryGetValueFrom(element, out instructionOrQuestion);
if (requiredQuestionsAnswered)
{
this.State = element;
this.IsRequestSatisfied = isRequestSatisfied;
this.IsInLoop = isInLoop;
this.IsProgressBeingMade = isProgressBeingMade;
this.NextSpeaker = nextSpeaker!;
this.InstructionOrQuestion = instructionOrQuestion!;
}
// TODO: To what extent do we want to enforce that the additional questions are also answered?
return requiredQuestionsAnswered;
}
[JsonInclude]
internal JsonElement? State;
/// <summary>
/// Specifies whether plan execution has started.
/// </summary>
[JsonIgnore]
public bool IsStarted => this.State != null;
/// <summary>
/// Specifies whether the task has been fully satisfied.
/// </summary>
[JsonIgnore]
public bool IsRequestSatisfied { get; private set; }
/// <summary>
/// Specifies whether the team is in a loop.
/// </summary>
[JsonIgnore]
public bool IsInLoop { get; private set; }
/// <summary>
/// Specifies whether the team is making progress on the task.
/// </summary>
[JsonIgnore]
public bool IsProgressBeingMade { get; private set; }
/// <summary>
/// Gets the next team member to take a turn.
/// </summary>
[JsonIgnore]
public string NextSpeaker { get; private set; } = string.Empty;
/// <summary>
/// Gets the instruction or question to send to the next team member.
/// </summary>
[JsonIgnore]
public string InstructionOrQuestion { get; private set; } = string.Empty;
[JsonIgnore]
internal IEnumerable<ProgressLedgerSlot> Slots =>
[
IsRequestSatisfiedSlot,
IsInLoopSlot,
IsProgressBeingMadeSlot,
this.NextSpeakerSlot,
InstructionOrQuestionSlot,
.. this.AdditionalQuestions
];
internal bool TryGetCurrentSlotValue<T>(ProgressLedgerSlot<T> slot, [NotNullWhen(true)] out T? value)
{
if (!this.State.HasValue)
{
value = default;
return false;
}
return slot.TryGetValueFrom(this.State.Value, out value);
}
private (string QuestionBlock, string AnswerSchema)? _questionFormatCache;
internal (string QuestionBlock, string AnswerSchema) FormatQuestions()
{
if (!this._questionFormatCache.HasValue)
{
StringBuilder questionBuilder = new(), schemaBuilder = new();
schemaBuilder.AppendLine("{");
foreach (ProgressLedgerSlot slot in this.Slots)
{
questionBuilder.AppendLine(slot.FormattedQuestion);
schemaBuilder.AppendLine($"\"{slot.Key}\": {{")
.AppendLine($" \"{ProgressLedgerSlot.ValueKey}\": {slot.SchemaType}{slot.SuffixString},")
.AppendLine($" \"{ProgressLedgerSlot.ReasonKey}\": string")
.AppendLine("}");
}
schemaBuilder.AppendLine("}");
this._questionFormatCache = (questionBuilder.ToString(), schemaBuilder.ToString());
}
return this._questionFormatCache.Value;
}
}
internal abstract record ProgressLedgerSlot(string Key, string Question, string? SchemaTypeSuffix = null)
{
public const string ValueKey = "answer";
public const string ReasonKey = "reason";
internal string SuffixString => this.SchemaTypeSuffix == null ? string.Empty : $"({this.SchemaTypeSuffix})";
protected internal abstract string SchemaType { get; }
public string FormattedQuestion
{
get
{
if (field == null)
{
IEnumerable<string> questionLines = this.Question.Split(['\r', '\n'], StringSplitOptions.RemoveEmptyEntries)
.Select(line => line.TrimEnd());
field = $" - {string.Join("\n ", questionLines)}";
}
return field;
}
}
}
internal abstract record ProgressLedgerSlot<T>(string Key, string Question, string? SchemaTypeSuffix = null, JsonSerializerOptions? SerializerOptions = null)
: ProgressLedgerSlot(Key, Question, SchemaTypeSuffix)
{
protected internal virtual JsonTypeInfo<T> GetJsonTypeInfo() =>
((this.SerializerOptions ?? WorkflowsJsonUtilities.DefaultOptions).TryGetTypeInfo(typeof(T), out JsonTypeInfo? typeInfo)
? typeInfo as JsonTypeInfo<T> : null)
?? throw new InvalidOperationException($"Cannot get TypeInfo for {typeof(T)} from {(this.SerializerOptions == null ? "provided" : "default")} SerializationOptions.");
public bool TryGetValueFrom(JsonElement answers, [NotNullWhen(true)] out T? value)
{
if (answers.TryGetProperty(this.Key, out JsonElement slotElement) &&
slotElement.ValueKind != JsonValueKind.Null &&
slotElement.TryGetProperty(ValueKey, out JsonElement answerValue))
{
try
{
T? result = answerValue.Deserialize(this.GetJsonTypeInfo());
if (result != null)
{
value = result;
return true;
}
}
catch
{
}
}
value = default;
return false;
}
public bool TryGetReasonFrom(JsonElement answers, [NotNullWhen(true)] out string? value)
{
if (answers.TryGetProperty(this.Key, out JsonElement slotElement) &&
slotElement.ValueKind != JsonValueKind.Null &&
slotElement.TryGetProperty(ReasonKey, out JsonElement reasonValue))
{
try
{
string? result = reasonValue.Deserialize(WorkflowsJsonUtilities.JsonContext.Default.String);
if (result != null)
{
value = result;
return true;
}
}
catch
{
}
}
value = default;
return false;
}
}
internal sealed record BooleanProgressLedgerSlot(string Key, string Question, string? SchemaTypeSuffix = null) : ProgressLedgerSlot<bool>(Key, Question, SchemaTypeSuffix)
{
// Since we know the type statically, we can directly return the JsonTypeInfo for string from our JsonContext,
// which is more efficient than looking it up via the options.
protected internal override JsonTypeInfo<bool> GetJsonTypeInfo() => WorkflowsJsonUtilities.JsonContext.Default.Boolean;
protected internal override string SchemaType => "boolean";
}
internal sealed record StringProgressLedgerSlot(string Key, string Question, string? SchemaTypeSuffix = null) : ProgressLedgerSlot<string>(Key, Question, SchemaTypeSuffix)
{
// Since we know the type statically, we can directly return the JsonTypeInfo for string from our JsonContext,
// which is more efficient than looking it up via the options.
protected internal override JsonTypeInfo<string> GetJsonTypeInfo() => WorkflowsJsonUtilities.JsonContext.Default.String;
protected internal override string SchemaType => "string";
}
@@ -0,0 +1,158 @@
// Copyright (c) Microsoft. All rights reserved.
using System.Collections.Generic;
using System.Threading.Tasks;
using Microsoft.Agents.AI.Workflows.Specialized.Magentic;
using ExecutorFactoryFunc = System.Func<Microsoft.Agents.AI.Workflows.ExecutorConfig<Microsoft.Agents.AI.Workflows.ExecutorOptions>,
string,
System.Threading.Tasks.ValueTask<Microsoft.Agents.AI.Workflows.Specialized.Magentic.MagenticOrchestrator>>;
namespace Microsoft.Agents.AI.Workflows;
/// <summary>
/// Fluent builder for creating Magentic One multi-agent orchestration workflows.
///
/// Magentic One workflows use an LLM-powered manager to coordinate multiple agents through dynamic task planning, progress tracking,
/// and adaptive replanning.The manager creates plans, selects agents, monitors progress, and determines when to replan or complete.
///
/// The builder provides a fluent API for configuring participants, the manager, optional plan review, checkpointing, and event
/// callbacks.
///
/// Human-in-the-loop Support: Magentic provides specialized HITL mechanisms via:
/// - `RequirePlanSignoff` - Review and approve/revise plans before execution
/// - Tool approval via `function_approval_request`: Approve individual tool calls on participating agents. Note that tool calls are
/// not supported on the ManagerAgent.
/// </summary>
/// <param name="managerAgent"></param>
public class MagenticWorkflowBuilder(AIAgent managerAgent)
{
private readonly List<AIAgent> _team = new();
private string? _name;
private string? _description;
private int _maxStalls = TaskLimits.DefaultMaxStallCount;
private int? _maxRounds;
private int? _maxResets;
private bool _requirePlanSignoff = true;
/// <inheritdoc cref="GroupChatWorkflowBuilder.AddParticipants(IEnumerable{AIAgent})"/>
public MagenticWorkflowBuilder AddParticipants(params IEnumerable<AIAgent> agents)
{
this._team.AddRange(agents);
return this;
}
/// <inheritdoc cref="WorkflowBuilder.WithName(string)"/>
public MagenticWorkflowBuilder WithName(string name)
{
this._name = name;
return this;
}
/// <inheritdoc cref="WorkflowBuilder.WithDescription(string)"/>
public MagenticWorkflowBuilder WithDescription(string description)
{
this._description = description;
return this;
}
/// <summary>
/// Set the maximum number of coordination rounds. <see langword="null"/> means unlimited.
/// </summary>
/// <returns></returns>
public MagenticWorkflowBuilder WithMaxRounds(int? maxRounds = null)
{
this._maxRounds = maxRounds;
return this;
}
/// <summary>
/// Set the maximum number ofnumber of resets allowed. <see langword="null"/> means unlimited.
/// </summary>
/// <returns></returns>
public MagenticWorkflowBuilder WithMaxResets(int? maxResets = null)
{
this._maxResets = maxResets;
return this;
}
/// <summary>
/// Set the maximum number of consecutive rounds without progress before replan (default 3).
/// </summary>
/// <returns></returns>
public MagenticWorkflowBuilder WithMaxStalls(int maxStalls = TaskLimits.DefaultMaxStallCount)
{
this._maxStalls = maxStalls;
return this;
}
/// <summary>
/// If <see langword="true"/>, requires human approval of the initial plan or any updates before proceeding. True by default.
/// </summary>
/// <param name="requirePlanSignoff"></param>
/// <returns></returns>
public MagenticWorkflowBuilder RequirePlanSignoff(bool requirePlanSignoff = true)
{
this._requirePlanSignoff = requirePlanSignoff;
return this;
}
private WorkflowBuilder ReduceToWorkflowBuilder()
{
// Create a copy of the team so that improper modifications by using the builder after .Build() do not affect the
// workflow in unexpected ways.
List<AIAgent> team = [.. this._team];
ExecutorBinding orchestrator = CreateOrchestratorBinding(managerAgent, team, this.Limits, this._requirePlanSignoff);
WorkflowBuilder result = new(orchestrator);
AIAgentHostOptions options = new()
{
ReassignOtherAgentsAsUsers = true,
ForwardIncomingMessages = false
};
List<ExecutorBinding> teamBindings = [];
foreach (AIAgent agent in team)
{
ExecutorBinding binding = agent.BindAsExecutor(options);
teamBindings.Add(binding);
result.AddEdge(binding, orchestrator);
}
result.AddFanOutEdge(orchestrator, teamBindings)
.WithOutputFrom(orchestrator);
if (!string.IsNullOrWhiteSpace(this._name))
{
result.WithName(this._name);
}
if (!string.IsNullOrWhiteSpace(this._description))
{
result.WithDescription(this._description);
}
return result;
}
/// <inheritdoc cref="WorkflowBuilder.Build"/>
public Workflow Build() => this.ReduceToWorkflowBuilder().Build();
private TaskLimits Limits => new(
MaxRoundCount: this._maxRounds,
MaxResetCount: this._maxResets,
MaxStallCount: this._maxStalls);
private static ExecutorBinding CreateOrchestratorBinding(AIAgent managerAgent, List<AIAgent> team, TaskLimits limits, bool requirePlanSignoff)
{
ExecutorFactoryFunc factory = CreateOrchestratorAsync;
return factory.BindExecutor(nameof(MagenticOrchestrator));
ValueTask<MagenticOrchestrator> CreateOrchestratorAsync(ExecutorConfig<ExecutorOptions> options, string sessionId)
{
return new(new MagenticOrchestrator(managerAgent, team, limits, requirePlanSignoff));
}
}
}
@@ -0,0 +1,9 @@
// Copyright (c) Microsoft. All rights reserved.
namespace Microsoft.Agents.AI.Workflows;
/// <summary>
/// Notifies an AIAgent-hosting executor that it should reset its conversation state, and start a new session, if appropriate.
/// Note that for Agent Orchestrations, only Magentic makes use of this functionality.
/// </summary>
public sealed record ResetChatSignal();
@@ -24,7 +24,7 @@ internal static class TurnExtensions
=> handoffState.TurnToken.ShouldEmitStreamingEvents(agentSetting);
}
internal sealed class AIAgentHostExecutor : ChatProtocolExecutor
internal class AIAgentHostExecutor : ChatProtocolExecutor
{
private readonly AIAgent _agent;
private readonly AIAgentHostOptions _options;
@@ -40,7 +40,9 @@ internal sealed class AIAgentHostExecutor : ChatProtocolExecutor
StringMessageChatRole = ChatRole.User
};
public AIAgentHostExecutor(AIAgent agent, AIAgentHostOptions options) : base(id: agent.GetDescriptiveId(),
public static string IdFor(AIAgent agent) => agent.GetDescriptiveId();
public AIAgentHostExecutor(AIAgent agent, AIAgentHostOptions options) : base(id: IdFor(agent),
s_defaultChatProtocolOptions,
declareCrossRunShareable: false) // Explicitly false, because we maintain turn state on the instance
{
@@ -67,7 +69,14 @@ internal sealed class AIAgentHostExecutor : ChatProtocolExecutor
protected override ProtocolBuilder ConfigureProtocol(ProtocolBuilder protocolBuilder)
{
return this.ConfigureUserInputHandling(base.ConfigureProtocol(protocolBuilder));
return this.ConfigureUserInputHandling(base.ConfigureProtocol(protocolBuilder))
.ConfigureRoutes(routeBuilder => routeBuilder.AddHandler<ResetChatSignal>(this.ResetChat));
}
internal void ResetChat(ResetChatSignal signal, IWorkflowContext context)
{
this._session = null;
this._currentTurnEmitEvents = null;
}
private ValueTask HandleUserInputResponseAsync(
@@ -0,0 +1,175 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Text;
using System.Text.Json;
using System.Text.RegularExpressions;
using Microsoft.Extensions.AI;
namespace Microsoft.Agents.AI.Workflows.Specialized.Magentic;
internal static partial class ChatMessageExtensions
{
private static void ProcessAIContents(StringBuilder resultBuilder, IEnumerable<AIContent> contents, StreamingToolCallResultPairMatcher? pairMatcher = null)
{
pairMatcher ??= new();
foreach (AIContent content in contents)
{
switch (content)
{
case TextContent textContent:
resultBuilder.AppendLine(textContent.Text);
break;
//case DataContent dataContent:
// // We really do not know how to deal with anything other than image data with descriptions, which is not
// // a well-defined concept in MEAI (as contrasted with AutoGen's ImageContent type)
// break;
case ErrorContent errorContent:
resultBuilder.AppendLine($"[ERROR{(errorContent.ErrorCode != null ? $"(Code={errorContent.ErrorCode})" : string.Empty)}]");
resultBuilder.AppendLine(errorContent.Message);
if (errorContent.Details != null)
{
resultBuilder.Append("Details:").AppendLine(errorContent.Details);
}
break;
case FunctionCallContent functionCallContent:
pairMatcher.CollectFunctionCall(functionCallContent);
break;
case FunctionResultContent functionResultContent:
pairMatcher.TryResolveFunctionCall(functionResultContent, out string? functionName);
string result = functionResultContent.Result?.ToString() ?? string.Empty;
resultBuilder.AppendLine($"[Tool Call '{functionName ?? functionResultContent.CallId}' Result]")
.AppendLine(result);
break;
case McpServerToolCallContent mstContent:
pairMatcher.CollectMcpServerToolCall(mstContent);
break;
case McpServerToolResultContent mstResultContent:
if (mstResultContent.Outputs?.Any() is true)
{
pairMatcher.TryResolveMcpServerToolCall(mstResultContent, out string? mcpServerToolName);
resultBuilder.AppendLine($"[Start MCP Server Tool Call '{mcpServerToolName ?? mstResultContent.CallId}' Results]");
ProcessAIContents(resultBuilder, mstResultContent.Outputs!);
resultBuilder.AppendLine($"[End MCP Server Tool Call '{mcpServerToolName ?? mstResultContent.CallId}']");
}
break;
case TextReasoningContent reasoningContent:
if (!string.IsNullOrWhiteSpace(reasoningContent.Text))
{
resultBuilder.Append("[Reasoning] ")
.AppendLine(reasoningContent.Text);
}
break;
case UriContent uriContent:
resultBuilder.AppendLine(uriContent.Uri.ToString());
break;
}
}
}
public static string GetText(this List<ChatMessage> messages)
{
if (messages.Count == 0)
{
return string.Empty;
}
StringBuilder builder = new();
StreamingToolCallResultPairMatcher pairMatcher = new();
foreach (ChatMessage message in messages)
{
ProcessAIContents(builder, message.Contents, pairMatcher);
}
return builder.ToString();
}
private const string FencedJsonRegexPattern = @"```(?<lang>[a-z]+)?\s*(?<json>\{[\s\S]*?\})\s*```";
#if NET
[GeneratedRegex(FencedJsonRegexPattern, RegexOptions.IgnoreCase | RegexOptions.CultureInvariant | RegexOptions.ExplicitCapture)]
public static partial Regex FencedJsonRegex();
#else
public static Regex FencedJsonRegex() => s_fencedJsonRegex;
private static readonly Regex s_fencedJsonRegex =
new(FencedJsonRegexPattern, RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.ExplicitCapture);
#endif
internal static JsonElement ExtractJson(string messageText)
{
Match match = FencedJsonRegex().Match(messageText);
if (match.Success)
{
return JsonElement.Parse(match.Groups["json"].Value);
}
int start = messageText.IndexOf('{'), scanHead = start;
int? end = null;
if (scanHead < 0)
{
throw new InvalidOperationException("No JSON object found.");
}
int depth = 0;
bool inQuotes = false, inEscape = false;
for (; scanHead < messageText.Length && end is null; scanHead++)
{
if (inEscape)
{
inEscape = false;
continue;
}
switch (messageText[scanHead])
{
case '{' when !inQuotes:
depth++;
break;
case '}' when !inQuotes:
depth--;
if (depth == 0)
{
end = scanHead;
}
break;
case '\"':
// We already handled inEscape, so we can always flip inQuotes here
inQuotes = !inQuotes;
break;
case '\\':
Debug.Assert(!inEscape);
inEscape = true;
break;
}
}
if (end is null)
{
throw new InvalidOperationException("Unbalanced JSON braces.");
}
return JsonElement.Parse(messageText.Substring(start, end.Value - start + 1));
}
public static JsonElement ExtractJson(this ChatMessage message) => ExtractJson(message.Text);
}
@@ -0,0 +1,72 @@
// Copyright (c) Microsoft. All rights reserved.
using System.Collections.Generic;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.AI;
namespace Microsoft.Agents.AI.Workflows.Specialized.Magentic;
internal sealed class ExecutorAgentHarness(AIAgent agent, AIAgentUnservicedRequestsCollector collector)
{
internal const string AgentSessionKey = nameof(AgentSession);
private AgentSession? _session;
private async ValueTask<AgentSession> EnsureSessionAsync(IWorkflowContext context, CancellationToken cancellationToken) =>
this._session ??= await agent.CreateSessionAsync(cancellationToken).ConfigureAwait(false);
public async ValueTask<AgentResponse> InvokeAgentAsync(IEnumerable<ChatMessage> messages, IWorkflowContext context, bool emitUpdateEvents, CancellationToken cancellationToken = default)
{
AgentResponse response;
if (emitUpdateEvents)
{
// Run the agent in streaming mode only when agent run update events are to be emitted.
IAsyncEnumerable<AgentResponseUpdate> agentStream = agent.RunStreamingAsync(
messages,
await this.EnsureSessionAsync(context, cancellationToken).ConfigureAwait(false),
cancellationToken: cancellationToken);
List<AgentResponseUpdate> updates = [];
await foreach (AgentResponseUpdate update in agentStream.ConfigureAwait(false))
{
await context.YieldOutputAsync(update, cancellationToken).ConfigureAwait(false);
collector.ProcessAgentResponseUpdate(update);
updates.Add(update);
}
response = updates.ToAgentResponse();
}
else
{
// Otherwise, run the agent in non-streaming mode.
response = await agent.RunAsync(messages,
await this.EnsureSessionAsync(context, cancellationToken).ConfigureAwait(false),
cancellationToken: cancellationToken)
.ConfigureAwait(false);
collector.ProcessAgentResponse(response);
}
return response;
}
public async ValueTask<JsonElement?> SerializeSessionAsync(CancellationToken cancellationToken)
=> this._session == null
? null
: await agent.SerializeSessionAsync(this._session, cancellationToken: cancellationToken).ConfigureAwait(false);
public async ValueTask DeserializeSessionAsync(JsonElement? serializedSession, CancellationToken cancellationToken)
{
this._session = serializedSession == null
? null
: await agent.DeserializeSessionAsync(serializedSession.Value, cancellationToken: cancellationToken)
.ConfigureAwait(false);
}
public void ResetSession()
{
this._session = null;
}
}
@@ -0,0 +1,8 @@
// Copyright (c) Microsoft. All rights reserved.
namespace Microsoft.Agents.AI.Workflows.Specialized.Magentic;
internal static class MagenticConstants
{
public const string MagenticTaskContextKey = nameof(MagenticTaskContextKey);
}
@@ -0,0 +1,122 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
using System.Collections.Generic;
using System.Runtime.ExceptionServices;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.AI;
namespace Microsoft.Agents.AI.Workflows.Specialized.Magentic;
internal class MagenticManager(AIAgent managerAgent)
{
private static async ValueTask<ChatMessage> CheckResponseAsync(Task<AgentResponse> responseTask, IWorkflowContext context, CancellationToken cancellationToken)
{
AgentResponse response = await responseTask.ConfigureAwait(false);
if (response.Messages.Count == 0)
{
throw new InvalidOperationException("Planner Agent did not return any messages.");
}
if (response.Messages.Count > 1)
{
await context.AddEventAsync(new WorkflowWarningEvent("Planner Agent returned multiple messages; using the last one."), cancellationToken)
.ConfigureAwait(false);
}
return response.Messages[response.Messages.Count - 1];
}
private ValueTask<ChatMessage> InvokeAgentAsync(IEnumerable<ChatMessage> messages, IWorkflowContext context, CancellationToken cancellationToken, AgentSession? session = null)
=> CheckResponseAsync(managerAgent.RunAsync(messages, session, cancellationToken: cancellationToken), context, cancellationToken);
public async ValueTask<TaskLedger> UpdatePlanAsync(MagenticTaskContext taskContext, IWorkflowContext context, CancellationToken cancellationToken)
{
// If we already have a TaskLedger, we need to update the facts based on the existing factset; otherwise, we use the initial facts construction
bool isReplan = taskContext.TaskLedger != null;
AgentSession localSession = await managerAgent.CreateSessionAsync(cancellationToken).ConfigureAwait(false);
ChatMessage factsRequest = new(ChatRole.User, isReplan ? taskContext.ToTaskLedgerFactsUpdatePrompt() : taskContext.ToTaskLedgerFactsPrompt());
ChatMessage updatedFacts = await this.InvokeAgentAsync(
messages: [.. taskContext.ChatHistory, factsRequest],
context,
cancellationToken,
localSession)
.ConfigureAwait(false);
ChatMessage planRequest = new(ChatRole.User, isReplan ? taskContext.ToTaskLedgerPlanUpdatePrompt() : taskContext.ToTaskLedgerPlanPrompt());
ChatMessage updatedPlan = await this.InvokeAgentAsync(
// We rely on the AgentSession to maintain the context of the conversation, so we don't include the
// history, facts request, or updated facts in the messages list.
messages: [planRequest],
context,
cancellationToken,
localSession)
.ConfigureAwait(false);
taskContext.ChatHistory.AddRange([factsRequest, updatedFacts, planRequest, updatedPlan]);
return new(updatedFacts, updatedPlan);
}
public async ValueTask UpdateProgressLedgerAsync(MagenticTaskContext taskContext, IWorkflowContext context, CancellationToken cancellationToken)
{
ChatMessage progressRequest = new(ChatRole.User, taskContext.ToProgressLedgerPrompt());
ExceptionDispatchInfo? lastException = null;
int maxRetryCount = taskContext.TaskLimits.MaxProgressLedgerRetryCount;
for (int attempts = 0; attempts < maxRetryCount; attempts++)
{
ChatMessage progressUpdateMessage = await this.InvokeAgentAsync(
messages: [.. taskContext.ChatHistory, progressRequest],
context,
cancellationToken)
.ConfigureAwait(false);
try
{
lastException = null;
JsonElement stateUpdateJson = progressUpdateMessage.ExtractJson();
if (!taskContext.ProgressLedger.TryUpdateState(stateUpdateJson))
{
throw new InvalidOperationException("Could not answer progress ledger questions with provided JSON.");
}
break;
}
catch (Exception e)
{
lastException = ExceptionDispatchInfo.Capture(e);
string warnString = $"Progress ledger JSON parse failed (attempt {attempts}/{maxRetryCount}): {e}";
await context.AddEventAsync(new WorkflowWarningEvent(warnString), cancellationToken).ConfigureAwait(false);
if (attempts < maxRetryCount)
{
await Task.Delay(250 * attempts, cancellationToken).ConfigureAwait(false);
}
}
}
lastException?.Throw();
}
public async ValueTask<ChatMessage> PrepareFinalAnswerAsync(MagenticTaskContext taskContext, IWorkflowContext context, CancellationToken cancellationToken)
{
ChatMessage finalAnswerRequest = new(ChatRole.User, taskContext.ToFinalAnswerPrompt());
ChatMessage finalAnswer = await this.InvokeAgentAsync([.. taskContext.ChatHistory, finalAnswerRequest], context, cancellationToken)
.ConfigureAwait(false);
return new(ChatRole.Assistant, finalAnswer.Text)
{
AuthorName = finalAnswer.AuthorName ?? nameof(MagenticManager),
MessageId = finalAnswer.MessageId ?? Guid.NewGuid().ToString("N"),
CreatedAt = finalAnswer.CreatedAt ?? DateTimeOffset.UtcNow,
RawRepresentation = finalAnswer.RawRepresentation,
};
}
}
@@ -0,0 +1,326 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.Json.Serialization;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.AI;
namespace Microsoft.Agents.AI.Workflows.Specialized.Magentic;
/// <summary>
/// Base type for Magentic Orchestration Events
/// </summary>
/// <param name="data"></param>
[JsonDerivedType(typeof(MagenticPlanCreatedEvent))]
[JsonDerivedType(typeof(MagenticReplannedEvent))]
[JsonDerivedType(typeof(MagenticProgressLedgerUpdatedEvent))]
public abstract class MagenticOrchestratorEvent(object? data) : WorkflowEvent(data)
{
}
/// <summary>
/// Represents the creation of the initial plan
/// </summary>
/// <param name="fullTaskLeger"></param>
public sealed class MagenticPlanCreatedEvent(ChatMessage fullTaskLeger) : MagenticOrchestratorEvent(fullTaskLeger)
{
/// <summary>
/// A <see cref="ChatMessage"/> containing the initial plan.
/// </summary>
public ChatMessage FullTaskLedger { get; } = fullTaskLeger;
}
/// <summary>
/// Represents the creation of a new plan in response to a stall.
/// </summary>
/// <param name="fullTaskLeger"></param>
public sealed class MagenticReplannedEvent(ChatMessage fullTaskLeger) : MagenticOrchestratorEvent(fullTaskLeger)
{
/// <summary>
/// A <see cref="ChatMessage"/> containing the new plan.
/// </summary>
public ChatMessage FullTaskLedger { get; } = fullTaskLeger;
}
/// <summary>
/// Represents an update to the <see cref="MagenticProgressLedger"/> when running a coordination round.
/// </summary>
/// <param name="progressLedger"></param>
public sealed class MagenticProgressLedgerUpdatedEvent(MagenticProgressLedger progressLedger) : MagenticOrchestratorEvent(progressLedger)
{
/// <summary>
/// The new state of the <see cref="MagenticProgressLedger"/>
/// </summary>
public MagenticProgressLedger ProgressLedger { get; } = progressLedger;
}
/// <summary>
/// Magentic orchestrator that defines the workflow structure.
///
/// This orchestrator manages the overall Magentic workflow in the following structure:
///
/// 1. Upon receiving the task(a list of messages), it creates the plan using the manager then runs the inner loop.
/// 2. The inner loop is distributed and implementation is decentralized. In the orchestrator, it is responsible for:
/// - Creating the progress ledger using the manager.
/// - Checking for task completion.
/// - Detecting stalling or looping and triggering replanning if needed.
/// - Sending requests to participants based on the progress ledger's next speaker.
/// - Issue requests for human intervention if enabled and needed.
/// 3. The inner loop waits for responses from the selected participant, then continues the loop.
/// 4. The orchestrator breaks out of the inner loop when the replanning or final answer conditions are met.
/// 5. The outer loop handles replanning and reenters the inner loop.
/// </summary>
/// <param name="managerAgent"></param>
/// <param name="team"></param>
/// <param name="limits"></param>
/// <param name="requirePlanSignoff"></param>
internal class MagenticOrchestrator(AIAgent managerAgent, List<AIAgent> team, TaskLimits limits, bool requirePlanSignoff)
: ChatProtocolExecutor(nameof(MagenticOrchestrator), s_options, declareCrossRunShareable: false)
{
private readonly MagenticManager _manager = new(managerAgent);
private static readonly ChatProtocolExecutorOptions s_options = new()
{
StringMessageChatRole = ChatRole.User,
AutoSendTurnToken = false
};
private MagenticTaskContext? _taskContext;
private PortBinding? _planReviewPort;
protected override ProtocolBuilder ConfigureProtocol(ProtocolBuilder protocolBuilder)
{
return base.ConfigureProtocol(protocolBuilder).ConfigureRoutes(ConfigureRoutes);
void ConfigureRoutes(RouteBuilder routeBuilder) => routeBuilder.AddPortHandler<MagenticPlanReviewRequest, MagenticPlanReviewResponse>(
"RequestPlanReview",
this.ProcessPlanReviewAsync,
out this._planReviewPort);
}
private ValueTask SubmitPlanReviewRequestAsync(MagenticTaskContext taskContext, IWorkflowContext workflowContext)
{
MagenticProgressLedger? progressLedger = taskContext.ProgressLedger;
if (progressLedger?.IsStarted is not true)
{
progressLedger = null;
}
MagenticPlanReviewRequest request = new(taskContext.TaskLedger!.CurrentPlan, progressLedger, taskContext.IsStalled);
return this._planReviewPort!.PostRequestAsync(request);
}
private async ValueTask ProcessPlanReviewAsync(MagenticPlanReviewResponse response, IWorkflowContext context, CancellationToken cancellationToken)
{
/*
Handle the human response to the plan review request.
Logic:
There are code paths which will trigger a plan review request to the human:
- Initial plan creation if `require_plan_signoff` is True.
- Potentially during the inner loop if stalling is detected (resetting and replanning).
The human can either approve the plan or request revisions with comments.
- If approved, proceed to run the outer loop, which simply adds the task ledger
to the conversation and enters the inner loop.
- If revision requested, append the review comments to the chat history,
trigger replanning via the manager, emit a REPLANNED event, then run the outer loop.
*/
if (this._taskContext == null || this._taskContext.TaskLedger == null)
{
throw new InvalidOperationException("Magentic Orchestration was not initialized correctly.");
}
if (this._taskContext.IsTerminated)
{
throw new InvalidOperationException("Magentic Orchestration has already been terminated and cannot process new messages. Please start a new session.");
}
if (response.IsApproved)
{
await this.DelegateToTeamAsync(this._taskContext, context, cancellationToken).ConfigureAwait(false);
}
else
{
this._taskContext.ChatHistory.AddRange(response.Review);
await this.UpdatePlanAndDelegateAsync(this._taskContext, context, cancellationToken).ConfigureAwait(false);
}
}
private async ValueTask UpdatePlanAndDelegateAsync(MagenticTaskContext taskContext, IWorkflowContext context, CancellationToken cancellationToken)
{
bool isReplan = taskContext.TaskLedger != null;
taskContext.TaskLedger = await this._manager.UpdatePlanAsync(taskContext, context, cancellationToken)
.ConfigureAwait(false);
this._fullTaskLedgerMessage = new(ChatRole.User, taskContext.ToTaskLedgerFullPrompt());
taskContext.ChatHistory.Add(this._fullTaskLedgerMessage);
await context.AddEventAsync(isReplan
? new MagenticReplannedEvent(this._fullTaskLedgerMessage)
: new MagenticPlanCreatedEvent(this._fullTaskLedgerMessage), cancellationToken).ConfigureAwait(false);
if (requirePlanSignoff)
{
await this.SubmitPlanReviewRequestAsync(taskContext, context).ConfigureAwait(false);
}
else
{
await this.DelegateToTeamAsync(taskContext, context, cancellationToken).ConfigureAwait(false);
}
}
protected override async ValueTask TakeTurnAsync(List<ChatMessage> messages, IWorkflowContext context, bool? emitEvents, CancellationToken cancellationToken = default)
{
// First Turn: Initialize the task context and send the initial messages to the planner agent
this._taskContext ??= new(messages, team, limits, emitEvents, []);
await this.UpdatePlanAndDelegateAsync(this._taskContext, context, cancellationToken).ConfigureAwait(false);
}
private ChatMessage? _fullTaskLedgerMessage;
private ValueTask DelegateToTeamAsync(MagenticTaskContext taskContext, IWorkflowContext context, CancellationToken cancellationToken)
{
return this.RunCoordinationRoundAsync(taskContext, context, cancellationToken);
}
private async ValueTask RunCoordinationRoundAsync(MagenticTaskContext taskContext, IWorkflowContext context, CancellationToken cancellationToken)
{
(bool hitRoundLimit, bool hitResetLimit) = taskContext.CheckLimits();
if (hitRoundLimit || hitResetLimit)
{
string limitType = hitRoundLimit ? "round" : "reset";
List<ChatMessage> messages = [new(ChatRole.Assistant, $"Task execution stopped due to hitting the maximum {limitType} count limit.")];
await context.YieldOutputAsync(messages, cancellationToken).ConfigureAwait(false);
taskContext.IsTerminated = true;
return;
}
taskContext.TaskCounters.RoundCount++;
// Update the Progress Ledger
try
{
await this._manager.UpdateProgressLedgerAsync(taskContext, context, cancellationToken).ConfigureAwait(false);
await context.AddEventAsync(new MagenticProgressLedgerUpdatedEvent(taskContext.ProgressLedger), cancellationToken)
.ConfigureAwait(false);
}
// Retry on exception to max retry count, unless it is OperationCancelledException - in that case exit the loop right away
catch (Exception ex) when (ex is not OperationCanceledException)
{
await context.AddEventAsync(new WorkflowWarningEvent($"Magentic Orchestrator: Progress ledger creation failed, triggering reset: {ex}"), cancellationToken)
.ConfigureAwait(false);
await this.ResetAndReplanAsync(taskContext, context, cancellationToken).ConfigureAwait(false);
return;
}
// Check and handle finish condition
if (taskContext.ProgressLedger.IsRequestSatisfied)
{
await this.PrepareFinalAnswerAsync(taskContext, context, cancellationToken).ConfigureAwait(false);
return;
}
// Check and handle stalls
if (taskContext.ProgressLedger.IsInLoop || !taskContext.ProgressLedger.IsProgressBeingMade)
{
taskContext.TaskCounters.StallCount++;
}
else
{
taskContext.TaskCounters.StallCount = Math.Max(0, taskContext.TaskCounters.StallCount - 1);
}
if (taskContext.IsStalled)
{
await this.ResetAndReplanAsync(taskContext, context, cancellationToken).ConfigureAwait(false);
return;
}
// Prepare to delegate to the next speaker
string nextSpeaker = taskContext.ProgressLedger.NextSpeaker;
if (string.IsNullOrEmpty(nextSpeaker))
{
await context.AddEventAsync(new WorkflowWarningEvent("Next speaker answer empty; selecting first participant as fallback"), cancellationToken)
.ConfigureAwait(false);
nextSpeaker = team.First().Name!;
}
AIAgent? nextAgent = team.FirstOrDefault(agent => agent.Name == nextSpeaker);
if (nextAgent == null)
{
await context.AddEventAsync(new WorkflowWarningEvent($"Invalid next speaker: {nextSpeaker}"), cancellationToken)
.ConfigureAwait(false);
await this.PrepareFinalAnswerAsync(taskContext, context, cancellationToken).ConfigureAwait(false);
return;
}
if (!string.IsNullOrWhiteSpace(taskContext.ProgressLedger.InstructionOrQuestion))
{
ChatMessage instruction = new(ChatRole.Assistant, taskContext.ProgressLedger.InstructionOrQuestion);
taskContext.ChatHistory.Add(instruction);
await context.SendMessageAsync(instruction, cancellationToken).ConfigureAwait(false);
}
string nextExecutorId = AIAgentHostExecutor.IdFor(nextAgent);
await context.SendMessageAsync(new TurnToken(taskContext.EmitUpdateEvents), nextExecutorId, cancellationToken).ConfigureAwait(false);
}
private async ValueTask ResetAndReplanAsync(MagenticTaskContext taskContext, IWorkflowContext context, CancellationToken cancellationToken)
{
taskContext.Reset();
await context.SendMessageAsync(new ResetChatSignal(), cancellationToken: cancellationToken).ConfigureAwait(false);
await this.UpdatePlanAndDelegateAsync(taskContext, context, cancellationToken).ConfigureAwait(false);
}
private async ValueTask PrepareFinalAnswerAsync(MagenticTaskContext taskContext, IWorkflowContext context, CancellationToken cancellationToken)
{
List<ChatMessage> messages = [await this._manager.PrepareFinalAnswerAsync(taskContext, context, cancellationToken).ConfigureAwait(false)];
await context.YieldOutputAsync(messages, cancellationToken).ConfigureAwait(false);
taskContext.IsTerminated = true;
}
private const string CurrentTurnEmitUpdateEventsKey = nameof(CurrentTurnEmitUpdateEventsKey);
protected internal override async ValueTask OnCheckpointingAsync(IWorkflowContext context, CancellationToken cancellationToken = default)
{
Task contextStateTask = this._taskContext == null
? Task.CompletedTask
: context.QueueStateUpdateAsync(MagenticConstants.MagenticTaskContextKey,
this._taskContext.ExportState(),
cancellationToken: cancellationToken)
.AsTask();
await Task.WhenAll(base.OnCheckpointingAsync(context, cancellationToken).AsTask(),
contextStateTask).ConfigureAwait(false);
}
protected internal override async ValueTask OnCheckpointRestoredAsync(IWorkflowContext context, CancellationToken cancellationToken = default)
{
await Task.WhenAll(base.OnCheckpointRestoredAsync(context, cancellationToken).AsTask(), LoadContextStateAsync())
.ConfigureAwait(false);
async Task LoadContextStateAsync()
{
MagenticTaskState? state = await context.ReadStateAsync<MagenticTaskState>(MagenticConstants.MagenticTaskContextKey, cancellationToken: cancellationToken)
.ConfigureAwait(false);
if (state != null)
{
this._taskContext = new MagenticTaskContext(state, team, limits, []);
}
}
}
}
@@ -0,0 +1,95 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.Json;
using Microsoft.Extensions.AI;
namespace Microsoft.Agents.AI.Workflows.Specialized.Magentic;
internal record TaskLimits(int MaxStallCount = TaskLimits.DefaultMaxStallCount,
int? MaxRoundCount = null,
int? MaxResetCount = null,
int MaxProgressLedgerRetryCount = TaskLimits.DefaultMaxProgressLedgerRetryCount)
{
public const int DefaultMaxStallCount = 3;
public const int DefaultMaxProgressLedgerRetryCount = 3;
}
internal record TaskLedger(ChatMessage CurrentFacts, ChatMessage CurrentPlan);
internal class TaskCounters
{
public int RoundCount { get; set; }
public int StallCount { get; set; }
public int ResetCount { get; set; }
}
internal record MagenticTaskState(List<ChatMessage> TaskDefinition, List<ChatMessage> ChatHistory, TaskLedger? TaskLedger, JsonElement? ProgressLedgerState, TaskCounters Counters, bool Terminated, bool? EmitUpdateEvents)
{
}
internal class MagenticTaskContext(List<ChatMessage> taskDefinition, List<AIAgent> team, TaskLimits limits, bool? emitUpdateEvents, IEnumerable<ProgressLedgerSlot> additionalProgressQuestions)
{
internal MagenticTaskContext(MagenticTaskState state, List<AIAgent> team, TaskLimits limits, IEnumerable<ProgressLedgerSlot> additionalProgressQuestions)
: this(state.TaskDefinition, team, limits, state.EmitUpdateEvents, additionalProgressQuestions)
{
this.TaskLedger = state.TaskLedger;
this.TaskCounters = state.Counters;
this.ChatHistory = state.ChatHistory;
this.IsTerminated = state.Terminated;
if (state.ProgressLedgerState.HasValue && !this.ProgressLedger.TryUpdateState(state.ProgressLedgerState.Value))
{
throw new InvalidOperationException("Could not load progress ledger state value");
}
}
public string Task { get; } = taskDefinition.GetText();
public string TeamDescription { get; } = GetTeamDescription(team);
public List<ChatMessage> ChatHistory { get; internal set; } = new();
public TaskLedger? TaskLedger { get; internal set; }
public TaskLimits TaskLimits => limits;
public bool IsTerminated { get; internal set; }
public bool IsStalled => this.TaskCounters.StallCount >= this.TaskLimits.MaxStallCount;
public (bool HitRoundLimit, bool HitResetLimit) CheckLimits()
{
return (this.TaskLimits.MaxRoundCount.HasValue && this.TaskLimits.MaxRoundCount.Value <= this.TaskCounters.RoundCount,
this.TaskLimits.MaxResetCount.HasValue && this.TaskLimits.MaxResetCount.Value <= this.TaskCounters.ResetCount);
}
public TaskCounters TaskCounters { get; internal set; } = new();
public MagenticProgressLedger ProgressLedger { get; } = new(GetTeamNames(team), additionalProgressQuestions);
public bool? EmitUpdateEvents => emitUpdateEvents;
public static string GetTeamDescription(IEnumerable<AIAgent> team)
{
return string.Join("\n", team.Select(agent => $"- {agent.Name}: {agent.Description}"));
}
public static string GetTeamNames(IEnumerable<AIAgent> team)
{
return string.Join(", ", team.Select(agent => agent.Name));
}
public MagenticTaskState ExportState()
{
return new(taskDefinition, this.ChatHistory, this.TaskLedger, this.ProgressLedger.State, this.TaskCounters, this.IsTerminated, this.EmitUpdateEvents);
}
internal void Reset()
{
this.ChatHistory.Clear();
this.TaskCounters.ResetCount++;
this.TaskCounters.StallCount = 0;
}
}
@@ -0,0 +1,151 @@
// Copyright (c) Microsoft. All rights reserved.
using Microsoft.Extensions.AI;
namespace Microsoft.Agents.AI.Workflows.Specialized.Magentic;
internal static class PromptTemplateExtensions
{
public static string ToTaskLedgerFactsPrompt(this MagenticTaskContext taskContext)
{
return $"""
Below I will present you a request.
Before we begin addressing the request, please answer the following pre-survey to the best of your ability.
Keep in mind that you are Ken Jennings-level with trivia, and Mensa-level with puzzles, so there should be
a deep well to draw from.
Here is the request:
{taskContext.Task}
Here is the pre-survey:
1. Please list any specific facts or figures that are GIVEN in the request itself.It is possible that
there are none.
2. Please list any facts that may need to be looked up, and WHERE SPECIFICALLY they might be found.
In some cases, authoritative sources are mentioned in the request itself.
3. Please list any facts that may need to be derived(e.g., via logical deduction, simulation, or computation)
4. Please list any facts that are recalled from memory, hunches, well-reasoned guesses, etc.
When answering this survey, keep in mind that "facts" will typically be specific names, dates, statistics, etc.
Your answer should use headings:
1. GIVEN OR VERIFIED FACTS
2. FACTS TO LOOK UP
3. FACTS TO DERIVE
4. EDUCATED GUESSES
DO NOT include any other headings or sections in your response.DO NOT list next steps or plans until asked to do so.
""";
}
public static string ToTaskLedgerFactsUpdatePrompt(this MagenticTaskContext taskContext)
{
return $"""
As a reminder, we are working to solve the following task:
{taskContext.Task}
It is clear we are not making as much progress as we would like, but we may have learned something new.
Please rewrite the following fact sheet, updating it to include anything new we have learned that may be helpful.
Example edits can include (but are not limited to) adding new guesses, moving educated guesses to verified facts
if appropriate, etc. Updates may be made to any section of the fact sheet, and more than one section of the fact
sheet can be edited. This is an especially good time to update educated guesses, so please at least add or update
one educated guess or hunch, and explain your reasoning.
Here is the old fact sheet:
{taskContext.TaskLedger?.CurrentFacts ?? new(ChatRole.Assistant, string.Empty)}
""";
}
public static string ToTaskLedgerPlanPrompt(this MagenticTaskContext taskContext)
{
return $"""
Fantastic. To address this request we have assembled the following team:
{taskContext.TeamDescription}
Based on the team composition, and known and unknown facts, please devise a short bullet-point plan for addressing the
original request. Remember, there is no requirement to involve all team members. A team member's particular expertise
may not be needed for this task.
""";
}
public static string ToTaskLedgerPlanUpdatePrompt(this MagenticTaskContext taskContext)
{
return $"""
Please briefly explain what went wrong on this last run
(the root cause of the failure), and then come up with a new plan that takes steps and includes hints to overcome prior
challenges and especially avoids repeating the same mistakes. As before, the new plan should be concise, expressed in
bullet-point form, and consider the following team composition:
{taskContext.TeamDescription}
""";
}
public static string ToTaskLedgerFullPrompt(this MagenticTaskContext taskContext)
{
return $"""
We are working to address the following user request:
{taskContext.Task}
To answer this request we have assembled the following team:
{taskContext.TeamDescription}
Here is an initial fact sheet to consider:
{taskContext.TaskLedger!.CurrentFacts ?? new(ChatRole.Assistant, string.Empty)}
Here is the plan to follow as best as possible:
{taskContext.TaskLedger!.CurrentPlan}
""";
}
public static string ToProgressLedgerPrompt(this MagenticTaskContext taskContext)
{
(string questions, string schema) = taskContext.ProgressLedger.FormatQuestions();
return $"""
Recall we are working on the following request:
{taskContext.Task}
And we have assembled the following team:
{taskContext.TeamDescription}
To make progress on the request, please answer the following questions, including necessary reasoning:
{questions}
Please output an answer in pure JSON format according to the following schema. The JSON object must be parsable as-is.
DO NOT OUTPUT ANYTHING OTHER THAN JSON, AND DO NOT DEVIATE FROM THIS SCHEMA:
{schema}
""";
}
public static string ToFinalAnswerPrompt(this MagenticTaskContext taskContext)
{
return $"""
We are working on the following task:
{taskContext.Task}
We have completed the task.
The above messages contain the conversation that took place to complete the task.
Based on the information gathered, provide the final answer to the original request.
The answer should be phrased as if you were speaking to the user.
""";
}
}
@@ -0,0 +1,77 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using Microsoft.Extensions.AI;
namespace Microsoft.Agents.AI.Workflows.Specialized.Magentic;
internal sealed class StreamingToolCallResultPairMatcher
{
private enum CallType
{
Function,
McpServerTool
}
private record CallSummaryKey(CallType Type, string CallId);
private struct ToolCallSummary(CallType callType, string callId, string name)
{
public CallType CallType => callType;
public string? CallId => callId;
public string Name => name;
}
private readonly Dictionary<CallSummaryKey, ToolCallSummary> _callSummaries = new();
private void Collect(CallType callType, string callId, string name, string callContentTypeName, string resultContentTypeName)
{
CallSummaryKey key = new(callType, callId);
if (this._callSummaries.ContainsKey(key))
{
throw new InvalidOperationException($"Duplicate {callContentTypeName} with CallId '{callId}' without corresponding {resultContentTypeName}.");
}
this._callSummaries[key] = new ToolCallSummary(callType, callId, name);
}
public void CollectFunctionCall(FunctionCallContent callContent)
{
const string FunctionCallContentTypeName = nameof(FunctionCallContent);
const string FunctionResultContentTypeName = nameof(FunctionResultContent);
this.Collect(CallType.Function, callContent.CallId, callContent.Name, FunctionCallContentTypeName, FunctionResultContentTypeName);
}
public void CollectMcpServerToolCall(McpServerToolCallContent callContent)
{
const string McpServerToolCallContentTypeName = nameof(McpServerToolCallContent);
const string McpServerToolResultContentTypeName = nameof(McpServerToolResultContent);
this.Collect(CallType.McpServerTool, callContent.CallId, callContent.Name, McpServerToolCallContentTypeName, McpServerToolResultContentTypeName);
}
private bool TryResolve(CallType callType, string callId, [NotNullWhen(true)] out string? name)
{
CallSummaryKey key = new(callType, callId);
bool hasMatchingCall = this._callSummaries.TryGetValue(key, out ToolCallSummary callSummary);
if (hasMatchingCall)
{
this._callSummaries.Remove(key);
}
name = hasMatchingCall ? callSummary.Name : null;
return hasMatchingCall;
}
public bool TryResolveFunctionCall(FunctionResultContent resultContent, [NotNullWhen(true)] out string? name)
=> this.TryResolve(CallType.Function, resultContent.CallId, out name);
public bool TryResolveMcpServerToolCall(McpServerToolResultContent resultContent, [NotNullWhen(true)] out string? name)
=> this.TryResolve(CallType.McpServerTool, resultContent.CallId, out name);
}
@@ -1,6 +1,7 @@
// Copyright (c) Microsoft. All rights reserved.
using System.Text.Json.Serialization;
using Microsoft.Agents.AI.Workflows.Specialized.Magentic;
namespace Microsoft.Agents.AI.Workflows;
@@ -14,6 +15,8 @@ namespace Microsoft.Agents.AI.Workflows;
[JsonDerivedType(typeof(WorkflowWarningEvent))]
[JsonDerivedType(typeof(WorkflowOutputEvent))]
[JsonDerivedType(typeof(RequestInfoEvent))]
[JsonDerivedType(typeof(MagenticOrchestratorEvent))]
public class WorkflowEvent(object? data = null)
{
/// <summary>
@@ -7,6 +7,7 @@ using System.Text.Json.Serialization;
using Microsoft.Agents.AI.Workflows.Checkpointing;
using Microsoft.Agents.AI.Workflows.Execution;
using Microsoft.Agents.AI.Workflows.Specialized;
using Microsoft.Agents.AI.Workflows.Specialized.Magentic;
using Microsoft.Extensions.AI;
namespace Microsoft.Agents.AI.Workflows;
@@ -97,6 +98,10 @@ internal static partial class WorkflowsJsonUtilities
[JsonSerializable(typeof(AIAgentHostState))]
[JsonSerializable(typeof(HandoffSharedState))]
[JsonSerializable(typeof(HandoffAgentHostState))]
[JsonSerializable(typeof(MagenticPlanReviewRequest))]
[JsonSerializable(typeof(MagenticPlanReviewResponse))]
[JsonSerializable(typeof(MagenticTaskState))]
[JsonSerializable(typeof(ResetChatSignal))]
// Event Types
//[JsonSerializable(typeof(WorkflowEvent))]
@@ -17,9 +17,6 @@ namespace AnthropicChatCompletion.IntegrationTests;
public class AnthropicChatCompletionFixture : IChatClientAgentFixture
{
// All tests for Anthropic are intended to be ran locally as the CI pipeline for Anthropic is not setup.
internal const string SkipReason = "Integrations tests for local execution only";
private readonly bool _useReasoningModel;
private readonly bool _useBeta;
@@ -105,7 +102,22 @@ public class AnthropicChatCompletionFixture : IChatClientAgentFixture
public async ValueTask InitializeAsync()
{
Assert.SkipWhen(SkipReason is not null, SkipReason ?? string.Empty);
// Temporarily disabled: Anthropic SDK has a binary incompatibility with the current
// Microsoft.Extensions.AI version (WebSearchToolResultContent.Results method not found).
// See: https://github.com/microsoft/agent-framework/pull/5515
Assert.Skip("Anthropic integration tests temporarily disabled due to SDK incompatibility with Microsoft.Extensions.AI");
try
{
_ = TestConfiguration.GetRequiredValue(TestSettings.AnthropicApiKey);
_ = TestConfiguration.GetRequiredValue(TestSettings.AnthropicChatModelName);
_ = TestConfiguration.GetRequiredValue(TestSettings.AnthropicReasoningModelName);
}
catch (InvalidOperationException ex)
{
Assert.Skip("Anthropic configuration could not be loaded. Error:" + ex.Message);
}
this._agent = await this.CreateChatClientAgentAsync();
}
@@ -1,5 +1,6 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
using System.Threading.Tasks;
using AgentConformance.IntegrationTests.Support;
using Anthropic;
@@ -17,19 +18,28 @@ namespace AnthropicChatCompletion.IntegrationTests;
/// Integration tests for Anthropic Skills functionality.
/// These tests are designed to be run locally with a valid Anthropic API key.
/// </summary>
/// <remarks>
/// Temporarily disabled due to Anthropic SDK binary incompatibility with
/// the current Microsoft.Extensions.AI version (WebSearchToolResultContent.Results).
/// </remarks>
[Trait("Category", "IntegrationDisabled")]
public sealed class AnthropicSkillsIntegrationTests
{
// All tests for Anthropic are intended to be ran locally as the CI pipeline for Anthropic is not setup.
private const string SkipReason = "Integrations tests for local execution only";
[Fact]
public async Task CreateAgentWithPptxSkillAsync()
{
Assert.SkipWhen(SkipReason is not null, SkipReason ?? string.Empty);
// Arrange
AnthropicClient anthropicClient = new() { ApiKey = TestConfiguration.GetRequiredValue(TestSettings.AnthropicApiKey) };
string model = TestConfiguration.GetRequiredValue(TestSettings.AnthropicChatModelName);
AnthropicClient? anthropicClient;
string? model;
try
{
anthropicClient = new() { ApiKey = TestConfiguration.GetRequiredValue(TestSettings.AnthropicApiKey) };
model = TestConfiguration.GetRequiredValue(TestSettings.AnthropicChatModelName);
}
catch (InvalidOperationException ex)
{
Assert.Skip("Anthropic configuration could not be loaded. Error:" + ex.Message);
return;
}
BetaSkillParams pptxSkill = new()
{
@@ -56,10 +66,16 @@ public sealed class AnthropicSkillsIntegrationTests
[Fact]
public async Task ListAnthropicManagedSkillsAsync()
{
Assert.SkipWhen(SkipReason is not null, SkipReason ?? string.Empty);
// Arrange
AnthropicClient anthropicClient = new() { ApiKey = TestConfiguration.GetRequiredValue(TestSettings.AnthropicApiKey) };
AnthropicClient? anthropicClient;
try
{
anthropicClient = new() { ApiKey = TestConfiguration.GetRequiredValue(TestSettings.AnthropicApiKey) };
}
catch (InvalidOperationException ex)
{
Assert.Skip("Anthropic configuration could not be loaded. Error:" + ex.Message);
return;
}
// Act
SkillListPage skills = await anthropicClient.Beta.Skills.List(
@@ -41,7 +41,14 @@ param(
[string] $Repository = "foundry-hosting-it",
[string] $TestContainerProject = "dotnet/tests/Foundry.Hosting.IntegrationTests.TestContainer"
[string] $TestContainerProject = "dotnet/tests/Foundry.Hosting.IntegrationTests.TestContainer",
# Explicit opt-in for the no-rebuild fast path. CI sets this after running the
# "Build Foundry hosted IT (and its deps)" step, which guarantees the prebuilt
# library DLLs match current source. Off by default so local invocations always
# let publish rebuild ProjectReferences and never produce an image whose tag is
# computed from current source while the contents come from a stale build.
[switch] $UsePrebuiltProjectReferences
)
$ErrorActionPreference = "Stop"
@@ -100,7 +107,60 @@ if (Test-Path $out) {
Remove-Item -Recurse -Force $out
}
dotnet publish $TestContainerProject -c Release -f net10.0 -r linux-musl-x64 --self-contained false -o $out --tl:off | Out-Host
# Conditionally tell publish to skip rebuilding ProjectReferences and consume the
# prebuilt library DLLs in place. This avoids two failure modes that arise when
# the CI workflow runs a `dotnet build` of the same library projects immediately
# before this script:
# 1) MSB3026 "file is being used by another process" when publish's MSBuild
# tries to overwrite src/<lib>/bin/Release/net10.0/<lib>.dll while the
# previous build's shared-compilation server still holds a file handle.
# 2) Publish needlessly rebuilding identical managed (RID-agnostic) library
# DLLs that prebuild already produced.
# Gated on -UsePrebuiltProjectReferences (a strict opt-in) instead of marker
# detection, because a developer machine may have a stale Release build of the
# libraries from days ago; using those would silently produce an image whose
# content is older than the source the tag is computed from.
$publishExtraArgs = @()
if ($UsePrebuiltProjectReferences) {
Write-Host "-UsePrebuiltProjectReferences: skipping ProjectReference rebuild." -ForegroundColor DarkGray
$publishExtraArgs += "-p:BuildProjectReferences=false"
} else {
# Preflight: in default (rebuild) mode, publish propagates RuntimeIdentifier=linux-musl-x64
# to library ProjectReferences and writes their intermediates to a RID-suffixed obj path
# (e.g. obj/Release/net10.0/linux-musl-x64/). DefaultItemExcludes follows the new
# IntermediateOutputPath, so any *.AssemblyInfo.cs left in obj/Release/net10.0/ from a
# prior `dotnet build` is no longer excluded and gets picked up by the **/*.cs Compile
# glob, producing CS0579 "duplicate attribute" errors. Detect that state up front and
# tell the user exactly how to recover.
$staleObjProbes = @(
"dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/obj/Release/net10.0",
"dotnet/src/Microsoft.Agents.AI.Foundry/obj/Release/net10.0",
"dotnet/src/Microsoft.Agents.AI/obj/Release/net10.0",
"dotnet/src/Microsoft.Agents.AI.Abstractions/obj/Release/net10.0"
)
$stale = @($staleObjProbes | Where-Object { Test-Path (Join-Path $_ "*.AssemblyInfo.cs") })
if ($stale.Count -gt 0) {
$msg = @(
"Detected prior Release/net10.0 build outputs in:"
($stale | ForEach-Object { " - $_" })
""
"Publish would propagate -r linux-musl-x64 to those ProjectReferences and the"
"leftover obj/Release/net10.0/*.AssemblyInfo.cs files would cause CS0579 duplicate"
"attribute errors. Pick one:"
" (a) Pass -UsePrebuiltProjectReferences (skips ProjectReference rebuild and"
" uses the existing src/<lib>/bin/Release/net10.0/*.dll outputs in place)."
" Only safe when you know those DLLs match current source - this is the path"
" CI uses immediately after its 'Build Foundry hosted IT (and its deps)' step."
" (b) Remove the stale obj/Release trees, e.g.:"
" Remove-Item -Recurse -Force dotnet/src/Microsoft.Agents.AI*/obj/Release"
" and re-run."
) -join "`n"
throw $msg
}
Write-Host "Letting publish build ProjectReferences (pass -UsePrebuiltProjectReferences in CI to skip)." -ForegroundColor DarkGray
}
dotnet publish $TestContainerProject -c Release -f net10.0 -r linux-musl-x64 --self-contained false -o $out @publishExtraArgs --tl:off | Out-Host
if ($LASTEXITCODE -ne 0) {
throw "dotnet publish failed with exit code $LASTEXITCODE."
}
@@ -13,8 +13,6 @@ namespace Microsoft.Agents.AI.DurableTask.IntegrationTests;
[Trait("Category", "SampleValidation")]
public sealed class ConsoleAppSamplesValidation(ITestOutputHelper outputHelper) : SamplesValidationBase(outputHelper)
{
private const string SkipFlakyTimingTest = "Flaky: timing-dependent LLM test, see https://github.com/microsoft/agent-framework/issues/4971";
private static readonly string s_samplesPath = Path.GetFullPath(
Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "..", "..", "..", "..", "..", "samples", "04-hosting", "DurableAgents", "ConsoleApps"));
@@ -69,7 +67,7 @@ public sealed class ConsoleAppSamplesValidation(ITestOutputHelper outputHelper)
});
}
[Fact]
[RetryFact(2, 5000)]
public async Task SingleAgentOrchestrationChainingSampleValidationAsync()
{
using CancellationTokenSource testTimeoutCts = this.CreateTestTimeoutCts();
@@ -105,7 +103,7 @@ public sealed class ConsoleAppSamplesValidation(ITestOutputHelper outputHelper)
});
}
[Fact]
[RetryFact(2, 5000)]
public async Task MultiAgentConcurrencySampleValidationAsync()
{
using CancellationTokenSource testTimeoutCts = this.CreateTestTimeoutCts();
@@ -160,7 +158,7 @@ public sealed class ConsoleAppSamplesValidation(ITestOutputHelper outputHelper)
});
}
[Fact]
[RetryFact(2, 5000)]
public async Task MultiAgentConditionalSampleValidationAsync()
{
using CancellationTokenSource testTimeoutCts = this.CreateTestTimeoutCts();
@@ -237,14 +235,14 @@ public sealed class ConsoleAppSamplesValidation(ITestOutputHelper outputHelper)
Assert.True(foundSuccess, "Orchestration did not complete successfully.");
}
[Fact(Skip = SkipFlakyTimingTest)]
[RetryFact(2, 5000)]
public async Task SingleAgentOrchestrationHITLSampleValidationAsync()
{
string samplePath = Path.Combine(s_samplesPath, "05_AgentOrchestration_HITL");
await this.RunSampleTestAsync(samplePath, async (process, logs) =>
{
using CancellationTokenSource testTimeoutCts = this.CreateTestTimeoutCts();
using CancellationTokenSource testTimeoutCts = this.CreateTestTimeoutCts(TimeSpan.FromSeconds(180));
// Start the HITL orchestration following the happy path from README
await this.WriteInputAsync(process, "The Future of Artificial Intelligence", testTimeoutCts.Token);
@@ -260,7 +258,7 @@ public sealed class ConsoleAppSamplesValidation(ITestOutputHelper outputHelper)
while ((line = this.ReadLogLine(logs, testTimeoutCts.Token)) != null)
{
// Look for notification that content is ready. The first time we see this, we should send a rejection.
// The second time we see this, we should send approval.
// Subsequent times we see this, we should send approval (LLM may produce extra review cycles).
if (line.Contains("Content is ready for review", StringComparison.OrdinalIgnoreCase))
{
if (!rejectionSent)
@@ -275,20 +273,15 @@ public sealed class ConsoleAppSamplesValidation(ITestOutputHelper outputHelper)
testTimeoutCts.Token);
rejectionSent = true;
}
else if (!approvalSent)
else
{
// Prompt: Approve? (y/n):
// Approve any subsequent draft (LLM non-determinism may produce extra review cycles)
await this.WriteInputAsync(process, "y", testTimeoutCts.Token);
// Prompt: Feedback (optional):
await this.WriteInputAsync(process, "Looks good!", testTimeoutCts.Token);
approvalSent = true;
}
else
{
// This should never happen
Assert.Fail("Unexpected message found.");
}
}
// Look for success message
@@ -311,14 +304,14 @@ public sealed class ConsoleAppSamplesValidation(ITestOutputHelper outputHelper)
});
}
[Fact(Skip = SkipFlakyTimingTest)]
[RetryFact(2, 5000)]
public async Task LongRunningToolsSampleValidationAsync()
{
string samplePath = Path.Combine(s_samplesPath, "06_LongRunningTools");
await this.RunSampleTestAsync(samplePath, async (process, logs) =>
{
// This test takes a bit longer to run due to the multiple agent interactions and the lengthy content generation.
using CancellationTokenSource testTimeoutCts = this.CreateTestTimeoutCts(TimeSpan.FromSeconds(90));
using CancellationTokenSource testTimeoutCts = this.CreateTestTimeoutCts(TimeSpan.FromSeconds(180));
// Test starting an agent that schedules a content generation orchestration
await this.WriteInputAsync(
@@ -335,7 +328,7 @@ public sealed class ConsoleAppSamplesValidation(ITestOutputHelper outputHelper)
while ((line = this.ReadLogLine(logs, testTimeoutCts.Token)) != null)
{
// Look for notification that content is ready. The first time we see this, we should send a rejection.
// The second time we see this, we should send approval.
// Subsequent times we see this, we should send approval (LLM may produce extra review cycles).
if (line.Contains("NOTIFICATION: Please review the following content for approval", StringComparison.OrdinalIgnoreCase))
{
// Wait for the notification to be fully written to the console
@@ -350,20 +343,15 @@ public sealed class ConsoleAppSamplesValidation(ITestOutputHelper outputHelper)
testTimeoutCts.Token);
rejectionSent = true;
}
else if (!approvalSent)
else
{
// Approve the content. Note that we need to send a newline character to the console first before sending the input.
// Approve any subsequent draft (LLM non-determinism may produce extra review cycles)
await this.WriteInputAsync(
process,
"\nApprove the content",
testTimeoutCts.Token);
approvalSent = true;
}
else
{
// This should never happen
Assert.Fail("Unexpected message found.");
}
}
// Look for success message
@@ -396,14 +384,14 @@ public sealed class ConsoleAppSamplesValidation(ITestOutputHelper outputHelper)
});
}
[Fact(Skip = SkipFlakyTimingTest)]
[RetryFact(2, 5000)]
public async Task ReliableStreamingSampleValidationAsync()
{
string samplePath = Path.Combine(s_samplesPath, "07_ReliableStreaming");
await this.RunSampleTestAsync(samplePath, async (process, logs) =>
{
// This test takes a bit longer to run due to the multiple agent interactions and the lengthy content generation.
using CancellationTokenSource testTimeoutCts = this.CreateTestTimeoutCts(TimeSpan.FromSeconds(90));
using CancellationTokenSource testTimeoutCts = this.CreateTestTimeoutCts(TimeSpan.FromSeconds(150));
// Test the agent endpoint with a simple prompt
await this.WriteInputAsync(process, "Plan a 5-day trip to Seattle. Include daily activities.", testTimeoutCts.Token);
@@ -19,11 +19,9 @@ namespace Microsoft.Agents.AI.DurableTask.IntegrationTests;
[Trait("Category", "Integration")]
public sealed class ExternalClientTests(ITestOutputHelper outputHelper) : IDisposable
{
private const string SkipFlakyTimingTest = "Flaky: timing-dependent LLM test, see https://github.com/microsoft/agent-framework/issues/4971";
private static readonly TimeSpan s_defaultTimeout = Debugger.IsAttached
? TimeSpan.FromMinutes(5)
: TimeSpan.FromSeconds(60);
: TimeSpan.FromSeconds(120);
private static readonly IConfiguration s_configuration =
new ConfigurationBuilder()
@@ -38,7 +36,7 @@ public sealed class ExternalClientTests(ITestOutputHelper outputHelper) : IDispo
public void Dispose() => this._cts.Dispose();
[Fact]
[RetryFact(2, 5000)]
public async Task SimplePromptAsync()
{
// Setup
@@ -77,7 +75,7 @@ public sealed class ExternalClientTests(ITestOutputHelper outputHelper) : IDispo
Assert.Contains(agentLogs, log => log.EventId.Name == "LogAgentResponse");
}
[Fact(Skip = SkipFlakyTimingTest)]
[RetryFact(2, 5000)]
public async Task CallFunctionToolsAsync()
{
int weatherToolInvocationCount = 0;
@@ -129,7 +127,7 @@ public sealed class ExternalClientTests(ITestOutputHelper outputHelper) : IDispo
Assert.Equal(1, packingListToolInvocationCount);
}
[Fact(Skip = SkipFlakyTimingTest)]
[RetryFact(2, 5000)]
public async Task CallLongRunningFunctionToolsAsync()
{
[Description("Starts a greeting workflow and returns the workflow instance ID")]
@@ -217,7 +217,7 @@ public abstract class SamplesValidationBase : IAsyncLifetime
/// </summary>
protected CancellationTokenSource CreateTestTimeoutCts(TimeSpan? timeout = null)
{
TimeSpan testTimeout = Debugger.IsAttached ? TimeSpan.FromMinutes(5) : timeout ?? TimeSpan.FromSeconds(60);
TimeSpan testTimeout = Debugger.IsAttached ? TimeSpan.FromMinutes(5) : timeout ?? TimeSpan.FromSeconds(120);
return new CancellationTokenSource(testTimeout);
}
@@ -22,7 +22,7 @@ public sealed class WorkflowConsoleAppSamplesValidation(ITestOutputHelper output
/// <inheritdoc />
protected override string TaskHubPrefix => "workflow";
[Fact]
[RetryFact(2, 5000)]
public async Task SequentialWorkflowSampleValidationAsync()
{
using CancellationTokenSource testTimeoutCts = this.CreateTestTimeoutCts(s_testTimeout);
@@ -71,7 +71,7 @@ public sealed class WorkflowConsoleAppSamplesValidation(ITestOutputHelper output
});
}
[Fact]
[RetryFact(2, 5000)]
public async Task ConcurrentWorkflowSampleValidationAsync()
{
using CancellationTokenSource testTimeoutCts = this.CreateTestTimeoutCts(s_testTimeout);
@@ -120,7 +120,7 @@ public sealed class WorkflowConsoleAppSamplesValidation(ITestOutputHelper output
});
}
[Fact]
[RetryFact(2, 5000)]
public async Task ConditionalEdgesWorkflowSampleValidationAsync()
{
using CancellationTokenSource testTimeoutCts = this.CreateTestTimeoutCts(s_testTimeout);
@@ -182,7 +182,7 @@ public sealed class WorkflowConsoleAppSamplesValidation(ITestOutputHelper output
}
}
[Fact]
[RetryFact(2, 5000)]
public async Task WorkflowEventsSampleValidationAsync()
{
using CancellationTokenSource testTimeoutCts = this.CreateTestTimeoutCts(s_testTimeout);
@@ -278,7 +278,7 @@ public sealed class WorkflowConsoleAppSamplesValidation(ITestOutputHelper output
});
}
[Fact]
[RetryFact(2, 5000)]
public async Task WorkflowSharedStateSampleValidationAsync()
{
using CancellationTokenSource testTimeoutCts = this.CreateTestTimeoutCts(s_testTimeout);
@@ -376,7 +376,7 @@ public sealed class WorkflowConsoleAppSamplesValidation(ITestOutputHelper output
});
}
[Fact]
[RetryFact(2, 5000)]
public async Task SubWorkflowsSampleValidationAsync()
{
using CancellationTokenSource testTimeoutCts = this.CreateTestTimeoutCts(s_testTimeout);
@@ -452,7 +452,7 @@ public sealed class WorkflowConsoleAppSamplesValidation(ITestOutputHelper output
});
}
[Fact]
[RetryFact(2, 5000)]
public async Task WorkflowHITLSampleValidationAsync()
{
using CancellationTokenSource testTimeoutCts = this.CreateTestTimeoutCts(s_testTimeout);
@@ -505,7 +505,7 @@ public sealed class WorkflowConsoleAppSamplesValidation(ITestOutputHelper output
});
}
[Fact]
[RetryFact(2, 5000)]
public async Task WorkflowAndAgentsSampleValidationAsync()
{
using CancellationTokenSource testTimeoutCts = this.CreateTestTimeoutCts(s_testTimeout);
@@ -37,7 +37,7 @@ public class AgentFrameworkResponseHandlerTelemetryTests
public async Task CreateAsync_DefaultAgent_EmitsInvokeAgentSpanAsync()
{
// Arrange
var activities = new List<Activity>();
var activities = new ConcurrentActivityList();
using var tracerProvider = Sdk.CreateTracerProviderBuilder()
.AddSource(ResponsesSourceName)
.AddInMemoryExporter(activities)
@@ -56,7 +56,7 @@ public class AgentFrameworkResponseHandlerTelemetryTests
await foreach (var _ in handler.CreateAsync(request, context, CancellationToken.None)) { }
// Assert — filter by agent name to isolate this test's span from any parallel test spans
var mySpan = Assert.Single(activities.Where(a => TelemetryTestAgent.AgentName.Equals(a.GetTagItem("gen_ai.agent.name"))).ToList());
var mySpan = Assert.Single(activities.Snapshot().Where(a => TelemetryTestAgent.AgentName.Equals(a.GetTagItem("gen_ai.agent.name"))).ToList());
Assert.Equal("invoke_agent", mySpan.GetTagItem("gen_ai.operation.name"));
Assert.NotNull(mySpan.GetTagItem("gen_ai.agent.id"));
}
@@ -65,7 +65,7 @@ public class AgentFrameworkResponseHandlerTelemetryTests
public async Task CreateAsync_KeyedAgent_EmitsInvokeAgentSpanAsync()
{
// Arrange
var activities = new List<Activity>();
var activities = new ConcurrentActivityList();
using var tracerProvider = Sdk.CreateTracerProviderBuilder()
.AddSource(ResponsesSourceName)
.AddInMemoryExporter(activities)
@@ -84,7 +84,7 @@ public class AgentFrameworkResponseHandlerTelemetryTests
await foreach (var _ in handler.CreateAsync(request, context, CancellationToken.None)) { }
// Assert — filter by agent name to isolate this test's span
var mySpan = Assert.Single(activities.Where(a => TelemetryTestAgent.AgentName.Equals(a.GetTagItem("gen_ai.agent.name"))).ToList());
var mySpan = Assert.Single(activities.Snapshot().Where(a => TelemetryTestAgent.AgentName.Equals(a.GetTagItem("gen_ai.agent.name"))).ToList());
Assert.Equal("invoke_agent", mySpan.GetTagItem("gen_ai.operation.name"));
}
@@ -95,8 +95,8 @@ public class AgentFrameworkResponseHandlerTelemetryTests
// If ApplyOpenTelemetry double-wraps, an extra span would appear on ResponsesSourceName.
// If it correctly skips wrapping, only the pre-wrap's unique source emits spans.
var preWrapSource = Guid.NewGuid().ToString();
var preWrapActivities = new List<Activity>();
var responsesActivities = new List<Activity>();
var preWrapActivities = new ConcurrentActivityList();
var responsesActivities = new ConcurrentActivityList();
using var preWrapProvider = Sdk.CreateTracerProviderBuilder()
.AddSource(preWrapSource)
@@ -125,18 +125,19 @@ public class AgentFrameworkResponseHandlerTelemetryTests
await foreach (var _ in handler.CreateAsync(request, context, CancellationToken.None)) { }
// Assert — pre-wrap source emits exactly 1 span (agent ran)
Assert.Single(preWrapActivities);
Assert.Equal("invoke_agent", preWrapActivities[0].GetTagItem("gen_ai.operation.name"));
var preWrapSnapshot = preWrapActivities.Snapshot();
Assert.Single(preWrapSnapshot);
Assert.Equal("invoke_agent", preWrapSnapshot[0].GetTagItem("gen_ai.operation.name"));
// ResponsesSourceName emits 0 spans — ApplyOpenTelemetry skipped wrapping the pre-instrumented agent
Assert.DoesNotContain(responsesActivities, a => TelemetryTestAgent.AgentName.Equals(a.GetTagItem("gen_ai.agent.name")));
Assert.DoesNotContain(responsesActivities.Snapshot(), a => TelemetryTestAgent.AgentName.Equals(a.GetTagItem("gen_ai.agent.name")));
}
[Fact]
public async Task CreateAsync_DefaultAgent_SpanDisplayNameContainsAgentNameAsync()
{
// Arrange
var activities = new List<Activity>();
var activities = new ConcurrentActivityList();
using var tracerProvider = Sdk.CreateTracerProviderBuilder()
.AddSource(ResponsesSourceName)
.AddInMemoryExporter(activities)
@@ -155,7 +156,7 @@ public class AgentFrameworkResponseHandlerTelemetryTests
await foreach (var _ in handler.CreateAsync(request, context, CancellationToken.None)) { }
// Assert — display name follows "invoke_agent {Name}({Id})" convention; filter by agent name to isolate
var mySpan = Assert.Single(activities.Where(a => TelemetryTestAgent.AgentName.Equals(a.GetTagItem("gen_ai.agent.name"))).ToList());
var mySpan = Assert.Single(activities.Snapshot().Where(a => TelemetryTestAgent.AgentName.Equals(a.GetTagItem("gen_ai.agent.name"))).ToList());
Assert.Contains("invoke_agent", mySpan.DisplayName, StringComparison.Ordinal);
Assert.Contains(TelemetryTestAgent.AgentName, mySpan.DisplayName, StringComparison.Ordinal);
}
@@ -231,4 +232,35 @@ public class AgentFrameworkResponseHandlerTelemetryTests
}
private sealed class TelemetryAgentSession : AgentSession;
/// <summary>
/// Thread-safe <see cref="ICollection{Activity}"/> used by OTel's InMemoryExporter to capture
/// activities emitted on globally-listened sources. Required because the exporter writes into
/// the supplied collection from background Activity completion callbacks while the test thread
/// may be enumerating it for assertions, and other tests in the same assembly may emit on the
/// same source concurrently. A plain <see cref="List{Activity}"/> trips
/// "Collection was modified; enumeration operation may not execute." in that scenario.
/// </summary>
private sealed class ConcurrentActivityList : ICollection<Activity>
{
private readonly List<Activity> _items = new();
private readonly object _gate = new();
public int Count { get { lock (this._gate) { return this._items.Count; } } }
public bool IsReadOnly => false;
public void Add(Activity item) { lock (this._gate) { this._items.Add(item); } }
public void Clear() { lock (this._gate) { this._items.Clear(); } }
public bool Contains(Activity item) { lock (this._gate) { return this._items.Contains(item); } }
public void CopyTo(Activity[] array, int arrayIndex) { lock (this._gate) { this._items.CopyTo(array, arrayIndex); } }
public bool Remove(Activity item) { lock (this._gate) { return this._items.Remove(item); } }
public Activity[] Snapshot()
{
lock (this._gate) { return this._items.ToArray(); }
}
public IEnumerator<Activity> GetEnumerator() => ((IEnumerable<Activity>)this.Snapshot()).GetEnumerator();
System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() => this.GetEnumerator();
}
}
@@ -15,8 +15,6 @@ namespace Microsoft.Agents.AI.Hosting.AzureFunctions.IntegrationTests;
[Trait("Category", "SampleValidation")]
public sealed class SamplesValidation(ITestOutputHelper outputHelper) : IAsyncLifetime
{
private const string SkipFlakyTimingTest = "Flaky: timing-dependent LLM test, see https://github.com/microsoft/agent-framework/issues/4971";
private const string AzureFunctionsPort = "7071";
private const string AzuritePort = "10000";
private const string DtsPort = "8080";
@@ -37,7 +35,7 @@ public sealed class SamplesValidation(ITestOutputHelper outputHelper) : IAsyncLi
.Build();
private static bool s_infrastructureStarted;
private static readonly TimeSpan s_orchestrationTimeout = TimeSpan.FromMinutes(2);
private static readonly TimeSpan s_orchestrationTimeout = TimeSpan.FromMinutes(3);
// In CI, `dotnet run` builds the Functions project from scratch before the host starts, so 60s is not enough.
private static readonly TimeSpan s_functionsReadyTimeout = TimeSpan.FromSeconds(180);
@@ -62,7 +60,7 @@ public sealed class SamplesValidation(ITestOutputHelper outputHelper) : IAsyncLi
await Task.CompletedTask;
}
[Fact]
[RetryFact(2, 5000)]
public async Task SingleAgentSampleValidationAsync()
{
string samplePath = Path.Combine(s_samplesPath, "01_SingleAgent");
@@ -107,7 +105,7 @@ public sealed class SamplesValidation(ITestOutputHelper outputHelper) : IAsyncLi
});
}
[Fact]
[Fact(Skip = "Flaky: LLM non-determinism can produce null orchestration results")]
public async Task SingleAgentOrchestrationChainingSampleValidationAsync()
{
string samplePath = Path.Combine(s_samplesPath, "02_AgentOrchestration_Chaining");
@@ -150,7 +148,7 @@ public sealed class SamplesValidation(ITestOutputHelper outputHelper) : IAsyncLi
});
}
[Fact]
[RetryFact(2, 5000)]
public async Task MultiAgentOrchestrationConcurrentSampleValidationAsync()
{
string samplePath = Path.Combine(s_samplesPath, "03_AgentOrchestration_Concurrency");
@@ -200,7 +198,7 @@ public sealed class SamplesValidation(ITestOutputHelper outputHelper) : IAsyncLi
});
}
[Fact]
[RetryFact(2, 5000)]
public async Task MultiAgentOrchestrationConditionalsSampleValidationAsync()
{
string samplePath = Path.Combine(s_samplesPath, "04_AgentOrchestration_Conditionals");
@@ -218,7 +216,7 @@ public sealed class SamplesValidation(ITestOutputHelper outputHelper) : IAsyncLi
});
}
[Fact]
[RetryFact(2, 5000)]
public async Task SingleAgentOrchestrationHITLSampleValidationAsync()
{
string samplePath = Path.Combine(s_samplesPath, "05_AgentOrchestration_HITL");
@@ -274,7 +272,7 @@ public sealed class SamplesValidation(ITestOutputHelper outputHelper) : IAsyncLi
});
}
[Fact(Skip = SkipFlakyTimingTest)]
[RetryFact(2, 5000)]
public async Task LongRunningToolsSampleValidationAsync()
{
string samplePath = Path.Combine(s_samplesPath, "06_LongRunningTools");
@@ -316,7 +314,7 @@ public sealed class SamplesValidation(ITestOutputHelper outputHelper) : IAsyncLi
}
},
message: "Orchestration is requesting human feedback",
timeout: TimeSpan.FromSeconds(60));
timeout: TimeSpan.FromSeconds(180));
// Approve the content
Uri approvalUri = new($"{runAgentUri}?thread_id={sessionId}");
@@ -336,7 +334,7 @@ public sealed class SamplesValidation(ITestOutputHelper outputHelper) : IAsyncLi
}
},
message: "Content published notification is logged",
timeout: TimeSpan.FromSeconds(60));
timeout: TimeSpan.FromSeconds(180));
// Verify the final orchestration status by asking the agent for the status
Uri statusUri = new($"{runAgentUri}?thread_id={sessionId}");
@@ -360,11 +358,11 @@ public sealed class SamplesValidation(ITestOutputHelper outputHelper) : IAsyncLi
return isCompleted && hasContent;
},
message: "Orchestration is completed",
timeout: TimeSpan.FromSeconds(60));
timeout: TimeSpan.FromSeconds(180));
});
}
[Fact]
[RetryFact(2, 5000)]
public async Task AgentAsMcpToolAsync()
{
string samplePath = Path.Combine(s_samplesPath, "07_AgentAsMcpTool");
@@ -404,7 +402,7 @@ public sealed class SamplesValidation(ITestOutputHelper outputHelper) : IAsyncLi
});
}
[Fact(Skip = SkipFlakyTimingTest)]
[RetryFact(2, 5000)]
public async Task ReliableStreamingSampleValidationAsync()
{
string samplePath = Path.Combine(s_samplesPath, "08_ReliableStreaming");
@@ -0,0 +1,153 @@
// Copyright (c) Microsoft. All rights reserved.
//using System.Collections.Generic;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using FluentAssertions;
using Microsoft.Agents.AI.Workflows.Specialized.Magentic;
using Microsoft.Extensions.AI;
namespace Microsoft.Agents.AI.Workflows.UnitTests;
public class MagenticManagerTests
{
private static void CheckMessage(ChatMessage message, string expectedText, bool runPropertySmokeTest = false, bool skipCreatedAt = true)
{
message.Text.Should().Be(expectedText);
if (runPropertySmokeTest)
{
message.AuthorName.Should().Be(nameof(MagenticOrchestrator));
if (!skipCreatedAt)
{
message.CreatedAt.Should().NotBeNull().And.NotBeBefore(DateTimeOffset.UtcNow.AddDays(-1));
}
message.Role.Should().Be(ChatRole.Assistant);
message.MessageId.Should().NotBeNull();
}
}
[Theory]
[InlineData(false)]
[InlineData(true)]
public async Task Test_MagenticManager_UpdatePlanAsync(bool hasExistingPlan)
{
TestReplayAgent testAgent = new(name: nameof(MagenticOrchestrator),
messages:
[
[new(ChatRole.Assistant, "Facts")],
[new(ChatRole.Assistant, "Plan")],
]);
TestEchoAgent participant = new(name: "Echo");
MagenticManager manager = new(testAgent);
MagenticTaskContext taskContext = new([new(ChatRole.User, "Task")], [participant], new TaskLimits(), null, []);
if (hasExistingPlan)
{
taskContext.TaskLedger = new(new(ChatRole.Assistant, "OldFacts"), new(ChatRole.Assistant, "OldPlan"));
}
TestRunContext runContext = new();
IWorkflowContext workflowContext = runContext.BindWorkflowContext(nameof(MagenticOrchestrator));
TaskLedger newPlan = await manager.UpdatePlanAsync(taskContext, workflowContext, CancellationToken.None);
CheckMessage(newPlan.CurrentFacts, "Facts");
CheckMessage(newPlan.CurrentPlan, "Plan");
taskContext.ChatHistory.Should().HaveCount(4);
if (hasExistingPlan)
{
ChatMessage factsRequest = taskContext.ChatHistory[0];
factsRequest.Text.Should().Contain("OldFacts");
}
ChatMessage facts = taskContext.ChatHistory[1];
facts.Should().Be(newPlan.CurrentFacts);
ChatMessage plan = taskContext.ChatHistory[3];
plan.Should().Be(newPlan.CurrentPlan);
}
[Theory]
[InlineData(0)]
[InlineData(1)]
[InlineData(2)]
[InlineData(3)]
[InlineData(4)]
public async Task Test_MagenticManager_UpdateProgressLedgerAsync(int failures)
{
List<List<ChatMessage>> turns =
TestProgressLedgerState.MissingRequired.Take(failures)
.Select<TestProgressLedgerState, List<ChatMessage>>(
state => [new ChatMessage(ChatRole.Assistant, state.ToJsonString())])
.ToList();
turns.Should().HaveCount(failures);
turns.Add([new ChatMessage(ChatRole.Assistant, TestProgressLedgerState.Default.ToJsonString())]);
TestReplayAgent testAgent = new(name: nameof(MagenticOrchestrator),
messages: turns);
TestEchoAgent participant = new(name: "Echo");
MagenticManager manager = new(testAgent);
MagenticTaskContext taskContext = new([new(ChatRole.User, "Task")], [participant], new TaskLimits(), null, []);
taskContext.TaskLedger = new(new(ChatRole.Assistant, "OldFacts"), new(ChatRole.Assistant, "OldPlan"));
TestRunContext runContext = new();
IWorkflowContext workflowContext = runContext.BindWorkflowContext(nameof(MagenticOrchestrator));
// Precondition check: ProgressLedger should be not "started"
taskContext.ProgressLedger.IsStarted.Should().BeFalse();
Func<Task> action = () => manager.UpdateProgressLedgerAsync(taskContext, workflowContext, CancellationToken.None).AsTask();
if (failures >= taskContext.TaskLimits.MaxProgressLedgerRetryCount)
{
// We expect to see an exception if the number of failures exceeds the maximum retry count
await action.Should().ThrowAsync();
taskContext.ProgressLedger.IsStarted.Should().BeFalse();
}
else
{
await action.Should().NotThrowAsync();
taskContext.ProgressLedger.IsStarted.Should().BeTrue();
TestProgressLedgerState.Default.Validate(taskContext.ProgressLedger);
}
int expectedWarnings = Math.Min(failures, 3);
runContext.Events.Should().HaveCount(expectedWarnings).And.AllBeOfType<WorkflowWarningEvent>();
}
[Fact]
public async Task Test_MagenticManager_PrepareFinalAnswerAsync()
{
TestReplayAgent testAgent = new(name: nameof(MagenticOrchestrator),
messages:
[
[
new(ChatRole.Assistant, "FinalAnswer")
],
]);
TestEchoAgent participant = new(name: "Echo");
MagenticManager manager = new(testAgent);
MagenticTaskContext taskContext = new([new(ChatRole.User, "Task")], [participant], new TaskLimits(), null, []);
TestRunContext runContext = new();
IWorkflowContext workflowContext = runContext.BindWorkflowContext(nameof(MagenticOrchestrator));
ChatMessage answer = await manager.PrepareFinalAnswerAsync(taskContext, workflowContext, CancellationToken.None);
CheckMessage(answer, "FinalAnswer", true, false);
}
}
@@ -0,0 +1,214 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
using System.Text.Json;
using FluentAssertions;
using Microsoft.Agents.AI.Workflows.Specialized.Magentic;
using Microsoft.Extensions.AI;
namespace Microsoft.Agents.AI.Workflows.UnitTests;
public class MagenticProgressLedgerTests
{
public record KVPair(string key);
public record AnswerReasonPair(bool answer, string reason);
[Theory]
[InlineData(false)]
[InlineData(true)]
public void Test_ExtractJson_SucceedsWhenInBlockQuote(bool isTagged)
{
// Arrange
string json = isTagged
? "```json\n{\"key\": \"value\"}\n```"
: "```{\"key\": \"value\"}```";
string embedded = $"Some text before the JSON block.\n{json}\nSome text after the JSON block.";
ChatMessage message = new(ChatRole.Assistant, embedded);
// Act
JsonElement element = message.ExtractJson();
// Assert
KVPair? result = element.Deserialize<KVPair>();
result.Should().NotBeNull();
result.key.Should().Be("value");
}
[Fact]
public void Test_ExtractJson_SucceedsWhenScanning()
{
// Arrange
ChatMessage message = new(ChatRole.Assistant,
"""
Some text before the JSON embed.
{"key": "value"}
Some text after the JSON embed.
""");
// Act
JsonElement element = message.ExtractJson();
// Assert
KVPair? result = element.Deserialize<KVPair>();
result.Should().NotBeNull();
result.key.Should().Be("value");
}
[Fact]
public void Test_ExtractJson_FailsWhenUnbalanced()
{
// Arrange
ChatMessage message = new(ChatRole.Assistant,
"""
Some text before the JSON embed.
{"key": { "key2": "value" }
Some text after the JSON embed.
""");
// Act
Func<JsonElement> action = () => message.ExtractJson();
// Assert
action.Should().Throw();
}
[Fact]
public void Test_ExtractJson_FailsWhenNoJson()
{
// Arrange
ChatMessage message = new(ChatRole.Assistant,
"""
Some text, without JSON
""");
// Act
Func<JsonElement> action = () => message.ExtractJson();
// Assert
action.Should().Throw();
}
[Fact]
public void Test_ExtractJson_SuceedsWithQuotesBrackets()
{
// Arrange
ChatMessage message = new(ChatRole.Assistant,
"""
{"reason":"the output contained }", "answer": false}
""");
// Act
JsonElement element = message.ExtractJson();
// Assert
AnswerReasonPair? result = element.Deserialize<AnswerReasonPair>();
result.Should().NotBeNull();
result.reason.Should().Be("the output contained }");
result.answer.Should().BeFalse();
}
public static readonly string TestTeamNames = string.Join(", ", ["CodingAgent", "CodeExecutor", "WebSurferAgent", "FileSurferAgent"]);
[Fact]
public void Test_ProgressLedgerState_IsEmptyWhenStarted()
{
// Arrange/Act
MagenticProgressLedger ledger = new(TestTeamNames, []);
// Assert
ledger.State.Should().BeNull();
ledger.IsStarted.Should().BeFalse();
ledger.TryGetCurrentSlotValue(TestProgressLedgerState.CustomSlot1, out _).Should().BeFalse();
ledger.TryGetCurrentSlotValue(TestProgressLedgerState.CustomSlot2, out _).Should().BeFalse();
}
[Theory]
[InlineData(0, "RequiredOnly")]
[InlineData(1, "IncludeCustom")]
public void Test_ProgressLedgerState_IsNotEmptyWhenRestored(int caseIndex, string _)
{
// Arrange
TestProgressLedgerState state = TestProgressLedgerState.Working[caseIndex];
JsonElement element = state.ToJson();
// Act
MagenticProgressLedger ledger = new(TestTeamNames, [], element);
// Assert
ledger.State.Should().Be(element);
state.Validate(ledger);
}
[Theory]
[InlineData(0, "RequiredOnly")]
[InlineData(1, "IncludeCustom")]
public void Test_ProgressLedgerState_SwitchesToStartedWhenStateUpdates(int caseIndex, string _)
{
// Arrange
MagenticProgressLedger ledger = new(TestTeamNames, []);
TestProgressLedgerState targetState = TestProgressLedgerState.Working[caseIndex];
JsonElement element = targetState.ToJson();
ledger.State.Should().BeNull();
// Act
ledger.TryUpdateState(element).Should().BeTrue();
// Assert
ledger.State.Should().Be(element);
targetState.Validate(ledger);
}
[Theory]
[InlineData(0, "is_request_satisfied")]
[InlineData(1, "is_in_loop")]
[InlineData(2, "is_progress_being_made")]
[InlineData(3, "instruction_or_question")]
[InlineData(4, "next_speaker")]
public void Test_ProgressLedgerState_FailsToUpdateWhenRequiredAnswersMissing(int caseIndex, string _)
{
// Arrange
MagenticProgressLedger ledger = new(TestTeamNames, []);
TestProgressLedgerState targetState = TestProgressLedgerState.MissingRequired[caseIndex];
JsonElement element = targetState.ToJson();
ledger.State.Should().BeNull();
// Act
ledger.TryUpdateState(element).Should().BeFalse();
ledger.State.Should().BeNull();
}
[Theory]
[InlineData(true)]
[InlineData(false)]
public void Test_ProgressLedgerState_GeneratesCorrectSchema(bool includeCustom)
{
// Arrange
MagenticProgressLedger ledger = new(TestTeamNames, includeCustom
? [TestProgressLedgerState.CustomSlot1, TestProgressLedgerState.CustomSlot2]
: []);
// Act
(string questionBlock, string answerSchema) = ledger.FormatQuestions();
foreach (ProgressLedgerSlot slot in ledger.Slots)
{
// Best-efforts validation: I do not want to make it super-brittle and check for 1:1: with the template
// since that is effectively checking that string formatting works right to some extent.
questionBlock.Should().Contain(slot.Question);
answerSchema.Should().Contain(slot.Key);
answerSchema.Should().Contain(slot.SchemaType);
if (!string.IsNullOrWhiteSpace(slot.SchemaTypeSuffix))
{
answerSchema.Should().Contain(slot.SchemaTypeSuffix);
}
}
}
}
@@ -0,0 +1,88 @@
// Copyright (c) Microsoft. All rights reserved.
using System.Text.Json;
using System.Text.Json.Serialization;
using FluentAssertions;
namespace Microsoft.Agents.AI.Workflows.UnitTests;
public sealed record Slot<T>(T? answer, string? reason);
public record TestProgressLedgerState(Slot<bool?>? is_request_satisfied = null,
Slot<bool?>? is_in_loop = null,
Slot<bool?>? is_progress_being_made = null,
Slot<string>? instruction_or_question = null,
Slot<string>? next_speaker = null,
Slot<bool?>? custom1 = null,
Slot<string>? custom2 = null)
{
public TestProgressLedgerState() : this(new Slot<bool?>(false, "is_request_satisfied_reason"),
new Slot<bool?>(false, "is_in_loop_reason"),
new Slot<bool?>(false, "is_progress_being_made_reason"),
new Slot<string>("Answer", "instruction_or_question_reason"),
new Slot<string>("Lorem Ipsum", "next_speaker_reason"),
new Slot<bool?>(false, "custom1_reason"),
new Slot<string>("Custom2", "custom2_reason"))
{ }
public string ToJsonString() => this.ToJson().ToString();
private static readonly JsonSerializerOptions s_options = new()
{
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull
};
public JsonElement ToJson() => JsonSerializer.SerializeToElement(this, s_options);
internal static BooleanProgressLedgerSlot CustomSlot1 = new("custom1", "Custom Slot 1");
internal static StringProgressLedgerSlot CustomSlot2 = new("custom2", "Custom Slot 2");
public static bool TryGetCustom1(MagenticProgressLedger state, out bool result)
=> state.TryGetCurrentSlotValue(CustomSlot1, out result);
public static bool TryGetCustom2(MagenticProgressLedger state, out string? result)
=> state.TryGetCurrentSlotValue(CustomSlot2, out result);
public void Validate(MagenticProgressLedger state)
{
state.IsRequestSatisfied.Should().Be(this.is_request_satisfied!.answer!.Value);
state.IsInLoop.Should().Be(this.is_in_loop!.answer!.Value);
state.IsProgressBeingMade.Should().Be(this.is_progress_being_made!.answer!.Value);
state.InstructionOrQuestion.Should().Be(this.instruction_or_question!.answer);
state.NextSpeaker.Should().Be(this.next_speaker!.answer);
if (this.custom1 != null)
{
TryGetCustom1(state, out bool custom1Value).Should().BeTrue();
custom1Value.Should().Be(this.custom1.answer!.Value);
}
else
{
TryGetCustom1(state, out _).Should().BeFalse();
}
if (this.custom2 != null)
{
TryGetCustom2(state, out string? custom2Value).Should().BeTrue();
custom2Value.Should().Be(this.custom2.answer);
}
else
{
TryGetCustom2(state, out _).Should().BeFalse();
}
}
public static readonly TestProgressLedgerState Default = new();
public static readonly TestProgressLedgerState RequiredOnly = Default with { custom1 = null, custom2 = null };
public static readonly TestProgressLedgerState[] Working = [RequiredOnly, Default];
public static readonly TestProgressLedgerState[] MissingRequired =
[
Default with { is_request_satisfied = null },
Default with { is_in_loop = null},
Default with { is_progress_being_made = null},
Default with { instruction_or_question = null},
Default with { next_speaker = null},
];
}
@@ -11,8 +11,14 @@ using Microsoft.Extensions.AI;
namespace Microsoft.Agents.AI.Workflows.UnitTests;
public class TestReplayAgent(List<ChatMessage>? messages = null, string? id = null, string? name = null) : AIAgent
public class TestReplayAgent(List<List<ChatMessage>> messages, string? id = null, string? name = null) : AIAgent
{
public TestReplayAgent(List<ChatMessage> messages, string? id = null, string? name = null) : this([messages ?? []], id, name)
{ }
public TestReplayAgent(string? id = null, string? name = null) : this([[]], id, name)
{ }
protected override string? IdCore => id;
public override string? Name => name;
@@ -57,46 +63,55 @@ public class TestReplayAgent(List<ChatMessage>? messages = null, string? id = nu
public static TestReplayAgent FromStrings(params string[] messages) =>
new(ToChatMessages(messages));
public List<ChatMessage> Messages { get; } = Validate(messages) ?? [];
public List<List<ChatMessage>> Messages { get; } = Validate(messages) ?? [];
protected override Task<AgentResponse> RunCoreAsync(IEnumerable<ChatMessage> messages, AgentSession? session = null, AgentRunOptions? options = null, CancellationToken cancellationToken = default)
=> this.RunStreamingAsync(messages, session, options, cancellationToken).ToAgentResponseAsync(cancellationToken);
public int Turn { get; set; }
protected override async IAsyncEnumerable<AgentResponseUpdate> RunCoreStreamingAsync(IEnumerable<ChatMessage> messages, AgentSession? session = null, AgentRunOptions? options = null, [EnumeratorCancellation] CancellationToken cancellationToken = default)
{
string responseId = Guid.NewGuid().ToString("N");
foreach (ChatMessage message in this.Messages)
if (this.Turn < this.Messages.Count)
{
foreach (AIContent content in message.Contents)
foreach (ChatMessage message in this.Messages[this.Turn++])
{
yield return new AgentResponseUpdate()
foreach (AIContent content in message.Contents)
{
AgentId = this.Id,
AuthorName = this.Name,
MessageId = message.MessageId,
ResponseId = responseId,
Contents = [content],
Role = message.Role,
};
yield return new AgentResponseUpdate()
{
AgentId = this.Id,
AuthorName = this.Name,
MessageId = message.MessageId,
ResponseId = responseId,
Contents = [content],
Role = message.Role,
};
}
}
}
}
private static List<ChatMessage>? Validate(List<ChatMessage>? candidateMessages)
private static List<List<ChatMessage>>? Validate(List<List<ChatMessage>>? candidateMessages)
{
string? currentMessageId = null;
string? lastMessageId = null;
if (candidateMessages is not null)
if (candidateMessages != null)
{
foreach (ChatMessage message in candidateMessages)
foreach (List<ChatMessage> candidateMessagesTurn in candidateMessages)
{
if (currentMessageId is null)
foreach (ChatMessage message in candidateMessagesTurn)
{
currentMessageId = message.MessageId;
}
else if (currentMessageId == message.MessageId)
{
throw new ArgumentException("Duplicate consecutive message ids");
if (lastMessageId is null || lastMessageId != message.MessageId)
{
lastMessageId = message.MessageId;
}
else if (lastMessageId == message.MessageId)
{
throw new ArgumentException("Duplicate consecutive message ids");
}
}
}
}
@@ -135,6 +135,7 @@ from ._sessions import (
from ._settings import SecretString, load_settings
from ._skills import (
AggregatingSkillsSource,
ClassSkill,
DeduplicatingSkillsSource,
DelegatingSkillsSource,
FileSkill,
@@ -345,6 +346,7 @@ __all__ = [
"ChatResponseUpdate",
"CheckResult",
"CheckpointStorage",
"ClassSkill",
"CompactionProvider",
"CompactionStrategy",
"Content",
+484 -41
View File
@@ -5,7 +5,7 @@
Defines the core data model classes for the agent skills system:
- **Skills:** :class:`Skill` (abstract base), :class:`InlineSkill` (code-defined),
and :class:`FileSkill` (filesystem-backed).
:class:`ClassSkill` (class-based), and :class:`FileSkill` (filesystem-backed).
- **Resources:** :class:`SkillResource` (abstract base), :class:`InlineSkillResource`
(static content or callable).
- **Scripts:** :class:`SkillScript` (abstract base), :class:`InlineSkillScript`
@@ -27,6 +27,9 @@ Skills can come from different sources:
Represented as :class:`FileSkill` instances.
- **Code-defined** — created as :class:`InlineSkill` instances in Python code,
with optional callable resources attached via the ``@skill.resource`` decorator.
- **Class-based** — created by subclassing :class:`ClassSkill` to define
self-contained, reusable skill types with ``create_resource()`` and
``create_script()`` factory methods.
- **Custom sources** — any :class:`SkillsSource` implementation that provides
skills from arbitrary origins (REST APIs, databases, etc.).
@@ -570,6 +573,65 @@ def _validate_skill_description(name: str, description: str) -> None:
)
def _build_skill_content(
name: str,
description: str,
instructions: str,
resources: Sequence[SkillResource] | None = None,
scripts: Sequence[SkillScript] | None = None,
) -> str:
"""Build XML-structured content for code-defined and class-based skills.
Produces an XML document containing name, description, instructions,
resources, and scripts elements. Used by both :class:`InlineSkill`
and :class:`ClassSkill` to generate their ``content`` property.
Args:
name: The skill name.
description: The skill description.
instructions: The raw instructions text.
resources: Optional resources associated with the skill.
scripts: Optional scripts associated with the skill.
Returns:
An XML-structured content string.
"""
result = (
f"<name>{xml_escape(name)}</name>\n"
f"<description>{xml_escape(description)}</description>\n"
"\n"
"<instructions>\n"
f"{instructions}\n"
"</instructions>"
)
if resources:
resource_lines = "\n".join(_create_resource_element(r) for r in resources)
result += f"\n\n<resources>\n{resource_lines}\n</resources>"
if scripts:
script_lines = "\n".join(_create_script_element(s) for s in scripts)
result += f"\n\n<scripts>\n{script_lines}\n</scripts>"
return result
def _create_resource_element(resource: SkillResource) -> str:
"""Create a self-closing ``<resource …/>`` XML element from a :class:`SkillResource`.
Args:
resource: The resource to create the element from.
Returns:
A single indented XML element string with ``name`` and optional
``description`` attributes.
"""
attrs = f'name="{xml_escape(resource.name, quote=True)}"'
if resource.description:
attrs += f' description="{xml_escape(resource.description, quote=True)}"'
return f" <resource {attrs}/>"
@experimental(feature_id=ExperimentalFeature.SKILLS)
class InlineSkill(Skill):
"""A skill defined entirely in code with resources and scripts.
@@ -634,25 +696,10 @@ class InlineSkill(Skill):
if self._cached_content is not None:
return self._cached_content
result = (
f"<name>{xml_escape(self.name)}</name>\n"
f"<description>{xml_escape(self.description)}</description>\n"
"\n"
"<instructions>\n"
f"{self.instructions}\n"
"</instructions>"
self._cached_content = _build_skill_content(
self.name, self.description, self.instructions, self._resources, self._scripts
)
if self._resources:
resource_lines = "\n".join(self._create_resource_element(r) for r in self._resources)
result += f"\n\n<resources>\n{resource_lines}\n</resources>"
if self._scripts:
script_lines = "\n".join(_create_script_element(s) for s in self._scripts)
result += f"\n\n<scripts>\n{script_lines}\n</scripts>"
self._cached_content = result
return result
return self._cached_content
@property
def resources(self) -> list[SkillResource]:
@@ -664,22 +711,6 @@ class InlineSkill(Skill):
"""Mutable list of :class:`SkillScript` instances."""
return self._scripts
@staticmethod
def _create_resource_element(resource: SkillResource) -> str:
"""Create a self-closing ``<resource …/>`` XML element from an :class:`SkillResource`.
Args:
resource: The resource to create the element from.
Returns:
A single indented XML element string with ``name`` and optional
``description`` attributes.
"""
attrs = f'name="{xml_escape(resource.name, quote=True)}"'
if resource.description:
attrs += f' description="{xml_escape(resource.description, quote=True)}"'
return f" <resource {attrs}/>"
def resource(
self,
func: Callable[..., Any] | None = None,
@@ -700,8 +731,7 @@ class InlineSkill(Skill):
Keyword Args:
name: Resource name override. Defaults to ``func.__name__``.
description: Resource description override. Defaults to the
function's docstring (via :func:`inspect.getdoc`).
description: Resource description override. Defaults to ``None``.
Returns:
The original function unchanged, or a secondary decorator when
@@ -727,7 +757,7 @@ class InlineSkill(Skill):
def decorator(f: Callable[..., Any]) -> Callable[..., Any]:
resource_name = name or f.__name__
resource_description = description or (inspect.getdoc(f) or None)
resource_description = description
self._resources.append(
InlineSkillResource(
name=resource_name,
@@ -761,8 +791,7 @@ class InlineSkill(Skill):
Keyword Args:
name: Script name override. Defaults to ``func.__name__``.
description: Script description override. Defaults to the
function's docstring (via :func:`inspect.getdoc`).
description: Script description override. Defaults to ``None``.
Returns:
The original function unchanged, or a secondary decorator when
@@ -789,7 +818,7 @@ class InlineSkill(Skill):
def decorator(f: Callable[..., Any]) -> Callable[..., Any]:
script_name = name or f.__name__
script_description = description or (inspect.getdoc(f) or None)
script_description = description
self._scripts.append(
InlineSkillScript(
name=script_name,
@@ -804,6 +833,420 @@ class InlineSkill(Skill):
return decorator(func)
def _make_method_name(method_name: str) -> str:
"""Convert a Python method name to a skill resource/script name.
Replaces underscores with hyphens to match the skill naming convention.
Args:
method_name: The Python method name (e.g. ``"conversion_table"``).
Returns:
The converted name (e.g. ``"conversion-table"``).
"""
return method_name.replace("_", "-").strip("-")
def _validate_member_name(name: str, kind: str) -> None:
"""Validate a resource or script name at decoration time.
Args:
name: The name to validate.
kind: ``"resource"`` or ``"script"`` — used in error messages.
Raises:
ValueError: If the name is empty, too long, or contains invalid characters.
"""
if not name or not name.strip():
raise ValueError(f"@ClassSkill.{kind} name cannot be empty.")
if len(name) > MAX_NAME_LENGTH or not VALID_NAME_RE.match(name):
raise ValueError(
f"Invalid @ClassSkill.{kind} name '{name}': Must be {MAX_NAME_LENGTH} characters or fewer, "
"using only lowercase letters, numbers, and hyphens, and must not start or end with a hyphen "
"or contain consecutive hyphens."
)
def _discover_marked_members(cls: type, marker_attr: str) -> list[tuple[str, dict[str, Any]]]:
"""Scan a class for methods or properties stamped with a marker attribute.
Checks both regular callable attributes (via ``dir``) and ``property``
descriptors (via ``cls.__dict__``) whose ``fget`` carries the marker.
Args:
cls: The class to scan.
marker_attr: The marker attribute name to look for (e.g.
``"_skill_resource_marker"``).
Returns:
A list of ``(member_name, marker_dict)`` tuples.
"""
results: list[tuple[str, dict[str, Any]]] = []
seen: set[str] = set()
# Walk the MRO so that property-resources defined on a parent class
# are also discovered. ``cls.__dict__`` only sees the leaf class.
for klass in cls.__mro__:
for attr_name, attr_value in klass.__dict__.items():
if attr_name in seen:
continue
if (
isinstance(attr_value, property)
and attr_value.fget is not None
and hasattr(attr_value.fget, marker_attr)
):
results.append((attr_name, getattr(attr_value.fget, marker_attr)))
seen.add(attr_name)
# Check regular callable attributes.
for attr_name in dir(cls):
if attr_name in seen:
continue
try:
attr = getattr(cls, attr_name, None)
except Exception:
# Some descriptors (e.g. abstract properties) may raise on access.
logger.warning("Skipping '%s' during skill discovery: descriptor raised on access", attr_name)
attr = None
if attr is not None and callable(attr) and hasattr(attr, marker_attr):
results.append((attr_name, getattr(attr, marker_attr)))
return results
@experimental(feature_id=ExperimentalFeature.SKILLS)
class ClassSkill(Skill, ABC):
"""Abstract base class for defining skills as reusable Python classes.
Inherit from this class to create a self-contained skill definition.
Override :attr:`instructions` to provide the skill body.
Resources and scripts can be defined in two ways:
- **Decorator-based (recommended):** Mark methods with
:meth:`ClassSkill.resource` and :meth:`ClassSkill.script` decorators
for automatic discovery.
- **Explicit override:** Override the :attr:`resources` and
:attr:`scripts` properties, constructing :class:`InlineSkillResource`
and :class:`InlineSkillScript` instances directly.
Class-based skills can be distributed via shared libraries or PyPI
packages, making them easy to reuse across projects.
Attributes:
name: Skill name (lowercase letters, numbers, hyphens only).
description: Human-readable description of the skill.
Examples:
Decorator-based (recommended):
.. code-block:: python
class UnitConverterSkill(ClassSkill):
def __init__(self) -> None:
super().__init__(
name="unit-converter",
description="Convert between common units.",
)
@property
def instructions(self) -> str:
return "Use this skill to convert units..."
@ClassSkill.resource(name="table")
def conversion_table(self) -> str:
return "| From | To | Factor |..."
@ClassSkill.script(name="convert")
def convert(self, value: float, factor: float) -> str:
return json.dumps({"result": round(value * factor, 4)})
Explicit override:
.. code-block:: python
class UnitConverterSkill(ClassSkill):
def __init__(self) -> None:
super().__init__(
name="unit-converter",
description="Convert between common units.",
)
@property
def instructions(self) -> str:
return "Use this skill to convert units..."
@property
def resources(self) -> list[SkillResource]:
return [
InlineSkillResource(name="table", content="| From | To | Factor |..."),
]
@property
def scripts(self) -> list[SkillScript]:
return [InlineSkillScript(name="convert", function=convert_fn)]
"""
def __init__(
self,
*,
name: str,
description: str,
) -> None:
"""Initialize a ClassSkill.
Args:
name: Skill name (lowercase letters, numbers, hyphens only;
max 64 characters).
description: Human-readable description of the skill
(≤1024 characters).
"""
super().__init__(name=name, description=description)
self._cached_content: str | None = None
self._cached_resources: list[SkillResource] | None = None
self._cached_scripts: list[SkillScript] | None = None
@staticmethod
def resource(
func: Callable[..., Any] | None = None,
*,
name: str | None = None,
description: str | None = None,
) -> Any:
"""Decorator that marks a method or property as a skill resource for auto-discovery.
When applied to a method or property on a :class:`ClassSkill` subclass,
it is automatically discovered and registered as an
:class:`InlineSkillResource`. Methods are invoked each time the
resource is read. Properties are evaluated via their getter.
Can be applied to a method directly, or stacked with ``@property``
(place ``@property`` first, ``@ClassSkill.resource`` second).
Supports bare usage (``@ClassSkill.resource``) and parameterized usage
(``@ClassSkill.resource(name="custom", description="...")``).
Args:
func: The function being decorated. Populated automatically when
the decorator is applied without parentheses.
Keyword Args:
name: Resource name override. Defaults to the method name with
underscores replaced by hyphens.
description: Resource description. Defaults to ``None``.
Examples:
On a method:
.. code-block:: python
@ClassSkill.resource(name="conversion-table")
def get_table(self) -> str:
return "..."
On a property:
.. code-block:: python
@property
@ClassSkill.resource
def conversion_table(self) -> str:
return "..."
"""
def decorator(f: Callable[..., Any]) -> Callable[..., Any]:
if isinstance(f, (property, classmethod, staticmethod)):
raise TypeError(
"@ClassSkill.resource must be applied before @property, @classmethod, or @staticmethod. "
"Place @property first, then @ClassSkill.resource."
)
if name is not None:
_validate_member_name(name, "resource")
f._skill_resource_marker = { # type: ignore[attr-defined]
"name": name,
"description": description,
}
return f
if func is None:
return decorator
return decorator(func)
@staticmethod
def script(
func: Callable[..., Any] | None = None,
*,
name: str | None = None,
description: str | None = None,
) -> Any:
"""Decorator that marks a method as a skill script for auto-discovery.
When applied to a method on a :class:`ClassSkill` subclass, the method is
automatically discovered and registered as an :class:`InlineSkillScript`.
The method's parameters (excluding ``self``) are used to generate a JSON
schema, and the method is invoked in-process when the script is run.
Supports bare usage (``@ClassSkill.script``) and parameterized usage
(``@ClassSkill.script(name="custom", description="...")``).
Args:
func: The function being decorated. Populated automatically when
the decorator is applied without parentheses.
Keyword Args:
name: Script name override. Defaults to the method name with
underscores replaced by hyphens.
description: Script description. Defaults to ``None``.
Examples:
.. code-block:: python
@ClassSkill.script(name="convert")
def convert(self, value: float, factor: float) -> str:
return json.dumps({"result": round(value * factor, 4)})
"""
def decorator(f: Callable[..., Any]) -> Callable[..., Any]:
if isinstance(f, (property, classmethod, staticmethod)):
raise TypeError(
"@ClassSkill.script must be applied before @property, @classmethod, or @staticmethod."
)
if name is not None:
_validate_member_name(name, "script")
f._skill_script_marker = { # type: ignore[attr-defined]
"name": name,
"description": description,
}
return f
if func is None:
return decorator
return decorator(func)
@property
@abstractmethod
def instructions(self) -> str:
"""The raw instructions text for this skill.
Subclasses must override this property to provide the skill body.
"""
...
@property
def resources(self) -> list[SkillResource]:
"""Resources discovered from :meth:`ClassSkill.resource`-decorated methods.
On first access, scans the class for methods marked with the
:meth:`ClassSkill.resource` decorator and instantiates
:class:`InlineSkillResource` instances from them.
The result is cached after the first access.
Override this property to provide resources explicitly instead of
using decorator-based discovery.
"""
if self._cached_resources is not None:
return list(self._cached_resources)
resources: list[SkillResource] = []
seen_names: set[str] = set()
for attr_name, attr in _discover_marked_members(type(self), "_skill_resource_marker"):
marker: dict[str, Any] = attr
resource_name = marker.get("name") or _make_method_name(attr_name)
if resource_name in seen_names:
raise ValueError(
f"Skill '{self.name}' already has a resource named '{resource_name}'. "
"Ensure each @ClassSkill.resource has a unique name."
)
seen_names.add(resource_name)
# Use inspect.getattr_static to check the descriptor type without
# triggering it, and walk the MRO so inherited properties are found.
static_attr = inspect.getattr_static(self, attr_name, None)
is_property = isinstance(static_attr, property)
resource_description = marker.get("description")
if is_property:
# Property — use a lambda that reads the property value each time.
# We capture attr_name to avoid late-binding issues.
# Do NOT call getattr here to avoid triggering the getter during discovery.
resource_func = (lambda name: lambda: getattr(self, name))(attr_name)
resources.append(
InlineSkillResource(
name=resource_name,
function=resource_func,
description=resource_description,
)
)
else:
# Regular method — use the bound method directly.
bound_method = getattr(self, attr_name)
resources.append(
InlineSkillResource(
name=resource_name,
function=bound_method,
description=resource_description,
)
)
self._cached_resources = resources
return list(self._cached_resources)
@property
def scripts(self) -> list[SkillScript]:
"""Scripts discovered from :meth:`ClassSkill.script`-decorated methods.
On first access, scans the class for methods marked with the
:meth:`ClassSkill.script` decorator and instantiates
:class:`InlineSkillScript` instances from them.
The result is cached after the first access.
Override this property to provide scripts explicitly instead of
using decorator-based discovery.
"""
if self._cached_scripts is not None:
return list(self._cached_scripts)
scripts: list[SkillScript] = []
seen_names: set[str] = set()
for attr_name, attr in _discover_marked_members(type(self), "_skill_script_marker"):
marker: dict[str, Any] = attr
script_name = marker.get("name") or _make_method_name(attr_name)
if script_name in seen_names:
raise ValueError(
f"Skill '{self.name}' already has a script named '{script_name}'. "
"Ensure each @ClassSkill.script has a unique name."
)
seen_names.add(script_name)
bound_method = getattr(self, attr_name)
script_description = marker.get("description")
scripts.append(
InlineSkillScript(
name=script_name,
function=bound_method,
description=script_description,
)
)
self._cached_scripts = scripts
return list(self._cached_scripts)
@property
def content(self) -> str:
"""Synthesized XML content containing name, description, instructions, resources, and scripts.
The result is cached after the first access.
"""
if self._cached_content is not None:
return self._cached_content
self._cached_content = _build_skill_content(
self.name, self.description, self.instructions, self.resources, self.scripts
)
return self._cached_content
@experimental(feature_id=ExperimentalFeature.SKILLS)
class FileSkill(Skill):
"""A :class:`Skill` discovered from a filesystem directory backed by a SKILL.md file.
+762 -8
View File
@@ -5,6 +5,7 @@
from __future__ import annotations
import os
from abc import ABC
from collections.abc import Sequence
from pathlib import Path
from typing import Any
@@ -14,6 +15,7 @@ import pytest
from agent_framework import (
AggregatingSkillsSource,
ClassSkill,
DeduplicatingSkillsSource,
FileSkill,
FileSkillScript,
@@ -32,6 +34,7 @@ from agent_framework._skills import (
DEFAULT_SCRIPT_EXTENSIONS,
InlineSkillResource,
InlineSkillScript,
_create_resource_element,
_create_script_element,
_FileSkillResource,
)
@@ -1004,7 +1007,7 @@ class TestInlineSkill:
assert len(skill.resources) == 1
assert skill.resources[0].name == "get_schema"
assert skill.resources[0].description == "Get the database schema."
assert skill.resources[0].description is None
assert isinstance(skill.resources[0], InlineSkillResource)
assert skill.resources[0].function is get_schema
@@ -1677,22 +1680,22 @@ class TestCreateResourceElement:
def test_name_only(self) -> None:
r = InlineSkillResource(name="my-ref", content="data")
elem = InlineSkill._create_resource_element(r)
elem = _create_resource_element(r)
assert elem == ' <resource name="my-ref"/>'
def test_with_description(self) -> None:
r = InlineSkillResource(name="my-ref", description="A reference.", content="data")
elem = InlineSkill._create_resource_element(r)
elem = _create_resource_element(r)
assert elem == ' <resource name="my-ref" description="A reference."/>'
def test_xml_escapes_name(self) -> None:
r = InlineSkillResource(name='ref"special', content="data")
elem = InlineSkill._create_resource_element(r)
elem = _create_resource_element(r)
assert "&quot;" in elem
def test_xml_escapes_description(self) -> None:
r = InlineSkillResource(name="ref", description='Uses <tags> & "quotes"', content="data")
elem = InlineSkill._create_resource_element(r)
elem = _create_resource_element(r)
assert "&lt;tags&gt;" in elem
assert "&amp;" in elem
assert "&quot;" in elem
@@ -2136,8 +2139,8 @@ class TestSkillResourceDecoratorEdgeCases:
return "data"
assert skill.resources[0].name == "custom-name"
# description falls back to docstring
assert skill.resources[0].description == "Some docs."
# description is None when not explicitly provided
assert skill.resources[0].description is None
def test_decorator_with_description_only(self) -> None:
skill = InlineSkill(name="my-skill", description="A skill.", instructions="Body")
@@ -2320,7 +2323,7 @@ class TestSkillScriptDecorator:
assert len(skill.scripts) == 1
assert skill.scripts[0].name == "analyze"
assert skill.scripts[0].description == "Run analysis."
assert skill.scripts[0].description is None
assert isinstance(skill.scripts[0], InlineSkillScript)
assert skill.scripts[0].function is analyze
@@ -3177,6 +3180,757 @@ class TestLoadSkillWithScripts:
result = provider._load_skill(_raw_skills(provider), "my-skill")
assert "<scripts>" not in result
# ---------------------------------------------------------------------------
# Tests: ClassSkill
# ---------------------------------------------------------------------------
class _MinimalClassSkill(ClassSkill):
"""A minimal class-based skill with no resources or scripts."""
def __init__(self) -> None:
super().__init__(name="minimal-skill", description="A minimal skill.")
@property
def instructions(self) -> str:
return "Do minimal things."
class _FullClassSkill(ClassSkill):
"""A class-based skill with resources and scripts."""
def __init__(self) -> None:
super().__init__(name="full-skill", description="A full skill.")
self._resources: list[SkillResource] | None = None
self._scripts: list[SkillScript] | None = None
@property
def instructions(self) -> str:
return "Use this skill for full tasks."
@property
def resources(self) -> list[SkillResource]:
if self._resources is None:
self._resources = [
InlineSkillResource(name="test-resource", content="Static resource content."),
]
return self._resources
@property
def scripts(self) -> list[SkillScript]:
if self._scripts is None:
self._scripts = [
InlineSkillScript(name="test-script", function=_class_skill_test_fn),
]
return self._scripts
def _class_skill_test_fn(value: float, factor: float) -> str:
"""Multiply value by factor."""
import json as _json
return _json.dumps({"result": round(value * factor, 4)})
class TestClassSkill:
"""Tests for ClassSkill abstract base class."""
def test_minimal_skill_has_no_resources(self) -> None:
skill = _MinimalClassSkill()
assert skill.resources == []
def test_minimal_skill_has_no_scripts(self) -> None:
skill = _MinimalClassSkill()
assert skill.scripts == []
def test_minimal_skill_content_contains_name(self) -> None:
skill = _MinimalClassSkill()
assert "<name>minimal-skill</name>" in skill.content
def test_minimal_skill_content_contains_description(self) -> None:
skill = _MinimalClassSkill()
assert "<description>A minimal skill.</description>" in skill.content
def test_minimal_skill_content_contains_instructions(self) -> None:
skill = _MinimalClassSkill()
assert "Do minimal things." in skill.content
def test_minimal_skill_content_no_resources_element(self) -> None:
skill = _MinimalClassSkill()
assert "<resources>" not in skill.content
def test_minimal_skill_content_no_scripts_element(self) -> None:
skill = _MinimalClassSkill()
assert "<scripts>" not in skill.content
def test_full_skill_has_resources(self) -> None:
skill = _FullClassSkill()
assert len(skill.resources) == 1
assert skill.resources[0].name == "test-resource"
def test_full_skill_has_scripts(self) -> None:
skill = _FullClassSkill()
assert len(skill.scripts) == 1
assert skill.scripts[0].name == "test-script"
def test_full_skill_content_contains_resources(self) -> None:
skill = _FullClassSkill()
assert "<resources>" in skill.content
assert 'name="test-resource"' in skill.content
def test_full_skill_content_contains_scripts(self) -> None:
skill = _FullClassSkill()
assert "<scripts>" in skill.content
assert 'name="test-script"' in skill.content
def test_content_is_cached(self) -> None:
skill = _MinimalClassSkill()
content1 = skill.content
content2 = skill.content
assert content1 is content2
def test_resources_are_lazy_cached(self) -> None:
skill = _FullClassSkill()
resources1 = skill.resources
resources2 = skill.resources
assert resources1 is resources2
def test_scripts_are_lazy_cached(self) -> None:
skill = _FullClassSkill()
scripts1 = skill.scripts
scripts2 = skill.scripts
assert scripts1 is scripts2
def test_script_has_parameters_schema(self) -> None:
skill = _FullClassSkill()
script = skill.scripts[0]
assert isinstance(script, InlineSkillScript)
schema = script.parameters_schema
assert schema is not None
assert "value" in schema.get("properties", {})
assert "factor" in schema.get("properties", {})
async def test_provider_with_class_skill(self) -> None:
skill = _FullClassSkill()
provider = SkillsProvider([skill])
await _init_provider(provider)
skills = _raw_skills(provider)
assert len(skills) == 1
assert skills[0].name == "full-skill"
async def test_provider_loads_class_skill_content(self) -> None:
skill = _FullClassSkill()
provider = SkillsProvider([skill])
await _init_provider(provider)
result = provider._load_skill(_raw_skills(provider), "full-skill")
assert "Use this skill for full tasks." in result
assert "<resources>" in result
assert "<scripts>" in result
async def test_in_memory_source_with_class_skill(self) -> None:
skill = _MinimalClassSkill()
source = InMemorySkillsSource([skill])
skills = await source.get_skills()
assert len(skills) == 1
assert skills[0].name == "minimal-skill"
async def test_mixed_inline_and_class_skills(self) -> None:
inline = InlineSkill(name="inline-skill", description="Inline", instructions="inline body")
class_skill = _MinimalClassSkill()
provider = SkillsProvider([inline, class_skill])
await _init_provider(provider)
skills = _raw_skills(provider)
names = {s.name for s in skills}
assert names == {"inline-skill", "minimal-skill"}
async def test_class_skill_script_runs(self) -> None:
skill = _FullClassSkill()
script = skill.scripts[0]
result = await script.run(skill, {"value": 10.0, "factor": 2.5})
import json as _json
parsed = _json.loads(result)
assert parsed["result"] == 25.0
async def test_class_skill_resource_reads(self) -> None:
skill = _FullClassSkill()
resource = skill.resources[0]
content = await resource.read()
assert content == "Static resource content."
# ---------------------------------------------------------------------------
# Tests: ClassSkill with decorator-based discovery
# ---------------------------------------------------------------------------
class _DecoratorClassSkill(ClassSkill):
"""A class-based skill using @ClassSkill.resource and @ClassSkill.script decorators."""
def __init__(self) -> None:
super().__init__(name="decorator-skill", description="A decorator-discovered skill.")
@property
def instructions(self) -> str:
return "Use this skill for decorator tests."
@ClassSkill.resource(name="lookup-table")
def get_table(self) -> str:
"""Conversion lookup table."""
return "| From | To | Factor |"
@ClassSkill.script(name="convert")
def run_convert(self, value: float, factor: float) -> str:
"""Convert a value."""
import json as _json
return _json.dumps({"result": round(value * factor, 4)})
class _BareDecoratorSkill(ClassSkill):
"""Skill using bare decorators (no arguments) — name/description from method."""
def __init__(self) -> None:
super().__init__(name="bare-skill", description="Bare decorator skill.")
@property
def instructions(self) -> str:
return "Bare instructions."
@ClassSkill.resource
def my_table(self) -> str:
"""The table docs."""
return "table content"
@ClassSkill.script
def my_script(self, x: int) -> int:
"""Double x."""
return x * 2
class _DuplicateResourceSkill(ClassSkill):
"""Skill with duplicate resource names — should raise."""
def __init__(self) -> None:
super().__init__(name="dup-skill", description="Dup.")
@property
def instructions(self) -> str:
return "x"
@ClassSkill.resource(name="same-name")
def res_a(self) -> str:
return "a"
@ClassSkill.resource(name="same-name")
def res_b(self) -> str:
return "b"
class _DuplicateScriptSkill(ClassSkill):
"""Skill with duplicate script names — should raise."""
def __init__(self) -> None:
super().__init__(name="dup-script-skill", description="Dup.")
@property
def instructions(self) -> str:
return "x"
@ClassSkill.script(name="same-name")
def script_a(self, x: int) -> int:
return x
@ClassSkill.script(name="same-name")
def script_b(self, x: int) -> int:
return x
class _SelfAccessSkill(ClassSkill):
"""Skill where resource/script access instance state via self."""
def __init__(self, multiplier: int = 10) -> None:
super().__init__(name="self-access", description="Self access skill.")
self.multiplier = multiplier
@property
def instructions(self) -> str:
return "Use multiplier."
@ClassSkill.resource(name="config")
def get_config(self) -> str:
return f"multiplier={self.multiplier}"
@ClassSkill.script(name="multiply")
def multiply(self, value: int) -> int:
return value * self.multiplier
class TestClassSkillDecoratorDiscovery:
"""Tests for decorator-based resource/script discovery on ClassSkill."""
def test_discovers_resources(self) -> None:
skill = _DecoratorClassSkill()
assert len(skill.resources) == 1
assert skill.resources[0].name == "lookup-table"
def test_discovers_scripts(self) -> None:
skill = _DecoratorClassSkill()
assert len(skill.scripts) == 1
assert skill.scripts[0].name == "convert"
def test_resource_description_from_decorator(self) -> None:
skill = _DecoratorClassSkill()
assert skill.resources[0].description is None
def test_script_description_from_decorator(self) -> None:
skill = _DecoratorClassSkill()
assert skill.scripts[0].description is None
def test_bare_decorator_name_from_method(self) -> None:
skill = _BareDecoratorSkill()
assert skill.resources[0].name == "my-table"
assert skill.scripts[0].name == "my-script"
def test_bare_decorator_description_is_none(self) -> None:
skill = _BareDecoratorSkill()
assert skill.resources[0].description is None
assert skill.scripts[0].description is None
async def test_resource_reads(self) -> None:
skill = _DecoratorClassSkill()
content = await skill.resources[0].read()
assert content == "| From | To | Factor |"
async def test_script_runs(self) -> None:
skill = _DecoratorClassSkill()
import json as _json
result = await skill.scripts[0].run(skill, {"value": 10.0, "factor": 2.5})
parsed = _json.loads(result)
assert parsed["result"] == 25.0
def test_script_schema_excludes_self(self) -> None:
skill = _DecoratorClassSkill()
script = skill.scripts[0]
assert isinstance(script, InlineSkillScript)
schema = script.parameters_schema
assert schema is not None
props = schema.get("properties", {})
assert "self" not in props
assert "value" in props
assert "factor" in props
def test_resources_cached(self) -> None:
skill = _DecoratorClassSkill()
r1 = skill.resources
r2 = skill.resources
assert r1 == r2
assert r1 is not r2 # defensive copy
def test_scripts_cached(self) -> None:
skill = _DecoratorClassSkill()
s1 = skill.scripts
s2 = skill.scripts
assert s1 == s2
assert s1 is not s2 # defensive copy
def test_content_includes_discovered_resources(self) -> None:
skill = _DecoratorClassSkill()
assert "<resources>" in skill.content
assert 'name="lookup-table"' in skill.content
def test_content_includes_discovered_scripts(self) -> None:
skill = _DecoratorClassSkill()
assert "<scripts>" in skill.content
assert 'name="convert"' in skill.content
def test_duplicate_resource_name_raises(self) -> None:
skill = _DuplicateResourceSkill()
with pytest.raises(ValueError, match="already has a resource named"):
_ = skill.resources
def test_duplicate_script_name_raises(self) -> None:
skill = _DuplicateScriptSkill()
with pytest.raises(ValueError, match="already has a script named"):
_ = skill.scripts
async def test_self_access_resource(self) -> None:
skill = _SelfAccessSkill(multiplier=42)
content = await skill.resources[0].read()
assert content == "multiplier=42"
async def test_self_access_script(self) -> None:
skill = _SelfAccessSkill(multiplier=3)
result = await skill.scripts[0].run(skill, {"value": 7})
assert result == 21
def test_no_decorators_yields_empty(self) -> None:
skill = _MinimalClassSkill()
assert skill.resources == []
assert skill.scripts == []
async def test_provider_with_decorator_skill(self) -> None:
skill = _DecoratorClassSkill()
provider = SkillsProvider([skill])
await _init_provider(provider)
skills = _raw_skills(provider)
assert len(skills) == 1
assert skills[0].name == "decorator-skill"
def test_manual_override_wins(self) -> None:
"""A subclass that overrides resources/scripts bypasses decorator discovery."""
skill = _FullClassSkill()
assert len(skill.resources) == 1
assert skill.resources[0].name == "test-resource"
async def test_property_resource_reads(self) -> None:
"""@ClassSkill.resource on a @property works correctly."""
skill = _PropertyResourceSkill()
assert len(skill.resources) == 1
assert skill.resources[0].name == "static-table"
content = await skill.resources[0].read()
assert "miles" in content
def test_property_resource_description_is_none_without_explicit(self) -> None:
skill = _PropertyResourceSkill()
assert skill.resources[0].description is None
def test_property_resource_in_content(self) -> None:
skill = _PropertyResourceSkill()
assert 'name="static-table"' in skill.content
async def test_mixed_property_and_method_resources(self) -> None:
"""Property and method resources can coexist."""
skill = _MixedPropertyMethodSkill()
names = {r.name for r in skill.resources}
assert names == {"prop-data", "method-data"}
for r in skill.resources:
content = await r.read()
assert "content" in content.lower()
def test_explicit_resource_description_in_object(self) -> None:
"""Explicit description= on @ClassSkill.resource is stored on the object."""
skill = _ExplicitDescriptionSkill()
res = next(r for r in skill.resources if r.name == "described-res")
assert res.description == "A described resource."
def test_explicit_script_description_in_object(self) -> None:
"""Explicit description= on @ClassSkill.script is stored on the object."""
skill = _ExplicitDescriptionSkill()
scr = next(s for s in skill.scripts if s.name == "described-scr")
assert scr.description == "A described script."
def test_explicit_description_in_content_xml(self) -> None:
"""Explicit descriptions appear in the skill content XML."""
skill = _ExplicitDescriptionSkill()
assert 'description="A described resource."' in skill.content
assert 'description="A described script."' in skill.content
def test_property_getter_not_called_during_discovery(self) -> None:
"""Property getter must NOT be evaluated when resources are discovered."""
skill = _PropertyCallCountSkill()
assert skill.getter_call_count == 0
_ = skill.resources # discovery should NOT call the getter
assert skill.getter_call_count == 0
async def test_property_getter_called_on_read(self) -> None:
"""Property getter IS evaluated when the resource is read."""
skill = _PropertyCallCountSkill()
_ = skill.resources
assert skill.getter_call_count == 0
await skill.resources[0].read()
assert skill.getter_call_count == 1
def test_make_method_name_strips_leading_trailing_hyphens(self) -> None:
"""_make_method_name strips leading/trailing underscores turned to hyphens."""
from agent_framework._skills import _make_method_name
assert _make_method_name("my_method") == "my-method"
assert _make_method_name("_private_method_") == "private-method"
assert _make_method_name("__dunder__") == "dunder"
assert _make_method_name("already_good") == "already-good"
def test_inherited_decorated_resources_are_discovered(self) -> None:
"""Decorated resources from a parent class are discovered on subclass."""
skill = _ChildSkill()
names = {r.name for r in skill.resources}
assert "parent-data" in names
def test_inherited_decorated_scripts_are_discovered(self) -> None:
"""Decorated scripts from a parent class are discovered on subclass."""
skill = _ChildSkill()
names = {s.name for s in skill.scripts}
assert "parent-action" in names
def test_child_can_add_own_resources(self) -> None:
"""A child class can add resources alongside inherited ones."""
skill = _ChildSkill()
names = {r.name for r in skill.resources}
assert "parent-data" in names
assert "child-data" in names
async def test_script_receives_kwargs(self) -> None:
"""ClassSkill scripts receive **kwargs forwarded from the runtime."""
skill = _KwargsSkill()
script = skill.scripts[0]
result = await script.run(skill, {"x": 5}, custom_key="hello")
assert result == "5-hello"
def test_wrong_decorator_order_resource_raises(self) -> None:
"""@ClassSkill.resource above @property raises TypeError at class definition."""
with pytest.raises(TypeError, match="must be applied before @property"):
class _BadOrder(ClassSkill):
def __init__(self) -> None:
super().__init__(name="bad", description="bad")
@property
def instructions(self) -> str:
return "x"
@ClassSkill.resource(name="oops") # wrong: should be below @property
@property
def bad_prop(self) -> str:
return "x"
def test_wrong_decorator_order_script_raises(self) -> None:
"""@ClassSkill.script on a property raises TypeError."""
with pytest.raises(TypeError, match="must be applied before"):
class _BadOrder(ClassSkill):
def __init__(self) -> None:
super().__init__(name="bad", description="bad")
@property
def instructions(self) -> str:
return "x"
@ClassSkill.script(name="oops")
@property
def bad_prop(self) -> str:
return "x"
def test_invalid_explicit_resource_name_raises(self) -> None:
"""Invalid name= on @ClassSkill.resource raises ValueError at decoration."""
with pytest.raises(ValueError, match="Invalid @ClassSkill.resource name"):
class _BadName(ClassSkill):
def __init__(self) -> None:
super().__init__(name="bad", description="bad")
@property
def instructions(self) -> str:
return "x"
@ClassSkill.resource(name="UPPER CASE!")
def res(self) -> str:
return "x"
def test_invalid_explicit_script_name_raises(self) -> None:
"""Invalid name= on @ClassSkill.script raises ValueError at decoration."""
with pytest.raises(ValueError, match="Invalid @ClassSkill.script name"):
class _BadName(ClassSkill):
def __init__(self) -> None:
super().__init__(name="bad", description="bad")
@property
def instructions(self) -> str:
return "x"
@ClassSkill.script(name="has spaces")
def scr(self, x: int) -> int:
return x
def test_empty_explicit_name_raises(self) -> None:
"""Empty name= on @ClassSkill.resource raises ValueError."""
with pytest.raises(ValueError, match="name cannot be empty"):
class _EmptyName(ClassSkill):
def __init__(self) -> None:
super().__init__(name="bad", description="bad")
@property
def instructions(self) -> str:
return "x"
@ClassSkill.resource(name="")
def res(self) -> str:
return "x"
def test_resources_copy_prevents_cache_mutation(self) -> None:
"""Mutating the returned resources list does not affect the cache."""
skill = _DecoratorClassSkill()
r1 = skill.resources
r1.clear()
r2 = skill.resources
assert len(r2) == 1 # original cached list is intact
def test_scripts_copy_prevents_cache_mutation(self) -> None:
"""Mutating the returned scripts list does not affect the cache."""
skill = _DecoratorClassSkill()
s1 = skill.scripts
s1.clear()
s2 = skill.scripts
assert len(s2) == 1 # original cached list is intact
async def test_inherited_property_resource_discovered(self) -> None:
"""A @property @ClassSkill.resource on a parent class is discovered on child."""
skill = _ChildWithInheritedPropertySkill()
names = {r.name for r in skill.resources}
assert "parent-prop" in names
content = await next(r for r in skill.resources if r.name == "parent-prop").read()
assert content == "parent property content"
# ---------------------------------------------------------------------------
# Helper skills for additional tests
# ---------------------------------------------------------------------------
class _ExplicitDescriptionSkill(ClassSkill):
"""Skill with explicit descriptions on decorator."""
def __init__(self) -> None:
super().__init__(name="desc-skill", description="Explicit desc.")
@property
def instructions(self) -> str:
return "x"
@ClassSkill.resource(name="described-res", description="A described resource.")
def res(self) -> str:
return "data"
@ClassSkill.script(name="described-scr", description="A described script.")
def scr(self, x: int) -> int:
return x
class _PropertyCallCountSkill(ClassSkill):
"""Tracks how many times the property getter is called."""
def __init__(self) -> None:
super().__init__(name="callcount-skill", description="Tracks calls.")
self.getter_call_count = 0
@property
def instructions(self) -> str:
return "x"
@property
@ClassSkill.resource(name="counted")
def counted_resource(self) -> str:
self.getter_call_count += 1
return "counted"
class _ParentSkill(ClassSkill, ABC):
"""Parent with decorated resources/scripts."""
@ClassSkill.resource(name="parent-data")
def parent_resource(self) -> str:
return "parent"
@ClassSkill.script(name="parent-action")
def parent_script(self, x: int) -> int:
return x
class _ChildSkill(_ParentSkill):
"""Child inheriting parent resources and adding its own."""
def __init__(self) -> None:
super().__init__(name="child-skill", description="Child.")
@property
def instructions(self) -> str:
return "child"
@ClassSkill.resource(name="child-data")
def child_resource(self) -> str:
return "child"
class _KwargsSkill(ClassSkill):
"""Skill that uses **kwargs from runtime."""
def __init__(self) -> None:
super().__init__(name="kwargs-skill", description="Kwargs.")
@property
def instructions(self) -> str:
return "x"
@ClassSkill.script(name="echo")
def echo(self, x: int, **kwargs: Any) -> str:
return f"{x}-{kwargs.get('custom_key', 'none')}"
class _ParentWithPropertyResource(ClassSkill, ABC):
"""Parent with a property-based resource."""
@property
@ClassSkill.resource(name="parent-prop")
def parent_property(self) -> str:
return "parent property content"
class _ChildWithInheritedPropertySkill(_ParentWithPropertyResource):
"""Child that should discover inherited property resource."""
def __init__(self) -> None:
super().__init__(name="child-prop-skill", description="Child prop.")
@property
def instructions(self) -> str:
return "x"
class _PropertyResourceSkill(ClassSkill):
"""Skill with a property-based resource."""
def __init__(self) -> None:
super().__init__(name="prop-skill", description="Property skill.")
@property
def instructions(self) -> str:
return "Use this skill."
@property
@ClassSkill.resource(name="static-table")
def conversion_table(self) -> str:
"""Static conversion table."""
return "| miles | km | 1.60934 |"
class _MixedPropertyMethodSkill(ClassSkill):
"""Skill with both property and method resources."""
def __init__(self) -> None:
super().__init__(name="mixed-prop", description="Mixed.")
@property
def instructions(self) -> str:
return "x"
@property
@ClassSkill.resource(name="prop-data")
def static_data(self) -> str:
"""Static content."""
return "Property Content"
@ClassSkill.resource(name="method-data")
def dynamic_data(self) -> str:
"""Dynamic content."""
return "Method Content"
async def test_code_skill_scripts_element_contains_parameters(self) -> None:
"""Scripts XML includes parameters schema when the function has typed parameters."""
+12 -10
View File
@@ -10,7 +10,8 @@ Start with file-based or code-defined skills, then explore combining them and ad
|--------|-------------|
| [**file_based_skill**](file_based_skill/) | Define skills as `SKILL.md` files on disk with reference documents and executable scripts. Uses the unit-converter skill. |
| [**code_defined_skill**](code_defined_skill/) | Define skills entirely in Python code using `Skill`, `@skill.resource`, and `@skill.script` decorators. Uses a code-defined unit-converter skill. |
| [**mixed_skills**](mixed_skills/) | Combine code-defined and file-based skills in a single agent. Uses a code-defined volume-converter and a file-based unit-converter. |
| [**class_based_skill**](class_based_skill/) | Define skills as Python classes using `ClassSkill` with `@ClassSkill.resource` and `@ClassSkill.script` decorators for auto-discovery. Uses a class-based unit-converter skill. |
| [**mixed_skills**](mixed_skills/) | Combine code-defined, class-based, and file-based skills in a single agent. Uses a code-defined volume-converter, a class-based temperature-converter, and a file-based unit-converter. |
| [**script_approval**](script_approval/) | Require human-in-the-loop approval before executing skill scripts |
## Key Concepts
@@ -23,17 +24,18 @@ Skills use a three-step interaction model to minimize token usage:
2. **Load** — Full instructions are loaded on-demand via the `load_skill` tool
3. **Access** — Resources are read via `read_skill_resource`; scripts are executed via `run_skill_script`
### File-Based vs Code-Defined Skills
### File-Based vs Code-Defined vs Class-Based Skills
| Aspect | File-Based | Code-Defined |
|--------|-----------|--------------|
| Definition | `SKILL.md` files on disk | `Skill` instances in Python |
| Resources | Static files in `references/` and `assets/` directories | Callable functions via `@skill.resource` decorator |
| Scripts | Python files in `scripts/` directory (executed via subprocess) | Callable functions via `@skill.script` decorator (executed in-process) |
| Discovery | Automatic via `skill_paths` parameter | Explicit via `skills` parameter |
| Dynamic content | No (static files only) | Yes (functions can generate content at runtime) |
| Aspect | File-Based | Code-Defined | Class-Based |
|--------|-----------|--------------|-------------|
| Definition | `SKILL.md` files on disk | `Skill` instances in Python | Classes extending `ClassSkill` |
| Resources | Static files in `references/` and `assets/` directories | Callable functions via `@skill.resource` decorator | `@ClassSkill.resource` decorator (auto-discovered) |
| Scripts | Python files in `scripts/` directory (executed via subprocess) | Callable functions via `@skill.script` decorator (executed in-process) | `@ClassSkill.script` decorator (executed in-process) |
| Discovery | Automatic via `skill_paths` parameter | Explicit via `skills` parameter | Explicit via `skills` parameter |
| Dynamic content | No (static files only) | Yes (functions can generate content at runtime) | Yes (functions can generate content at runtime) |
| Sharing pattern | Copy skill directory | Inline or shared instances | Package in shared libraries/PyPI |
Both types can be combined in a single `SkillsProvider` — see the [mixed_skills](mixed_skills/) sample.
All three types can be combined in a single `SkillsProvider` — see the [mixed_skills](mixed_skills/) sample.
### Script Execution
@@ -0,0 +1,71 @@
# Class-Based Agent Skills
This sample demonstrates how to define **Agent Skills as Python classes** using `ClassSkill`.
## What's Demonstrated
- Creating skills as classes that extend `ClassSkill`
- Bundling name, description, instructions, resources, and scripts into a single class
- Using `@ClassSkill.resource` decorator for automatic resource discovery
- Using `@ClassSkill.script` decorator for automatic script discovery
- Lazy-loading and caching of resources and scripts
- Registering class-based skills with `SkillsProvider`
## Skills Included
### unit-converter (class-based)
A `UnitConverterSkill` class that converts between common units. Defined in `class_based_skill.py`:
- `conversion-table` — Static resource with factor table
- `convert` — Script that performs `value × factor` conversion
## Project Structure
```
class_based_skill/
├── class_based_skill.py
└── README.md
```
## Running the Sample
### Prerequisites
- An [Azure AI Foundry](https://ai.azure.com/) project with a deployed model (e.g. `gpt-4o-mini`)
### Environment Variables
Set the required environment variables in a `.env` file (see `python/.env.example`):
- `FOUNDRY_PROJECT_ENDPOINT`: Your Azure AI Foundry project endpoint
- `FOUNDRY_MODEL`: The name of your model deployment (defaults to `gpt-4o-mini`)
### Authentication
This sample uses `AzureCliCredential` for authentication. Run `az login` in your terminal before running the sample.
### Run
```bash
cd python
uv run samples/02-agents/skills/class_based_skill/class_based_skill.py
```
### Expected Output
```
Converting units with class-based skills
------------------------------------------------------------
Agent: Here are your conversions:
1. **26.2 miles → 42.16 km** (a marathon distance)
2. **75 kg → 165.35 lbs**
```
## Learn More
- [Agent Skills Specification](https://agentskills.io/)
- [Code-Defined Skills Sample](../code_defined_skill/)
- [Mixed Skills Sample](../mixed_skills/)
- [Microsoft Agent Framework Documentation](../../../../../docs/)
@@ -0,0 +1,145 @@
# Copyright (c) Microsoft. All rights reserved.
import asyncio
import json
import os
# Uncomment this filter to suppress the experimental Skills warning before
# using the sample's Skills APIs.
# import warnings # isort: skip
# warnings.filterwarnings("ignore", message=r"\[SKILLS\].*", category=FutureWarning)
from textwrap import dedent
from agent_framework import Agent, ClassSkill, SkillsProvider
from agent_framework.foundry import FoundryChatClient
from azure.identity import AzureCliCredential
from dotenv import load_dotenv
"""
Class-Based Agent Skills — Define skills as Python classes
This sample demonstrates how to define Agent Skills as reusable Python classes
by subclassing ``ClassSkill``. Class-based skills bundle all components (name,
description, instructions, resources, scripts) into a single class, making
them easy to package and distribute via shared libraries or PyPI.
Key concepts shown:
- Subclassing ``ClassSkill`` to create a self-contained skill
- Using ``@property`` + ``@ClassSkill.resource`` (bare) — name defaults to method name
- Using ``@ClassSkill.script(name=..., description=...)`` — explicit name and description
- Lazy-loading and caching of resources and scripts
"""
# Load environment variables from .env file
load_dotenv()
# ---------------------------------------------------------------------------
# Class-Based Skill: UnitConverterSkill
# ---------------------------------------------------------------------------
class UnitConverterSkill(ClassSkill):
"""A unit-converter skill defined as a Python class.
Converts between common units (miles↔km, pounds↔kg) using a
conversion factor. Resources and scripts are discovered automatically
via decorators.
"""
def __init__(self) -> None:
super().__init__(
name="unit-converter",
description=(
"Convert between common units using a multiplication factor. "
"Use when asked to convert miles, kilometers, pounds, or kilograms."
),
)
@property
def instructions(self) -> str:
return dedent("""\
Use this skill when the user asks to convert between units.
1. Review the conversion-table resource to find the factor for the requested conversion.
2. Use the convert script, passing the value and factor from the table.
3. Present the result clearly with both units.
""")
# 1. Property with bare decorator — name defaults to the method name
# ("conversion_table" → "conversion-table"), no description.
# Place @property first, then @ClassSkill.resource.
@property
@ClassSkill.resource
def conversion_table(self) -> str:
"""Lookup table of multiplication factors for common unit conversions."""
return dedent("""\
# Conversion Tables
Formula: **result = value × factor**
| From | To | Factor |
|-------------|-------------|----------|
| miles | kilometers | 1.60934 |
| kilometers | miles | 0.621371 |
| pounds | kilograms | 0.453592 |
| kilograms | pounds | 2.20462 |
""")
# 2. Explicit name — overrides the method name
# 3. Explicit description — provides a description for the script
@ClassSkill.script(name="convert", description="Multiplies a value by a conversion factor.")
def convert_units(self, value: float, factor: float) -> str:
"""Convert a value using a multiplication factor: result = value × factor.
Args:
value: The numeric value to convert.
factor: Conversion factor from the conversion table.
Returns:
JSON string with the inputs and converted result.
"""
result = round(value * factor, 4)
return json.dumps({"value": value, "factor": factor, "result": result})
async def main() -> None:
"""Run the class-based skills demo."""
endpoint = os.environ["FOUNDRY_PROJECT_ENDPOINT"]
deployment = os.environ.get("FOUNDRY_MODEL", "gpt-4o-mini")
client = FoundryChatClient(
project_endpoint=endpoint,
model=deployment,
credential=AzureCliCredential(),
)
# Instantiate the class-based skill and pass it to the provider
unit_converter = UnitConverterSkill()
async with Agent(
client=client,
instructions="You are a helpful assistant that can convert units.",
context_providers=[SkillsProvider(unit_converter)],
) as agent:
print("Converting units with class-based skills")
print("-" * 60)
response = await agent.run(
"How many kilometers is a marathon (26.2 miles)? And how many pounds is 75 kilograms?"
)
print(f"Agent: {response}\n")
if __name__ == "__main__":
asyncio.run(main())
"""
Sample output:
Converting units with class-based skills
------------------------------------------------------------
Agent: Here are your conversions:
1. **26.2 miles → 42.16 km** (a marathon distance)
2. **75 kg → 165.35 lbs**
"""
@@ -1,17 +1,18 @@
# Mixed Skills — Code Skills and File Skills
# Mixed Skills — Code, Class, and File Skills
This sample demonstrates how to combine **code-defined skills** and
**file-based skills** in a single agent using a `SkillScriptRunner` callable
and `SkillsProvider`.
This sample demonstrates how to combine **code-defined skills**,
**class-based skills**, and **file-based skills** in a single agent using
`SkillsProvider`.
## Concepts
| Concept | Description |
|---------|-------------|
| **Code skill** | A `Skill` created in Python with `@skill.script` decorators for in-process callable functions and `@skill.resource` for dynamic content |
| **Class skill** | A self-contained skill class extending `ClassSkill`, bundling instructions, resources, and scripts |
| **File skill** | A skill discovered from a `SKILL.md` file on disk, with reference documents and executable script files |
| **`script_runner`** | A callable (sync or async) satisfying the `SkillScriptRunner` protocol — required when file skills have scripts |
| **`SkillsProvider`** | Registers both code-defined and file-based skills in a single provider |
| **`SkillsProvider`** | Registers code-defined, class-based, and file-based skills in a single provider |
## Skills in This Sample
@@ -24,6 +25,15 @@ Defined entirely in Python code using decorators:
Code scripts run **in-process** — no subprocess or external runner needed.
### temperature-converter (class skill)
Defined as a `TemperatureConverterSkill` class extending `ClassSkill`:
- **`@ClassSkill.resource`** — `temperature-conversion-formulas`: °F↔°C↔K formulas
- **`@ClassSkill.script`** — `convert-temperature`: converts between temperature scales
Class-based scripts run **in-process** — no subprocess or external runner needed.
### unit-converter (file skill)
Discovered from `skills/unit-converter/SKILL.md`:
@@ -43,7 +53,10 @@ File scripts are executed as **local Python subprocesses** via the
│ AggregatingSkillsSource([ │
│ FileSkillsSource("./skills", # file skills │
│ script_runner=runner), │
│ InMemorySkillsSource([skill]), # code skills
│ InMemorySkillsSource([
│ volume_skill, # code skill │
│ temp_converter, # class skill │
│ ]), │
│ ]) │
│ ) │
│ ) │
@@ -54,6 +67,7 @@ File scripts are executed as **local Python subprocesses** via the
│ script_runner(skill, script, args) │
│ │
│ • Code scripts (@skill.script) → in-process call │
│ • Class scripts (@ClassSkill.script) → in-process call │
│ • File scripts (scripts/*.py) → subprocess via │
│ the callback function │
└─────────────────────────────────────────────────────────────┘
@@ -16,6 +16,7 @@ from typing import Any
from agent_framework import (
Agent,
AggregatingSkillsSource,
ClassSkill,
DeduplicatingSkillsSource,
FileSkillsSource,
InlineSkill,
@@ -34,28 +35,32 @@ if _SKILLS_ROOT not in sys.path:
from subprocess_script_runner import subprocess_script_runner # noqa: E402
"""
Mixed Skills — Code skills and file skills in a single agent
Mixed Skills — Code, class, and file skills in a single agent
This sample demonstrates how to combine **code-defined skills** (with
``@skill.script`` and ``@skill.resource`` decorators) and **file-based skills**
(discovered from ``SKILL.md`` files on disk) in a single agent using
``SkillsProvider`` and a ``SkillScriptRunner`` callable.
``@skill.script`` and ``@skill.resource`` decorators), **class-based skills**
(subclassing ``ClassSkill``), and **file-based skills** (discovered from
``SKILL.md`` files on disk) in a single agent using ``SkillsProvider`` and
a ``SkillScriptRunner`` callable.
Key concepts shown:
- Code skills with ``@skill.script``: executable Python functions the agent
can invoke directly in-process.
- Code skills with ``@skill.resource``: dynamic content the agent can read
on demand.
- Class skills: self-contained skill classes extending ``ClassSkill``.
- File skills from disk: ``SKILL.md`` files with reference documents and
executable script files.
- ``script_runner``: routes **file-based** script execution
through a callback, enabling custom handling (e.g. subprocess calls).
Code-defined scripts (``@skill.script``) run in-process automatically.
Code-defined and class-based scripts run in-process automatically.
The sample registers two skills:
The sample registers three skills:
1. **volume-converter** (code skill) — converts between gallons and liters using
``@skill.script`` for conversion and ``@skill.resource`` for the factor table.
2. **unit-converter** (file skill) — converts between common units (miles↔km,
2. **temperature-converter** (class skill) — converts between temperature scales
(°F↔°C↔K) using a ``ClassSkill`` subclass.
3. **unit-converter** (file skill) — converts between common units (miles↔km,
pounds↔kg) via a subprocess-executed Python script discovered from
``skills/unit-converter/SKILL.md``.
"""
@@ -110,9 +115,68 @@ def convert_volume(value: float, factor: float) -> str:
# ---------------------------------------------------------------------------
# 2. Wire everything together and run the agent
# 2. Define a class-based skill for temperature conversion
# ---------------------------------------------------------------------------
class TemperatureConverterSkill(ClassSkill):
"""A temperature-converter skill defined as a Python class.
Converts between temperature scales (Fahrenheit, Celsius, Kelvin).
Resources and scripts are discovered automatically via decorators.
"""
def __init__(self) -> None:
super().__init__(
name="temperature-converter",
description="Convert between temperature scales (Fahrenheit, Celsius, Kelvin).",
)
@property
def instructions(self) -> str:
return dedent("""\
Use this skill when the user asks to convert temperatures.
1. Read the temperature-conversion-formulas resource to find the factor and offset
for the requested conversion.
2. Use the convert-temperature script, passing value, factor, and offset.
3. Present the result clearly with both temperature scales.
""")
@ClassSkill.resource(name="temperature-conversion-formulas")
def formulas(self) -> str:
"""Temperature conversion formulas reference table."""
return dedent("""\
# Temperature Conversion Formulas
Formula: **result = value × factor + offset**
| From | To | Factor | Offset |
|-------------|-------------|----------|-----------|
| Fahrenheit | Celsius | 0.555556 | -17.7778 |
| Celsius | Fahrenheit | 1.8 | 32 |
| Celsius | Kelvin | 1 | 273.15 |
| Kelvin | Celsius | 1 | -273.15 |
""")
@ClassSkill.script(name="convert-temperature")
def convert_temperature(self, value: float, factor: float, offset: float = 0) -> str:
"""Convert a temperature value using factor and offset from the formulas resource.
Args:
value: The numeric temperature value to convert.
factor: Conversion factor from the formulas resource.
offset: Offset to add after multiplying (default 0).
Returns:
JSON string with the conversion result.
"""
result = round(value * factor + offset, 4)
return json.dumps({"value": value, "factor": factor, "offset": offset, "result": result})
# ---------------------------------------------------------------------------
# 3. Wire everything together and run the agent
# ---------------------------------------------------------------------------
async def main() -> None:
"""Run the combined skills demo."""
@@ -126,9 +190,11 @@ async def main() -> None:
credential=AzureCliCredential(),
)
# Create the SkillsProvider with both code and file skills.
# The script_runner handles file-based scripts; code-defined scripts
# (@skill.script) run in-process automatically.
# Create the SkillsProvider with code, class, and file skills.
# The script_runner handles file-based scripts; code-defined and
# class-based scripts run in-process automatically.
temperature_converter = TemperatureConverterSkill()
skills_provider = SkillsProvider(
DeduplicatingSkillsSource(
AggregatingSkillsSource([
@@ -136,7 +202,7 @@ async def main() -> None:
str(Path(__file__).parent / "skills"),
script_runner=subprocess_script_runner,
),
InMemorySkillsSource([volume_converter_skill]),
InMemorySkillsSource([volume_converter_skill, temperature_converter]),
])
)
)
@@ -144,14 +210,17 @@ async def main() -> None:
# Run the agent
async with Agent(
client=client,
instructions="You are a helpful assistant that can convert units.",
instructions="You are a helpful assistant that can convert units, volumes, and temperatures.",
context_providers=[skills_provider],
) as agent:
# Ask the agent to use both skills
print("Converting units")
# Ask the agent to use all three skills
print("Converting with mixed skills (file + code + class)")
print("-" * 60)
response = await agent.run(
"How many kilometers is a marathon (26.2 miles)? And how many liters is a 5-gallon bucket?"
"I need three conversions: "
"1) How many kilometers is a marathon (26.2 miles)? "
"2) How many liters is a 5-gallon bucket? "
"3) What is 98.6°F in Celsius?"
)
print(f"Agent: {response}\n")
@@ -162,12 +231,11 @@ if __name__ == "__main__":
"""
Sample output:
Converting units
Converting with mixed skills (file + code + class)
------------------------------------------------------------
Agent: Here are your conversions:
1. **26.2 miles → 42.16 km** (a marathon distance)
2. **5 gallons → 18.93 liters**
I used the conversion factors from each skill's reference table.
3. **98.6°F → 37.0°C**
"""
@@ -2,16 +2,18 @@
"""Aggregate per-provider JUnit XML test results and generate a trend report.
Parses ``pytest.xml`` (JUnit XML) files produced by each CI job, merges them
into a single run, combines with historical data, and generates a markdown
trend table — the same pattern used by ``scripts/sample_validation/aggregate.py``.
Parses JUnit XML files produced by CI jobs — both ``pytest.xml`` (Python) and
xunit v3 ``*.junit`` (dotnet) — merges them into a single run, combines
with historical data, and generates a markdown trend table.
Usage (from CI):
python aggregate.py <reports-dir> <history-file> <output-file>
The reports directory is expected to contain subdirectories named
``test-results-<provider>/`` each containing a ``pytest.xml`` file
(created by ``actions/download-artifact``).
The reports directory is expected to contain artifact subdirectories. Two
layouts are supported:
- **Python (pytest):** ``test-results-<provider>/pytest.xml``
- **Dotnet (xunit):** ``dotnet-test-results-<tfm>-<os>/*.junit``
"""
from __future__ import annotations
@@ -46,9 +48,21 @@ def _format_run_label(timestamp: str) -> str:
def _derive_provider(directory_name: str) -> str:
"""Derive a provider label from a report directory name.
``test-results-openai`` → ``OpenAI``
``test-results-azure-openai`` → ``Azure OpenAI``
Handles both Python and dotnet naming conventions:
- ``test-results-openai`` → ``OpenAI``
- ``test-results-azure-openai`` → ``Azure OpenAI``
- ``dotnet-test-results-net10.0-ubuntu-latest`` → ``net10.0 (ubuntu)``
"""
# Dotnet convention: dotnet-test-results-<framework>-<os>
if directory_name.startswith("dotnet-test-results-"):
raw = directory_name.replace("dotnet-test-results-", "")
# e.g. "net10.0-ubuntu-latest" → framework="net10.0", os="ubuntu-latest"
parts = raw.split("-", 1)
framework = parts[0]
os_label = parts[1].split("-")[0] if len(parts) > 1 else ""
return f"{framework} ({os_label})" if os_label else framework
# Python convention: test-results-<provider>
raw = directory_name.replace("test-results-", "")
known = {
"openai": "OpenAI",
@@ -102,11 +116,21 @@ def _parse_junit_xml(xml_path: Path) -> list[dict[str, str]]:
# it appends the class name, e.g.:
# "packages.foundry.tests.foundry.test_foundry_embedding_client.TestFoundryEmbeddingIntegration"
# We want the file-level module: "test_foundry_embedding_client"
#
# xunit (dotnet) writes classname as the full C# type, e.g.:
# "OpenAIChatCompletion.IntegrationTests.ChatCompletionTests"
# We want the project prefix: "OpenAIChatCompletion"
if classname:
parts = classname.rsplit(".", 2)
# If the last segment starts with uppercase it's a class name — take the one before it
if len(parts) >= 2 and parts[-1][0:1].isupper():
module = parts[-2]
# For dotnet: if the penultimate part is "IntegrationTests" or "UnitTests",
# use the part before that (the project name) instead
if parts[-2] in ("IntegrationTests", "UnitTests") and len(parts) >= 3:
# parts[0] may contain dots — take the last segment of it
module = parts[0].rsplit(".", 1)[-1]
else:
module = parts[-2]
else:
module = parts[-1]
else:
@@ -148,28 +172,61 @@ def _parse_junit_xml(xml_path: Path) -> list[dict[str, str]]:
# ---------------------------------------------------------------------------
def _discover_xml_files(reports_dir: Path) -> list[tuple[str, Path]]:
"""Discover JUnit XML test result files in artifact subdirectories.
Handles two directory layouts:
- **Python (pytest):** ``test-results-<provider>/pytest.xml``
- **Dotnet (xunit):** ``dotnet-test-results-<tfm>-<os>/*.junit``
Returns:
List of ``(directory_name, xml_path)`` tuples.
"""
xml_files: list[tuple[str, Path]] = []
if not reports_dir.is_dir():
return xml_files
for subdir in sorted(reports_dir.iterdir()):
if not subdir.is_dir():
continue
# Python layout: single pytest.xml per artifact
pytest_xml = subdir / "pytest.xml"
if pytest_xml.exists():
xml_files.append((subdir.name, pytest_xml))
continue
# Dotnet layout: multiple *.junit files per artifact
junit_files = sorted(subdir.rglob("*.junit"))
for jf in junit_files:
xml_files.append((subdir.name, jf))
# Fallback: any .xml file that looks like JUnit (not .trx, not cobertura)
if not junit_files:
for xf in sorted(subdir.rglob("*.xml")):
if xf.suffix == ".xml" and not xf.name.endswith(".cobertura.xml"):
xml_files.append((subdir.name, xf))
return xml_files
def load_current_run(reports_dir: Path) -> dict[str, Any]:
"""Load per-provider JUnit XML reports from the current CI run and merge.
Supports both pytest (Python) and xunit v3 (dotnet) JUnit XML formats.
Args:
reports_dir: Directory containing ``test-results-<provider>/`` subdirs.
reports_dir: Directory containing artifact subdirectories with XML reports.
Returns:
Merged run dict with ``timestamp``, ``summary``, ``results``.
"""
combined_results: dict[str, dict[str, str]] = {} # nodeid → {status, provider}
# actions/download-artifact creates: reports_dir/test-results-openai/pytest.xml
xml_files: list[tuple[str, Path]] = []
if reports_dir.is_dir():
for subdir in sorted(reports_dir.iterdir()):
if subdir.is_dir():
xml_file = subdir / "pytest.xml"
if xml_file.exists():
xml_files.append((subdir.name, xml_file))
xml_files = _discover_xml_files(reports_dir)
if not xml_files:
print(f"Warning: No pytest.xml files found in {reports_dir}")
print(f"Warning: No JUnit XML files found in {reports_dir}")
return {
"timestamp": datetime.now(timezone.utc).isoformat(),
"summary": {
@@ -181,19 +238,42 @@ def load_current_run(reports_dir: Path) -> dict[str, Any]:
"results": {},
}
# Dotnet tests always run under multiple frameworks, so we always
# qualify their keys with the provider to ensure deterministic,
# stable keys across runs regardless of file parse order.
is_dotnet = any(d.startswith("dotnet-test-results-") for d, _ in xml_files)
for dir_name, xml_file in xml_files:
print(f" Loading: {xml_file}")
provider = _derive_provider(dir_name)
tests = _parse_junit_xml(xml_file)
for test in tests:
combined_results[test["nodeid"]] = {
raw_id = test["nodeid"]
key = f"{provider}::{raw_id}" if is_dotnet else raw_id
combined_results[key] = {
"status": test["status"],
"provider": provider,
"module": test.get("module", ""),
}
# Build summary counts using mutually exclusive status buckets.
# Errors are folded into the failed count for display purposes.
# Build per-provider summary counts so the report can show one row per
# framework (dotnet) or per provider (Python).
provider_counts: dict[str, dict[str, int]] = {}
for r in combined_results.values():
prov = r.get("provider", "Unknown")
if prov not in provider_counts:
provider_counts[prov] = {"total": 0, "passed": 0, "failed": 0, "skipped": 0}
provider_counts[prov]["total"] += 1
st = r["status"]
if st == "passed":
provider_counts[prov]["passed"] += 1
elif st in ("failed", "error"):
provider_counts[prov]["failed"] += 1
elif st == "skipped":
provider_counts[prov]["skipped"] += 1
# Overall summary (sum across all providers).
statuses = [r["status"] for r in combined_results.values()]
summary = {
"total": len(statuses),
@@ -205,6 +285,7 @@ def load_current_run(reports_dir: Path) -> dict[str, Any]:
return {
"timestamp": datetime.now(timezone.utc).isoformat(),
"summary": summary,
"provider_summaries": provider_counts,
"results": combined_results,
}
@@ -253,7 +334,29 @@ def generate_trend_report(runs: list[dict[str, Any]]) -> str:
"",
]
# --- Overall status table (most recent first) ---
# Detect whether this is a dotnet report (provider-qualified keys).
is_dotnet = False
for run in runs:
provider_sums = run.get("provider_summaries", {})
if any(p.startswith("net") for p in provider_sums):
is_dotnet = True
break
if is_dotnet:
_generate_dotnet_report(lines, runs)
else:
_generate_python_report(lines, runs)
lines.append("")
lines.append("**Legend:** ✅ Passed · ❌ Failed · ⏭️ Skipped · ⚠️ Expected Failure (xfail) · N/A Not available")
lines.append("")
return "\n".join(lines)
def _generate_python_report(lines: list[str], runs: list[dict[str, Any]]) -> None:
"""Generate the original single-table Python report format."""
# --- Overall status table ---
lines.append("## Overall Status (Last 5 Runs)")
lines.append("")
lines.append("| Run | Total | ✅ Passed | ❌ Failed | ⏭️ Skipped |")
@@ -276,27 +379,91 @@ def generate_trend_report(runs: list[dict[str, Any]]) -> str:
lines.append("")
# --- Per-test results table ---
lines.append("## Per-Test Results")
lines.append("")
# --- Single per-test results table ---
_generate_per_test_table(lines, runs, "## Per-Test Results")
# Collect all test nodeids, providers, and modules across all runs
all_tests: dict[str, str] = {} # nodeid → provider (from most recent run)
all_modules: dict[str, str] = {} # nodeid → module (from most recent run)
def _generate_dotnet_report(lines: list[str], runs: list[dict[str, Any]]) -> None:
"""Generate per-framework tables for dotnet (net10.0, net472, etc.)."""
# Collect all providers seen across all runs, sorted for stable ordering
all_providers: set[str] = set()
for run in runs:
all_providers.update(run.get("provider_summaries", {}).keys())
providers = sorted(all_providers)
for provider in providers:
lines.append(f"## {provider}")
lines.append("")
# --- Per-provider summary table ---
lines.append("| Run | Total | ✅ Passed | ❌ Failed | ⏭️ Skipped |")
lines.append("|-----|-------|-----------|-----------|------------|")
for run in reversed(runs):
ps = run.get("provider_summaries", {}).get(provider, {})
total = ps.get("total", 0)
label = _format_run_label(run["timestamp"])
if total == 0:
lines.append(f"| {label} | N/A | N/A | N/A | N/A |")
else:
lines.append(
f"| {label} "
f"| {total} "
f"| {ps.get('passed', 0)}/{total} "
f"| {ps.get('failed', 0)}/{total} "
f"| {ps.get('skipped', 0)}/{total} |"
)
for _ in range(MAX_HISTORY - len(runs)):
lines.append("| N/A | N/A | N/A | N/A | N/A |")
lines.append("")
# --- Per-test table filtered to this provider ---
_generate_per_test_table(
lines, runs,
heading=None,
provider_filter=provider,
)
def _generate_per_test_table(
lines: list[str],
runs: list[dict[str, Any]],
heading: str | None = None,
provider_filter: str | None = None,
) -> None:
"""Emit a per-test trend table, optionally filtered to a single provider."""
if heading:
lines.append(heading)
lines.append("")
# Collect all test nodeids (and metadata) across all runs
all_tests: dict[str, str] = {} # nodeid → provider
all_modules: dict[str, str] = {} # nodeid → module
for run in runs:
for nodeid, info in run.get("results", {}).items():
provider = info.get("provider", "Unknown") if isinstance(info, dict) else "Unknown"
module = info.get("module", "") if isinstance(info, dict) else ""
all_tests[nodeid] = provider
if not isinstance(info, dict):
continue
prov = info.get("provider", "Unknown")
if provider_filter and prov != provider_filter:
continue
module = info.get("module", "")
all_tests[nodeid] = prov
all_modules[nodeid] = module
if not all_tests:
lines.append("*No test results available.*")
return "\n".join(lines)
lines.append("")
return
# Build header (most recent run first)
header = "| Test | File | Provider |"
separator = "|------|------|----------|"
# Build header
if provider_filter:
header = "| Test | File |"
separator = "|------|------|"
else:
header = "| Test | File | Provider |"
separator = "|------|------|----------|"
for run in reversed(runs):
label = _format_run_label(run["timestamp"])
header += f" {label} |"
@@ -308,12 +475,15 @@ def generate_trend_report(runs: list[dict[str, Any]]) -> str:
lines.append(header)
lines.append(separator)
# Sort by provider then test name
for nodeid in sorted(all_tests, key=lambda n: (all_tests[n], n)):
provider = all_tests[nodeid]
# Sort by module then test name
for nodeid in sorted(all_tests, key=lambda n: (all_modules.get(n, ""), n)):
module = all_modules.get(nodeid, "")
short = _short_name(nodeid)
row = f"| `{short}` | `{module}` | {provider} |"
if provider_filter:
row = f"| `{short}` | `{module}` |"
else:
provider = all_tests[nodeid]
row = f"| `{short}` | `{module}` | {provider} |"
for run in reversed(runs):
result = run.get("results", {}).get(nodeid)
@@ -330,10 +500,6 @@ def generate_trend_report(runs: list[dict[str, Any]]) -> str:
lines.append(row)
lines.append("")
lines.append("**Legend:** ✅ Passed · ❌ Failed · ⏭️ Skipped · ⚠️ Expected Failure (xfail) · N/A Not available")
lines.append("")
return "\n".join(lines)
# ---------------------------------------------------------------------------