diff --git a/docs/decisions/0021-agent-skills-design.md b/docs/decisions/0021-agent-skills-design.md new file mode 100644 index 0000000000..d63f38c734 --- /dev/null +++ b/docs/decisions/0021-agent-skills-design.md @@ -0,0 +1,960 @@ +status: proposed +date: 2026-03-23 +contact: sergeymenshykh +deciders: rbarreto, westey-m, eavanvalkenburg +--- + +# Agent Skills: Multi-Source Architecture + +## Context and Problem Statement + +The Agent Framework needs a skills system that lets agents discover and use domain-specific knowledge, reference documents, and executable scripts. Skills can originate from different sources — filesystem directories (SKILL.md files), inline C# code, or reusable class libraries — and the framework must support all three uniformly while allowing extensibility, composition, and filtering. + +## Decision Drivers + +- Skills must be definable from multiple sources: filesystem, inline code, reusable classes, etc +- Common abstractions are needed so the provider and builder work uniformly regardless of skill origin +- File-based scripts must support user-defined executors, enabling custom runtimes and languages; code/class-based scripts execute in-process as C# delegates +- Skills must be filterable so consumers can include or exclude specific skills based on defined criteria +- Multiple skill sources must be composable into a single provider +- It must be possible to add custom skill sources (e.g., databases, REST APIs, package registries) by implementing a common abstraction + +## Architecture + +### Model-Facing Tools + +Skills are presented to the model as up to three tools that progressively disclose skill content. The system prompt lists available skill names and descriptions; the model then calls these tools on demand: + +- **`load_skill(skillName)`** — returns the full skill body (instructions, listed resources, listed scripts) +- **`read_skill_resource(skillName, resourceName)`** — reads a supplementary resource (file-based or code-defined) associated with a skill +- **`run_skill_script(skillName, scriptName, arguments?)`** — executes a script associated with a skill; only registered when at least one skill contains scripts + +Each tool delegates to the corresponding method on the resolved `AgentSkill` — calling `Resource.ReadAsync()` or `Script.RunAsync()` respectively. + +If skills have no scripts defined, the `run_skill_script` tool is **not advertised** to the model and instructions related to script execution are **not included** in the default skills instructions. + +### Abstract Base Types + +The architecture defines four abstract base types that all skill variants implement: + +```csharp +public abstract class AgentSkill +{ + public abstract AgentSkillFrontmatter Frontmatter { get; } + public abstract string Content { get; } + public abstract IReadOnlyList? Resources { get; } + public abstract IReadOnlyList? Scripts { get; } +} + +public abstract class AgentSkillResource +{ + public string Name { get; } + public string? Description { get; } + public abstract Task ReadAsync(IServiceProvider? serviceProvider = null, CancellationToken cancellationToken = default); +} + +public abstract class AgentSkillScript +{ + public string Name { get; } + public string? Description { get; } + public abstract Task RunAsync(AgentSkill skill, AIFunctionArguments arguments, CancellationToken cancellationToken = default); +} + +public abstract class AgentSkillsSource +{ + public abstract Task> GetSkillsAsync(CancellationToken cancellationToken = default); +} +``` + +Skill metadata is captured via `AgentSkillFrontmatter`: + +```csharp +public sealed class AgentSkillFrontmatter +{ + public AgentSkillFrontmatter(string name, string description) { ... } + + public string Name { get; } + public string Description { get; } + public string? License { get; set; } + public string? Compatibility { get; set; } + public string? AllowedTools { get; set; } + public AdditionalPropertiesDictionary? Metadata { get; set; } +} +``` + +The type hierarchy at a glance: + +``` +AgentSkill (abstract) AgentSkillsSource (abstract) +├── AgentFileSkill ├── AgentFileSkillsSource (public) +└── [Programmatic] ├── AgentInMemorySkillsSource (public) + ├── AgentInlineSkill ├── AggregatingAgentSkillsSource (public) + └── AgentClassSkill (abstract) └── DelegatingAgentSkillsSource (abstract, public) + ├── FilteringAgentSkillsSource (public) +AgentSkillResource (abstract) ├── CachingAgentSkillsSource (public) +├── AgentFileSkillResource └── DeduplicatingAgentSkillsSource (public) +└── AgentInlineSkillResource + AgentSkillScript (abstract) + ├── AgentFileSkillScript + └── AgentInlineSkillScript +``` + +There are two top-level categories of skills: + +1. **File-Based Skills** — discovered from `SKILL.md` files on the filesystem. Resources and scripts are files in subdirectories. +2. **Programmatic Skills** — defined in C# code. These are further divided into: + - **Inline Skills** — built at runtime via the `AgentInlineSkill` class and its fluent API. Ideal for quick, agent-specific skill definitions. + - **Class-Based Skills** — defined as reusable C# classes that subclass `AgentClassSkill`. Ideal for packaging skills as shared libraries or NuGet packages. + +Both programmatic skill types use `AgentInlineSkillResource` and `AgentInlineSkillScript` for their resources and scripts. They are typically served by `AgentInMemorySkillsSource`, which accepts any `AgentSkill` and is not limited to programmatic skills. + +### File-Based Skills + +File-based skills are authored as `SKILL.md` files on disk. Resources and scripts are discovered from corresponding subfolders within the skill directory. + +**`AgentFileSkill`** — A filesystem-based skill discovered from a directory containing a `SKILL.md` file. Parsed from YAML frontmatter; content is the raw markdown body. Resources and scripts are discovered from files in corresponding subfolders: + +```csharp +public sealed class AgentFileSkill : AgentSkill +{ + internal AgentFileSkill( + AgentSkillFrontmatter frontmatter, string content, string path, + IReadOnlyList? resources = null, + IReadOnlyList? scripts = null) { ... } +} +``` + +**`AgentFileSkillResource`** — A file-based skill resource. Reads content from a file on disk relative to the skill directory: + +```csharp +internal sealed class AgentFileSkillResource : AgentSkillResource +{ + public AgentFileSkillResource(string name, string fullPath) { ... } + + public string FullPath { get; } + + public override Task ReadAsync(IServiceProvider? serviceProvider = null, CancellationToken cancellationToken = default) + { + return File.ReadAllTextAsync(FullPath, Encoding.UTF8, cancellationToken); + } +} +``` + +**`AgentFileSkillScript`** — A file-based skill script that represents a script file on disk. Delegates execution to an external `AgentFileSkillScriptRunner` callback (e.g., runs Python/shell via `Process.Start`). Throws `NotSupportedException` if no executor is configured: + +```csharp +public delegate Task AgentFileSkillScriptRunner( + AgentFileSkill skill, AgentFileSkillScript script, + AIFunctionArguments arguments, CancellationToken cancellationToken); + +public sealed class AgentFileSkillScript : AgentSkillScript +{ + private readonly AgentFileSkillScriptRunner _executor; + + internal AgentFileSkillScript(string name, string fullPath, AgentFileSkillScriptRunner executor) + : base(name) { ... } + + public override async Task RunAsync(AgentSkill skill, AIFunctionArguments arguments, ...) + { + + return await _executor(fileSkill, this, arguments, cancellationToken); + } +} +``` + +The executor can be provided at the **provider level** via `AgentSkillsProviderBuilder.UseFileScriptRunner(executor)` and optionally overridden for a **particular file skill** or for a **set of skills** at the file skill source level, giving fine-grained control over how different scripts are executed. + +**`AgentFileSkillsSource`** — A skill source that discovers skills from filesystem directories containing `SKILL.md` files. Recursively scans directories (max 2 levels), validates frontmatter, and enforces path traversal and symlink security checks: + +```csharp +public sealed partial class AgentFileSkillsSource : AgentSkillsSource +{ + public AgentFileSkillsSource( + IEnumerable skillPaths, + AgentFileSkillScriptRunner scriptRunner, + AgentFileSkillsSourceOptions? options = null, + ILoggerFactory? loggerFactory = null) { ... } +} +``` + +**`AgentFileSkillsSourceOptions`** — Configuration options for `AgentFileSkillsSource`. Allows customizing the allowed file extensions for resources and scripts without adding constructor parameters: + +```csharp +public sealed class AgentFileSkillsSourceOptions +{ + public IEnumerable? AllowedResourceExtensions { get; set; } + public IEnumerable? AllowedScriptExtensions { get; set; } +} +``` + +**Example** — A file-based skill on disk and how it is added to a source: + +``` +skills/ +└── unit-converter/ + ├── SKILL.md # frontmatter + instructions + ├── resources/ + │ └── conversion-table.csv # discovered as a resource + └── scripts/ + └── convert.py # discovered as a script +``` + +```csharp +var source = new AgentFileSkillsSource(skillPaths: ["./skills"], scriptRunner: SubprocessScriptRunner.RunAsync); + +var provider = new AgentSkillsProvider(source); + +AIAgent agent = chatClient.AsAIAgent(new ChatClientAgentOptions +{ + AIContextProviders = [provider], +}); +``` + +### Programmatic Skills + +Programmatic skills are defined in C# code rather than discovered from the filesystem. There are two kinds: **inline** and **class-based**. Both use `AgentInlineSkillResource` and `AgentInlineSkillScript` for resources and scripts, and are held by a single `AgentInMemorySkillsSource`. + +**`AgentInMemorySkillsSource`** — A general-purpose skill source that holds any `AgentSkill` instances in memory. Although commonly used for programmatic skills (`AgentInlineSkill` and `AgentClassSkill`), it accepts any `AgentSkill` subclass and is not restricted to code-defined skills: + +```csharp +public sealed class AgentInMemorySkillsSource : AgentSkillsSource +{ + public AgentInMemorySkillsSource( + IEnumerable skills, + ILoggerFactory? loggerFactory = null) { ... } +} +``` + +#### Inline Skills + +Inline skills are built at runtime via the `AgentInlineSkill` class and its fluent API. They are ideal for quick, agent-specific skill definitions where a full class hierarchy would be overkill. + +**`AgentInlineSkill`** — A skill defined entirely in code. Resources can be static values or functions; scripts are always functions. Constructed with name, description, and instructions, then extended with resources and scripts: + +```csharp +public sealed class AgentInlineSkill : AgentSkill +{ + public AgentInlineSkill(string name, string description, string instructions, string? license = null, string? compatibility = null, ...) { ... } + public AgentInlineSkill(AgentSkillFrontmatter frontmatter, string instructions) { ... } + + public AgentInlineSkill AddResource(object value, string name, string? description = null); + public AgentInlineSkill AddResource(Delegate handler, string name, string? description = null); + public AgentInlineSkill AddScript(Delegate handler, string name, string? description = null); +} +``` + +**`AgentInlineSkillResource`** — A skill resource that wraps a static value: + +```csharp +public sealed class AgentInlineSkillResource : AgentSkillResource +{ + public AgentInlineSkillResource(object value, string name, string? description = null) + : base(name, description) + { + _value = value; + } + + public override Task ReadAsync(IServiceProvider? serviceProvider = null, CancellationToken cancellationToken = default) + { + return Task.FromResult(_value); + } +} +``` + +**`AgentInlineSkillResource`** — A skill resource backed by a delegate. The delegate is invoked via an `AIFunction` each time `ReadAsync` is called, producing a dynamic (computed) value: + +```csharp +public sealed class AgentInlineSkillResource : AgentSkillResource +{ + public AgentInlineSkillResource(Delegate handler, string name, string? description = null) + : base(name, description) + { + _function = AIFunctionFactory.Create(handler, name: name); + } + + public override async Task ReadAsync(IServiceProvider? serviceProvider = null, CancellationToken cancellationToken = default) + { + return await _function.InvokeAsync(new AIFunctionArguments() { Services = serviceProvider }, cancellationToken); + } +} +``` + +**`AgentInlineSkillScript`** — A skill script backed by a delegate via an `AIFunction`: + +```csharp +public sealed class AgentInlineSkillScript : AgentSkillScript +{ + private readonly AIFunction _function; + + public AgentInlineSkillScript(Delegate handler, string name, string? description = null) + : base(name, description) + { + _function = AIFunctionFactory.Create(handler, name: name); + } + + public JsonElement? ParametersSchema => _function.JsonSchema; + + public override async Task RunAsync(AgentSkill skill, AIFunctionArguments arguments, ...) + { + return await _function.InvokeAsync(arguments, cancellationToken); + } +} +``` + +**Example** — Creating an inline skill with a resource and script, then adding it to a source: + +```csharp +var skill = new AgentInlineSkill( + name: "unit-converter", + description: "Converts between measurement units.", + instructions: """ + Use this skill to convert values between metric and imperial units. + Refer to the conversion-table resource for supported unit pairs. + Run the convert script to perform conversions. + """ + ) + .AddResource("kg=2.205lb, m=3.281ft, L=0.264gal", "conversion-table", "Supported unit pairs") + .AddScript(Convert, "convert", "Converts a value between units"); + +var source = new AgentInMemorySkillsSource([skill]); + +var provider = new AgentSkillsProvider(source); + +AIAgent agent = chatClient.AsAIAgent(new ChatClientAgentOptions +{ + AIContextProviders = [provider], +}); + +static string Convert(double value, double factor) + => JsonSerializer.Serialize(new { result = Math.Round(value * factor, 4) }); +``` + +#### Class-Based Skills + +Class-based skills are designed for packaging skills as reusable libraries. Users subclass `AgentClassSkill` and override properties. Unlike inline skills, class-based skills are self-contained, can live in shared libraries or NuGet packages, and are well-suited for dependency injection. + +**`AgentClassSkill`** — An abstract base class for defining skills as reusable C# classes that bundle all skill components (frontmatter, instructions, resources, scripts) together. Designed for packaging skills as distributable libraries: + +```csharp +public abstract class AgentClassSkill : AgentSkill +{ + public abstract string Instructions { get; } + + // Content is auto-synthesized from Frontmatter + Instructions + Resources + Scripts + public override string Content => + SkillContentBuilder.BuildContent(Frontmatter.Name, Frontmatter.Description, + SkillContentBuilder.BuildBody(Instructions, Resources, Scripts)); +} +``` + +**Example** — Defining a class-based skill and adding it to a source: + +```csharp +public class UnitConverterSkill : AgentClassSkill +{ + public override AgentSkillFrontmatter Frontmatter { get; } = + new("unit-converter", "Converts between measurement units."); + + public override string Instructions => """ + Use this skill to convert values between metric and imperial units. + Refer to the conversion-table resource for supported unit pairs. + Run the convert script to perform conversions. + """; + + public override IReadOnlyList? Resources { get; } = + [ + new AgentInlineSkillResource("kg=2.205lb, m=3.281ft", "conversion-table"), + ]; + + public override IReadOnlyList? Scripts { get; } = + [ + new AgentInlineSkillScript(Convert, "convert"), + ]; + + private static string Convert(double value, double factor) + => JsonSerializer.Serialize(new { result = Math.Round(value * factor, 4) }); +} + +var source = new AgentInMemorySkillsSource([new UnitConverterSkill()]); + +var provider = new AgentSkillsProvider(source); + +AIAgent agent = chatClient.AsAIAgent(new ChatClientAgentOptions +{ + AIContextProviders = [provider], +}); +``` + +## Filtering, Caching, and Deduplication + +The following subsections present alternative approaches for handling filtering, caching, and deduplication of skills across multiple sources. + +### Via Composition + +In this approach, the `AgentSkillsProvider` accepts a **single** `AgentSkillsSource`. Multiple sources are composed externally via an aggregate source, and cross-cutting concerns like filtering, caching, and deduplication are implemented as **source decorators** — subclasses of `DelegatingAgentSkillsSource` that intercept `GetSkillsAsync()`. + +**`FilteringAgentSkillsSource`** — A decorator that applies filter logic before returning results. The decorator pattern keeps filtering orthogonal to source implementations and allows composing multiple filters: + +```csharp +public sealed class FilteringAgentSkillsSource : DelegatingAgentSkillsSource +{ + private readonly Func _predicate; + + public FilteringAgentSkillsSource(AgentSkillsSource innerSource, Func predicate) + : base(innerSource) + { + _predicate = predicate; + } + + public override async Task> GetSkillsAsync(CancellationToken cancellationToken = default) + { + var skills = await this.InnerSource.GetSkillsAsync(cancellationToken); + return skills.Where(_predicate).ToList(); + } +} +``` + +**`CachingAgentSkillsSource`** — A decorator that caches skills after the first load, keeping the provider stateless and giving consumers control over caching granularity per source. For example, file-based skills (expensive to discover) can be cached while code-defined skills remain uncached: + +```csharp +public sealed class CachingAgentSkillsSource : DelegatingAgentSkillsSource +{ + private IList? _cached; + + public CachingAgentSkillsSource(AgentSkillsSource innerSource) + : base(innerSource) + { + } + + public override async Task> GetSkillsAsync(CancellationToken cancellationToken = default) + { + return _cached ??= await this.InnerSource.GetSkillsAsync(cancellationToken); + } +} +``` + +**Deduplication** is similarly implemented as a decorator (`DeduplicatingAgentSkillsSource`) that deduplicates by name (case-insensitive, first-one-wins) and logs a warning for skipped duplicates. + +**Example** — Combining file-based and code-defined sources with filtering and caching: + +```csharp +var fileSource = new CachingAgentSkillsSource(new AgentFileSkillsSource(["./skills"])); +var codeSource = new AgentInMemorySkillsSource([myCodeSkill]); + +var compositeSource = new FilteringAgentSkillsSource( + new AggregatingAgentSkillsSource([fileSource, codeSource]), + filter: s => s.Frontmatter.Name != "internal"); + +var provider = new AgentSkillsProvider(compositeSource); + +AIAgent agent = chatClient.AsAIAgent(new ChatClientAgentOptions +{ + AIContextProviders = [provider], +}); +``` + +**Pros:** +- Clean single-responsibility: the provider serves skills, sources provide them. +- Caching, filtering, and deduplication are composable as source decorators — each concern is a separate, testable wrapper. + +**Cons:** +- DI is less flexible: multiple `AgentSkillsSource` implementations registered in the container cannot be auto-injected into the provider. The consumer must manually compose them via an aggregate source. +- Increased public API surface: requires additional public classes (aggregate source, caching decorators, filtering decorators) that consumers need to learn and use. + +### Via AgentSkillsProvider + +In this approach, the `AgentSkillsProvider` accepts **`IEnumerable`** and handles aggregation, filtering, caching, and deduplication internally. + +The provider aggregates skills from all registered sources, deduplicates by name (case-insensitive, first-one-wins), caches the result after the first load, and optionally applies filtering via a predicate on `AgentSkillsProviderOptions`. Duplicate skill names are logged as warnings. + +**Example** — Registering multiple sources directly with the provider: + +```csharp +// Conceptual example — in practice, use AgentSkillsProviderBuilder +var fileSource = new AgentFileSkillsSource(["./skills"]); +var codeSource = new AgentInMemorySkillsSource([myCodeSkill]); + +var provider = new AgentSkillsProvider( + sources: [fileSource, codeSource], + options: new AgentSkillsProviderOptions + { + Filter = s => s.Frontmatter.Name != "internal", + }); + +AIAgent agent = chatClient.AsAIAgent(new ChatClientAgentOptions +{ + AIContextProviders = [provider], +}); +``` + +**Pros:** +- DI-friendly: register multiple `AgentSkillsSource` implementations in the container, and they are all auto-injected into `AgentSkillsProvider` via `IEnumerable`. +- Smaller public API surface: no need for aggregate source, caching decorators, or filtering decorator classes — these concerns are handled internally by the provider. + +**Cons:** +- The provider takes on multiple responsibilities — aggregation, caching, deduplication, and filtering. +- Less granular caching control: caching is all-or-nothing across sources rather than per-source as with decorators. +- Less extensible: new behaviors (e.g., ordering, TTL expiration) require modifying the provider rather than adding a decorator. + +### Builder Pattern + +**`AgentSkillsProviderBuilder`** provides a fluent API for composing skills from multiple sources. The builder centralizes configuration — script executors, approval callbacks, prompt templates, and filtering — so consumers don't need to know the underlying source types. + +The builder internally decides how to wire up the object graph: it creates the appropriate source instances, applies caching and filtering, and returns a fully configured `AgentSkillsProvider`. This keeps the setup code concise while still allowing fine-grained control when needed. + +**Example** — Using the builder to combine multiple source types with configuration: + +```csharp +var provider = new AgentSkillsProviderBuilder() + .UseFileSkill("./skills") // file-based source + .UseInlineSkills(codeSkill) // code-defined source + .UseClassSkills(new ClassSkill()) // class-based source + .UseFileScriptRunner(SubprocessScriptRunner.RunAsync) // script runner + .UseScriptApproval() // optional human-in-the-loop + .UsePromptTemplate(customTemplate) // optional prompt customization + .UseFilter(s => s.Frontmatter.Name != "internal") // optional skill filtering + .Build(); + +AIAgent agent = chatClient.AsAIAgent(new ChatClientAgentOptions +{ + AIContextProviders = [provider], +}); +``` + +## Adding a Custom Skill Type + +The skills framework is designed for extensibility. While file-based and inline skills cover common +scenarios, you can introduce entirely new skill types by subclassing the four base classes: + +| Base class | Purpose | +|-----------------------|-----------------------------------------------------| +| `AgentSkillsSource` | Discovers and loads skills from a particular origin | +| `AgentSkill` | Holds metadata, content, resources, and scripts | +| `AgentSkillResource` | Provides supplementary content to a skill | +| `AgentSkillScript` | Represents an executable action within a skill | + +The example below implements a **cloud-based skill type** where skills, resources, and scripts are +all stored in and executed through a remote cloud service (e.g., Azure Blob Storage + Azure Functions). + +### Step 1 — Define a custom resource + +A `CloudSkillResource` reads resource content from a cloud storage endpoint instead of the local +filesystem: + +```csharp +/// +/// A skill resource backed by a cloud storage endpoint. +/// +public sealed class CloudSkillResource : AgentSkillResource +{ + private readonly HttpClient _httpClient; + + public CloudSkillResource(string name, Uri blobUri, HttpClient httpClient, string? description = null) + : base(name, description) + { + BlobUri = blobUri ?? throw new ArgumentNullException(nameof(blobUri)); + _httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient)); + } + + /// + /// Gets the URI of the cloud blob that holds this resource's content. + /// + public Uri BlobUri { get; } + + /// + public override async Task ReadAsync( + IServiceProvider? serviceProvider = null, + CancellationToken cancellationToken = default) + { + return await _httpClient.GetStringAsync(BlobUri, cancellationToken).ConfigureAwait(false); + } +} +``` + +### Step 2 — Define a custom script + +A `CloudSkillScript` executes a script by calling a cloud function endpoint, passing arguments as +the request body: + +```csharp +/// +/// A skill script executed via a cloud function endpoint. +/// +public sealed class CloudSkillScript : AgentSkillScript +{ + private readonly HttpClient _httpClient; + + public CloudSkillScript(string name, Uri functionUri, HttpClient httpClient, string? description = null) + : base(name, description) + { + FunctionUri = functionUri ?? throw new ArgumentNullException(nameof(functionUri)); + _httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient)); + } + + /// + /// Gets the URI of the cloud function that runs this script. + /// + public Uri FunctionUri { get; } + + /// + public override async Task RunAsync( + AgentSkill skill, + AIFunctionArguments arguments, + CancellationToken cancellationToken = default) + { + var json = JsonSerializer.Serialize(arguments); + using var content = new StringContent(json, Encoding.UTF8, "application/json"); + var response = await _httpClient.PostAsync(FunctionUri, content, cancellationToken) + .ConfigureAwait(false); + response.EnsureSuccessStatusCode(); + return await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + } +} +``` + +### Step 3 — Define a custom skill + +A `CloudSkill` bundles cloud-specific metadata (e.g., the base endpoint) with the standard skill +shape: + +```csharp +/// +/// An whose content, resources, and scripts are stored in a cloud service. +/// +public sealed class CloudSkill : AgentSkill +{ + public CloudSkill( + AgentSkillFrontmatter frontmatter, + string content, + Uri endpoint, + IReadOnlyList? resources = null, + IReadOnlyList? scripts = null) + { + Frontmatter = frontmatter ?? throw new ArgumentNullException(nameof(frontmatter)); + Content = content ?? throw new ArgumentNullException(nameof(content)); + Endpoint = endpoint ?? throw new ArgumentNullException(nameof(endpoint)); + Resources = resources; + Scripts = scripts; + } + + /// + public override AgentSkillFrontmatter Frontmatter { get; } + + /// + public override string Content { get; } + + /// + /// Gets the base cloud endpoint for this skill. + /// + public Uri Endpoint { get; } + + /// + public override IReadOnlyList? Resources { get; } + + /// + public override IReadOnlyList? Scripts { get; } +} +``` + +### Step 4 — Define a custom source + +A `CloudSkillsSource` discovers skills from a cloud catalog API and constructs `CloudSkill` +instances with their associated resources and scripts: + +```csharp +/// +/// A skill source that discovers and loads skills from a cloud catalog API. +/// +public sealed class CloudSkillsSource : AgentSkillsSource +{ + private readonly Uri _catalogUri; + private readonly HttpClient _httpClient; + + public CloudSkillsSource(Uri catalogUri, HttpClient httpClient) + { + _catalogUri = catalogUri ?? throw new ArgumentNullException(nameof(catalogUri)); + _httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient)); + } + + /// + public override async Task> GetSkillsAsync( + CancellationToken cancellationToken = default) + { + // Fetch the skill catalog from the cloud service. + var json = await _httpClient.GetStringAsync(_catalogUri, cancellationToken) + .ConfigureAwait(false); + var catalog = JsonSerializer.Deserialize(json)!; + + var skills = new List(); + + foreach (var entry in catalog.Skills) + { + var frontmatter = new AgentSkillFrontmatter(entry.Name, entry.Description); + + // Build cloud-backed resources. + var resources = entry.Resources + .Select(r => new CloudSkillResource(r.Name, r.BlobUri, _httpClient, r.Description)) + .ToList(); + + // Build cloud-backed scripts. + var scripts = entry.Scripts + .Select(s => new CloudSkillScript(s.Name, s.FunctionUri, _httpClient, s.Description)) + .ToList(); + + skills.Add(new CloudSkill(frontmatter, entry.Content, entry.Endpoint, resources, scripts)); + } + + return skills; + } +} +``` + +### Step 5 — Register with the builder + +Use `UseSource` to wire the custom source into the provider: + +```csharp +var httpClient = new HttpClient(); + +var provider = new AgentSkillsProviderBuilder() + .UseSource(new CloudSkillsSource( + new Uri("https://my-service.example.com/skills/catalog"), + httpClient)) + // Mix with other source types if needed: + .UseFileSkill("/local/skills", scriptRunner) + .UseInlineSkills(someInlineSkill) + .Build(); +``` + +The `AgentSkillsProvider` handles all skill types uniformly — any combination of file-based, inline, +class-based, and custom skills can coexist in the same provider. Custom skills automatically +participate in the model-facing tools (`load_skill`, `read_skill_resource`, `run_skill_script`), +filtering, deduplication, and caching — no additional integration work is required. + +## Script Representation: `AgentSkillScript` vs `AIFunction` + +Two approaches were considered for representing executable scripts within skills: + +### Option A — Custom `AgentSkillScript` abstract base class (original design) + +Scripts are modeled as a custom `AgentSkillScript` abstract class with `Name`, `Description`, and +`RunAsync(AgentSkill, AIFunctionArguments, CancellationToken)`. Concrete implementations: +`AgentInlineSkillScript` (wraps a delegate/`AIFunction`) and `AgentFileSkillScript` (wraps a file path + executor delegate). + +```csharp +// Base type +public abstract class AgentSkillScript +{ + public string Name { get; } + public string? Description { get; } + public abstract Task RunAsync(AgentSkill skill, AIFunctionArguments arguments, CancellationToken cancellationToken = default); +} + +// AgentSkill exposes scripts as: +public abstract IReadOnlyList? Scripts { get; } + +// Inline script wraps an AIFunction internally +var script = new AgentInlineSkillScript(ConvertUnits, "convert"); + +// Pre-built AIFunction must be wrapped +var script = new AgentInlineSkillScript(myAIFunction); + +// Class-based skill declares scripts as: +public override IReadOnlyList? Scripts { get; } = +[ + new AgentInlineSkillScript(ConvertUnits, "convert"), +]; + +// Provider executes scripts by passing the owning skill: +await script.RunAsync(skill, arguments, cancellationToken); +``` + +**Pros:** + +- **Explicit skill context at execution time.** `RunAsync` receives the owning `AgentSkill`, so any script can access skill metadata or resources during execution without requiring construction-time wiring. +- **Self-contained abstraction.** A dedicated type communicates clearly that scripts are a skills-framework concept, separate from general-purpose AI functions. +- **Easier extensibility for custom script types.** Third-party implementations can subclass `AgentSkillScript` and access the owning skill in `RunAsync` without special setup. + +**Cons:** + +- **Wrapper overhead.** `AgentInlineSkillScript` is a thin pass-through around `AIFunction` — it adds a class, a constructor, and an indirection layer for no behavioral difference. +- **Parallel abstraction.** `AgentSkillScript` and `AIFunction` serve overlapping purposes (named callable with arguments), creating two parallel hierarchies for the same concept. +- **Friction for consumers.** Users who already have `AIFunction` instances must wrap them in `AgentInlineSkillScript` to use them as scripts, adding ceremony. + +### Option B — Reuse `AIFunction` directly + +Scripts are represented as `AIFunction` (from `Microsoft.Extensions.AI`). `AgentSkill.Scripts` returns +`IReadOnlyList?`. `AgentInlineSkillScript` is eliminated entirely — callers use +`AIFunctionFactory.Create(delegate, name: ...)` or pass `AIFunction` instances directly. +`AgentFileSkillScript` becomes an `AIFunction` subclass that captures its owning `AgentFileSkill` via +an internal back-reference set during construction. + +```csharp +// AgentSkill exposes scripts as AIFunction directly: +public abstract IReadOnlyList? Scripts { get; } + +// Inline scripts use AIFunctionFactory — no wrapper class needed +var skill = new AgentInlineSkill("my-skill", "desc", "instructions"); +skill.AddScript(ConvertUnits, "convert"); // delegate +skill.AddScript(myAIFunction); // pre-built AIFunction — no wrapping + +// Class-based skill declares scripts as: +public override IReadOnlyList? Scripts { get; } = +[ + AIFunctionFactory.Create(ConvertUnits, name: "convert"), +]; + +// Provider executes scripts via standard AIFunction invocation: +await script.InvokeAsync(arguments, cancellationToken); + +// File-based scripts extend AIFunction and capture the owning skill internally: +public sealed class AgentFileSkillScript : AIFunction +{ + internal AgentFileSkill? Skill { get; set; } // set by AgentFileSkill constructor + + protected override async ValueTask InvokeCoreAsync( + AIFunctionArguments arguments, CancellationToken cancellationToken) + { + return await _executor(Skill!, this, arguments, cancellationToken); + } +} +``` + +**Pros:** + +- **Fewer types.** Eliminates `AgentSkillScript` and `AgentInlineSkillScript`, reducing the public API surface by two classes. +- **Seamless interop.** Any `AIFunction` — whether from `AIFunctionFactory`, a custom subclass, or an external library — can be used as a skill script with zero wrapping. +- **Consistent with `Microsoft.Extensions.AI` ecosystem.** Scripts share the same type as tool functions used by `IChatClient` and `FunctionInvokingChatClient`, reducing conceptual overhead for developers already familiar with the ecosystem. + +**Cons:** + +- **No owning-skill context in invocation signature.** `AIFunction.InvokeAsync` does not accept an `AgentSkill` parameter, so `AgentFileSkillScript` must capture its owning skill via an internal setter during construction. This adds a construction-order dependency: the skill must set the back-reference on its scripts. +- **Custom script types lose automatic skill access.** Third-party `AIFunction` subclasses that need the owning skill must implement their own mechanism (e.g., constructor injection, closure capture) instead of receiving it as a method parameter. +- **Semantic overloading.** `AIFunction` now means both "a tool the model can call" and "a script within a skill", which could blur the distinction for framework users. + +## Resource Representation: `AgentSkillResource` vs `AIFunction` + +Two approaches were considered for representing skill resources (supplementary content such as references, assets, or dynamic data): + +### Option A — Custom `AgentSkillResource` abstract base class (original design) + +Resources are modeled as a custom `AgentSkillResource` abstract class with `Name`, `Description`, and +`ReadAsync(IServiceProvider?, CancellationToken)`. Concrete implementations: +`AgentInlineSkillResource` (static value, delegate, or `AIFunction` wrapper) and `AgentFileSkillResource` (reads file content from disk). + +```csharp +// Base type +public abstract class AgentSkillResource +{ + public string Name { get; } + public string? Description { get; } + public abstract Task ReadAsync(IServiceProvider? serviceProvider = null, CancellationToken cancellationToken = default); +} + +// AgentSkill exposes resources as: +public abstract IReadOnlyList? Resources { get; } + +// Static resource +var resource = new AgentInlineSkillResource("static content", "my-resource"); + +// Dynamic resource (delegate) +var resource = new AgentInlineSkillResource((IServiceProvider sp) => GetData(sp), "my-resource"); + +// Pre-built AIFunction must be wrapped +var resource = new AgentInlineSkillResource(myAIFunction); + +// Class-based skill declares resources as: +public override IReadOnlyList? Resources { get; } = +[ + new AgentInlineSkillResource("# Conversion Tables\n...", "conversion-table"), +]; + +// Provider reads resources via: +await resource.ReadAsync(serviceProvider, cancellationToken); +``` + +**Pros:** + +- **Clear semantic distinction.** A dedicated `AgentSkillResource` type distinguishes resources (data providers) from scripts (executable actions), making the API self-documenting. +- **Purpose-built API.** `ReadAsync` communicates intent better than `InvokeAsync` for a data-access operation. + +**Cons:** + +- **Wrapper overhead.** `AgentInlineSkillResource` wraps `AIFunction` internally for delegate/function cases — adding a class and indirection for no behavioral difference. +- **Parallel abstraction.** `AgentSkillResource` and `AIFunction` serve overlapping purposes (named callable that returns data), creating two parallel hierarchies. +- **Friction for consumers.** Users who already have `AIFunction` instances must wrap them in `AgentInlineSkillResource`, adding ceremony. + +### Option B — Reuse `AIFunction` directly + +Resources are represented as `AIFunction`. `AgentSkill.Resources` returns `IReadOnlyList?`. +`AgentInlineSkillResource` becomes an `AIFunction` subclass (retained as a convenience for the static-value +pattern: `new AgentInlineSkillResource("data", "name")`). `AgentFileSkillResource` becomes an `AIFunction` +subclass that reads file content. + +```csharp +// AgentSkill exposes resources as AIFunction directly: +public abstract IReadOnlyList? Resources { get; } + +// Static resource — AgentInlineSkillResource is retained as a convenience AIFunction subclass +var resource = new AgentInlineSkillResource("static content", "my-resource"); + +// Dynamic resource — AgentInlineSkillResource wraps delegate as AIFunction +var resource = new AgentInlineSkillResource((IServiceProvider sp) => GetData(sp), "my-resource"); + +// Pre-built AIFunction can be used directly — no wrapping needed +skill.AddResource(myAIFunction); + +// Class-based skill declares resources as: +public override IReadOnlyList? Resources { get; } = +[ + new AgentInlineSkillResource("# Conversion Tables\n...", "conversion-table"), +]; + +// Provider reads resources via standard AIFunction invocation: +await resource.InvokeAsync(arguments, cancellationToken); + +// File-based resources extend AIFunction directly: +internal sealed class AgentFileSkillResource : AIFunction +{ + public string FullPath { get; } + + protected override async ValueTask InvokeCoreAsync( + AIFunctionArguments arguments, CancellationToken cancellationToken) + { + return await File.ReadAllTextAsync(FullPath, Encoding.UTF8, cancellationToken); + } +} +``` + +**Pros:** + +- **Fewer base types.** Eliminates the `AgentSkillResource` abstract class, reducing the public API surface. +- **Seamless interop.** Any `AIFunction` can be used as a skill resource with zero wrapping. + +**Cons:** + +- **Loss of semantic distinction.** Resources and scripts are now both `AIFunction`, which could make it less obvious which list a function belongs to when reading code. +- **Static values require a wrapper.** Unlike the original `ReadAsync` which could return a stored value directly, `AIFunction.InvokeAsync` implies invocation. `AgentInlineSkillResource` is retained as a convenience subclass to handle the static-value case, so this is not eliminated — just moved to a different class. + +## Decision Outcome + +### 1. Keep `AgentSkillResource` and `AgentSkillScript` (Option A for both sections) + +We are staying with the custom `AgentSkillResource` and `AgentSkillScript` model classes instead of reusing `AIFunction`: + +- **Resources have no parameters.** If a consumer provides an `AIFunction` with parameters, those parameters will never be advertised to the LLM, and the resulting call will fail. +- **Approval breaks for `AIFunction`-based representations.** When a resource or script represented by an `AIFunction` is configured with approval, the second approval invocation will not work correctly. +- **Injecting the owning skill into an `AIFunction`-based script is problematic.** Constructor injection would introduce a circular reference between the skill and the script. An internal property setter is possible but adds coupling. + +### 2. Make all agent skill classes internal + +All agent-skill-related classes are made `internal` to minimize the public API surface while the feature matures. We can reconsider and promote types to `public` later based on community signal. + +This leaves two public entry points: + +- **`AgentSkillsProvider`** — use directly when all skills come from a single source and filtering is not needed. +- **`AgentSkillsProviderBuilder`** — use when mixing skill types or when filtering support is required. + +### 3. Caching at provider level + +Caching of tools and instructions is implemented inside `AgentSkillsProvider` rather than as an external decorator. Recreating tools and instructions on every provider call is wasteful, and a caching decorator sitting outside the provider would not have the information needed to cache them effectively. diff --git a/docs/decisions/0022-chat-history-persistence-consistency.md b/docs/decisions/0022-chat-history-persistence-consistency.md new file mode 100644 index 0000000000..2c760e6614 --- /dev/null +++ b/docs/decisions/0022-chat-history-persistence-consistency.md @@ -0,0 +1,116 @@ +--- +status: accepted +contact: westey-m +date: 2026-03-23 +deciders: sergeymenshykh, markwallace, rbarreto, dmytrostruk, westey-m, eavanvalkenburg, stephentoub +consulted: +informed: +--- + +# Chat History Persistence Consistency + +## Context and Problem Statement + +When using `ChatClientAgent` with tools, the `FunctionInvokingChatClient` (FIC) loops multiple times — service call → tool execution → service call → … — before producing a final response. There are two points of discrepancy between how chat history is stored by the framework's `ChatHistoryProvider` and how the underlying AI service stores chat history (e.g., OpenAI Responses with `store=true`): + +1. **Persistence timing**: The AI service persists messages after *each* service call within the FIC loop. The `ChatHistoryProvider` currently persists messages only once, at the *end* of the full agent run (after all FIC loop iterations complete). + +2. **Trailing `FunctionResultContent` storage**: When tool calling is terminated mid-loop (e.g., via `FunctionInvokingChatClient` termination filters), the final response from the agent may contain `FunctionResultContent` that was never sent to a subsequent service call. The AI service never stores this trailing `FunctionResultContent`, but the `ChatHistoryProvider` currently stores all response content, including the trailing `FunctionResultContent`. + +These discrepancies mean that a `ChatHistoryProvider`-managed conversation and a service-managed conversation can diverge in content and structure, even when processing the same interactions. + +### Practical Impact: Resuming After Tool-Call Termination + +Today, users of `AIAgent` get different behaviors depending on whether chat history is stored service-side or in a `ChatHistoryProvider`. This creates concrete challenges — for example, when the function call loop is terminated and the user wants to resume the conversation in a subsequent run. With service-stored history, the trailing `FunctionResultContent` is never persisted, so the last stored message is the `FunctionCallContent` from the service. With `ChatHistoryProvider`-stored history, the trailing `FunctionResultContent` *is* persisted. The user cannot know whether the last `FunctionResultContent` is in the chat history or not without inspecting the storage mechanism, making it difficult to write resumption logic that works correctly regardless of the storage backend. + +### Relationship Between the Two Discrepancies + +The persistence timing and `FunctionResultContent` trimming behaviors are interrelated: + +- **Per-service-call persistence**: When messages are persisted after each individual service call, trailing `FunctionResultContent` trimming is unnecessary. If tool calling is terminated, the `FunctionResultContent` from the terminated call was never sent to a subsequent service call, so it is never persisted. The per-service-call approach naturally matches the service's behavior. + +- **Per-run persistence**: When messages are batched and persisted at the end of the full run, trailing `FunctionResultContent` trimming becomes necessary to match the service's behavior. Without trimming, the stored history contains `FunctionResultContent` that the service would never have stored. + +This means the trimming feature (introduced in [PR #4792](https://github.com/microsoft/agent-framework/pull/4792)) is primarily needed as a complement to per-run persistence. The `PersistChatHistoryAtEndOfRun` setting (introduced in [PR #4762](https://github.com/microsoft/agent-framework/pull/4762)) inverts the default so that per-service-call persistence is the standard behavior, and per-run persistence is opt-in. + +## Decision Drivers + +- **A. Consistency**: The default behavior of `ChatHistoryProvider` should produce stored history that closely matches what the underlying AI service would store, minimizing surprise when switching between framework-managed and service-managed chat history. +- **B. Atomicity**: A run that fails mid-way through a multi-step tool-calling loop should not leave chat history in a partially-updated state, unless the user explicitly opts into that behavior. +- **C. Recoverability**: For long-running tool-calling loops, it should be possible to recover intermediate progress if the process is interrupted, rather than losing all work from the current run. +- **D. Simplicity**: The default behavior should be easy to understand and predict for most users, without requiring knowledge of the FIC loop internals. +- **E. Flexibility**: Regardless of the chosen default, users should be able to opt into the alternative behavior. + +## Considered Options + +- Option 1: Default to per-run persistence with `FunctionResultContent` trimming (opt-in to per-service-call) +- Option 2: Default to per-service-call persistence (opt-in to per-run) + +## Pros and Cons of the Options + +### Option 1: Default to per-run persistence with `FunctionResultContent` trimming + +Keep the current default behavior of persisting chat history only at the end of the full agent run. Add `FunctionResultContent` trimming as the default to improve consistency with service storage. Provide an opt-in setting for users who want per-service-call persistence. + +Settings: +- `PersistChatHistoryAtEndOfRun` = `true` + +- Good, because runs are atomic — chat history is only updated when the full run succeeds, satisfying driver B. +- Good, because the mental model is simple: one run = one history update, satisfying driver D. +- Good, because trimming trailing `FunctionResultContent` improves consistency with service storage, partially satisfying driver A. +- Good, because users can opt in to per-service-call persistence for checkpointing/recovery scenarios, satisfying drivers C and E. +- Bad, because the default persistence timing still differs from the service's behavior (per-run vs. per-service-call), only partially satisfying driver A. +- Bad, because if the process crashes mid-loop, all intermediate progress from the current run is lost, not satisfying driver C by default. + +### Option 2: Default to per-service-call persistence + +Change the default to persist chat history after each individual service call within the FIC loop, matching the AI service's behavior. Trailing `FunctionResultContent` trimming is unnecessary with this approach (it is naturally handled). Provide an opt-in setting for users who want per-run atomicity with trimming. + +Settings: +- `PersistChatHistoryAtEndOfRun` = `false` (default) + +- Good, because the stored history matches the service's behavior by default for both timing and content, fully satisfying driver A. +- Good, because intermediate progress is preserved if the process is interrupted, satisfying driver C. +- Good, because no separate `FunctionResultContent` trimming logic is needed, reducing complexity. +- Bad, because chat history may be left in an incomplete state if the run fails mid-loop (e.g., `FunctionCallContent` stored without corresponding `FunctionResultContent`), not satisfying driver B. A subsequent run cannot proceed without manually providing the missing `FunctionResultContent`. +- Bad, because the mental model is more complex: a single run may produce multiple history updates, partially failing driver D. +- Neutral, because users can opt out to per-run persistence if they prefer atomicity, satisfying driver E. + +## Decision Outcome + +Chosen option: **Option 2 — Default to per-service-call persistence**, because it fully satisfies the consistency driver (A), naturally handles `FunctionResultContent` trimming without additional logic, and provides better recoverability for long-running tool-calling loops. Per-run persistence remains available via the `PersistChatHistoryAtEndOfRun` setting for users who prefer atomic run semantics. + +### Configuration Matrix + +The behavior depends on the combination of `UseProvidedChatClientAsIs` and `PersistChatHistoryAtEndOfRun`: + +| `UseProvidedChatClientAsIs` | `PersistChatHistoryAtEndOfRun` | Behavior | +|---|---|---| +| `false` (default) | `false` (default) | **Per-service-call persistence.** A `ChatHistoryPersistingChatClient` middleware is automatically injected into the chat client pipeline between `FunctionInvokingChatClient` and the leaf `IChatClient`. Messages are persisted after each service call. | +| `true` | `false` | **User responsibility.** No middleware is injected because the user has provided a custom chat client stack. The user is responsible for ensuring correct persistence behavior (e.g., by including their own persisting middleware). | +| `false` | `true` | **Per-run persistence with marking.** A `ChatHistoryPersistingChatClient` middleware is injected, but configured to *mark* messages with metadata rather than store them immediately. At the end of the run, marked messages are stored. Trailing `FunctionResultContent` is trimmed. | +| `true` | `true` | **Per-run persistence with warning.** The system checks whether the custom chat client stack includes a `ChatHistoryPersistingChatClient`. If not, a warning is emitted (particularly relevant for workflow handoff scenarios where trimming cannot be guaranteed). If no `ChatHistoryPersistingChatClient` is preset, all messages are stored at the end of the run, otherwise marked messages are stored. | + +### Consequences + +- Good, because the stored history matches the service's behavior by default for both timing and content, fully satisfying consistency (driver A). +- Good, because intermediate progress is preserved if the process is interrupted, satisfying recoverability (driver C). +- Good, because no separate `FunctionResultContent` trimming logic is needed in the default path, reducing complexity. +- Good, because marking persisted messages with metadata enables deduplication and aids debugging. +- Good, because warnings for custom chat client configurations without the persisting middleware help prevent silent failures in workflow handoff scenarios. +- Bad, because chat history may be left in an incomplete state if the run fails mid-loop (e.g., `FunctionCallContent` stored without corresponding `FunctionResultContent`), requiring manual recovery in rare cases. +- Bad, because the mental model is more complex for the default path: a single run may produce multiple history updates. +- Neutral, because users who prefer atomic run semantics can opt in to per-run persistence via `PersistChatHistoryAtEndOfRun = true`. +- Neutral, because increased write frequency from per-service-call persistence may impact performance for some storage backends; this can be mitigated with a caching decorator. + +### Implementation Notes + +#### Conversation ID Consistency + +The `ChatHistoryPersistingChatClient` middleware must also update the session's `ConversationId` consistently for both response-based and conversation-based service interactions, ensuring the session always reflects the latest service-provided identifier. + +## More Information + +- [PR #4762: Persist messages during function call loop](https://github.com/microsoft/agent-framework/pull/4762) — introduces `PersistChatHistoryAfterEachServiceCall` option and `ChatHistoryPersistingChatClient` decorator +- [PR #4792: Trim final FRC to match service storage](https://github.com/microsoft/agent-framework/pull/4792) — introduces `StoreFinalFunctionResultContent` option and `FilterFinalFunctionResultContent` logic +- [Issue #2889](https://github.com/microsoft/agent-framework/issues/2889) — original issue tracking chat history persistence during function call loops diff --git a/dotnet/src/Microsoft.Agents.AI.AzureAI/ProjectResponsesClientExtensions.cs b/dotnet/src/Microsoft.Agents.AI.AzureAI/ProjectResponsesClientExtensions.cs new file mode 100644 index 0000000000..5a899d5076 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.AzureAI/ProjectResponsesClientExtensions.cs @@ -0,0 +1,60 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Diagnostics.CodeAnalysis; +using Microsoft.Extensions.AI; +using Microsoft.Shared.DiagnosticIds; +using Microsoft.Shared.Diagnostics; +using OpenAI.Responses; + +namespace Azure.AI.Extensions.OpenAI; + +/// +/// Provides extension methods for +/// to simplify the creation of AI agents that work with Azure AI services. +/// +[Experimental(DiagnosticIds.Experiments.AIOpenAIResponses)] +public static class ProjectResponsesClientExtensions +{ + /// + /// Gets an for use with this that does not store responses for later retrieval. + /// + /// + /// This corresponds to setting the "store" property in the JSON representation to false. + /// + /// The client. + /// Optional deployment name (model) to use for requests. + /// + /// Includes an encrypted version of reasoning tokens in reasoning item outputs. + /// This enables reasoning items to be used in multi-turn conversations when using the Responses API statelessly + /// (like when the store parameter is set to false, or when an organization is enrolled in the zero data retention program). + /// Defaults to . + /// + /// An that can be used to converse via the that does not store responses for later retrieval. + /// is . + [Experimental(DiagnosticIds.Experiments.AgentsAIExperiments)] + public static IChatClient AsIChatClientWithStoredOutputDisabled(this ProjectResponsesClient responseClient, string? deploymentName = null, bool includeReasoningEncryptedContent = true) + { + return Throw.IfNull(responseClient) + .AsIChatClient(deploymentName) + .AsBuilder() + .ConfigureOptions(x => + { + var previousFactory = x.RawRepresentationFactory; + x.RawRepresentationFactory = state => + { + var responseOptions = previousFactory?.Invoke(state) as CreateResponseOptions ?? new CreateResponseOptions(); + + responseOptions.StoredOutputEnabled = false; + + if (includeReasoningEncryptedContent && + !responseOptions.IncludedProperties.Contains(IncludedResponseProperty.ReasoningEncryptedContent)) + { + responseOptions.IncludedProperties.Add(IncludedResponseProperty.ReasoningEncryptedContent); + } + + return responseOptions; + }; + }) + .Build(); + } +} diff --git a/dotnet/src/Microsoft.Agents.AI.OpenAI/Extensions/OpenAIResponseClientExtensions.cs b/dotnet/src/Microsoft.Agents.AI.OpenAI/Extensions/OpenAIResponseClientExtensions.cs index 5aee8eb046..4ceff75743 100644 --- a/dotnet/src/Microsoft.Agents.AI.OpenAI/Extensions/OpenAIResponseClientExtensions.cs +++ b/dotnet/src/Microsoft.Agents.AI.OpenAI/Extensions/OpenAIResponseClientExtensions.cs @@ -105,7 +105,7 @@ public static class OpenAIResponseClientExtensions /// This corresponds to setting the "store" property in the JSON representation to false. /// /// The client. - /// Optional default model ID to use for requests. Required when using a plain (not via Azure OpenAI). + /// Optional default model ID to use for requests. /// /// Includes an encrypted version of reasoning tokens in reasoning item outputs. /// This enables reasoning items to be used in multi-turn conversations when using the Responses API statelessly diff --git a/dotnet/tests/AzureAI.IntegrationTests/AIProjectClientCreateTests.cs b/dotnet/tests/AzureAI.IntegrationTests/AIProjectClientCreateTests.cs index e1cef1f1aa..425197853e 100644 --- a/dotnet/tests/AzureAI.IntegrationTests/AIProjectClientCreateTests.cs +++ b/dotnet/tests/AzureAI.IntegrationTests/AIProjectClientCreateTests.cs @@ -188,6 +188,134 @@ public class AIProjectClientCreateTests } } + /// + /// Validates that an agent version created with an OpenAPI tool definition via the native + /// Azure.AI.Projects SDK and then wrapped with AsAIAgent(agentVersion) correctly + /// invokes the server-side OpenAPI function through RunAsync. + /// Regression test for https://github.com/microsoft/agent-framework/issues/4883. + /// + [RetryFact(Constants.RetryCount, Constants.RetryDelay, Skip = "For manual testing only")] + public async Task AsAIAgent_WithOpenAPITool_NativeSDKCreation_InvokesServerSideToolAsync() + { + // Arrange — create agent version with OpenAPI tool using native Azure.AI.Projects SDK types. + string AgentName = AIProjectClientFixture.GenerateUniqueAgentName("OpenAPITestAgent"); + const string AgentInstructions = "You are a helpful assistant that can use the countries API to retrieve information about countries by their currency code."; + + const string CountriesOpenApiSpec = """ + { + "openapi": "3.1.0", + "info": { + "title": "REST Countries API", + "description": "Retrieve information about countries by currency code", + "version": "v3.1" + }, + "servers": [ + { + "url": "https://restcountries.com/v3.1" + } + ], + "paths": { + "/currency/{currency}": { + "get": { + "description": "Get countries that use a specific currency code (e.g., USD, EUR, GBP)", + "operationId": "GetCountriesByCurrency", + "parameters": [ + { + "name": "currency", + "in": "path", + "description": "Currency code (e.g., USD, EUR, GBP)", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Successful response with list of countries", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "type": "object" + } + } + } + } + }, + "404": { + "description": "No countries found for the currency" + } + } + } + } + } + } + """; + + // Step 1: Create the OpenAPI function definition and agent version using native SDK types. + var openApiFunction = new OpenApiFunctionDefinition( + "get_countries", + BinaryData.FromString(CountriesOpenApiSpec), + new OpenAPIAnonymousAuthenticationDetails()) + { + Description = "Retrieve information about countries by currency code" + }; + + var definition = new PromptAgentDefinition(model: TestConfiguration.GetRequiredValue(TestSettings.AzureAIModelDeploymentName)) + { + Instructions = AgentInstructions, + Tools = { (ResponseTool)AgentTool.CreateOpenApiTool(openApiFunction) } + }; + + AgentVersionCreationOptions creationOptions = new(definition); + AgentVersion agentVersion = await this._client.Agents.CreateAgentVersionAsync(AgentName, creationOptions); + + try + { + // Step 2: Wrap the agent version using AsAIAgent extension. + ChatClientAgent agent = this._client.AsAIAgent(agentVersion); + + // Assert the agent was created correctly and retains version metadata. + Assert.NotNull(agent); + Assert.Equal(AgentName, agent.Name); + var retrievedVersion = agent.GetService(); + Assert.NotNull(retrievedVersion); + + // Step 3: Call RunAsync to trigger the server-side OpenAPI function. + var result = await agent.RunAsync("What countries use the Euro (EUR) as their currency? Please list them."); + + // Step 4: Validate the OpenAPI tool was invoked server-side. + // Note: Server-side OpenAPI tools (executed within the Responses API via AgentReference) + // do not surface as FunctionCallContent in the MEAI abstraction — the API handles the full + // tool loop internally. We validate tool invocation by asserting the response contains + // multiple specific country names that the model would need API data to enumerate accurately. + var text = result.ToString(); + Assert.NotEmpty(text); + + // The response must mention multiple well-known Eurozone countries — requiring several + // correct entries makes it highly unlikely the model answered purely from parametric knowledge. + int matchCount = 0; + foreach (var country in new[] { "Germany", "France", "Italy", "Spain", "Portugal", "Netherlands", "Belgium", "Austria", "Ireland", "Finland" }) + { + if (text.Contains(country, StringComparison.OrdinalIgnoreCase)) + { + matchCount++; + } + } + + Assert.True( + matchCount >= 3, + $"Expected response to list at least 3 Eurozone countries from the OpenAPI tool, but found {matchCount}. Response: {text}"); + } + finally + { + // Cleanup. + await this._client.Agents.DeleteAgentAsync(AgentName); + } + } + [Theory] [InlineData("CreateWithChatClientAgentOptionsAsync")] public async Task CreateAgent_CreatesAgentWithAIFunctionToolsAsync(string createMechanism) diff --git a/dotnet/tests/Microsoft.Agents.AI.AzureAI.UnitTests/ProjectResponsesClientExtensionsTests.cs b/dotnet/tests/Microsoft.Agents.AI.AzureAI.UnitTests/ProjectResponsesClientExtensionsTests.cs new file mode 100644 index 0000000000..d10ef861c9 --- /dev/null +++ b/dotnet/tests/Microsoft.Agents.AI.AzureAI.UnitTests/ProjectResponsesClientExtensionsTests.cs @@ -0,0 +1,169 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Reflection; +using Azure.AI.Extensions.OpenAI; +using Microsoft.Extensions.AI; +using OpenAI.Responses; + +namespace Microsoft.Agents.AI.AzureAI.UnitTests; + +/// +/// Unit tests for the class. +/// +public sealed class ProjectResponsesClientExtensionsTests +{ + private static ProjectResponsesClient CreateTestClient() + { + return new ProjectResponsesClient(new FakeAuthenticationTokenProvider()); + } + + /// + /// Verify that AsIChatClientWithStoredOutputDisabled throws ArgumentNullException when client is null. + /// + [Fact] + public void AsIChatClientWithStoredOutputDisabled_WithNullClient_ThrowsArgumentNullException() + { + // Act & Assert + var exception = Assert.Throws(() => + ((ProjectResponsesClient)null!).AsIChatClientWithStoredOutputDisabled()); + + Assert.Equal("responseClient", exception.ParamName); + } + + /// + /// Verify that AsIChatClientWithStoredOutputDisabled wraps the original ProjectResponsesClient, + /// which remains accessible via the service chain. + /// + [Fact] + public void AsIChatClientWithStoredOutputDisabled_InnerResponsesClientIsAccessible() + { + // Arrange + var responseClient = CreateTestClient(); + + // Act + var chatClient = responseClient.AsIChatClientWithStoredOutputDisabled(); + + // Assert - the inner ProjectResponsesClient should be accessible via GetService + var innerClient = chatClient.GetService(); + Assert.NotNull(innerClient); + Assert.Same(responseClient, innerClient); + } + + /// + /// Verify that AsIChatClientWithStoredOutputDisabled with includeReasoningEncryptedContent false + /// wraps the original ProjectResponsesClient, which remains accessible via the service chain. + /// + [Fact] + public void AsIChatClientWithStoredOutputDisabled_WithIncludeReasoningFalse_InnerResponsesClientIsAccessible() + { + // Arrange + var responseClient = CreateTestClient(); + + // Act + var chatClient = responseClient.AsIChatClientWithStoredOutputDisabled(includeReasoningEncryptedContent: false); + + // Assert - the inner ProjectResponsesClient should be accessible via GetService + var innerClient = chatClient.GetService(); + Assert.NotNull(innerClient); + Assert.Same(responseClient, innerClient); + } + + /// + /// Verify that AsIChatClientWithStoredOutputDisabled with default parameter (includeReasoningEncryptedContent = true) + /// configures StoredOutputEnabled to false and includes ReasoningEncryptedContent in IncludedProperties. + /// + [Fact] + public void AsIChatClientWithStoredOutputDisabled_Default_ConfiguresStoredOutputDisabledWithReasoningEncryptedContent() + { + // Arrange + var responseClient = CreateTestClient(); + + // Act + var chatClient = responseClient.AsIChatClientWithStoredOutputDisabled(); + + // Assert + var createResponseOptions = GetCreateResponseOptionsFromPipeline(chatClient); + Assert.NotNull(createResponseOptions); + Assert.False(createResponseOptions.StoredOutputEnabled); + Assert.Contains(IncludedResponseProperty.ReasoningEncryptedContent, createResponseOptions.IncludedProperties); + } + + /// + /// Verify that AsIChatClientWithStoredOutputDisabled with includeReasoningEncryptedContent explicitly set to true + /// configures StoredOutputEnabled to false and includes ReasoningEncryptedContent in IncludedProperties. + /// + [Fact] + public void AsIChatClientWithStoredOutputDisabled_WithIncludeReasoningTrue_ConfiguresStoredOutputDisabledWithReasoningEncryptedContent() + { + // Arrange + var responseClient = CreateTestClient(); + + // Act + var chatClient = responseClient.AsIChatClientWithStoredOutputDisabled(includeReasoningEncryptedContent: true); + + // Assert + var createResponseOptions = GetCreateResponseOptionsFromPipeline(chatClient); + Assert.NotNull(createResponseOptions); + Assert.False(createResponseOptions.StoredOutputEnabled); + Assert.Contains(IncludedResponseProperty.ReasoningEncryptedContent, createResponseOptions.IncludedProperties); + } + + /// + /// Verify that AsIChatClientWithStoredOutputDisabled with includeReasoningEncryptedContent set to false + /// configures StoredOutputEnabled to false and does not include ReasoningEncryptedContent in IncludedProperties. + /// + [Fact] + public void AsIChatClientWithStoredOutputDisabled_WithIncludeReasoningFalse_ConfiguresStoredOutputDisabledWithoutReasoningEncryptedContent() + { + // Arrange + var responseClient = CreateTestClient(); + + // Act + var chatClient = responseClient.AsIChatClientWithStoredOutputDisabled(includeReasoningEncryptedContent: false); + + // Assert + var createResponseOptions = GetCreateResponseOptionsFromPipeline(chatClient); + Assert.NotNull(createResponseOptions); + Assert.False(createResponseOptions.StoredOutputEnabled); + Assert.DoesNotContain(IncludedResponseProperty.ReasoningEncryptedContent, createResponseOptions.IncludedProperties); + } + + /// + /// Verify that AsIChatClientWithStoredOutputDisabled works with an optional deployment name. + /// + [Fact] + public void AsIChatClientWithStoredOutputDisabled_WithDeploymentName_ConfiguresStoredOutputDisabled() + { + // Arrange + var responseClient = CreateTestClient(); + + // Act + var chatClient = responseClient.AsIChatClientWithStoredOutputDisabled(deploymentName: "my-deployment"); + + // Assert + var createResponseOptions = GetCreateResponseOptionsFromPipeline(chatClient); + Assert.NotNull(createResponseOptions); + Assert.False(createResponseOptions.StoredOutputEnabled); + Assert.Contains(IncludedResponseProperty.ReasoningEncryptedContent, createResponseOptions.IncludedProperties); + } + + /// + /// Extracts the produced by the ConfigureOptions pipeline + /// by using reflection to access the configure action and invoking it on a test . + /// + private static CreateResponseOptions? GetCreateResponseOptionsFromPipeline(IChatClient chatClient) + { + var configureField = chatClient.GetType().GetField("_configureOptions", BindingFlags.NonPublic | BindingFlags.Instance); + Assert.NotNull(configureField); + + var configureAction = configureField.GetValue(chatClient) as Action; + Assert.NotNull(configureAction); + + var options = new ChatOptions(); + configureAction(options); + + Assert.NotNull(options.RawRepresentationFactory); + return options.RawRepresentationFactory(chatClient) as CreateResponseOptions; + } +}