Compare commits

..
Author SHA1 Message Date
Tao Chen 20ae71251d remove temp 2026-04-27 20:54:01 -07:00
Tao Chen 48edeb3f67 Fix int tests 2026-04-27 20:51:15 -07:00
Tao Chen 8940b45d5b Address comments 2026-04-27 09:48:05 -07:00
Tao Chen 78c7d5fc84 Fix README 2026-04-24 17:27:02 -07:00
Tao Chen 9cf4c9169a Fix file content and add more tests 2026-04-24 17:18:21 -07:00
Tao Chen 2c6aa98b18 Add file data type support 2026-04-24 12:04:58 -07:00
Tao Chen 334ce4dfe0 Update foundry hosting samples 2026-04-24 11:48:17 -07:00
158 changed files with 1719 additions and 13292 deletions
-4
View File
@@ -242,7 +242,3 @@ python/dotnet-ref
# Generated filtered solution files (created by eng/scripts/New-FilteredSolution.ps1)
dotnet/filtered-*.slnx
**/*.lscache
# Local tool state
.omc/
.omx/
+9 -9
View File
@@ -56,15 +56,15 @@
<PackageVersion Include="System.Threading.Tasks.Extensions" Version="4.6.3" />
<PackageVersion Include="System.Net.Security" Version="4.3.2" />
<!-- OpenTelemetry -->
<PackageVersion Include="OpenTelemetry" Version="1.15.3" />
<PackageVersion Include="OpenTelemetry.Api" Version="1.15.3" />
<PackageVersion Include="OpenTelemetry.Exporter.Console" Version="1.15.3" />
<PackageVersion Include="OpenTelemetry.Exporter.InMemory" Version="1.15.3" />
<PackageVersion Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" Version="1.15.3" />
<PackageVersion Include="OpenTelemetry.Extensions.Hosting" Version="1.15.3" />
<PackageVersion Include="OpenTelemetry.Instrumentation.AspNetCore" Version="1.15.2" />
<PackageVersion Include="OpenTelemetry.Instrumentation.Http" Version="1.15.1" />
<PackageVersion Include="OpenTelemetry.Instrumentation.Runtime" Version="1.15.1" />
<PackageVersion Include="OpenTelemetry" Version="1.15.0" />
<PackageVersion Include="OpenTelemetry.Api" Version="1.15.0" />
<PackageVersion Include="OpenTelemetry.Exporter.Console" Version="1.15.0" />
<PackageVersion Include="OpenTelemetry.Exporter.InMemory" Version="1.15.0" />
<PackageVersion Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" Version="1.15.0" />
<PackageVersion Include="OpenTelemetry.Extensions.Hosting" Version="1.14.0" />
<PackageVersion Include="OpenTelemetry.Instrumentation.AspNetCore" Version="1.14.0" />
<PackageVersion Include="OpenTelemetry.Instrumentation.Http" Version="1.14.0" />
<PackageVersion Include="OpenTelemetry.Instrumentation.Runtime" Version="1.14.0" />
<!-- Microsoft.AspNetCore.* -->
<PackageVersion Include="Microsoft.AspNetCore.Authentication.JwtBearer" Version="10.0.0" />
<PackageVersion Include="Microsoft.AspNetCore.Authentication.OpenIdConnect" Version="10.0.0" />
@@ -5,16 +5,16 @@
// This is provided for demonstration purposes only.
using System.Diagnostics;
using System.Text.Json;
using Microsoft.Agents.AI;
using Microsoft.Extensions.AI;
/// <summary>
/// Executes file-based skill scripts as local subprocesses.
/// </summary>
/// <remarks>
/// This runner uses the script's absolute path and converts the arguments
/// to CLI arguments. When the LLM sends a JSON array, each element is used
/// as a positional argument. It is intended for demonstration purposes only.
/// This runner uses the script's absolute path, converts the arguments
/// to CLI flags, and returns captured output. It is intended for
/// demonstration purposes only.
/// </remarks>
internal static class SubprocessScriptRunner
{
@@ -24,8 +24,7 @@ internal static class SubprocessScriptRunner
public static async Task<object?> RunAsync(
AgentFileSkill skill,
AgentFileSkillScript script,
JsonElement? arguments,
IServiceProvider? serviceProvider,
AIFunctionArguments arguments,
CancellationToken cancellationToken)
{
if (!File.Exists(script.FullPath))
@@ -62,27 +61,24 @@ internal static class SubprocessScriptRunner
startInfo.FileName = script.FullPath;
}
if (arguments is { ValueKind: JsonValueKind.Array } json)
if (arguments is not null)
{
// Positional CLI arguments
foreach (var element in json.EnumerateArray())
foreach (var (key, value) in arguments)
{
if (element.ValueKind != JsonValueKind.String)
if (value is bool boolValue)
{
throw new InvalidOperationException(
$"File-based skill scripts only accept string CLI arguments but received a JSON element of kind '{element.ValueKind}'. " +
"All array elements must be JSON strings.");
if (boolValue)
{
startInfo.ArgumentList.Add(NormalizeKey(key));
}
}
else if (value is not null)
{
startInfo.ArgumentList.Add(NormalizeKey(key));
startInfo.ArgumentList.Add(value.ToString()!);
}
startInfo.ArgumentList.Add(element.GetString()!);
}
}
else if (arguments is not null && arguments.Value.ValueKind != JsonValueKind.Null && arguments.Value.ValueKind != JsonValueKind.Undefined)
{
throw new InvalidOperationException(
$"Expected a JSON array of CLI arguments but received {arguments.Value.ValueKind}. " +
"File-based skill scripts expect positional arguments as a JSON array of strings.");
}
Process? process = null;
try
@@ -132,4 +128,10 @@ internal static class SubprocessScriptRunner
process?.Dispose();
}
}
/// <summary>
/// Normalizes a parameter key to a consistent --flag format.
/// Models may return keys with or without leading dashes (e.g., "value" vs "--value").
/// </summary>
private static string NormalizeKey(string key) => "--" + key.TrimStart('-');
}
@@ -65,53 +65,6 @@ Workflow orchestration started for CancelOrder. Orchestration runId: abc123def45
>
> If not provided, a unique run ID is auto-generated.
### Wait for the Workflow Result
By default, the HTTP endpoint returns `202 Accepted` immediately with the run ID. If you want to wait for the workflow to complete and get the result in the response, add the `x-ms-wait-for-response: true` header:
Bash (Linux/macOS/WSL):
```bash
curl -X POST http://localhost:7071/api/workflows/CancelOrder/run \
-H "Content-Type: text/plain" \
-H "x-ms-wait-for-response: true" \
-d "12345"
```
PowerShell:
```powershell
Invoke-RestMethod -Method Post `
-Uri http://localhost:7071/api/workflows/CancelOrder/run `
-ContentType text/plain `
-Headers @{ "x-ms-wait-for-response" = "true" } `
-Body "12345"
```
The response will contain the workflow result as plain text (200 OK):
```text
Cancellation email sent for order 12345 to jerry@example.com.
```
To get the result as JSON, also include the `Accept: application/json` header:
```bash
curl -X POST http://localhost:7071/api/workflows/CancelOrder/run \
-H "Content-Type: text/plain" \
-H "x-ms-wait-for-response: true" \
-H "Accept: application/json" \
-d "12345"
```
```json
{
"runId": "abc123def456",
"workflowStatus": "Completed",
"result": "Cancellation email sent for order 12345 to jerry@example.com."
}
```
In the function app logs, you will see the sequential execution of each executor:
```text
@@ -7,21 +7,6 @@ Content-Type: text/plain
12345
### Cancel an order and wait for the result
POST {{authority}}/api/workflows/CancelOrder/run
Content-Type: text/plain
x-ms-wait-for-response: true
12345
### Cancel an order and wait for the result (JSON response)
POST {{authority}}/api/workflows/CancelOrder/run
Content-Type: text/plain
Accept: application/json
x-ms-wait-for-response: true
12345
### Cancel an order with a custom run ID
POST {{authority}}/api/workflows/CancelOrder/run?runId=my-custom-id-123
Content-Type: text/plain
@@ -34,13 +19,6 @@ Content-Type: text/plain
12345
### Get order status and wait for the result
POST {{authority}}/api/workflows/OrderStatus/run
Content-Type: text/plain
x-ms-wait-for-response: true
12345
### Batch cancel orders with a complex JSON input
POST {{authority}}/api/workflows/BatchCancelOrders/run
Content-Type: application/json
@@ -13,8 +13,6 @@
<ItemGroup>
<PackageReference Include="Azure.AI.AgentServer.Invocations" />
<PackageReference Include="DotNetEnv" />
<PackageReference Include="OpenTelemetry.Api" />
<PackageReference Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" />
</ItemGroup>
<!-- For contributors: uses ProjectReference to build against local source -->
@@ -34,7 +34,6 @@
<PackageReference Include="Azure.AI.Projects" VersionOverride="2.1.0-beta.1" />
<PackageReference Include="Azure.Identity" />
<PackageReference Include="ModelContextProtocol" />
<PackageReference Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" />
</ItemGroup>
<ItemGroup>
@@ -21,8 +21,6 @@ internal static class BuiltInFunctions
internal const string HttpPrefix = "http-";
internal const string McpToolPrefix = "mcptool-";
private const string WaitForResponseHeaderName = "x-ms-wait-for-response";
internal static readonly string RunAgentHttpFunctionEntryPoint = $"{typeof(BuiltInFunctions).FullName!}.{nameof(RunAgentHttpAsync)}";
internal static readonly string RunAgentEntityFunctionEntryPoint = $"{typeof(BuiltInFunctions).FullName!}.{nameof(InvokeAgentAsync)}";
internal static readonly string RunAgentMcpToolFunctionEntryPoint = $"{typeof(BuiltInFunctions).FullName!}.{nameof(RunMcpToolAsync)}";
@@ -64,11 +62,6 @@ internal static class BuiltInFunctions
StartOrchestrationOptions? options = instanceId is not null ? new StartOrchestrationOptions(instanceId) : null;
string resolvedInstanceId = await client.ScheduleNewOrchestrationInstanceAsync(orchestrationFunctionName, orchestrationInput, options);
if (ShouldWaitForResponse(req, defaultValue: false))
{
return await WaitForWorkflowCompletionAsync(req, client, context, resolvedInstanceId);
}
HttpResponseData response = req.CreateResponse(HttpStatusCode.Accepted);
await response.WriteStringAsync($"Workflow orchestration started for {workflowName}. Orchestration runId: {resolvedInstanceId}");
return response;
@@ -311,7 +304,15 @@ internal static class BuiltInFunctions
}
// Check if we should wait for response (default is true)
bool waitForResponse = ShouldWaitForResponse(req, defaultValue: true);
bool waitForResponse = true;
if (req.Headers.TryGetValues("x-ms-wait-for-response", out IEnumerable<string>? waitForResponseValues))
{
string? waitForResponseValue = waitForResponseValues.FirstOrDefault();
if (!string.IsNullOrEmpty(waitForResponseValue) && bool.TryParse(waitForResponseValue, out bool parsedValue))
{
waitForResponse = parsedValue;
}
}
AIAgent agentProxy = client.AsDurableAgentProxy(context, agentName);
@@ -427,95 +428,6 @@ internal static class BuiltInFunctions
return metadata.ReadOutputAs<DurableWorkflowResult>()?.Result;
}
/// <summary>
/// Waits for a workflow orchestration to complete and returns an appropriate HTTP response.
/// </summary>
private static async Task<HttpResponseData> WaitForWorkflowCompletionAsync(
HttpRequestData req,
DurableTaskClient client,
FunctionContext context,
string instanceId)
{
bool acceptsJson = AcceptsJson(req);
OrchestrationMetadata? metadata = await client.WaitForInstanceCompletionAsync(
instanceId,
getInputsAndOutputs: true,
cancellation: context.CancellationToken);
if (metadata is null)
{
return await CreateErrorResponseAsync(req, context, HttpStatusCode.NotFound,
$"No workflow orchestration with ID '{instanceId}' was found.", acceptsJson);
}
if (metadata.RuntimeStatus is OrchestrationRuntimeStatus.Failed)
{
string errorMessage = metadata.FailureDetails?.ErrorMessage ?? "Unknown error";
HttpResponseData failedResponse = req.CreateResponse(HttpStatusCode.OK);
if (acceptsJson)
{
await failedResponse.WriteAsJsonAsync(
new WorkflowRunResponse(instanceId, metadata.RuntimeStatus.ToString(), Result: null, Error: errorMessage),
context.CancellationToken);
}
else
{
failedResponse.Headers.Add("Content-Type", "text/plain");
await failedResponse.WriteStringAsync(errorMessage, context.CancellationToken);
}
return failedResponse;
}
if (metadata.RuntimeStatus is not OrchestrationRuntimeStatus.Completed)
{
return await CreateErrorResponseAsync(req, context, HttpStatusCode.InternalServerError,
$"Workflow orchestration '{instanceId}' ended with unexpected status '{metadata.RuntimeStatus}'.", acceptsJson);
}
string? result = metadata.ReadOutputAs<DurableWorkflowResult>()?.Result;
HttpResponseData response = req.CreateResponse(HttpStatusCode.OK);
if (acceptsJson)
{
JsonElement? resultElement = null;
if (!string.IsNullOrEmpty(result))
{
try
{
using JsonDocument doc = JsonDocument.Parse(result);
resultElement = doc.RootElement.Clone();
}
catch (JsonException)
{
// Result is a plain string (not valid JSON) — serialize it as a JSON string element.
var buffer = new System.Buffers.ArrayBufferWriter<byte>();
using (var writer = new Utf8JsonWriter(buffer))
{
writer.WriteStringValue(result);
}
using JsonDocument fallbackDoc = JsonDocument.Parse(buffer.WrittenMemory);
resultElement = fallbackDoc.RootElement.Clone();
}
}
await response.WriteAsJsonAsync(
new WorkflowRunResponse(instanceId, metadata.RuntimeStatus.ToString(), resultElement),
context.CancellationToken);
}
else
{
response.Headers.Add("Content-Type", "text/plain");
await response.WriteStringAsync(result ?? string.Empty, context.CancellationToken);
}
return response;
}
/// <summary>
/// Creates an error response with the specified status code and error message.
/// </summary>
@@ -523,18 +435,18 @@ internal static class BuiltInFunctions
/// <param name="context">The function context.</param>
/// <param name="statusCode">The HTTP status code.</param>
/// <param name="errorMessage">The error message.</param>
/// <param name="acceptsJson">Optional pre-computed value indicating whether the client accepts JSON. When <see langword="null"/>, the value is determined from the request's <c>Accept</c> header.</param>
/// <returns>The HTTP response data containing the error.</returns>
private static async Task<HttpResponseData> CreateErrorResponseAsync(
HttpRequestData req,
FunctionContext context,
HttpStatusCode statusCode,
string errorMessage,
bool? acceptsJson = null)
string errorMessage)
{
HttpResponseData response = req.CreateResponse(statusCode);
bool acceptsJson = req.Headers.TryGetValues("Accept", out IEnumerable<string>? acceptValues) &&
acceptValues.Contains("application/json", StringComparer.OrdinalIgnoreCase);
if (acceptsJson ?? AcceptsJson(req))
if (acceptsJson)
{
ErrorResponse errorResponse = new((int)statusCode, errorMessage);
await response.WriteAsJsonAsync(errorResponse, context.CancellationToken);
@@ -567,7 +479,10 @@ internal static class BuiltInFunctions
HttpResponseData response = req.CreateResponse(statusCode);
response.Headers.Add("x-ms-thread-id", sessionId);
if (AcceptsJson(req))
bool acceptsJson = req.Headers.TryGetValues("Accept", out IEnumerable<string>? acceptValues) &&
acceptValues.Contains("application/json", StringComparer.OrdinalIgnoreCase);
if (acceptsJson)
{
AgentRunSuccessResponse successResponse = new((int)statusCode, sessionId, agentResponse);
await response.WriteAsJsonAsync(successResponse, context.CancellationToken);
@@ -596,7 +511,10 @@ internal static class BuiltInFunctions
HttpResponseData response = req.CreateResponse(HttpStatusCode.Accepted);
response.Headers.Add("x-ms-thread-id", sessionId);
if (AcceptsJson(req))
bool acceptsJson = req.Headers.TryGetValues("Accept", out IEnumerable<string>? acceptValues) &&
acceptValues.Contains("application/json", StringComparer.OrdinalIgnoreCase);
if (acceptsJson)
{
AgentRunAcceptedResponse acceptedResponse = new((int)HttpStatusCode.Accepted, sessionId);
await response.WriteAsJsonAsync(acceptedResponse, context.CancellationToken);
@@ -610,34 +528,6 @@ internal static class BuiltInFunctions
return response;
}
/// <summary>
/// Returns <see langword="true"/> when the caller has requested waiting for the workflow/agent to complete,
/// as indicated by the <c>x-ms-wait-for-response</c> header. Falls back to <paramref name="defaultValue"/>
/// when the header is absent or not a valid boolean.
/// </summary>
private static bool ShouldWaitForResponse(HttpRequestData req, bool defaultValue)
{
if (req.Headers.TryGetValues(WaitForResponseHeaderName, out IEnumerable<string>? values) &&
bool.TryParse(values.FirstOrDefault(), out bool parsed))
{
return parsed;
}
return defaultValue;
}
/// <summary>
/// Returns <see langword="true"/> when the request accepts the <c>application/json</c> media type.
/// </summary>
private static bool AcceptsJson(HttpRequestData req)
{
return req.Headers.TryGetValues("Accept", out IEnumerable<string>? acceptValues) &&
acceptValues
.SelectMany(v => v.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries))
.Select(v => v.Split(';', 2)[0].Trim())
.Contains("application/json", StringComparer.OrdinalIgnoreCase);
}
private static string GetAgentName(FunctionContext context)
{
// Check if the function name starts with the HttpPrefix
@@ -701,19 +591,6 @@ internal static class BuiltInFunctions
[property: JsonPropertyName("eventName")] string? EventName,
[property: JsonPropertyName("response")] JsonElement Response);
/// <summary>
/// Represents a workflow run response when waiting for completion.
/// </summary>
/// <param name="RunId">The orchestration run ID.</param>
/// <param name="WorkflowStatus">The orchestration runtime status (e.g., "Completed", "Failed").</param>
/// <param name="Result">The workflow result as a JSON element so POCOs serialize as nested objects rather than escaped strings.</param>
/// <param name="Error">An optional error message when the workflow has failed.</param>
private sealed record WorkflowRunResponse(
[property: JsonPropertyName("runId")] string RunId,
[property: JsonPropertyName("workflowStatus")] string WorkflowStatus,
[property: JsonPropertyName("result")] JsonElement? Result,
[property: JsonPropertyName("error"), JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] string? Error = null);
/// <summary>
/// A service provider that combines the original service provider with an additional DurableTaskClient instance.
/// </summary>
@@ -2,7 +2,6 @@
## [Unreleased]
- Support returning workflow results from HTTP trigger endpoint ([#5321](https://github.com/microsoft/agent-framework/pull/5321))
- Added MCP tool trigger support for durable workflows ([#4768](https://github.com/microsoft/agent-framework/pull/4768))
- Added Azure Functions hosting support for durable workflows ([#4436](https://github.com/microsoft/agent-framework/pull/4436))
@@ -26,12 +26,6 @@ public sealed class DeclarativeWorkflowOptions(ResponseAgentProvider agentProvid
/// </summary>
public IMcpToolHandler? McpToolHandler { get; init; }
/// <summary>
/// Gets or sets the HTTP request handler for executing <c>HttpRequestAction</c> actions within workflows.
/// If not set, HTTP request actions will fail with an appropriate error message.
/// </summary>
public IHttpRequestHandler? HttpRequestHandler { get; init; }
/// <summary>
/// Defines the configuration settings for the workflow.
/// </summary>
@@ -1,289 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net.Http;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
namespace Microsoft.Agents.AI.Workflows.Declarative;
/// <summary>
/// Default implementation of <see cref="IHttpRequestHandler"/> built on <see cref="HttpClient"/>.
/// </summary>
/// <remarks>
/// <para>
/// This handler supports per-request authentication via an optional <c>httpClientProvider</c> callback that
/// returns a pre-configured <see cref="HttpClient"/> for a given request (e.g. authenticated, custom handler).
/// When the provider returns <see langword="null"/>, or no provider is supplied, a shared internal <see cref="HttpClient"/>
/// is used.
/// </para>
/// <para>
/// The handler applies the per-request <see cref="HttpRequestInfo.Timeout"/> using a linked <see cref="CancellationTokenSource"/>
/// so it does not mutate <see cref="HttpClient.Timeout"/> on shared instances.
/// </para>
/// </remarks>
public sealed class DefaultHttpRequestHandler : IHttpRequestHandler, IAsyncDisposable
{
private readonly Func<HttpRequestInfo, CancellationToken, Task<HttpClient?>>? _httpClientProvider;
private readonly Lazy<HttpClient> _ownedHttpClient;
/// <summary>
/// Initializes a new instance of the <see cref="DefaultHttpRequestHandler"/> class that uses an
/// internally owned <see cref="HttpClient"/> for all requests. The internal client is disposed
/// when <see cref="DisposeAsync"/> is called.
/// </summary>
public DefaultHttpRequestHandler()
: this(httpClientProvider: null)
{
}
/// <summary>
/// Initializes a new instance of the <see cref="DefaultHttpRequestHandler"/> class that uses the
/// supplied <see cref="HttpClient"/> for all requests.
/// </summary>
/// <param name="httpClient">
/// The <see cref="HttpClient"/> to use for all requests. The caller retains ownership of this
/// instance; it is not disposed by <see cref="DisposeAsync"/>.
/// </param>
/// <exception cref="ArgumentNullException"><paramref name="httpClient"/> is <see langword="null"/>.</exception>
public DefaultHttpRequestHandler(HttpClient httpClient)
: this(CreateSingleClientProvider(httpClient))
{
}
/// <summary>
/// Initializes a new instance of the <see cref="DefaultHttpRequestHandler"/> class that selects
/// an <see cref="HttpClient"/> per request via a caller-supplied callback — for example, to route
/// different URLs through differently authenticated clients.
/// </summary>
/// <param name="httpClientProvider">
/// An optional callback invoked for each request. The callback receives the <see cref="HttpRequestInfo"/>
/// and should return a pre-configured <see cref="HttpClient"/> (e.g. with authentication or a custom
/// transport). Return <see langword="null"/> to fall back to the handler's shared internal
/// <see cref="HttpClient"/>.
/// </param>
/// <remarks>
/// <para>
/// <b>Ownership</b>: the caller is solely responsible for the lifetime of clients returned by this
/// callback. <see cref="DefaultHttpRequestHandler"/> will <b>not</b> dispose provider-returned
/// clients; only the handler's internally owned fallback client is disposed by <see cref="DisposeAsync"/>.
/// </para>
/// <para>
/// <b>Reuse</b>: callers are expected to cache and reuse clients (for example, keyed by base URL or
/// auth scope) across requests. Returning a newly allocated <see cref="HttpClient"/> on every
/// invocation will leak sockets and handler resources.
/// </para>
/// </remarks>
public DefaultHttpRequestHandler(Func<HttpRequestInfo, CancellationToken, Task<HttpClient?>>? httpClientProvider)
{
this._httpClientProvider = httpClientProvider;
this._ownedHttpClient = new Lazy<HttpClient>(() => new HttpClient(), LazyThreadSafetyMode.ExecutionAndPublication);
}
private static Func<HttpRequestInfo, CancellationToken, Task<HttpClient?>> CreateSingleClientProvider(HttpClient httpClient)
{
if (httpClient is null)
{
throw new ArgumentNullException(nameof(httpClient));
}
return (_, _) => Task.FromResult<HttpClient?>(httpClient);
}
/// <inheritdoc/>
public async Task<HttpRequestResult> SendAsync(HttpRequestInfo request, CancellationToken cancellationToken = default)
{
if (request is null)
{
throw new ArgumentNullException(nameof(request));
}
if (string.IsNullOrWhiteSpace(request.Url))
{
throw new ArgumentException("Request URL must be provided.", nameof(request));
}
if (string.IsNullOrWhiteSpace(request.Method))
{
throw new ArgumentException("Request method must be provided.", nameof(request));
}
HttpClient? providedClient = null;
if (this._httpClientProvider is not null)
{
providedClient = await this._httpClientProvider(request, cancellationToken).ConfigureAwait(false);
}
HttpClient client = providedClient ?? this._ownedHttpClient.Value;
using HttpRequestMessage httpRequest = BuildHttpRequestMessage(request);
using CancellationTokenSource? timeoutCts = request.Timeout is { } timeout && timeout > TimeSpan.Zero
? CancellationTokenSource.CreateLinkedTokenSource(cancellationToken)
: null;
timeoutCts?.CancelAfter(request.Timeout!.Value);
CancellationToken effectiveToken = timeoutCts?.Token ?? cancellationToken;
using HttpResponseMessage httpResponse = await client
.SendAsync(httpRequest, HttpCompletionOption.ResponseContentRead, effectiveToken)
.ConfigureAwait(false);
string? body = httpResponse.Content is null
? null
#if NET
: await httpResponse.Content.ReadAsStringAsync(effectiveToken).ConfigureAwait(false);
#else
: await httpResponse.Content.ReadAsStringAsync().ConfigureAwait(false);
#endif
Dictionary<string, IReadOnlyList<string>> headers = new(StringComparer.OrdinalIgnoreCase);
AppendHeaders(headers, httpResponse.Headers);
if (httpResponse.Content is not null)
{
AppendHeaders(headers, httpResponse.Content.Headers);
}
return new HttpRequestResult
{
StatusCode = (int)httpResponse.StatusCode,
IsSuccessStatusCode = httpResponse.IsSuccessStatusCode,
Body = body,
Headers = headers,
};
}
/// <inheritdoc/>
public ValueTask DisposeAsync()
{
if (this._ownedHttpClient.IsValueCreated)
{
this._ownedHttpClient.Value.Dispose();
}
return default;
}
private static HttpRequestMessage BuildHttpRequestMessage(HttpRequestInfo request)
{
HttpMethod method = ResolveMethod(request.Method);
string requestUri = ResolveRequestUri(request);
HttpRequestMessage httpRequest = new(method, requestUri);
if (request.Body is not null)
{
string contentType = string.IsNullOrWhiteSpace(request.BodyContentType)
? "text/plain"
: request.BodyContentType!;
httpRequest.Content = new StringContent(request.Body, Encoding.UTF8);
// Replace the default content-type header (including charset) with the declared type.
httpRequest.Content.Headers.Remove("Content-Type");
httpRequest.Content.Headers.TryAddWithoutValidation("Content-Type", contentType);
}
if (request.Headers is not null)
{
foreach (KeyValuePair<string, string> header in request.Headers)
{
if (string.IsNullOrEmpty(header.Key))
{
continue;
}
// Content-* headers belong on HttpContent; all others belong on the request.
if (header.Key.StartsWith("Content-", StringComparison.OrdinalIgnoreCase) && httpRequest.Content is not null)
{
httpRequest.Content.Headers.Remove(header.Key);
httpRequest.Content.Headers.TryAddWithoutValidation(header.Key, header.Value);
continue;
}
if (!httpRequest.Headers.TryAddWithoutValidation(header.Key, header.Value))
{
httpRequest.Content?.Headers.TryAddWithoutValidation(header.Key, header.Value);
}
}
}
return httpRequest;
}
private static HttpMethod ResolveMethod(string method)
{
string normalized = method.Trim().ToUpperInvariant();
return normalized switch
{
"GET" => HttpMethod.Get,
"POST" => HttpMethod.Post,
"PUT" => HttpMethod.Put,
"DELETE" => HttpMethod.Delete,
#if NET
"PATCH" => HttpMethod.Patch,
#else
"PATCH" => new HttpMethod("PATCH"),
#endif
_ => new HttpMethod(normalized),
};
}
private static string ResolveRequestUri(HttpRequestInfo request)
{
string baseUrl = request.Url;
if (request.QueryParameters is null || request.QueryParameters.Count == 0)
{
return baseUrl;
}
StringBuilder queryBuilder = new();
foreach (KeyValuePair<string, string> parameter in request.QueryParameters)
{
if (string.IsNullOrEmpty(parameter.Key))
{
continue;
}
if (queryBuilder.Length > 0)
{
queryBuilder.Append('&');
}
queryBuilder.Append(Uri.EscapeDataString(parameter.Key))
.Append('=')
.Append(Uri.EscapeDataString(parameter.Value ?? string.Empty));
}
if (queryBuilder.Length == 0)
{
return baseUrl;
}
char separator = baseUrl.Contains('?') ? '&' : '?';
return string.Concat(baseUrl, separator.ToString(), queryBuilder.ToString());
}
private static void AppendHeaders(
Dictionary<string, IReadOnlyList<string>> target,
System.Net.Http.Headers.HttpHeaders source)
{
foreach (KeyValuePair<string, IEnumerable<string>> header in source)
{
string[] values = header.Value.ToArray();
if (target.TryGetValue(header.Key, out IReadOnlyList<string>? existing))
{
List<string> combined = new(existing);
combined.AddRange(values);
target[header.Key] = combined;
}
else
{
target[header.Key] = values;
}
}
}
}
@@ -1,103 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using System.Threading;
using System.Threading.Tasks;
namespace Microsoft.Agents.AI.Workflows.Declarative;
/// <summary>
/// Defines the contract for executing HTTP requests emitted by <c>HttpRequestAction</c> within declarative workflows.
/// </summary>
/// <remarks>
/// This interface allows the HTTP request dispatch to be abstracted, enabling different implementations
/// for local development, hosted workflows, authenticated scenarios, and testing.
/// </remarks>
public interface IHttpRequestHandler
{
/// <summary>
/// Sends an HTTP request and returns the response.
/// </summary>
/// <param name="request">The HTTP request to send.</param>
/// <param name="cancellationToken">A token to observe cancellation.</param>
/// <returns>The <see cref="HttpRequestResult"/> describing the HTTP response.</returns>
Task<HttpRequestResult> SendAsync(
HttpRequestInfo request,
CancellationToken cancellationToken = default);
}
/// <summary>
/// Describes an HTTP request to be sent by an <see cref="IHttpRequestHandler"/>.
/// </summary>
[SuppressMessage("Design", "CA1056:URI-like properties should not be strings", Justification = "URL is carried as a string to preserve the declarative expression result and to avoid forcing handler implementations to construct a Uri eagerly.")]
public sealed class HttpRequestInfo
{
/// <summary>
/// Gets the HTTP method to use (GET, POST, PUT, PATCH, DELETE).
/// </summary>
public string Method { get; init; } = "GET";
/// <summary>
/// Gets the absolute URL to send the request to.
/// </summary>
public string Url { get; init; } = string.Empty;
/// <summary>
/// Gets the headers to include on the request, excluding the <c>Content-Type</c> header (which is supplied via <see cref="BodyContentType"/>).
/// </summary>
public IReadOnlyDictionary<string, string>? Headers { get; init; }
/// <summary>
/// Gets the <c>Content-Type</c> of the request body, or <see langword="null"/> if no body is sent.
/// </summary>
public string? BodyContentType { get; init; }
/// <summary>
/// Gets the serialized request body, or <see langword="null"/> if no body is sent.
/// </summary>
public string? Body { get; init; }
/// <summary>
/// Gets the maximum amount of time to wait for the request to complete, or <see langword="null"/> to use the handler default.
/// </summary>
public TimeSpan? Timeout { get; init; }
/// <summary>
/// Gets the query parameters to append to the request URL, with values already formatted as strings.
/// </summary>
public IReadOnlyDictionary<string, string>? QueryParameters { get; init; }
/// <summary>
/// Gets the name of the declared remote connection, or <see langword="null"/> if no connection is declared.
/// This maps to the Foundry project connection Id and is only used when running in foundry service.
/// </summary>
public string? ConnectionName { get; init; }
}
/// <summary>
/// Represents the result of an HTTP request executed by an <see cref="IHttpRequestHandler"/>.
/// </summary>
public sealed class HttpRequestResult
{
/// <summary>
/// Gets the HTTP status code returned by the server.
/// </summary>
public int StatusCode { get; init; }
/// <summary>
/// Gets a value indicating whether the status code is in the range 200-299.
/// </summary>
public bool IsSuccessStatusCode { get; init; }
/// <summary>
/// Gets the response body, or <see langword="null"/> if no body was returned.
/// </summary>
public string? Body { get; init; }
/// <summary>
/// Gets the response headers keyed by header name. Each header may have multiple values.
/// </summary>
public IReadOnlyDictionary<string, IReadOnlyList<string>>? Headers { get; init; }
}
@@ -529,18 +529,6 @@ internal sealed class WorkflowActionVisitor : DialogActionVisitor
this._workflowModel.AddNode(new DelegateActionExecutor(postId, this._workflowState, action.CompleteAsync), action.ParentId);
}
protected override void Visit(HttpRequestAction item)
{
this.Trace(item);
if (this._workflowOptions.HttpRequestHandler is null)
{
throw new DeclarativeModelException("HTTP request handler not configured. Set HttpRequestHandler in DeclarativeWorkflowOptions to use HttpRequestAction actions.");
}
this.ContinueWith(new HttpRequestExecutor(item, this._workflowOptions.HttpRequestHandler, this._workflowOptions.AgentProvider, this._workflowState));
}
#region Not supported
protected override void Visit(AnswerQuestionWithAI item) => this.NotSupported(item);
@@ -585,6 +573,8 @@ internal sealed class WorkflowActionVisitor : DialogActionVisitor
protected override void Visit(GetConversationMembers item) => this.NotSupported(item);
protected override void Visit(HttpRequestAction item) => this.NotSupported(item);
protected override void Visit(RecognizeIntent item) => this.NotSupported(item);
protected override void Visit(TransferConversation item) => this.NotSupported(item);
@@ -1,346 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
using System.Collections.Generic;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Agents.AI.Workflows.Declarative.Extensions;
using Microsoft.Agents.AI.Workflows.Declarative.Interpreter;
using Microsoft.Agents.AI.Workflows.Declarative.Kit;
using Microsoft.Agents.AI.Workflows.Declarative.PowerFx;
using Microsoft.Agents.ObjectModel;
using Microsoft.Extensions.AI;
using Microsoft.PowerFx.Types;
using Microsoft.Shared.Diagnostics;
namespace Microsoft.Agents.AI.Workflows.Declarative.ObjectModel;
/// <summary>
/// Executor for the <see cref="HttpRequestAction"/> action.
/// Dispatches the request through the configured <see cref="IHttpRequestHandler"/> and assigns
/// the response body and headers to the declared property paths.
/// </summary>
internal sealed class HttpRequestExecutor(
HttpRequestAction model,
IHttpRequestHandler httpRequestHandler,
ResponseAgentProvider agentProvider,
WorkflowFormulaState state) :
DeclarativeActionExecutor<HttpRequestAction>(model, state)
{
/// <inheritdoc/>
protected override async ValueTask<object?> ExecuteAsync(IWorkflowContext context, CancellationToken cancellationToken = default)
{
string method = this.GetMethod();
string url = this.GetUrl();
Dictionary<string, string>? headers = this.GetHeaders();
Dictionary<string, string>? queryParameters = this.GetQueryParameters();
(string? body, string? contentType) = this.GetBody();
TimeSpan? timeout = this.GetTimeout();
string? conversationId = this.GetConversationId();
string? connectionName = this.GetConnectionName();
HttpRequestInfo requestInfo = new()
{
Method = method,
Url = url,
Headers = headers,
QueryParameters = queryParameters,
Body = body,
BodyContentType = contentType,
Timeout = timeout,
ConnectionName = connectionName,
};
HttpRequestResult result;
try
{
result = await httpRequestHandler.SendAsync(requestInfo, cancellationToken).ConfigureAwait(false);
}
catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested)
{
throw this.Exception($"HTTP request to '{url}' timed out.");
}
catch (Exception exception) when (exception is not DeclarativeActionException)
{
throw this.Exception($"HTTP request to '{url}' failed: {exception.Message}", exception);
}
if (result.IsSuccessStatusCode)
{
await this.AssignResponseAsync(context, result.Body).ConfigureAwait(false);
await this.AssignResponseHeadersAsync(context, result.Headers).ConfigureAwait(false);
await this.AddResponseToConversationAsync(conversationId, result.Body, cancellationToken).ConfigureAwait(false);
return default;
}
// Non-success status code - throw.
// Also publish response headers for diagnostic purposes.
await this.AssignResponseHeadersAsync(context, result.Headers).ConfigureAwait(false);
string bodyPreview = FormatBodyForDiagnostics(result.Body);
string message = bodyPreview.Length == 0
? $"HTTP request to '{url}' failed with status code {result.StatusCode}."
: $"HTTP request to '{url}' failed with status code {result.StatusCode}. Body: '{bodyPreview}'";
throw this.Exception(message);
}
// Response bodies can echo secrets (tokens, PII) and may be very large (multi-MB HTML error pages).
// Exception messages are often logged and persisted, so we clip the body to bound both exposure
// and message size. Full bodies are still available via the success path (assigned to Response).
private const int MaxBodyDiagnosticLength = 256;
private const string BodyTruncationSuffix = " \u2026 [truncated]";
private static string FormatBodyForDiagnostics(string? body)
{
if (string.IsNullOrEmpty(body))
{
return string.Empty;
}
int sourceLen = body!.Length;
bool truncated = sourceLen > MaxBodyDiagnosticLength;
int copyLen = truncated ? MaxBodyDiagnosticLength : sourceLen;
int finalLen = copyLen + (truncated ? BodyTruncationSuffix.Length : 0);
// Size the buffer for the final string so we only allocate once for the chars
// and once for the string itself. For a 10 KB error body we touch 256 chars instead of 10,000.
char[] buffer = new char[finalLen];
for (int i = 0; i < copyLen; i++)
{
char c = body[i];
buffer[i] = c is '\r' or '\n' or '\t' ? ' ' : c;
}
if (truncated)
{
BodyTruncationSuffix.CopyTo(0, buffer, copyLen, BodyTruncationSuffix.Length);
}
return new string(buffer);
}
private async ValueTask AddResponseToConversationAsync(string? conversationId, string? responseBody, CancellationToken cancellationToken)
{
if (conversationId is null || string.IsNullOrEmpty(responseBody))
{
return;
}
ChatMessage message = new(ChatRole.Assistant, responseBody);
await agentProvider.CreateMessageAsync(conversationId, message, cancellationToken).ConfigureAwait(false);
}
private async ValueTask AssignResponseAsync(IWorkflowContext context, string? responseBody)
{
if (this.Model.Response is not { Path: { } responsePath })
{
return;
}
await this.AssignAsync(responsePath, ParseResponseBody(responseBody), context).ConfigureAwait(false);
}
private async ValueTask AssignResponseHeadersAsync(IWorkflowContext context, IReadOnlyDictionary<string, IReadOnlyList<string>>? responseHeaders)
{
if (this.Model.ResponseHeaders is not { Path: { } headersPath })
{
return;
}
if (responseHeaders is null || responseHeaders.Count == 0)
{
await this.AssignAsync(headersPath, FormulaValue.NewBlank(), context).ConfigureAwait(false);
return;
}
// Flatten multi-value headers by joining with commas (standard HTTP header folding).
Dictionary<string, object?> flattened = new(StringComparer.OrdinalIgnoreCase);
foreach (KeyValuePair<string, IReadOnlyList<string>> header in responseHeaders)
{
flattened[header.Key] = string.Join(",", header.Value);
}
await this.AssignAsync(headersPath, flattened.ToFormula(), context).ConfigureAwait(false);
}
private static FormulaValue ParseResponseBody(string? responseBody)
{
if (string.IsNullOrEmpty(responseBody))
{
return FormulaValue.NewBlank();
}
// Attempt to parse as JSON so records/tables are exposed naturally to the workflow.
try
{
using JsonDocument jsonDocument = JsonDocument.Parse(responseBody);
object? parsedValue = jsonDocument.RootElement.ValueKind switch
{
JsonValueKind.Object => jsonDocument.ParseRecord(VariableType.RecordType),
JsonValueKind.Array => jsonDocument.ParseList(jsonDocument.RootElement.GetListTypeFromJson()),
JsonValueKind.String => jsonDocument.RootElement.GetString(),
JsonValueKind.Number => jsonDocument.RootElement.TryGetInt64(out long l)
? l
: jsonDocument.RootElement.GetDouble(),
JsonValueKind.True => true,
JsonValueKind.False => false,
JsonValueKind.Null => null,
_ => responseBody,
};
return parsedValue.ToFormula();
}
catch (JsonException)
{
// Not valid JSON — return the raw string.
return FormulaValue.New(responseBody);
}
}
private string GetMethod()
{
EnumExpression<HttpMethodTypeWrapper>? methodExpression = this.Model.Method;
if (methodExpression is null)
{
return "GET";
}
HttpMethodTypeWrapper wrapper = this.Evaluator.GetValue(methodExpression).Value;
return !string.IsNullOrEmpty(wrapper.UnknownValue) ? wrapper.UnknownValue! : wrapper.Value.ToString().ToUpperInvariant();
}
private string GetUrl() =>
this.Evaluator.GetValue(
Throw.IfNull(
this.Model.Url,
$"{nameof(this.Model)}.{nameof(this.Model.Url)}")).Value;
private Dictionary<string, string>? GetHeaders()
{
if (this.Model.Headers is null || this.Model.Headers.Count == 0)
{
return null;
}
Dictionary<string, string> result = new(StringComparer.OrdinalIgnoreCase);
foreach (KeyValuePair<string, StringExpression> header in this.Model.Headers)
{
string value = this.Evaluator.GetValue(header.Value).Value;
if (!string.IsNullOrEmpty(value))
{
result[header.Key] = value;
}
}
return result.Count == 0 ? null : result;
}
private (string? Body, string? ContentType) GetBody()
{
switch (this.Model.Body)
{
case null:
case NoRequestContent:
return (null, null);
case JsonRequestContent jsonContent when jsonContent.Content is not null:
{
FormulaValue formula = this.Evaluator.GetValue(jsonContent.Content).Value.ToFormula();
string json = formula.ToJson().ToJsonString();
return (json, "application/json");
}
case RawRequestContent rawContent:
{
string? content = rawContent.Content is null
? null
: this.Evaluator.GetValue(rawContent.Content).Value;
string? contentType = rawContent.ContentType is null
? null
: this.Evaluator.GetValue(rawContent.ContentType).Value;
return (content, string.IsNullOrEmpty(contentType) ? null : contentType);
}
default:
return (null, null);
}
}
private TimeSpan? GetTimeout()
{
if (this.Model.RequestTimeoutInMilliseconds is null || this.Model.RequestTimeoutInMillisecondsIsDefaultValue)
{
return null;
}
long value = this.Evaluator.GetValue(this.Model.RequestTimeoutInMilliseconds).Value;
return value > 0 ? TimeSpan.FromMilliseconds(value) : null;
}
private Dictionary<string, string>? GetQueryParameters()
{
if (this.Model.QueryParameters is null || this.Model.QueryParameters.Count == 0)
{
return null;
}
Dictionary<string, string> result = new(StringComparer.Ordinal);
foreach (KeyValuePair<string, ValueExpression> parameter in this.Model.QueryParameters)
{
if (string.IsNullOrEmpty(parameter.Key) || parameter.Value is null)
{
continue;
}
object? rawValue = this.Evaluator.GetValue(parameter.Value).Value.ToObject();
string? formatted = FormatQueryValue(rawValue);
if (formatted is not null)
{
result[parameter.Key] = formatted;
}
}
return result.Count == 0 ? null : result;
}
private static string? FormatQueryValue(object? value) =>
value switch
{
null => null,
string s => s,
bool b => b ? "true" : "false",
IFormattable formattable => formattable.ToString(null, System.Globalization.CultureInfo.InvariantCulture),
_ => value.ToString(),
};
private string? GetConversationId()
{
if (this.Model.ConversationId is null)
{
return null;
}
string value = this.Evaluator.GetValue(this.Model.ConversationId).Value;
return value.Length == 0 ? null : value;
}
private string? GetConnectionName()
{
RemoteConnection? connection = this.Model.Connection;
if (connection is null)
{
return null;
}
string? name = connection.Name is null
? null
: this.Evaluator.GetValue(connection.Name).Value;
return string.IsNullOrEmpty(name) ? null : name;
}
}
@@ -1,20 +1,6 @@
<?xml version="1.0" encoding="utf-8"?>
<!-- https://learn.microsoft.com/dotnet/fundamentals/package-validation/diagnostic-ids -->
<Suppressions xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema">
<Suppression>
<DiagnosticId>CP0002</DiagnosticId>
<Target>M:Microsoft.Agents.AI.AgentFileSkillScriptRunner.BeginInvoke(Microsoft.Agents.AI.AgentFileSkill,Microsoft.Agents.AI.AgentFileSkillScript,Microsoft.Extensions.AI.AIFunctionArguments,System.Threading.CancellationToken,System.AsyncCallback,System.Object)</Target>
<Left>lib/net10.0/Microsoft.Agents.AI.dll</Left>
<Right>lib/net10.0/Microsoft.Agents.AI.dll</Right>
<IsBaselineSuppression>true</IsBaselineSuppression>
</Suppression>
<Suppression>
<DiagnosticId>CP0002</DiagnosticId>
<Target>M:Microsoft.Agents.AI.AgentFileSkillScriptRunner.Invoke(Microsoft.Agents.AI.AgentFileSkill,Microsoft.Agents.AI.AgentFileSkillScript,Microsoft.Extensions.AI.AIFunctionArguments,System.Threading.CancellationToken)</Target>
<Left>lib/net10.0/Microsoft.Agents.AI.dll</Left>
<Right>lib/net10.0/Microsoft.Agents.AI.dll</Right>
<IsBaselineSuppression>true</IsBaselineSuppression>
</Suppression>
<Suppression>
<DiagnosticId>CP0002</DiagnosticId>
<Target>M:Microsoft.Agents.AI.AgentInlineSkill.#ctor(Microsoft.Agents.AI.AgentSkillFrontmatter,System.String)</Target>
@@ -43,13 +29,6 @@
<Right>lib/net10.0/Microsoft.Agents.AI.dll</Right>
<IsBaselineSuppression>true</IsBaselineSuppression>
</Suppression>
<Suppression>
<DiagnosticId>CP0002</DiagnosticId>
<Target>M:Microsoft.Agents.AI.AgentSkillScript.RunAsync(Microsoft.Agents.AI.AgentSkill,Microsoft.Extensions.AI.AIFunctionArguments,System.Threading.CancellationToken)</Target>
<Left>lib/net10.0/Microsoft.Agents.AI.dll</Left>
<Right>lib/net10.0/Microsoft.Agents.AI.dll</Right>
<IsBaselineSuppression>true</IsBaselineSuppression>
</Suppression>
<Suppression>
<DiagnosticId>CP0002</DiagnosticId>
<Target>M:Microsoft.Agents.AI.AgentSkillsProvider.#ctor(Microsoft.Agents.AI.AgentInlineSkill[])</Target>
@@ -64,20 +43,6 @@
<Right>lib/net10.0/Microsoft.Agents.AI.dll</Right>
<IsBaselineSuppression>true</IsBaselineSuppression>
</Suppression>
<Suppression>
<DiagnosticId>CP0002</DiagnosticId>
<Target>M:Microsoft.Agents.AI.AgentFileSkillScriptRunner.BeginInvoke(Microsoft.Agents.AI.AgentFileSkill,Microsoft.Agents.AI.AgentFileSkillScript,Microsoft.Extensions.AI.AIFunctionArguments,System.Threading.CancellationToken,System.AsyncCallback,System.Object)</Target>
<Left>lib/net472/Microsoft.Agents.AI.dll</Left>
<Right>lib/net472/Microsoft.Agents.AI.dll</Right>
<IsBaselineSuppression>true</IsBaselineSuppression>
</Suppression>
<Suppression>
<DiagnosticId>CP0002</DiagnosticId>
<Target>M:Microsoft.Agents.AI.AgentFileSkillScriptRunner.Invoke(Microsoft.Agents.AI.AgentFileSkill,Microsoft.Agents.AI.AgentFileSkillScript,Microsoft.Extensions.AI.AIFunctionArguments,System.Threading.CancellationToken)</Target>
<Left>lib/net472/Microsoft.Agents.AI.dll</Left>
<Right>lib/net472/Microsoft.Agents.AI.dll</Right>
<IsBaselineSuppression>true</IsBaselineSuppression>
</Suppression>
<Suppression>
<DiagnosticId>CP0002</DiagnosticId>
<Target>M:Microsoft.Agents.AI.AgentInlineSkill.#ctor(Microsoft.Agents.AI.AgentSkillFrontmatter,System.String)</Target>
@@ -106,13 +71,6 @@
<Right>lib/net472/Microsoft.Agents.AI.dll</Right>
<IsBaselineSuppression>true</IsBaselineSuppression>
</Suppression>
<Suppression>
<DiagnosticId>CP0002</DiagnosticId>
<Target>M:Microsoft.Agents.AI.AgentSkillScript.RunAsync(Microsoft.Agents.AI.AgentSkill,Microsoft.Extensions.AI.AIFunctionArguments,System.Threading.CancellationToken)</Target>
<Left>lib/net472/Microsoft.Agents.AI.dll</Left>
<Right>lib/net472/Microsoft.Agents.AI.dll</Right>
<IsBaselineSuppression>true</IsBaselineSuppression>
</Suppression>
<Suppression>
<DiagnosticId>CP0002</DiagnosticId>
<Target>M:Microsoft.Agents.AI.AgentSkillsProvider.#ctor(Microsoft.Agents.AI.AgentInlineSkill[])</Target>
@@ -127,20 +85,6 @@
<Right>lib/net472/Microsoft.Agents.AI.dll</Right>
<IsBaselineSuppression>true</IsBaselineSuppression>
</Suppression>
<Suppression>
<DiagnosticId>CP0002</DiagnosticId>
<Target>M:Microsoft.Agents.AI.AgentFileSkillScriptRunner.BeginInvoke(Microsoft.Agents.AI.AgentFileSkill,Microsoft.Agents.AI.AgentFileSkillScript,Microsoft.Extensions.AI.AIFunctionArguments,System.Threading.CancellationToken,System.AsyncCallback,System.Object)</Target>
<Left>lib/net8.0/Microsoft.Agents.AI.dll</Left>
<Right>lib/net8.0/Microsoft.Agents.AI.dll</Right>
<IsBaselineSuppression>true</IsBaselineSuppression>
</Suppression>
<Suppression>
<DiagnosticId>CP0002</DiagnosticId>
<Target>M:Microsoft.Agents.AI.AgentFileSkillScriptRunner.Invoke(Microsoft.Agents.AI.AgentFileSkill,Microsoft.Agents.AI.AgentFileSkillScript,Microsoft.Extensions.AI.AIFunctionArguments,System.Threading.CancellationToken)</Target>
<Left>lib/net8.0/Microsoft.Agents.AI.dll</Left>
<Right>lib/net8.0/Microsoft.Agents.AI.dll</Right>
<IsBaselineSuppression>true</IsBaselineSuppression>
</Suppression>
<Suppression>
<DiagnosticId>CP0002</DiagnosticId>
<Target>M:Microsoft.Agents.AI.AgentInlineSkill.#ctor(Microsoft.Agents.AI.AgentSkillFrontmatter,System.String)</Target>
@@ -169,13 +113,6 @@
<Right>lib/net8.0/Microsoft.Agents.AI.dll</Right>
<IsBaselineSuppression>true</IsBaselineSuppression>
</Suppression>
<Suppression>
<DiagnosticId>CP0002</DiagnosticId>
<Target>M:Microsoft.Agents.AI.AgentSkillScript.RunAsync(Microsoft.Agents.AI.AgentSkill,Microsoft.Extensions.AI.AIFunctionArguments,System.Threading.CancellationToken)</Target>
<Left>lib/net8.0/Microsoft.Agents.AI.dll</Left>
<Right>lib/net8.0/Microsoft.Agents.AI.dll</Right>
<IsBaselineSuppression>true</IsBaselineSuppression>
</Suppression>
<Suppression>
<DiagnosticId>CP0002</DiagnosticId>
<Target>M:Microsoft.Agents.AI.AgentSkillsProvider.#ctor(Microsoft.Agents.AI.AgentInlineSkill[])</Target>
@@ -190,20 +127,6 @@
<Right>lib/net8.0/Microsoft.Agents.AI.dll</Right>
<IsBaselineSuppression>true</IsBaselineSuppression>
</Suppression>
<Suppression>
<DiagnosticId>CP0002</DiagnosticId>
<Target>M:Microsoft.Agents.AI.AgentFileSkillScriptRunner.BeginInvoke(Microsoft.Agents.AI.AgentFileSkill,Microsoft.Agents.AI.AgentFileSkillScript,Microsoft.Extensions.AI.AIFunctionArguments,System.Threading.CancellationToken,System.AsyncCallback,System.Object)</Target>
<Left>lib/net9.0/Microsoft.Agents.AI.dll</Left>
<Right>lib/net9.0/Microsoft.Agents.AI.dll</Right>
<IsBaselineSuppression>true</IsBaselineSuppression>
</Suppression>
<Suppression>
<DiagnosticId>CP0002</DiagnosticId>
<Target>M:Microsoft.Agents.AI.AgentFileSkillScriptRunner.Invoke(Microsoft.Agents.AI.AgentFileSkill,Microsoft.Agents.AI.AgentFileSkillScript,Microsoft.Extensions.AI.AIFunctionArguments,System.Threading.CancellationToken)</Target>
<Left>lib/net9.0/Microsoft.Agents.AI.dll</Left>
<Right>lib/net9.0/Microsoft.Agents.AI.dll</Right>
<IsBaselineSuppression>true</IsBaselineSuppression>
</Suppression>
<Suppression>
<DiagnosticId>CP0002</DiagnosticId>
<Target>M:Microsoft.Agents.AI.AgentInlineSkill.#ctor(Microsoft.Agents.AI.AgentSkillFrontmatter,System.String)</Target>
@@ -232,13 +155,6 @@
<Right>lib/net9.0/Microsoft.Agents.AI.dll</Right>
<IsBaselineSuppression>true</IsBaselineSuppression>
</Suppression>
<Suppression>
<DiagnosticId>CP0002</DiagnosticId>
<Target>M:Microsoft.Agents.AI.AgentSkillScript.RunAsync(Microsoft.Agents.AI.AgentSkill,Microsoft.Extensions.AI.AIFunctionArguments,System.Threading.CancellationToken)</Target>
<Left>lib/net9.0/Microsoft.Agents.AI.dll</Left>
<Right>lib/net9.0/Microsoft.Agents.AI.dll</Right>
<IsBaselineSuppression>true</IsBaselineSuppression>
</Suppression>
<Suppression>
<DiagnosticId>CP0002</DiagnosticId>
<Target>M:Microsoft.Agents.AI.AgentSkillsProvider.#ctor(Microsoft.Agents.AI.AgentInlineSkill[])</Target>
@@ -253,20 +169,6 @@
<Right>lib/net9.0/Microsoft.Agents.AI.dll</Right>
<IsBaselineSuppression>true</IsBaselineSuppression>
</Suppression>
<Suppression>
<DiagnosticId>CP0002</DiagnosticId>
<Target>M:Microsoft.Agents.AI.AgentFileSkillScriptRunner.BeginInvoke(Microsoft.Agents.AI.AgentFileSkill,Microsoft.Agents.AI.AgentFileSkillScript,Microsoft.Extensions.AI.AIFunctionArguments,System.Threading.CancellationToken,System.AsyncCallback,System.Object)</Target>
<Left>lib/netstandard2.0/Microsoft.Agents.AI.dll</Left>
<Right>lib/netstandard2.0/Microsoft.Agents.AI.dll</Right>
<IsBaselineSuppression>true</IsBaselineSuppression>
</Suppression>
<Suppression>
<DiagnosticId>CP0002</DiagnosticId>
<Target>M:Microsoft.Agents.AI.AgentFileSkillScriptRunner.Invoke(Microsoft.Agents.AI.AgentFileSkill,Microsoft.Agents.AI.AgentFileSkillScript,Microsoft.Extensions.AI.AIFunctionArguments,System.Threading.CancellationToken)</Target>
<Left>lib/netstandard2.0/Microsoft.Agents.AI.dll</Left>
<Right>lib/netstandard2.0/Microsoft.Agents.AI.dll</Right>
<IsBaselineSuppression>true</IsBaselineSuppression>
</Suppression>
<Suppression>
<DiagnosticId>CP0002</DiagnosticId>
<Target>M:Microsoft.Agents.AI.AgentInlineSkill.#ctor(Microsoft.Agents.AI.AgentSkillFrontmatter,System.String)</Target>
@@ -295,13 +197,6 @@
<Right>lib/netstandard2.0/Microsoft.Agents.AI.dll</Right>
<IsBaselineSuppression>true</IsBaselineSuppression>
</Suppression>
<Suppression>
<DiagnosticId>CP0002</DiagnosticId>
<Target>M:Microsoft.Agents.AI.AgentSkillScript.RunAsync(Microsoft.Agents.AI.AgentSkill,Microsoft.Extensions.AI.AIFunctionArguments,System.Threading.CancellationToken)</Target>
<Left>lib/netstandard2.0/Microsoft.Agents.AI.dll</Left>
<Right>lib/netstandard2.0/Microsoft.Agents.AI.dll</Right>
<IsBaselineSuppression>true</IsBaselineSuppression>
</Suppression>
<Suppression>
<DiagnosticId>CP0002</DiagnosticId>
<Target>M:Microsoft.Agents.AI.AgentSkillsProvider.#ctor(Microsoft.Agents.AI.AgentInlineSkill[])</Target>
@@ -316,39 +211,4 @@
<Right>lib/netstandard2.0/Microsoft.Agents.AI.dll</Right>
<IsBaselineSuppression>true</IsBaselineSuppression>
</Suppression>
<Suppression>
<DiagnosticId>CP0005</DiagnosticId>
<Target>M:Microsoft.Agents.AI.AgentSkillScript.RunAsync(Microsoft.Agents.AI.AgentSkill,System.Nullable{System.Text.Json.JsonElement},System.IServiceProvider,System.Threading.CancellationToken)</Target>
<Left>lib/net10.0/Microsoft.Agents.AI.dll</Left>
<Right>lib/net10.0/Microsoft.Agents.AI.dll</Right>
<IsBaselineSuppression>true</IsBaselineSuppression>
</Suppression>
<Suppression>
<DiagnosticId>CP0005</DiagnosticId>
<Target>M:Microsoft.Agents.AI.AgentSkillScript.RunAsync(Microsoft.Agents.AI.AgentSkill,System.Nullable{System.Text.Json.JsonElement},System.IServiceProvider,System.Threading.CancellationToken)</Target>
<Left>lib/net472/Microsoft.Agents.AI.dll</Left>
<Right>lib/net472/Microsoft.Agents.AI.dll</Right>
<IsBaselineSuppression>true</IsBaselineSuppression>
</Suppression>
<Suppression>
<DiagnosticId>CP0005</DiagnosticId>
<Target>M:Microsoft.Agents.AI.AgentSkillScript.RunAsync(Microsoft.Agents.AI.AgentSkill,System.Nullable{System.Text.Json.JsonElement},System.IServiceProvider,System.Threading.CancellationToken)</Target>
<Left>lib/net8.0/Microsoft.Agents.AI.dll</Left>
<Right>lib/net8.0/Microsoft.Agents.AI.dll</Right>
<IsBaselineSuppression>true</IsBaselineSuppression>
</Suppression>
<Suppression>
<DiagnosticId>CP0005</DiagnosticId>
<Target>M:Microsoft.Agents.AI.AgentSkillScript.RunAsync(Microsoft.Agents.AI.AgentSkill,System.Nullable{System.Text.Json.JsonElement},System.IServiceProvider,System.Threading.CancellationToken)</Target>
<Left>lib/net9.0/Microsoft.Agents.AI.dll</Left>
<Right>lib/net9.0/Microsoft.Agents.AI.dll</Right>
<IsBaselineSuppression>true</IsBaselineSuppression>
</Suppression>
<Suppression>
<DiagnosticId>CP0005</DiagnosticId>
<Target>M:Microsoft.Agents.AI.AgentSkillScript.RunAsync(Microsoft.Agents.AI.AgentSkill,System.Nullable{System.Text.Json.JsonElement},System.IServiceProvider,System.Threading.CancellationToken)</Target>
<Left>lib/netstandard2.0/Microsoft.Agents.AI.dll</Left>
<Right>lib/netstandard2.0/Microsoft.Agents.AI.dll</Right>
<IsBaselineSuppression>true</IsBaselineSuppression>
</Suppression>
</Suppressions>
@@ -35,8 +35,7 @@ public abstract class AgentSkill
/// Gets the full skill content.
/// </summary>
/// <remarks>
/// For file-based skills this is the raw SKILL.md file content, optionally
/// augmented with a synthesized scripts block when scripts are present.
/// For file-based skills this is the raw SKILL.md file content.
/// For code-defined skills this is a synthesized XML document
/// containing name, description, and body (instructions, resources, scripts).
/// </remarks>
@@ -1,10 +1,10 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
using System.Diagnostics.CodeAnalysis;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.AI;
using Microsoft.Shared.DiagnosticIds;
using Microsoft.Shared.Diagnostics;
@@ -46,9 +46,8 @@ public abstract class AgentSkillScript
/// Runs the script with the given arguments.
/// </summary>
/// <param name="skill">The skill that owns this script.</param>
/// <param name="arguments">Raw JSON arguments for script execution, preserving the original format (object or array) sent by the caller.</param>
/// <param name="serviceProvider">Optional service provider for dependency injection.</param>
/// <param name="arguments">Arguments for script execution.</param>
/// <param name="cancellationToken">Cancellation token.</param>
/// <returns>The script execution result.</returns>
public abstract Task<object?> RunAsync(AgentSkill skill, JsonElement? arguments, IServiceProvider? serviceProvider, CancellationToken cancellationToken = default);
public abstract Task<object?> RunAsync(AgentSkill skill, AIFunctionArguments arguments, CancellationToken cancellationToken = default);
}
@@ -6,7 +6,6 @@ using System.Diagnostics.CodeAnalysis;
using System.Linq;
using System.Security;
using System.Text;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.AI;
@@ -244,7 +243,7 @@ public sealed partial class AgentSkillsProvider : AIContextProvider
}
AIFunction scriptFunction = AIFunctionFactory.Create(
(string skillName, string scriptName, JsonElement? arguments = null, IServiceProvider? serviceProvider = null, CancellationToken cancellationToken = default) =>
(string skillName, string scriptName, IDictionary<string, object?>? arguments = null, IServiceProvider? serviceProvider = null, CancellationToken cancellationToken = default) =>
this.RunSkillScriptAsync(skills, skillName, scriptName, arguments, serviceProvider, cancellationToken),
name: "run_skill_script",
description: "Runs a script associated with a skill.");
@@ -341,7 +340,7 @@ public sealed partial class AgentSkillsProvider : AIContextProvider
}
}
private async Task<object?> RunSkillScriptAsync(IList<AgentSkill> skills, string skillName, string scriptName, JsonElement? arguments = null, IServiceProvider? serviceProvider = null, CancellationToken cancellationToken = default)
private async Task<object?> RunSkillScriptAsync(IList<AgentSkill> skills, string skillName, string scriptName, IDictionary<string, object?>? arguments = null, IServiceProvider? serviceProvider = null, CancellationToken cancellationToken = default)
{
if (string.IsNullOrWhiteSpace(skillName))
{
@@ -367,7 +366,7 @@ public sealed partial class AgentSkillsProvider : AIContextProvider
try
{
return await script.RunAsync(skill, arguments, serviceProvider, cancellationToken).ConfigureAwait(false);
return await script.RunAsync(skill, new AIFunctionArguments(arguments) { Services = serviceProvider }, cancellationToken).ConfigureAwait(false);
}
catch (Exception ex)
{
@@ -15,8 +15,6 @@ public sealed class AgentFileSkill : AgentSkill
{
private readonly IReadOnlyList<AgentSkillResource> _resources;
private readonly IReadOnlyList<AgentSkillScript> _scripts;
private readonly string _originalContent;
private string? _content;
/// <summary>
/// Initializes a new instance of the <see cref="AgentFileSkill"/> class.
@@ -34,7 +32,7 @@ public sealed class AgentFileSkill : AgentSkill
IReadOnlyList<AgentSkillScript>? scripts = null)
{
this.Frontmatter = Throw.IfNull(frontmatter);
this._originalContent = Throw.IfNull(content);
this.Content = Throw.IfNull(content);
this.Path = Throw.IfNullOrWhitespace(path);
this._resources = resources ?? [];
this._scripts = scripts ?? [];
@@ -44,18 +42,7 @@ public sealed class AgentFileSkill : AgentSkill
public override AgentSkillFrontmatter Frontmatter { get; }
/// <inheritdoc/>
/// <remarks>
/// Returns the raw SKILL.md content. When the skill has scripts, a
/// <c>&lt;scripts&gt;&lt;script name="..."&gt;&lt;parameters_schema&gt;...&lt;/parameters_schema&gt;&lt;/script&gt;&lt;/scripts&gt;</c>
/// block is appended with a per-script entry describing the expected argument format.
/// The result is cached after the first access.
/// </remarks>
public override string Content
{
get => this._content ??= this._scripts is { Count: > 0 }
? this._originalContent + AgentInlineSkillContentBuilder.BuildScriptsBlock(this._scripts)
: this._originalContent;
}
public override string Content { get; }
/// <summary>
/// Gets the directory path where the skill was discovered.
@@ -2,9 +2,9 @@
using System;
using System.Diagnostics.CodeAnalysis;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.AI;
using Microsoft.Shared.DiagnosticIds;
using Microsoft.Shared.Diagnostics;
@@ -16,11 +16,6 @@ namespace Microsoft.Agents.AI;
[Experimental(DiagnosticIds.Experiments.AgentsAIExperiments)]
public sealed class AgentFileSkillScript : AgentSkillScript
{
/// <summary>
/// Cached JSON schema element describing the expected argument format: a string array of CLI arguments.
/// </summary>
private static readonly JsonElement s_defaultSchema = CreateDefaultSchema();
private readonly AgentFileSkillScriptRunner? _runner;
/// <summary>
@@ -42,14 +37,7 @@ public sealed class AgentFileSkillScript : AgentSkillScript
public string FullPath { get; }
/// <inheritdoc/>
/// <remarks>
/// Returns a fixed schema describing a string array of CLI arguments:
/// <c>{"type":"array","items":{"type":"string"}}</c>.
/// </remarks>
public override JsonElement? ParametersSchema => s_defaultSchema;
/// <inheritdoc/>
public override async Task<object?> RunAsync(AgentSkill skill, JsonElement? arguments, IServiceProvider? serviceProvider, CancellationToken cancellationToken = default)
public override async Task<object?> RunAsync(AgentSkill skill, AIFunctionArguments arguments, CancellationToken cancellationToken = default)
{
if (skill is not AgentFileSkill fileSkill)
{
@@ -63,12 +51,6 @@ public sealed class AgentFileSkillScript : AgentSkillScript
$"Supply a script runner when constructing {nameof(AgentFileSkillsSource)} to enable script execution.");
}
return await this._runner(fileSkill, this, arguments, serviceProvider, cancellationToken).ConfigureAwait(false);
}
private static JsonElement CreateDefaultSchema()
{
using JsonDocument document = JsonDocument.Parse("""{"type":"array","items":{"type":"string"}}""");
return document.RootElement.Clone();
return await this._runner(fileSkill, this, arguments, cancellationToken).ConfigureAwait(false);
}
}
@@ -1,10 +1,9 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
using System.Diagnostics.CodeAnalysis;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.AI;
using Microsoft.Shared.DiagnosticIds;
namespace Microsoft.Agents.AI;
@@ -14,19 +13,15 @@ namespace Microsoft.Agents.AI;
/// </summary>
/// <remarks>
/// Implementations determine the execution strategy (e.g., local subprocess, hosted code execution environment).
/// The <paramref name="arguments"/> parameter preserves the raw JSON sent by the caller, in the shape
/// described by <see cref="AgentFileSkillScript.ParametersSchema"/>.
/// </remarks>
/// <param name="skill">The skill that owns the script.</param>
/// <param name="script">The file-based script to run.</param>
/// <param name="arguments">Raw JSON arguments for the script, in the shape described by <see cref="AgentFileSkillScript.ParametersSchema"/>.</param>
/// <param name="serviceProvider">Optional service provider for dependency injection.</param>
/// <param name="arguments">Optional arguments for the script, provided by the agent/LLM.</param>
/// <param name="cancellationToken">Cancellation token.</param>
/// <returns>The script execution result.</returns>
[Experimental(DiagnosticIds.Experiments.AgentsAIExperiments)]
public delegate Task<object?> AgentFileSkillScriptRunner(
AgentFileSkill skill,
AgentFileSkillScript script,
JsonElement? arguments,
IServiceProvider? serviceProvider,
AIFunctionArguments arguments,
CancellationToken cancellationToken);
@@ -59,56 +59,32 @@ internal static class AgentInlineSkillContentBuilder
if (scripts is { Count: > 0 })
{
sb.Append('\n');
sb.Append(BuildScriptsBlock(scripts));
}
return sb.ToString();
}
/// <summary>
/// Builds a <c>&lt;scripts&gt;...&lt;/scripts&gt;</c> XML block for the given scripts.
/// Each script is emitted as a <c>&lt;script name="..."&gt;</c> element with optional
/// <c>description</c> attribute and <c>&lt;parameters_schema&gt;</c> child element.
/// </summary>
/// <param name="scripts">The scripts to include in the block.</param>
/// <returns>An XML string starting with <c>\n&lt;scripts&gt;</c>, or an empty string if the list is empty.</returns>
public static string BuildScriptsBlock(IReadOnlyList<AgentSkillScript> scripts)
{
_ = Throw.IfNull(scripts);
if (scripts.Count == 0)
{
return string.Empty;
}
var sb = new StringBuilder();
sb.Append("\n<scripts>\n");
foreach (var script in scripts)
{
var parametersSchema = script.ParametersSchema;
if (script.Description is null && parametersSchema is null)
sb.Append("\n\n<scripts>\n");
foreach (var script in scripts)
{
sb.Append($" <script name=\"{EscapeXmlString(script.Name)}\"/>\n");
}
else
{
sb.Append(script.Description is not null
? $" <script name=\"{EscapeXmlString(script.Name)}\" description=\"{EscapeXmlString(script.Description)}\">\n"
: $" <script name=\"{EscapeXmlString(script.Name)}\">\n");
var parametersSchema = script.ParametersSchema;
if (parametersSchema is not null)
if (script.Description is null && parametersSchema is null)
{
sb.Append($" <parameters_schema>{EscapeXmlString(parametersSchema.Value.GetRawText(), preserveQuotes: true)}</parameters_schema>\n");
sb.Append($" <script name=\"{EscapeXmlString(script.Name)}\"/>\n");
}
else
{
sb.Append(script.Description is not null
? $" <script name=\"{EscapeXmlString(script.Name)}\" description=\"{EscapeXmlString(script.Description)}\">\n"
: $" <script name=\"{EscapeXmlString(script.Name)}\">\n");
sb.Append(" </script>\n");
if (parametersSchema is not null)
{
sb.Append($" <parameters_schema>{EscapeXmlString(parametersSchema.Value.GetRawText(), preserveQuotes: true)}</parameters_schema>\n");
}
sb.Append(" </script>\n");
}
}
}
sb.Append("</scripts>");
sb.Append("</scripts>");
}
return sb.ToString();
}
@@ -1,7 +1,6 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using System.Reflection;
using System.Text.Json;
@@ -68,42 +67,8 @@ internal sealed class AgentInlineSkillScript : AgentSkillScript
public override JsonElement? ParametersSchema => this._function.JsonSchema;
/// <inheritdoc/>
public override async Task<object?> RunAsync(AgentSkill skill, JsonElement? arguments, IServiceProvider? serviceProvider, CancellationToken cancellationToken = default)
public override async Task<object?> RunAsync(AgentSkill skill, AIFunctionArguments arguments, CancellationToken cancellationToken = default)
{
var funcArgs = ConvertToFunctionArguments(arguments);
funcArgs.Services = serviceProvider;
return await this._function.InvokeAsync(funcArgs, cancellationToken).ConfigureAwait(false);
}
/// <summary>
/// Converts a raw <see cref="JsonElement"/> to <see cref="AIFunctionArguments"/> for delegate invocation.
/// </summary>
/// <exception cref="InvalidOperationException">
/// Thrown when <paramref name="arguments"/> is provided but is not a JSON object.
/// Inline skill scripts expect arguments as a JSON object whose properties map to the delegate's parameters.
/// </exception>
private static AIFunctionArguments ConvertToFunctionArguments(JsonElement? arguments)
{
if (arguments is null ||
arguments.Value.ValueKind == JsonValueKind.Null ||
arguments.Value.ValueKind == JsonValueKind.Undefined)
{
return [];
}
if (arguments.Value.ValueKind != JsonValueKind.Object)
{
throw new InvalidOperationException(
$"Inline skill scripts expect arguments as a JSON object but received a JSON element of kind '{arguments.Value.ValueKind}'.");
}
var dict = new Dictionary<string, object?>();
foreach (var property in arguments.Value.EnumerateObject())
{
dict[property.Name] = property.Value;
}
return new AIFunctionArguments(dict);
return await this._function.InvokeAsync(arguments, cancellationToken).ConfigureAwait(false);
}
}
@@ -3,7 +3,6 @@
using System.Diagnostics;
using System.Reflection;
using System.Text;
using System.Text.Json;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Logging;
using ModelContextProtocol.Client;
@@ -126,45 +125,6 @@ public sealed class WorkflowSamplesValidation(ITestOutputHelper outputHelper) :
},
message: "OrderStatus workflow completed",
timeout: s_orchestrationTimeout);
// Test the CancelOrder workflow with x-ms-wait-for-response header
this._outputHelper.WriteLine("Starting CancelOrder workflow with x-ms-wait-for-response: true...");
using HttpRequestMessage waitRequest = new(HttpMethod.Post, cancelOrderUri);
waitRequest.Content = new StringContent("55555", Encoding.UTF8, "text/plain");
waitRequest.Headers.Add("x-ms-wait-for-response", "true");
using HttpResponseMessage waitResponse = await s_sharedHttpClient.SendAsync(waitRequest);
Assert.True(waitResponse.IsSuccessStatusCode, $"CancelOrder wait-for-response request failed with status: {waitResponse.StatusCode}");
string waitResponseText = await waitResponse.Content.ReadAsStringAsync();
this._outputHelper.WriteLine($"CancelOrder wait-for-response result: {waitResponseText}");
// The response should contain the workflow result (not just "started for CancelOrder")
Assert.DoesNotContain("Workflow orchestration started", waitResponseText);
Assert.Contains("55555", waitResponseText);
// Test the wait-for-response with Accept: application/json header
this._outputHelper.WriteLine("Starting CancelOrder workflow with x-ms-wait-for-response and Accept: application/json...");
using HttpRequestMessage jsonWaitRequest = new(HttpMethod.Post, cancelOrderUri);
jsonWaitRequest.Content = new StringContent("77777", Encoding.UTF8, "text/plain");
jsonWaitRequest.Headers.Add("x-ms-wait-for-response", "true");
jsonWaitRequest.Headers.Add("Accept", "application/json");
using CancellationTokenSource jsonWaitCts = new(s_orchestrationTimeout);
using HttpResponseMessage jsonWaitResponse = await s_sharedHttpClient.SendAsync(jsonWaitRequest, jsonWaitCts.Token);
Assert.True(jsonWaitResponse.IsSuccessStatusCode, $"CancelOrder JSON wait-for-response request failed with status: {jsonWaitResponse.StatusCode}");
string jsonWaitResponseText = await jsonWaitResponse.Content.ReadAsStringAsync();
this._outputHelper.WriteLine($"CancelOrder JSON wait-for-response result: {jsonWaitResponseText}");
using JsonDocument jsonDoc = JsonDocument.Parse(jsonWaitResponseText);
JsonElement root = jsonDoc.RootElement;
Assert.True(root.TryGetProperty("runId", out _), "JSON response missing 'runId' property");
Assert.True(root.TryGetProperty("workflowStatus", out JsonElement statusEl), "JSON response missing 'workflowStatus' property");
Assert.Equal("Completed", statusEl.GetString());
Assert.True(root.TryGetProperty("result", out JsonElement resultEl), "JSON response missing 'result' property");
Assert.Contains("77777", resultEl.GetString());
});
}
@@ -8,6 +8,7 @@ using System.Reflection;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.AI;
using Microsoft.Extensions.DependencyInjection;
namespace Microsoft.Agents.AI.UnitTests.AgentSkills;
@@ -127,9 +128,8 @@ public sealed class AgentClassSkillTests
// Act — script with custom type deserialization
var script = skill.Scripts![0];
var inputJson = JsonSerializer.SerializeToElement(new LookupRequest { Query = "test", MaxResults = 5 }, jso);
using var argsDoc = JsonDocument.Parse($$"""{ "request": {{inputJson.GetRawText()}} }""");
var args = argsDoc.RootElement;
var scriptResult = await script.RunAsync(skill, args, null, CancellationToken.None);
var args = new AIFunctionArguments { ["request"] = inputJson };
var scriptResult = await script.RunAsync(skill, args, CancellationToken.None);
// Assert
Assert.NotNull(scriptResult);
@@ -173,14 +173,12 @@ public sealed class AgentClassSkillTests
// Act & Assert — static method
var doWorkScript = skill.Scripts!.First(s => s.Name == "do-work");
using var doWorkDoc = JsonDocument.Parse("""{"input":"hello"}""");
var doWorkResult = await doWorkScript.RunAsync(skill, doWorkDoc.RootElement, null, CancellationToken.None);
var doWorkResult = await doWorkScript.RunAsync(skill, new AIFunctionArguments { ["input"] = "hello" }, CancellationToken.None);
Assert.Equal("HELLO", doWorkResult?.ToString());
// Act & Assert — instance method
var appendScript = skill.Scripts!.First(s => s.Name == "append");
using var appendDoc = JsonDocument.Parse("""{"input":"test"}""");
var appendResult = await appendScript.RunAsync(skill, appendDoc.RootElement, null, CancellationToken.None);
var appendResult = await appendScript.RunAsync(skill, new AIFunctionArguments { ["input"] = "test" }, CancellationToken.None);
Assert.Equal("test-suffix", appendResult?.ToString());
}
@@ -369,7 +367,7 @@ public sealed class AgentClassSkillTests
// Act & Assert — all scripts produce values
foreach (var script in skill.Scripts!)
{
var result = await script.RunAsync(skill, null, null, CancellationToken.None);
var result = await script.RunAsync(skill, new AIFunctionArguments(), CancellationToken.None);
Assert.NotNull(result);
}
}
@@ -384,9 +382,8 @@ public sealed class AgentClassSkillTests
// Act & Assert — script with custom JSO
var script = skill.Scripts![0];
var inputJson = JsonSerializer.SerializeToElement(new LookupRequest { Query = "test", MaxResults = 3 }, jso);
using var argsDoc = JsonDocument.Parse($$"""{ "request": {{inputJson.GetRawText()}} }""");
var args = argsDoc.RootElement;
var scriptResult = await script.RunAsync(skill, args, null, CancellationToken.None);
var args = new AIFunctionArguments { ["request"] = inputJson };
var scriptResult = await script.RunAsync(skill, args, CancellationToken.None);
Assert.NotNull(scriptResult);
Assert.Contains("test", scriptResult!.ToString()!);
Assert.Contains("3", scriptResult!.ToString()!);
@@ -500,9 +497,8 @@ public sealed class AgentClassSkillTests
var script = skill.Scripts!.First(s => s.Name == "Lookup");
var jso = SkillTestJsonContext.Default.Options;
var inputJson = JsonSerializer.SerializeToElement(new LookupRequest { Query = "fallback", MaxResults = 7 }, jso);
using var argsDoc = JsonDocument.Parse($$"""{ "request": {{inputJson.GetRawText()}} }""");
var args = argsDoc.RootElement;
var result = await script.RunAsync(skill, args, null, CancellationToken.None);
var args = new AIFunctionArguments { ["request"] = inputJson };
var result = await script.RunAsync(skill, args, CancellationToken.None);
// Assert
Assert.NotNull(result);
@@ -535,9 +531,8 @@ public sealed class AgentClassSkillTests
var script = skill.Scripts!.First(s => s.Name == "Lookup");
var jso = SkillTestJsonContext.Default.Options;
var inputJson = JsonSerializer.SerializeToElement(new LookupRequest { Query = "explicit", MaxResults = 2 }, jso);
using var argsDoc = JsonDocument.Parse($$"""{ "request": {{inputJson.GetRawText()}} }""");
var args = argsDoc.RootElement;
var result = await script.RunAsync(skill, args, null, CancellationToken.None);
var args = new AIFunctionArguments { ["request"] = inputJson };
var result = await script.RunAsync(skill, args, CancellationToken.None);
// Assert
Assert.NotNull(result);
@@ -1,9 +1,9 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.AI;
namespace Microsoft.Agents.AI.UnitTests.AgentSkills;
@@ -16,13 +16,13 @@ public sealed class AgentFileSkillScriptTests
public async Task RunAsync_SkillIsNotAgentFileSkill_ThrowsInvalidOperationExceptionAsync()
{
// Arrange
static Task<object?> RunnerAsync(AgentFileSkill s, AgentFileSkillScript sc, JsonElement? a, IServiceProvider? sp, CancellationToken ct) => Task.FromResult<object?>("result");
static Task<object?> RunnerAsync(AgentFileSkill s, AgentFileSkillScript sc, AIFunctionArguments a, CancellationToken ct) => Task.FromResult<object?>("result");
var script = CreateScript("test-script", "/path/to/script.py", RunnerAsync);
var nonFileSkill = new TestAgentSkill("my-skill", "A skill", "Instructions.");
// Act & Assert
await Assert.ThrowsAsync<InvalidOperationException>(
() => script.RunAsync(nonFileSkill, null, null, CancellationToken.None));
() => script.RunAsync(nonFileSkill, new AIFunctionArguments(), CancellationToken.None));
}
[Fact]
@@ -30,7 +30,7 @@ public sealed class AgentFileSkillScriptTests
{
// Arrange
var runnerCalled = false;
Task<object?> runnerAsync(AgentFileSkill skill, AgentFileSkillScript scriptArg, JsonElement? args, IServiceProvider? sp, CancellationToken ct)
Task<object?> runnerAsync(AgentFileSkill skill, AgentFileSkillScript scriptArg, AIFunctionArguments args, CancellationToken ct)
{
runnerCalled = true;
return Task.FromResult<object?>("executed");
@@ -42,7 +42,7 @@ public sealed class AgentFileSkillScriptTests
"/skills/my-skill");
// Act
var result = await script.RunAsync(fileSkill, null, null, CancellationToken.None);
var result = await script.RunAsync(fileSkill, new AIFunctionArguments(), CancellationToken.None);
// Assert
Assert.True(runnerCalled);
@@ -55,7 +55,7 @@ public sealed class AgentFileSkillScriptTests
// Arrange
AgentFileSkill? capturedSkill = null;
AgentFileSkillScript? capturedScript = null;
Task<object?> runnerAsync(AgentFileSkill skill, AgentFileSkillScript scriptArg, JsonElement? args, IServiceProvider? sp, CancellationToken ct)
Task<object?> runnerAsync(AgentFileSkill skill, AgentFileSkillScript scriptArg, AIFunctionArguments args, CancellationToken ct)
{
capturedSkill = skill;
capturedScript = scriptArg;
@@ -68,7 +68,7 @@ public sealed class AgentFileSkillScriptTests
"/skills/owner-skill");
// Act
await script.RunAsync(fileSkill, null, null, CancellationToken.None);
await script.RunAsync(fileSkill, new AIFunctionArguments(), CancellationToken.None);
// Assert
Assert.Same(fileSkill, capturedSkill);
@@ -79,7 +79,7 @@ public sealed class AgentFileSkillScriptTests
public void Script_HasCorrectNameAndPath()
{
// Arrange & Act
static Task<object?> RunnerAsync(AgentFileSkill s, AgentFileSkillScript sc, JsonElement? a, IServiceProvider? sp, CancellationToken ct) => Task.FromResult<object?>(null);
static Task<object?> RunnerAsync(AgentFileSkill s, AgentFileSkillScript sc, AIFunctionArguments a, CancellationToken ct) => Task.FromResult<object?>(null);
var script = CreateScript("my-script", "/path/to/my-script.py", RunnerAsync);
// Assert
@@ -87,173 +87,10 @@ public sealed class AgentFileSkillScriptTests
Assert.Equal("/path/to/my-script.py", script.FullPath);
}
[Fact]
public void ParametersSchema_ReturnsExpectedArraySchema()
{
// Arrange
static Task<object?> RunnerAsync(AgentFileSkill s, AgentFileSkillScript sc, JsonElement? a, IServiceProvider? sp, CancellationToken ct) => Task.FromResult<object?>(null);
var script = CreateScript("my-script", "/path/to/script.py", RunnerAsync);
// Act
var schema = script.ParametersSchema;
// Assert
Assert.NotNull(schema);
var raw = schema!.Value.GetRawText();
Assert.Contains("\"type\":\"array\"", raw);
Assert.Contains("\"items\":{\"type\":\"string\"}", raw);
}
[Fact]
public void Content_WithScripts_AppendsPerScriptEntries()
{
// Arrange
static Task<object?> RunnerAsync(AgentFileSkill s, AgentFileSkillScript sc, JsonElement? a, IServiceProvider? sp, CancellationToken ct) => Task.FromResult<object?>(null);
var script1 = CreateScript("build", "/scripts/build.sh", RunnerAsync);
var script2 = CreateScript("deploy", "/scripts/deploy.sh", RunnerAsync);
var fileSkill = new AgentFileSkill(
new AgentSkillFrontmatter("my-skill", "A skill"),
"Original content",
"/skills/my-skill",
scripts: [script1, script2]);
// Act
var content = fileSkill.Content;
// Assert — content starts with original and appends per-script entries
Assert.StartsWith("Original content", content);
Assert.Contains("<scripts>", content);
Assert.Contains("<script name=\"build\">", content);
Assert.Contains("<script name=\"deploy\">", content);
Assert.Contains("<parameters_schema>", content);
Assert.Contains("</scripts>", content);
}
[Fact]
public void Content_WithoutScripts_ReturnsOriginalContent()
{
// Arrange
var fileSkill = new AgentFileSkill(
new AgentSkillFrontmatter("my-skill", "A skill"),
"Original content only",
"/skills/my-skill");
// Act
var content = fileSkill.Content;
// Assert
Assert.Equal("Original content only", content);
}
[Fact]
public void Content_WithScripts_IsCached()
{
// Arrange
static Task<object?> RunnerAsync(AgentFileSkill s, AgentFileSkillScript sc, JsonElement? a, IServiceProvider? sp, CancellationToken ct) => Task.FromResult<object?>(null);
var script = CreateScript("test", "/scripts/test.sh", RunnerAsync);
var fileSkill = new AgentFileSkill(
new AgentSkillFrontmatter("my-skill", "A skill"),
"Content",
"/skills/my-skill",
scripts: [script]);
// Act
var content1 = fileSkill.Content;
var content2 = fileSkill.Content;
// Assert
Assert.Same(content1, content2);
}
[Fact]
public async Task RunAsync_ForwardsJsonArrayArgumentsToRunnerAsync()
{
// Arrange
JsonElement? capturedArgs = null;
Task<object?> runnerAsync(AgentFileSkill skill, AgentFileSkillScript scriptArg, JsonElement? args, IServiceProvider? sp, CancellationToken ct)
{
capturedArgs = args;
return Task.FromResult<object?>("done");
}
var script = CreateScript("array-test", "/scripts/test.sh", runnerAsync);
var fileSkill = new AgentFileSkill(
new AgentSkillFrontmatter("my-skill", "A skill"),
"Content",
"/skills/my-skill");
using var arrayArgsDoc = JsonDocument.Parse("""["arg1","arg2","arg3"]""");
var arrayArgs = arrayArgsDoc.RootElement;
// Act
await script.RunAsync(fileSkill, arrayArgs, null, CancellationToken.None);
// Assert — the raw JSON array is forwarded unchanged
Assert.NotNull(capturedArgs);
Assert.Equal(JsonValueKind.Array, capturedArgs!.Value.ValueKind);
Assert.Equal("""["arg1","arg2","arg3"]""", capturedArgs.Value.GetRawText());
}
[Fact]
public async Task RunAsync_ForwardsServiceProviderToRunnerAsync()
{
// Arrange
IServiceProvider? capturedProvider = null;
Task<object?> runnerAsync(AgentFileSkill skill, AgentFileSkillScript scriptArg, JsonElement? args, IServiceProvider? sp, CancellationToken ct)
{
capturedProvider = sp;
return Task.FromResult<object?>("done");
}
var script = CreateScript("sp-test", "/scripts/test.sh", runnerAsync);
var fileSkill = new AgentFileSkill(
new AgentSkillFrontmatter("my-skill", "A skill"),
"Content",
"/skills/my-skill");
var mockProvider = new TestServiceProvider();
// Act
await script.RunAsync(fileSkill, null, mockProvider, CancellationToken.None);
// Assert
Assert.Same(mockProvider, capturedProvider);
}
[Fact]
public async Task RunAsync_NoRunner_ThrowsInvalidOperationExceptionAsync()
{
// Arrange — create script without a runner
var script = CreateScript("no-runner", "/scripts/test.sh", runner: null);
var fileSkill = new AgentFileSkill(
new AgentSkillFrontmatter("my-skill", "A skill"),
"Content",
"/skills/my-skill");
// Act & Assert
await Assert.ThrowsAsync<InvalidOperationException>(
() => script.RunAsync(fileSkill, null, null, CancellationToken.None));
}
[Fact]
public void Content_WithScripts_ContainsDefaultParametersSchema()
{
// Arrange
static Task<object?> RunnerAsync(AgentFileSkill s, AgentFileSkillScript sc, JsonElement? a, IServiceProvider? sp, CancellationToken ct) => Task.FromResult<object?>(null);
var script = CreateScript("test", "/scripts/test.sh", RunnerAsync);
var fileSkill = new AgentFileSkill(
new AgentSkillFrontmatter("my-skill", "A skill"),
"Original content",
"/skills/my-skill",
scripts: [script]);
// Act
var content = fileSkill.Content;
// Assert — the appended block contains the actual default schema from AgentFileSkillScript
Assert.Contains("""{"type":"array","items":{"type":"string"}}""", content);
}
/// <summary>
/// Helper to create an <see cref="AgentFileSkillScript"/> via reflection since the constructor is internal.
/// </summary>
private static AgentFileSkillScript CreateScript(string name, string fullPath, AgentFileSkillScriptRunner? runner)
private static AgentFileSkillScript CreateScript(string name, string fullPath, AgentFileSkillScriptRunner executor)
{
var ctor = typeof(AgentFileSkillScript).GetConstructor(
System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance,
@@ -261,14 +98,6 @@ public sealed class AgentFileSkillScriptTests
[typeof(string), typeof(string), typeof(AgentFileSkillScriptRunner)],
null) ?? throw new InvalidOperationException("Could not find internal constructor.");
return (AgentFileSkillScript)ctor.Invoke([name, fullPath, runner]);
}
/// <summary>
/// Minimal <see cref="IServiceProvider"/> for testing service forwarding.
/// </summary>
private sealed class TestServiceProvider : IServiceProvider
{
public object? GetService(Type serviceType) => null;
return (AgentFileSkillScript)ctor.Invoke([name, fullPath, executor]);
}
}
@@ -3,9 +3,9 @@
using System;
using System.IO;
using System.Linq;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.AI;
namespace Microsoft.Agents.AI.UnitTests.AgentSkills;
@@ -15,7 +15,7 @@ namespace Microsoft.Agents.AI.UnitTests.AgentSkills;
public sealed class AgentFileSkillsSourceScriptTests : IDisposable
{
private static readonly string[] s_rubyExtension = new[] { ".rb" };
private static readonly AgentFileSkillScriptRunner s_noOpExecutor = (skill, script, args, sp, ct) => Task.FromResult<object?>(null);
private static readonly AgentFileSkillScriptRunner s_noOpExecutor = (skill, script, args, ct) => Task.FromResult<object?>(null);
private readonly string _testRoot;
@@ -139,7 +139,7 @@ public sealed class AgentFileSkillsSourceScriptTests : IDisposable
var executorCalled = false;
var source = new AgentFileSkillsSource(
this._testRoot,
(skill, script, args, sp, ct) =>
(skill, script, args, ct) =>
{
executorCalled = true;
Assert.Equal("exec-skill", skill.Frontmatter.Name);
@@ -150,7 +150,7 @@ public sealed class AgentFileSkillsSourceScriptTests : IDisposable
// Act
var skills = await source.GetSkillsAsync(CancellationToken.None);
var scriptResult = await skills[0].Scripts![0].RunAsync(skills[0], null, null, CancellationToken.None);
var scriptResult = await skills[0].Scripts![0].RunAsync(skills[0], new AIFunctionArguments(), CancellationToken.None);
// Assert
Assert.True(executorCalled);
@@ -178,7 +178,7 @@ public sealed class AgentFileSkillsSourceScriptTests : IDisposable
var script = skills[0].Scripts![0];
// Assert — running the script throws because no runner was provided
await Assert.ThrowsAsync<InvalidOperationException>(() => script.RunAsync(skills[0], null, null, CancellationToken.None));
await Assert.ThrowsAsync<InvalidOperationException>(() => script.RunAsync(skills[0], new AIFunctionArguments(), CancellationToken.None));
}
[Fact]
@@ -204,10 +204,10 @@ public sealed class AgentFileSkillsSourceScriptTests : IDisposable
{
// Arrange
CreateSkillWithScript(this._testRoot, "args-skill", "Args test", "Body.", "scripts/test.py", "print('ok')");
JsonElement? capturedArgs = null;
AIFunctionArguments? capturedArgs = null;
var source = new AgentFileSkillsSource(
this._testRoot,
(skill, script, args, sp, ct) =>
(skill, script, args, ct) =>
{
capturedArgs = args;
return Task.FromResult<object?>("done");
@@ -215,15 +215,17 @@ public sealed class AgentFileSkillsSourceScriptTests : IDisposable
// Act
var skills = await source.GetSkillsAsync(CancellationToken.None);
using var argumentsDoc = JsonDocument.Parse("""{"value":26.2,"factor":1.60934}""");
var arguments = argumentsDoc.RootElement;
await skills[0].Scripts![0].RunAsync(skills[0], arguments, null, CancellationToken.None);
var arguments = new AIFunctionArguments
{
["value"] = 26.2,
["factor"] = 1.60934
};
await skills[0].Scripts![0].RunAsync(skills[0], arguments, CancellationToken.None);
// Assert
Assert.NotNull(capturedArgs);
Assert.Equal(JsonValueKind.Object, capturedArgs!.Value.ValueKind);
Assert.Equal(26.2, capturedArgs.Value.GetProperty("value").GetDouble());
Assert.Equal(1.60934, capturedArgs.Value.GetProperty("factor").GetDouble());
Assert.Equal(26.2, capturedArgs["value"]);
Assert.Equal(1.60934, capturedArgs["factor"]);
}
[Fact]
@@ -5,6 +5,7 @@ using System.Reflection;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.AI;
namespace Microsoft.Agents.AI.UnitTests.AgentSkills;
@@ -21,7 +22,7 @@ public sealed class AgentInlineSkillScriptTests
var skill = new AgentInlineSkill("test-skill", "Test.", "Instructions.");
// Act
var result = await script.RunAsync(skill, null, null, CancellationToken.None);
var result = await script.RunAsync(skill, new AIFunctionArguments(), CancellationToken.None);
// Assert
Assert.Equal("hello", result?.ToString());
@@ -33,11 +34,10 @@ public sealed class AgentInlineSkillScriptTests
// Arrange
var script = new AgentInlineSkillScript("add", (int a, int b) => a + b);
var skill = new AgentInlineSkill("calc-skill", "Calc.", "Instructions.");
using var argsDoc = JsonDocument.Parse("""{"a":3,"b":7}""");
var args = argsDoc.RootElement;
var args = new AIFunctionArguments { ["a"] = 3, ["b"] = 7 };
// Act
var result = await script.RunAsync(skill, args, null, CancellationToken.None);
var result = await script.RunAsync(skill, args, CancellationToken.None);
// Assert
Assert.Equal(10, int.Parse(result?.ToString()!));
@@ -129,11 +129,10 @@ public sealed class AgentInlineSkillScriptTests
}, serializerOptions: jso);
var skill = new AgentInlineSkill("test-skill", "Test.", "Instructions.");
var inputJson = JsonSerializer.SerializeToElement(new LookupRequest { Query = "test", MaxResults = 5 }, jso);
using var argsDoc = JsonDocument.Parse($$"""{ "request": {{inputJson.GetRawText()}} }""");
var args = argsDoc.RootElement;
var args = new AIFunctionArguments { ["request"] = inputJson };
// Act
var result = await script.RunAsync(skill, args, null, CancellationToken.None);
var result = await script.RunAsync(skill, args, CancellationToken.None);
// Assert — the custom input type was deserialized and the response was produced
Assert.NotNull(result);
@@ -146,11 +145,10 @@ public sealed class AgentInlineSkillScriptTests
// Arrange
var script = new AgentInlineSkillScript("echo", (string message) => message);
var skill = new AgentInlineSkill("test-skill", "Test.", "Instructions.");
using var argsDoc = JsonDocument.Parse("""{"message":"hello world"}""");
var args = argsDoc.RootElement;
var args = new AIFunctionArguments { ["message"] = "hello world" };
// Act
var result = await script.RunAsync(skill, args, null, CancellationToken.None);
var result = await script.RunAsync(skill, args, CancellationToken.None);
// Assert
Assert.Equal("hello world", result?.ToString());
@@ -177,11 +175,10 @@ public sealed class AgentInlineSkillScriptTests
var method = typeof(AgentInlineSkillScriptTests).GetMethod(nameof(StaticScriptHelper), BindingFlags.NonPublic | BindingFlags.Static)!;
var script = new AgentInlineSkillScript("static-method-script", method, target: null);
var skill = new AgentInlineSkill("test-skill", "Test.", "Instructions.");
using var argsDoc = JsonDocument.Parse("""{"input":"hello"}""");
var args = argsDoc.RootElement;
var args = new AIFunctionArguments { ["input"] = "hello" };
// Act
var result = await script.RunAsync(skill, args, null, CancellationToken.None);
var result = await script.RunAsync(skill, args, CancellationToken.None);
// Assert
Assert.Equal("HELLO", result?.ToString());
@@ -194,11 +191,10 @@ public sealed class AgentInlineSkillScriptTests
var method = typeof(AgentInlineSkillScriptTests).GetMethod(nameof(InstanceScriptHelper), BindingFlags.NonPublic | BindingFlags.Instance)!;
var script = new AgentInlineSkillScript("instance-method-script", method, target: this);
var skill = new AgentInlineSkill("test-skill", "Test.", "Instructions.");
using var argsDoc2 = JsonDocument.Parse("""{"input":"test"}""");
var args2 = argsDoc2.RootElement;
var args = new AIFunctionArguments { ["input"] = "test" };
// Act
var result = await script.RunAsync(skill, args2, null, CancellationToken.None);
var result = await script.RunAsync(skill, args, CancellationToken.None);
// Assert
Assert.Equal("test-suffix", result?.ToString());
@@ -227,63 +223,7 @@ public sealed class AgentInlineSkillScriptTests
Assert.Contains("input", schema!.Value.GetRawText());
}
[Fact]
public async Task RunAsync_WithNonObjectArguments_ThrowsInvalidOperationExceptionAsync()
{
// Arrange — inline scripts require a JSON object for arguments
var script = new AgentInlineSkillScript("noop", () => "ok");
var skill = new AgentInlineSkill("test-skill", "Test.", "Instructions.");
using var arrayArgsDoc = JsonDocument.Parse("""["a","b"]""");
var arrayArgs = arrayArgsDoc.RootElement;
// Act & Assert — non-object JSON should fail fast rather than silently dropping arguments
await Assert.ThrowsAsync<InvalidOperationException>(
() => script.RunAsync(skill, arrayArgs, null, CancellationToken.None));
}
[Fact]
public async Task RunAsync_WithNullArguments_TreatsAsNoArgumentsAsync()
{
// Arrange — a parameterless delegate should succeed when given null arguments
var script = new AgentInlineSkillScript("noop", () => "ok");
var skill = new AgentInlineSkill("test-skill", "Test.", "Instructions.");
// Act
var result = await script.RunAsync(skill, null, null, CancellationToken.None);
// Assert
Assert.Equal("ok", result?.ToString());
}
[Fact]
public async Task RunAsync_ServiceProviderIsForwardedAsync()
{
// Arrange — delegate that resolves a service from the IServiceProvider
IServiceProvider? capturedProvider = null;
var script = new AgentInlineSkillScript("svc-test", (IServiceProvider sp) =>
{
capturedProvider = sp;
return "done";
});
var skill = new AgentInlineSkill("test-skill", "Test.", "Instructions.");
var mockProvider = new TestServiceProvider();
// Act
await script.RunAsync(skill, null, mockProvider, CancellationToken.None);
// Assert
Assert.Same(mockProvider, capturedProvider);
}
private static string StaticScriptHelper(string input) => input.ToUpperInvariant();
private string InstanceScriptHelper(string input) => input + "-suffix";
/// <summary>
/// Minimal <see cref="IServiceProvider"/> for testing service forwarding.
/// </summary>
private sealed class TestServiceProvider : IServiceProvider
{
public object? GetService(Type serviceType) => null;
}
}
@@ -433,11 +433,10 @@ public sealed class AgentInlineSkillTests
TotalCount = request.MaxResults,
});
var inputJson = JsonSerializer.SerializeToElement(new LookupRequest { Query = "test", MaxResults = 3 }, jso);
using var argsDoc = JsonDocument.Parse($$"""{ "request": {{inputJson.GetRawText()}} }""");
var args = argsDoc.RootElement;
var args = new AIFunctionArguments { ["request"] = inputJson };
// Act
var result = await skill.Scripts![0].RunAsync(skill, args, null, CancellationToken.None);
var result = await skill.Scripts![0].RunAsync(skill, args, CancellationToken.None);
// Assert — the custom input was deserialized via skill-level JSO and response was produced
Assert.NotNull(result);
@@ -457,11 +456,10 @@ public sealed class AgentInlineSkillTests
TotalCount = request.MaxResults,
}, serializerOptions: scriptJso);
var inputJson = JsonSerializer.SerializeToElement(new LookupRequest { Query = "override", MaxResults = 7 }, scriptJso);
using var argsDoc = JsonDocument.Parse($$"""{ "request": {{inputJson.GetRawText()}} }""");
var args = argsDoc.RootElement;
var args = new AIFunctionArguments { ["request"] = inputJson };
// Act
var result = await skill.Scripts![0].RunAsync(skill, args, null, CancellationToken.None);
var result = await skill.Scripts![0].RunAsync(skill, args, CancellationToken.None);
// Assert — per-script JSO takes effect and custom types are properly marshaled
Assert.NotNull(result);
@@ -4,7 +4,6 @@ using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.AI;
@@ -16,7 +15,7 @@ namespace Microsoft.Agents.AI.UnitTests.AgentSkills;
/// </summary>
public sealed class AgentSkillsProviderTests : IDisposable
{
private static readonly AgentFileSkillScriptRunner s_noOpExecutor = (skill, script, args, sp, ct) => Task.FromResult<object?>(null);
private static readonly AgentFileSkillScriptRunner s_noOpExecutor = (skill, script, args, ct) => Task.FromResult<object?>(null);
private readonly string _testRoot;
private readonly TestAIAgent _agent = new();
@@ -463,7 +462,7 @@ public sealed class AgentSkillsProviderTests : IDisposable
// Act — call UseFileScriptRunner AFTER UseFileSkill (the bug scenario)
var provider = new AgentSkillsProviderBuilder()
.UseFileSkill(this._testRoot)
.UseFileScriptRunner((skill, script, args, sp, ct) =>
.UseFileScriptRunner((skill, script, args, ct) =>
{
executorCalled = true;
return Task.FromResult<object?>("executed");
@@ -488,62 +487,6 @@ public sealed class AgentSkillsProviderTests : IDisposable
Assert.True(executorCalled);
}
[Fact]
public async Task RunSkillScript_ForwardsJsonArgumentsAndServiceProviderToRunnerAsync()
{
// Arrange — create a skill with a script file
string skillDir = Path.Combine(this._testRoot, "fwd-skill");
Directory.CreateDirectory(Path.Combine(skillDir, "scripts"));
File.WriteAllText(
Path.Combine(skillDir, "SKILL.md"),
"---\nname: fwd-skill\ndescription: Forwarding test\n---\nBody.");
File.WriteAllText(
Path.Combine(skillDir, "scripts", "run.py"),
"print('ok')");
JsonElement? capturedArgs = null;
IServiceProvider? capturedServiceProvider = null;
var provider = new AgentSkillsProviderBuilder()
.UseFileSkill(this._testRoot)
.UseFileScriptRunner((skill, script, args, sp, ct) =>
{
capturedArgs = args;
capturedServiceProvider = sp;
return Task.FromResult<object?>("executed");
})
.Build();
var mockServiceProvider = new TestServiceProvider();
var invokingContext = new AIContextProvider.InvokingContext(this._agent, session: null, new AIContext());
var result = await provider.InvokingAsync(invokingContext, CancellationToken.None);
var runScriptTool = result.Tools!.First(t => t.Name == "run_skill_script") as AIFunction;
// Act — invoke with JsonElement arguments and a service provider
using var argsJsonDoc = JsonDocument.Parse("""["arg1","arg2"]""");
var argsJson = argsJsonDoc.RootElement;
await runScriptTool!.InvokeAsync(new AIFunctionArguments(new Dictionary<string, object?>
{
["skillName"] = "fwd-skill",
["scriptName"] = "scripts/run.py",
["arguments"] = argsJson,
})
{
Services = mockServiceProvider,
});
// Assert — JsonElement arguments and service provider are forwarded to the runner
Assert.NotNull(capturedArgs);
Assert.Equal(JsonValueKind.Array, capturedArgs!.Value.ValueKind);
Assert.Equal("""["arg1","arg2"]""", capturedArgs.Value.GetRawText());
Assert.Same(mockServiceProvider, capturedServiceProvider);
}
private sealed class TestServiceProvider : IServiceProvider
{
public object? GetService(Type serviceType) => null;
}
private static void CreateSkillIn(string root, string name, string description, string body)
{
string skillDir = Path.Combine(root, name);
@@ -15,7 +15,7 @@ public sealed class FileAgentSkillLoaderTests : IDisposable
private static readonly string[] s_customExtensions = [".custom"];
private static readonly string[] s_validExtensions = [".md", ".json", ".custom"];
private static readonly string[] s_mixedValidInvalidExtensions = [".md", "json"];
private static readonly AgentFileSkillScriptRunner s_noOpExecutor = (skill, script, args, sp, ct) => Task.FromResult<object?>(null);
private static readonly AgentFileSkillScriptRunner s_noOpExecutor = (skill, script, args, ct) => Task.FromResult<object?>(null);
private readonly string _testRoot;
@@ -60,20 +60,10 @@ public abstract class IntegrationTest : IDisposable
protected async ValueTask<DeclarativeWorkflowOptions> CreateOptionsAsync(bool externalConversation = false, params IEnumerable<AIFunction> functionTools)
{
return await this.CreateOptionsAsync(externalConversation, mcpToolProvider: null, httpRequestHandler: null, functionTools).ConfigureAwait(false);
return await this.CreateOptionsAsync(externalConversation, mcpToolProvider: null, functionTools).ConfigureAwait(false);
}
protected async ValueTask<DeclarativeWorkflowOptions> CreateOptionsAsync(bool externalConversation, IMcpToolHandler? mcpToolProvider, params IEnumerable<AIFunction> functionTools)
{
return await this.CreateOptionsAsync(externalConversation, mcpToolProvider, httpRequestHandler: null, functionTools).ConfigureAwait(false);
}
protected async ValueTask<DeclarativeWorkflowOptions> CreateOptionsAsync(bool externalConversation, IHttpRequestHandler? httpRequestHandler, params IEnumerable<AIFunction> functionTools)
{
return await this.CreateOptionsAsync(externalConversation, mcpToolProvider: null, httpRequestHandler, functionTools).ConfigureAwait(false);
}
protected async ValueTask<DeclarativeWorkflowOptions> CreateOptionsAsync(bool externalConversation, IMcpToolHandler? mcpToolProvider, IHttpRequestHandler? httpRequestHandler, params IEnumerable<AIFunction> functionTools)
{
AzureAgentProvider agentProvider =
new(this.TestEndpoint, TestAzureCliCredentials.CreateAzureCliCredential())
@@ -92,8 +82,7 @@ public abstract class IntegrationTest : IDisposable
{
ConversationId = conversationId,
LoggerFactory = this.Output,
McpToolHandler = mcpToolProvider,
HttpRequestHandler = httpRequestHandler,
McpToolHandler = mcpToolProvider
};
}
@@ -45,15 +45,6 @@ public sealed class InvokeToolWorkflowTest(ITestOutputHelper output) : Integrati
#endregion
#region InvokeHttpRequest Tests
[RetryTheory(3, 5000)]
[InlineData("HttpRequest.yaml", "visibility: public")]
public Task ValidateHttpRequestAsync(string workflowFileName, string? expectedResultContains) =>
this.RunHttpRequestTestAsync(workflowFileName, expectedResultContains);
#endregion
#region InvokeFunctionTool Test Helpers
/// <summary>
@@ -259,40 +250,6 @@ public sealed class InvokeToolWorkflowTest(ITestOutputHelper output) : Integrati
#endregion
#region InvokeHttpRequest Test Helpers
/// <summary>
/// Runs an HttpRequestAction workflow test with the specified configuration.
/// </summary>
private async Task RunHttpRequestTestAsync(
string workflowFileName,
string? expectedResultContains = null)
{
// Arrange
string workflowPath = GetWorkflowPath(workflowFileName);
await using DefaultHttpRequestHandler httpRequestHandler = new();
DeclarativeWorkflowOptions workflowOptions = await this.CreateOptionsAsync(
externalConversation: false,
httpRequestHandler: httpRequestHandler);
Workflow workflow = DeclarativeWorkflowBuilder.Build<string>(workflowPath, workflowOptions);
WorkflowHarness harness = new(workflow, runId: Path.GetFileNameWithoutExtension(workflowPath));
// Act
WorkflowEvents workflowEvents = await harness.RunWorkflowAsync("start").ConfigureAwait(false);
// Assert - Verify executor and action events
AssertWorkflowEventsEmitted(workflowEvents);
// Assert - Verify expected result if specified
if (expectedResultContains is not null)
{
AssertResultContains(workflowEvents, expectedResultContains);
}
}
#endregion
#region Shared Helpers
private static void AssertWorkflowEventsEmitted(WorkflowEvents workflowEvents)
@@ -1,32 +0,0 @@
#
# This workflow tests invoking HttpRequestAction end-to-end.
# Uses the public GitHub API (unauthenticated) to fetch repo metadata.
#
kind: Workflow
trigger:
kind: OnConversationStart
id: workflow_http_request_test
actions:
# Set the repo owner used to form the request URL.
- kind: SetVariable
id: set_repo_owner
variable: Local.RepoOwner
value: dotnet
# Invoke the GitHub repo API.
- kind: HttpRequestAction
id: fetch_repo_info
conversationId: =System.ConversationId
method: GET
url: =Concatenate("https://api.github.com/repos/", Local.RepoOwner, "/runtime")
headers:
Accept: application/vnd.github+json
User-Agent: agent-framework-integration-test
response: Local.RepoInfo
# Surface the Repo visibility field from the parsed JSON response.
- kind: SendMessage
id: show_visibility
message: "visibility: {Local.RepoInfo.visibility}"
@@ -181,7 +181,6 @@ public sealed class DeclarativeWorkflowTest(ITestOutputHelper output) : Workflow
[InlineData("ResetVariable.yaml", 2, "clear_var")]
[InlineData("MixedScopes.yaml", 2, "activity_input")]
[InlineData("CaseInsensitive.yaml", 6, "end_when_match")]
[InlineData("HttpRequest.yaml", 1, "http_request")]
public async Task ExecuteActionAsync(string workflowFile, int expectedCount, string expectedId)
{
await this.RunWorkflowAsync(workflowFile);
@@ -201,6 +200,7 @@ public sealed class DeclarativeWorkflowTest(ITestOutputHelper output) : Workflow
[InlineData(typeof(EmitEvent.Builder))]
[InlineData(typeof(GetActivityMembers.Builder))]
[InlineData(typeof(GetConversationMembers.Builder))]
[InlineData(typeof(HttpRequestAction.Builder))]
[InlineData(typeof(InvokeAIBuilderModelAction.Builder))]
[InlineData(typeof(InvokeConnectorAction.Builder))]
[InlineData(typeof(InvokeCustomModelAction.Builder))]
@@ -266,7 +266,6 @@ public sealed class DeclarativeWorkflowTest(ITestOutputHelper output) : Workflow
[InlineData("SendActivity.yaml", "activity_input")]
[InlineData("SetVariable.yaml", "set_var")]
[InlineData("SetTextVariable.yaml", "set_text")]
[InlineData("HttpRequest.yaml", "http_request")]
public async Task CancelRunAsync(string workflowPath, string expectedExecutedId)
{
// Arrange
@@ -375,12 +374,7 @@ public sealed class DeclarativeWorkflowTest(ITestOutputHelper output) : Workflow
{
using StreamReader yamlReader = File.OpenText(Path.Combine("Workflows", workflowPath));
Mock<ResponseAgentProvider> mockAgentProvider = CreateMockProvider($"{workflowInput}");
DeclarativeWorkflowOptions workflowContext =
new(mockAgentProvider.Object)
{
LoggerFactory = this.Output,
HttpRequestHandler = CreateMockHttpRequestHandler().Object,
};
DeclarativeWorkflowOptions workflowContext = new(mockAgentProvider.Object) { LoggerFactory = this.Output };
return DeclarativeWorkflowBuilder.Build<TInput>(yamlReader, workflowContext);
}
@@ -391,18 +385,4 @@ public sealed class DeclarativeWorkflowTest(ITestOutputHelper output) : Workflow
mockAgentProvider.Setup(provider => provider.CreateMessageAsync(It.IsAny<string>(), It.IsAny<ChatMessage>(), It.IsAny<CancellationToken>())).Returns(Task.FromResult(new ChatMessage(ChatRole.Assistant, input)));
return mockAgentProvider;
}
private static Mock<IHttpRequestHandler> CreateMockHttpRequestHandler()
{
Mock<IHttpRequestHandler> mockHandler = new(MockBehavior.Loose);
mockHandler
.Setup(handler => handler.SendAsync(It.IsAny<HttpRequestInfo>(), It.IsAny<CancellationToken>()))
.Returns(() => Task.FromResult(new HttpRequestResult
{
StatusCode = 200,
IsSuccessStatusCode = true,
Body = "{\"ok\":true}",
}));
return mockHandler;
}
}
@@ -1,510 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
using System.Collections.Generic;
using System.Net;
using System.Net.Http;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using FluentAssertions;
namespace Microsoft.Agents.AI.Workflows.Declarative.UnitTests;
/// <summary>
/// Unit tests for <see cref="DefaultHttpRequestHandler"/>.
/// </summary>
public sealed class DefaultHttpRequestHandlerTests
{
private static readonly string[] s_setCookieValues = ["a=1", "b=2"];
private const string TestUrl = "https://api.example.test/resource";
#region Constructor Tests
[Fact]
public async Task ConstructorWithNoParametersCreatesInstanceAsync()
{
// Act
await using DefaultHttpRequestHandler handler = new();
// Assert
handler.Should().NotBeNull();
}
[Fact]
public async Task ConstructorWithNullProviderCreatesInstanceAsync()
{
// Act
await using DefaultHttpRequestHandler handler = new(httpClientProvider: null);
// Assert
handler.Should().NotBeNull();
}
[Fact]
public void ConstructorWithNullHttpClientThrows()
{
// Act
Action act = () => _ = new DefaultHttpRequestHandler((HttpClient)null!);
// Assert
act.Should().Throw<ArgumentNullException>();
}
[Fact]
public async Task ConstructorWithHttpClientUsesSuppliedClientForAllRequestsAsync()
{
// Arrange
TestHttpMessageHandler messageHandler = new((req, _) =>
Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK)
{
Content = new StringContent("ok", Encoding.UTF8, "text/plain"),
}));
using HttpClient suppliedClient = new(messageHandler);
await using DefaultHttpRequestHandler handler = new(suppliedClient);
HttpRequestInfo request = new() { Method = "GET", Url = TestUrl };
// Act
HttpRequestResult result = await handler.SendAsync(request);
// Assert - the supplied HttpClient's underlying handler saw the request
messageHandler.LastRequest.Should().NotBeNull();
messageHandler.LastRequest!.RequestUri!.ToString().Should().Be(TestUrl);
result.Body.Should().Be("ok");
}
[Fact]
public async Task DisposeAsyncDoesNotDisposeCallerSuppliedHttpClientAsync()
{
// Arrange
TestHttpMessageHandler messageHandler = new((req, _) =>
Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK)));
using HttpClient suppliedClient = new(messageHandler);
// Act
DefaultHttpRequestHandler handler = new(suppliedClient);
await handler.DisposeAsync();
// Assert - supplied client remains usable (not disposed)
Func<Task> act = async () => await suppliedClient.GetAsync(new Uri(TestUrl));
await act.Should().NotThrowAsync<ObjectDisposedException>();
}
#endregion
#region Argument Validation Tests
[Fact]
public async Task SendAsyncWithNullRequestThrowsAsync()
{
// Arrange
await using DefaultHttpRequestHandler handler = new();
// Act
Func<Task> act = async () => await handler.SendAsync(null!);
// Assert
await act.Should().ThrowAsync<ArgumentNullException>();
}
[Fact]
public async Task SendAsyncWithEmptyUrlThrowsAsync()
{
// Arrange
await using DefaultHttpRequestHandler handler = new();
HttpRequestInfo request = new() { Method = "GET", Url = "" };
// Act
Func<Task> act = async () => await handler.SendAsync(request);
// Assert
await act.Should().ThrowAsync<ArgumentException>();
}
[Fact]
public async Task SendAsyncWithEmptyMethodThrowsAsync()
{
// Arrange
await using DefaultHttpRequestHandler handler = new();
HttpRequestInfo request = new() { Method = "", Url = TestUrl };
// Act
Func<Task> act = async () => await handler.SendAsync(request);
// Assert
await act.Should().ThrowAsync<ArgumentException>();
}
#endregion
#region Send Behavior Tests
[Fact]
public async Task SendAsyncUsesProvidedHttpClientAsync()
{
// Arrange
TestHttpMessageHandler messageHandler = new((req, _) =>
Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK)
{
Content = new StringContent("hello", Encoding.UTF8, "text/plain"),
}));
await using DefaultHttpRequestHandler handler = new((_, _) => Task.FromResult<HttpClient?>(new HttpClient(messageHandler)));
HttpRequestInfo request = new() { Method = "GET", Url = TestUrl };
// Act
HttpRequestResult result = await handler.SendAsync(request);
// Assert
messageHandler.LastRequest.Should().NotBeNull();
messageHandler.LastRequest!.Method.Should().Be(HttpMethod.Get);
messageHandler.LastRequest.RequestUri!.ToString().Should().Be(TestUrl);
result.StatusCode.Should().Be(200);
result.IsSuccessStatusCode.Should().BeTrue();
result.Body.Should().Be("hello");
}
[Fact]
public async Task SendAsyncMapsAllKnownMethodsAsync()
{
// Arrange
TestHttpMessageHandler messageHandler = new((req, _) =>
Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK)));
await using DefaultHttpRequestHandler handler = new((_, _) => Task.FromResult<HttpClient?>(new HttpClient(messageHandler)));
foreach (string method in new[] { "GET", "POST", "PUT", "PATCH", "DELETE", "CUSTOM" })
{
HttpRequestInfo request = new() { Method = method, Url = TestUrl };
// Act
await handler.SendAsync(request);
// Assert
messageHandler.LastRequest!.Method.Method.Should().Be(method);
}
}
[Fact]
public async Task SendAsyncNormalizesWhitespaceAroundCustomMethodAsync()
{
// Arrange
TestHttpMessageHandler messageHandler = new((req, _) =>
Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK)));
await using DefaultHttpRequestHandler handler = new((_, _) => Task.FromResult<HttpClient?>(new HttpClient(messageHandler)));
HttpRequestInfo request = new() { Method = " custom ", Url = TestUrl };
// Act
await handler.SendAsync(request);
// Assert - fallback path should apply the same Trim/ToUpperInvariant normalization.
messageHandler.LastRequest!.Method.Method.Should().Be("CUSTOM");
}
[Fact]
public async Task SendAsyncAppliesBodyAndContentTypeAsync()
{
// Arrange
TestHttpMessageHandler messageHandler = new((req, _) =>
Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK)));
await using DefaultHttpRequestHandler handler = new((_, _) => Task.FromResult<HttpClient?>(new HttpClient(messageHandler)));
HttpRequestInfo request = new()
{
Method = "POST",
Url = TestUrl,
Body = "{\"hello\":\"world\"}",
BodyContentType = "application/json",
};
// Act
await handler.SendAsync(request);
// Assert
messageHandler.LastRequestBody.Should().Be("{\"hello\":\"world\"}");
messageHandler.LastRequestContentType.Should().Be("application/json");
}
[Fact]
public async Task SendAsyncAppliesRequestHeadersAsync()
{
// Arrange
TestHttpMessageHandler messageHandler = new((req, _) =>
Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK)));
await using DefaultHttpRequestHandler handler = new((_, _) => Task.FromResult<HttpClient?>(new HttpClient(messageHandler)));
HttpRequestInfo request = new()
{
Method = "GET",
Url = TestUrl,
Headers = new Dictionary<string, string>
{
["Authorization"] = "Bearer secret",
["Accept"] = "application/json",
},
};
// Act
await handler.SendAsync(request);
// Assert
messageHandler.LastRequest!.Headers.Authorization!.ToString().Should().Be("Bearer secret");
messageHandler.LastRequest.Headers.Accept.Should().Contain(mediaType => mediaType.MediaType == "application/json");
}
[Fact]
public async Task SendAsyncRoutesContentHeadersToBodyAsync()
{
// Arrange
TestHttpMessageHandler messageHandler = new((req, _) =>
Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK)));
await using DefaultHttpRequestHandler handler = new((_, _) => Task.FromResult<HttpClient?>(new HttpClient(messageHandler)));
HttpRequestInfo request = new()
{
Method = "POST",
Url = TestUrl,
Body = "raw",
BodyContentType = "text/plain",
Headers = new Dictionary<string, string>
{
["Content-Language"] = "en-US",
},
};
// Act
await handler.SendAsync(request);
// Assert
messageHandler.LastRequest!.Content!.Headers.ContentLanguage.Should().Contain("en-US");
}
[Fact]
public async Task SendAsyncCapturesResponseHeadersAsync()
{
// Arrange
TestHttpMessageHandler messageHandler = new((req, _) =>
{
#pragma warning disable CA2025
HttpResponseMessage response = new(HttpStatusCode.OK)
{
Content = new StringContent("ok", Encoding.UTF8, "text/plain"),
};
response.Headers.Add("X-Request-Id", "request-1");
response.Headers.Add("Set-Cookie", s_setCookieValues);
return Task.FromResult(response);
#pragma warning restore CA2025
});
await using DefaultHttpRequestHandler handler = new((_, _) => Task.FromResult<HttpClient?>(new HttpClient(messageHandler)));
HttpRequestInfo request = new() { Method = "GET", Url = TestUrl };
// Act
HttpRequestResult result = await handler.SendAsync(request);
// Assert
result.Headers.Should().NotBeNull();
result.Headers!.Should().ContainKey("X-Request-Id");
result.Headers!["Set-Cookie"].Should().BeEquivalentTo(s_setCookieValues);
// Content headers also flattened in.
result.Headers!.Should().ContainKey("Content-Type");
}
[Fact]
public async Task SendAsyncReturnsFailureStatusWithoutThrowingAsync()
{
// Arrange
TestHttpMessageHandler messageHandler = new((req, _) =>
Task.FromResult(new HttpResponseMessage(HttpStatusCode.BadRequest)
{
Content = new StringContent("bad request", Encoding.UTF8, "text/plain"),
}));
await using DefaultHttpRequestHandler handler = new((_, _) => Task.FromResult<HttpClient?>(new HttpClient(messageHandler)));
HttpRequestInfo request = new() { Method = "GET", Url = TestUrl };
// Act
HttpRequestResult result = await handler.SendAsync(request);
// Assert
result.IsSuccessStatusCode.Should().BeFalse();
result.StatusCode.Should().Be(400);
result.Body.Should().Be("bad request");
}
[Fact]
public async Task SendAsyncTimeoutCancelsRequestAsync()
{
// Arrange
TestHttpMessageHandler messageHandler = new(async (req, ct) =>
{
await Task.Delay(TimeSpan.FromSeconds(5), ct).ConfigureAwait(false);
return new HttpResponseMessage(HttpStatusCode.OK);
});
await using DefaultHttpRequestHandler handler = new((_, _) => Task.FromResult<HttpClient?>(new HttpClient(messageHandler)));
HttpRequestInfo request = new()
{
Method = "GET",
Url = TestUrl,
Timeout = TimeSpan.FromMilliseconds(50),
};
// Act
Func<Task> act = async () => await handler.SendAsync(request);
// Assert
await act.Should().ThrowAsync<OperationCanceledException>();
}
[Fact]
public async Task SendAsyncFallsBackToOwnedClientWhenProviderReturnsNullAsync()
{
// Arrange
int providerCallCount = 0;
await using DefaultHttpRequestHandler handler = new((_, _) =>
{
providerCallCount++;
return Task.FromResult<HttpClient?>(null);
});
HttpRequestInfo request = new() { Method = "GET", Url = "http://127.0.0.1:1/" };
// Act - owned client will attempt real network and fail, but provider path should have been consulted first.
Func<Task> act = async () => await handler.SendAsync(request);
// Assert
await act.Should().ThrowAsync<Exception>();
providerCallCount.Should().Be(1);
}
#endregion
#region DisposeAsync
[Fact]
public async Task DisposeAsyncCompletesAsync()
{
// Arrange
DefaultHttpRequestHandler handler = new();
// Act
Func<Task> act = async () => await handler.DisposeAsync();
// Assert
await act.Should().NotThrowAsync();
}
[Fact]
public async Task DisposeAsyncCalledMultipleTimesSucceedsAsync()
{
// Arrange
DefaultHttpRequestHandler handler = new();
// Act
await handler.DisposeAsync();
Func<Task> second = async () => await handler.DisposeAsync();
// Assert
await second.Should().NotThrowAsync();
}
#endregion
#region Query Parameters and Connection Tests
[Fact]
public async Task QueryParametersAreAppendedToUrlAsync()
{
// Arrange
TestHttpMessageHandler fake = new(static (req, _) =>
Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent(string.Empty) }));
await using DefaultHttpRequestHandler handler = new((_, _) => Task.FromResult<HttpClient?>(new HttpClient(fake)));
HttpRequestInfo info = new()
{
Method = "GET",
Url = TestUrl,
QueryParameters = new Dictionary<string, string>
{
["filter"] = "active items",
["ids"] = "1,2,3",
},
};
// Act
await handler.SendAsync(info);
// Assert
fake.LastRequest.Should().NotBeNull();
string? query = fake.LastRequest!.RequestUri!.Query;
query.Should().Contain("filter=active%20items");
query.Should().Contain("ids=1%2C2%2C3");
}
[Fact]
public async Task QueryParametersPreserveExistingQueryStringAsync()
{
// Arrange
TestHttpMessageHandler fake = new(static (req, _) =>
Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent(string.Empty) }));
await using DefaultHttpRequestHandler handler = new((_, _) => Task.FromResult<HttpClient?>(new HttpClient(fake)));
HttpRequestInfo info = new()
{
Method = "GET",
Url = TestUrl + "?existing=yes",
QueryParameters = new Dictionary<string, string>
{
["added"] = "true",
},
};
// Act
await handler.SendAsync(info);
// Assert
fake.LastRequest!.RequestUri!.Query.Should().Be("?existing=yes&added=true");
}
#endregion
private sealed class TestHttpMessageHandler : HttpMessageHandler
{
private readonly Func<HttpRequestMessage, CancellationToken, Task<HttpResponseMessage>> _responseFactory;
public TestHttpMessageHandler(Func<HttpRequestMessage, CancellationToken, Task<HttpResponseMessage>> responseFactory)
{
this._responseFactory = responseFactory;
}
public HttpRequestMessage? LastRequest { get; private set; }
public string? LastRequestBody { get; private set; }
public string? LastRequestContentType { get; private set; }
protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
{
this.LastRequest = request;
if (request.Content is not null)
{
#if NET
this.LastRequestBody = await request.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false);
#else
this.LastRequestBody = await request.Content.ReadAsStringAsync().ConfigureAwait(false);
#endif
this.LastRequestContentType = request.Content.Headers.ContentType?.MediaType;
}
return await this._responseFactory(request, cancellationToken).ConfigureAwait(false);
}
}
}
@@ -1,759 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
using System;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Agents.AI.Workflows.Declarative.ObjectModel;
using Microsoft.Agents.AI.Workflows.Declarative.PowerFx;
using Microsoft.Agents.ObjectModel;
using Microsoft.Extensions.AI;
using Microsoft.PowerFx.Types;
using Moq;
namespace Microsoft.Agents.AI.Workflows.Declarative.UnitTests.ObjectModel;
/// <summary>
/// Tests for <see cref="HttpRequestExecutor"/>.
/// </summary>
public sealed class HttpRequestExecutorTest(ITestOutputHelper output) : WorkflowActionExecutorTest(output)
{
private const string TestUrl = "https://api.example.com/data";
private readonly Mock<ResponseAgentProvider> _agentProvider = new(MockBehavior.Loose);
[Fact]
public void InvalidModel()
{
// Arrange
Mock<IHttpRequestHandler> mockHandler = new();
// Act & Assert
Assert.Throws<DeclarativeModelException>(() => new HttpRequestExecutor(
new HttpRequestAction(),
mockHandler.Object,
this._agentProvider.Object,
this.State));
}
[Fact]
public void HttpRequestIsDiscreteAction()
{
// Arrange
Mock<IHttpRequestHandler> mockHandler = new();
HttpRequestAction model = this.CreateModel(
displayName: nameof(HttpRequestIsDiscreteAction),
url: TestUrl,
method: HttpMethodType.Get);
HttpRequestExecutor action = new(model, mockHandler.Object, this._agentProvider.Object, this.State);
// Act & Assert — IsDiscreteAction should be true for HttpRequest (single-step action).
VerifyIsDiscrete(action, isDiscrete: true);
}
[Fact]
public async Task HttpGetReturnsJsonObjectAsync()
{
// Arrange
this.State.InitializeSystem();
const string ResponseVar = "Result";
HttpRequestAction model = this.CreateModel(
displayName: nameof(HttpGetReturnsJsonObjectAsync),
url: TestUrl,
method: HttpMethodType.Get,
responseVariable: ResponseVar);
MockHttpRequestHandler handler = new(HttpRequestResult("{\"key\":\"value\",\"number\":42}"));
HttpRequestExecutor action = new(model, handler.Object, this._agentProvider.Object, this.State);
// Act
await this.ExecuteAsync(action);
// Assert
VerifyModel(model, action);
Assert.IsType<RecordValue>(this.State.Get(ResponseVar), exactMatch: false);
handler.VerifySent(info => info.Method == "GET" && info.Url == TestUrl);
}
[Fact]
public async Task HttpGetReturnsPlainStringAsync()
{
// Arrange
this.State.InitializeSystem();
const string ResponseVar = "Result";
HttpRequestAction model = this.CreateModel(
displayName: nameof(HttpGetReturnsPlainStringAsync),
url: TestUrl,
method: HttpMethodType.Get,
responseVariable: ResponseVar);
MockHttpRequestHandler handler = new(HttpRequestResult("not-json content"));
HttpRequestExecutor action = new(model, handler.Object, this._agentProvider.Object, this.State);
// Act
await this.ExecuteAsync(action);
// Assert
VerifyModel(model, action);
this.VerifyState(ResponseVar, FormulaValue.New("not-json content"));
}
[Fact]
public async Task HttpGetWithEmptyBodyYieldsBlankAsync()
{
// Arrange
this.State.InitializeSystem();
const string ResponseVar = "Result";
HttpRequestAction model = this.CreateModel(
displayName: nameof(HttpGetWithEmptyBodyYieldsBlankAsync),
url: TestUrl,
method: HttpMethodType.Get,
responseVariable: ResponseVar);
MockHttpRequestHandler handler = new(HttpRequestResult(null));
HttpRequestExecutor action = new(model, handler.Object, this._agentProvider.Object, this.State);
// Act
await this.ExecuteAsync(action);
// Assert
VerifyModel(model, action);
this.VerifyUndefined(ResponseVar);
}
[Fact]
public async Task HttpGetForwardsHeadersAsync()
{
// Arrange
this.State.InitializeSystem();
HttpRequestAction model = this.CreateModel(
displayName: nameof(HttpGetForwardsHeadersAsync),
url: TestUrl,
method: HttpMethodType.Get,
headers: new Dictionary<string, string>
{
["Authorization"] = "Bearer token",
["Accept"] = "application/json",
});
MockHttpRequestHandler handler = new(HttpRequestResult("{}"));
HttpRequestExecutor action = new(model, handler.Object, this._agentProvider.Object, this.State);
// Act
await this.ExecuteAsync(action);
// Assert
VerifyModel(model, action);
handler.VerifySent(info =>
info.Headers?["Authorization"] == "Bearer token" &&
info.Headers?["Accept"] == "application/json");
}
[Fact]
public async Task HttpPostWithJsonBodyAsync()
{
// Arrange
this.State.InitializeSystem();
HttpRequestAction model = this.CreateModel(
displayName: nameof(HttpPostWithJsonBodyAsync),
url: TestUrl,
method: HttpMethodType.Post,
jsonBody: new StringDataValue("hello"));
MockHttpRequestHandler handler = new(HttpRequestResult("{}"));
HttpRequestExecutor action = new(model, handler.Object, this._agentProvider.Object, this.State);
// Act
await this.ExecuteAsync(action);
// Assert
VerifyModel(model, action);
handler.VerifySent(info =>
info.Method == "POST" &&
info.BodyContentType == "application/json" &&
info.Body == "\"hello\"");
}
[Fact]
public async Task HttpPostWithRawBodyAsync()
{
// Arrange
this.State.InitializeSystem();
HttpRequestAction model = this.CreateModel(
displayName: nameof(HttpPostWithRawBodyAsync),
url: TestUrl,
method: HttpMethodType.Post,
rawBody: "raw body content",
rawContentType: "text/plain");
MockHttpRequestHandler handler = new(HttpRequestResult(""));
HttpRequestExecutor action = new(model, handler.Object, this._agentProvider.Object, this.State);
// Act
await this.ExecuteAsync(action);
// Assert
VerifyModel(model, action);
handler.VerifySent(info =>
info.BodyContentType == "text/plain" &&
info.Body == "raw body content");
}
[Fact]
public async Task HttpRequestRaisesOnErrorByDefaultAsync()
{
// Arrange
this.State.InitializeSystem();
HttpRequestAction model = this.CreateModel(
displayName: nameof(HttpRequestRaisesOnErrorByDefaultAsync),
url: TestUrl,
method: HttpMethodType.Get);
MockHttpRequestHandler handler = new(HttpRequestResult("server error", statusCode: 500, isSuccess: false));
HttpRequestExecutor action = new(model, handler.Object, this._agentProvider.Object, this.State);
// Act & Assert
await Assert.ThrowsAsync<DeclarativeActionException>(() => this.ExecuteAsync(action));
}
[Fact]
public async Task HttpRequestFailureExceptionTruncatesLongBodyAsync()
{
// Arrange
this.State.InitializeSystem();
HttpRequestAction model = this.CreateModel(
displayName: nameof(HttpRequestFailureExceptionTruncatesLongBodyAsync),
url: TestUrl,
method: HttpMethodType.Get);
string longBody = new('x', 10_000);
MockHttpRequestHandler handler = new(HttpRequestResult(longBody, statusCode: 500, isSuccess: false));
HttpRequestExecutor action = new(model, handler.Object, this._agentProvider.Object, this.State);
// Act
DeclarativeActionException exception =
await Assert.ThrowsAsync<DeclarativeActionException>(() => this.ExecuteAsync(action));
// Assert - message contains status and truncation marker, bounded in length, never the full body.
Assert.Contains("500", exception.Message);
Assert.Contains("[truncated]", exception.Message);
Assert.DoesNotContain(longBody, exception.Message);
Assert.True(exception.Message.Length < 512, $"Exception message too long: {exception.Message.Length} chars.");
}
[Fact]
public async Task HttpRequestFailureExceptionOmitsEmptyBodyAsync()
{
// Arrange
this.State.InitializeSystem();
HttpRequestAction model = this.CreateModel(
displayName: nameof(HttpRequestFailureExceptionOmitsEmptyBodyAsync),
url: TestUrl,
method: HttpMethodType.Get);
MockHttpRequestHandler handler = new(HttpRequestResult(body: null, statusCode: 404, isSuccess: false));
HttpRequestExecutor action = new(model, handler.Object, this._agentProvider.Object, this.State);
// Act
DeclarativeActionException exception =
await Assert.ThrowsAsync<DeclarativeActionException>(() => this.ExecuteAsync(action));
// Assert - status present, no stray "Body: ''" noise.
Assert.Contains("404", exception.Message);
Assert.DoesNotContain("Body:", exception.Message);
}
[Fact]
public async Task HttpRequestFailureExceptionSanitizesControlCharsAsync()
{
// Arrange
this.State.InitializeSystem();
HttpRequestAction model = this.CreateModel(
displayName: nameof(HttpRequestFailureExceptionSanitizesControlCharsAsync),
url: TestUrl,
method: HttpMethodType.Get);
MockHttpRequestHandler handler = new(HttpRequestResult("line1\r\nline2\tend", statusCode: 400, isSuccess: false));
HttpRequestExecutor action = new(model, handler.Object, this._agentProvider.Object, this.State);
// Act
DeclarativeActionException exception =
await Assert.ThrowsAsync<DeclarativeActionException>(() => this.ExecuteAsync(action));
// Assert - CR/LF/TAB collapsed to spaces so the message stays on one line.
Assert.DoesNotContain("\r", exception.Message);
Assert.DoesNotContain("\n", exception.Message);
Assert.DoesNotContain("\t", exception.Message);
Assert.Contains("line1", exception.Message);
Assert.Contains("line2", exception.Message);
}
[Fact]
public async Task HttpRequestPassesTimeoutToHandlerAsync()
{
// Arrange
this.State.InitializeSystem();
HttpRequestAction model = this.CreateModel(
displayName: nameof(HttpRequestPassesTimeoutToHandlerAsync),
url: TestUrl,
method: HttpMethodType.Get,
timeoutMilliseconds: 1500);
MockHttpRequestHandler handler = new(HttpRequestResult("{}"));
HttpRequestExecutor action = new(model, handler.Object, this._agentProvider.Object, this.State);
// Act
await this.ExecuteAsync(action);
// Assert
VerifyModel(model, action);
handler.VerifySent(info =>
info.Timeout is not null &&
info.Timeout.Value == TimeSpan.FromMilliseconds(1500));
}
[Fact]
public async Task HttpRequestTimeoutRaisesDeclarativeExceptionAsync()
{
// Arrange
this.State.InitializeSystem();
HttpRequestAction model = this.CreateModel(
displayName: nameof(HttpRequestTimeoutRaisesDeclarativeExceptionAsync),
url: TestUrl,
method: HttpMethodType.Get);
MockHttpRequestHandler handler = new(
HttpRequestResult("{}"),
throwOnSend: new OperationCanceledException());
HttpRequestExecutor action = new(model, handler.Object, this._agentProvider.Object, this.State);
// Act & Assert
await Assert.ThrowsAsync<DeclarativeActionException>(() => this.ExecuteAsync(action));
}
[Fact]
public async Task HttpRequestTransportFailureRaisesDeclarativeExceptionAsync()
{
// Arrange
this.State.InitializeSystem();
HttpRequestAction model = this.CreateModel(
displayName: nameof(HttpRequestTransportFailureRaisesDeclarativeExceptionAsync),
url: TestUrl,
method: HttpMethodType.Get);
MockHttpRequestHandler handler = new(
HttpRequestResult("{}"),
throwOnSend: new InvalidOperationException("transport failure"));
HttpRequestExecutor action = new(model, handler.Object, this._agentProvider.Object, this.State);
// Act & Assert
await Assert.ThrowsAsync<DeclarativeActionException>(() => this.ExecuteAsync(action));
}
[Fact]
public async Task HttpRequestStoresResponseHeadersAsync()
{
// Arrange
this.State.InitializeSystem();
const string HeaderVar = "Headers";
HttpRequestAction model = this.CreateModel(
displayName: nameof(HttpRequestStoresResponseHeadersAsync),
url: TestUrl,
method: HttpMethodType.Get,
responseHeadersVariable: HeaderVar);
Dictionary<string, IReadOnlyList<string>> responseHeaders = new(StringComparer.OrdinalIgnoreCase)
{
["X-Request-Id"] = ["abc-123"],
["Set-Cookie"] = ["a=1", "b=2"],
};
MockHttpRequestHandler handler = new(HttpRequestResult("{}", headers: responseHeaders));
HttpRequestExecutor action = new(model, handler.Object, this._agentProvider.Object, this.State);
// Act
await this.ExecuteAsync(action);
// Assert
VerifyModel(model, action);
FormulaValue storedHeaders = this.State.Get(HeaderVar);
Assert.IsType<RecordValue>(storedHeaders, exactMatch: false);
}
[Fact]
public async Task HttpRequestForwardsQueryParametersAsync()
{
// Arrange
this.State.InitializeSystem();
HttpRequestAction model = this.CreateModel(
displayName: nameof(HttpRequestForwardsQueryParametersAsync),
url: TestUrl,
method: HttpMethodType.Get,
queryParameters: new Dictionary<string, DataValue>
{
["filter"] = StringDataValue.Create("active"),
["limit"] = NumberDataValue.Create(10),
["includeDeleted"] = BooleanDataValue.Create(false),
});
MockHttpRequestHandler handler = new(HttpRequestResult("{}"));
HttpRequestExecutor action = new(model, handler.Object, this._agentProvider.Object, this.State);
// Act
await this.ExecuteAsync(action);
// Assert
VerifyModel(model, action);
handler.VerifySent(info =>
info.QueryParameters?.Count == 3 &&
info.QueryParameters["filter"] == "active" &&
info.QueryParameters["limit"] == "10" &&
info.QueryParameters["includeDeleted"] == "false");
}
[Fact]
public async Task HttpRequestAddsResponseToConversationAsync()
{
// Arrange
this.State.InitializeSystem();
const string ConversationId = "conv-12345";
const string ResponseBody = "response-text";
this._agentProvider
.Setup(p => p.CreateMessageAsync(It.IsAny<string>(), It.IsAny<ChatMessage>(), It.IsAny<CancellationToken>()))
.Returns<string, ChatMessage, CancellationToken>((_, message, _) => Task.FromResult(message));
HttpRequestAction model = this.CreateModel(
displayName: nameof(HttpRequestAddsResponseToConversationAsync),
url: TestUrl,
method: HttpMethodType.Get,
conversationId: ConversationId);
MockHttpRequestHandler handler = new(HttpRequestResult(ResponseBody));
HttpRequestExecutor action = new(model, handler.Object, this._agentProvider.Object, this.State);
// Act
await this.ExecuteAsync(action);
// Assert
VerifyModel(model, action);
this._agentProvider.Verify(
p => p.CreateMessageAsync(
ConversationId,
It.Is<ChatMessage>(m => m.Role == ChatRole.Assistant && m.Text == ResponseBody),
It.IsAny<CancellationToken>()),
Times.Once);
}
[Fact]
public async Task HttpRequestWithoutConversationIdSkipsConversationAsync()
{
// Arrange
this.State.InitializeSystem();
HttpRequestAction model = this.CreateModel(
displayName: nameof(HttpRequestWithoutConversationIdSkipsConversationAsync),
url: TestUrl,
method: HttpMethodType.Get);
MockHttpRequestHandler handler = new(HttpRequestResult("response"));
HttpRequestExecutor action = new(model, handler.Object, this._agentProvider.Object, this.State);
// Act
await this.ExecuteAsync(action);
// Assert
VerifyModel(model, action);
this._agentProvider.Verify(
p => p.CreateMessageAsync(It.IsAny<string>(), It.IsAny<ChatMessage>(), It.IsAny<CancellationToken>()),
Times.Never);
}
[Fact]
public async Task HttpRequestForwardsConnectionNameAsync()
{
// Arrange
this.State.InitializeSystem();
const string ConnectionName = "my-connection";
HttpRequestAction model = this.CreateModel(
displayName: nameof(HttpRequestForwardsConnectionNameAsync),
url: TestUrl,
method: HttpMethodType.Get,
connectionName: ConnectionName);
MockHttpRequestHandler handler = new(HttpRequestResult("{}"));
HttpRequestExecutor action = new(model, handler.Object, this._agentProvider.Object, this.State);
// Act
await this.ExecuteAsync(action);
// Assert
VerifyModel(model, action);
handler.VerifySent(info => info.ConnectionName == ConnectionName);
}
[Fact]
public async Task HttpRequestEmptyConversationIdSkipsConversationAsync()
{
// Arrange - empty-string conversationId should be treated as unset.
this.State.InitializeSystem();
HttpRequestAction model = this.CreateModel(
displayName: nameof(HttpRequestEmptyConversationIdSkipsConversationAsync),
url: TestUrl,
method: HttpMethodType.Get,
conversationId: "");
MockHttpRequestHandler handler = new(HttpRequestResult("response"));
HttpRequestExecutor action = new(model, handler.Object, this._agentProvider.Object, this.State);
// Act
await this.ExecuteAsync(action);
// Assert
VerifyModel(model, action);
this._agentProvider.Verify(
p => p.CreateMessageAsync(It.IsAny<string>(), It.IsAny<ChatMessage>(), It.IsAny<CancellationToken>()),
Times.Never);
}
[Fact]
public async Task HttpRequestEmptyResponseBodySkipsConversationAsync()
{
// Arrange - conversationId set, but empty body should not produce a conversation message.
this.State.InitializeSystem();
HttpRequestAction model = this.CreateModel(
displayName: nameof(HttpRequestEmptyResponseBodySkipsConversationAsync),
url: TestUrl,
method: HttpMethodType.Get,
conversationId: "conv-1");
MockHttpRequestHandler handler = new(HttpRequestResult(""));
HttpRequestExecutor action = new(model, handler.Object, this._agentProvider.Object, this.State);
// Act
await this.ExecuteAsync(action);
// Assert
VerifyModel(model, action);
this._agentProvider.Verify(
p => p.CreateMessageAsync(It.IsAny<string>(), It.IsAny<ChatMessage>(), It.IsAny<CancellationToken>()),
Times.Never);
}
[Fact]
public async Task HttpGetReturnsJsonArrayAsync()
{
// Arrange - exercises JsonValueKind.Array branch of ParseResponseBody.
this.State.InitializeSystem();
const string ResponseVar = "Result";
HttpRequestAction model = this.CreateModel(
displayName: nameof(HttpGetReturnsJsonArrayAsync),
url: TestUrl,
method: HttpMethodType.Get,
responseVariable: ResponseVar);
MockHttpRequestHandler handler = new(HttpRequestResult("[1, 2, 3]"));
HttpRequestExecutor action = new(model, handler.Object, this._agentProvider.Object, this.State);
// Act
await this.ExecuteAsync(action);
// Assert
VerifyModel(model, action);
FormulaValue stored = this.State.Get(ResponseVar);
Assert.IsType<TableValue>(stored, exactMatch: false);
}
[Fact]
public async Task HttpGetWithEmptyHeaderValueDropsHeaderAsync()
{
// Arrange - empty header values should be filtered out (matches GetHeaders guard).
this.State.InitializeSystem();
HttpRequestAction model = this.CreateModel(
displayName: nameof(HttpGetWithEmptyHeaderValueDropsHeaderAsync),
url: TestUrl,
method: HttpMethodType.Get,
headers: new Dictionary<string, string>
{
["X-Trace"] = "trace-1",
["X-Empty"] = "",
});
MockHttpRequestHandler handler = new(HttpRequestResult("{}"));
HttpRequestExecutor action = new(model, handler.Object, this._agentProvider.Object, this.State);
// Act
await this.ExecuteAsync(action);
// Assert
VerifyModel(model, action);
handler.VerifySent(info =>
info.Headers?.ContainsKey("X-Trace") == true &&
info.Headers?.ContainsKey("X-Empty") == false);
}
[Fact]
public async Task HttpRequestZeroTimeoutNotForwardedAsync()
{
// Arrange - non-positive timeouts should not be forwarded (handler default applies).
this.State.InitializeSystem();
HttpRequestAction model = this.CreateModel(
displayName: nameof(HttpRequestZeroTimeoutNotForwardedAsync),
url: TestUrl,
method: HttpMethodType.Get,
timeoutMilliseconds: 0);
MockHttpRequestHandler handler = new(HttpRequestResult("{}"));
HttpRequestExecutor action = new(model, handler.Object, this._agentProvider.Object, this.State);
// Act
await this.ExecuteAsync(action);
// Assert
VerifyModel(model, action);
handler.VerifySent(info => info.Timeout is null);
}
private static HttpRequestResult HttpRequestResult(
string? body,
int statusCode = 200,
bool isSuccess = true,
IReadOnlyDictionary<string, IReadOnlyList<string>>? headers = null) =>
new()
{
StatusCode = statusCode,
IsSuccessStatusCode = isSuccess,
Body = body,
Headers = headers,
};
private HttpRequestAction CreateModel(
string displayName,
string url,
HttpMethodType method,
string? responseVariable = null,
string? responseHeadersVariable = null,
IReadOnlyDictionary<string, string>? headers = null,
IReadOnlyDictionary<string, DataValue>? queryParameters = null,
string? conversationId = null,
string? connectionName = null,
DataValue? jsonBody = null,
string? rawBody = null,
string? rawContentType = null,
long? timeoutMilliseconds = null,
string? continueOnErrorStatusVariable = null,
string? continueOnErrorBodyVariable = null)
{
HttpRequestAction.Builder builder = new()
{
Id = this.CreateActionId(),
DisplayName = this.FormatDisplayName(displayName),
Url = new StringExpression.Builder(StringExpression.Literal(url)),
Method = new EnumExpression<HttpMethodTypeWrapper>.Builder(
EnumExpression<HttpMethodTypeWrapper>.Literal(HttpMethodTypeWrapper.Get(method))),
};
if (responseVariable is not null)
{
builder.Response = PropertyPath.Create(FormatVariablePath(responseVariable));
}
if (responseHeadersVariable is not null)
{
builder.ResponseHeaders = PropertyPath.Create(FormatVariablePath(responseHeadersVariable));
}
if (headers is not null)
{
foreach (KeyValuePair<string, string> header in headers)
{
builder.Headers.Add(header.Key, new StringExpression.Builder(StringExpression.Literal(header.Value)));
}
}
if (queryParameters is not null)
{
foreach (KeyValuePair<string, DataValue> parameter in queryParameters)
{
builder.QueryParameters.Add(parameter.Key, new ValueExpression.Builder(ValueExpression.Literal(parameter.Value)));
}
}
if (conversationId is not null)
{
builder.ConversationId = new StringExpression.Builder(StringExpression.Literal(conversationId));
}
if (connectionName is not null)
{
builder.Connection = new RemoteConnection.Builder
{
Name = new StringExpression.Builder(StringExpression.Literal(connectionName)),
};
}
if (jsonBody is not null)
{
builder.Body = new JsonRequestContent.Builder()
{
Content = new ValueExpression.Builder(ValueExpression.Literal(jsonBody)),
};
}
else if (rawBody is not null)
{
RawRequestContent.Builder rawBuilder = new()
{
Content = new StringExpression.Builder(StringExpression.Literal(rawBody)),
};
if (rawContentType is not null)
{
rawBuilder.ContentType = new StringExpression.Builder(StringExpression.Literal(rawContentType));
}
builder.Body = rawBuilder;
}
if (timeoutMilliseconds is not null)
{
builder.RequestTimeoutInMilliseconds = new IntExpression.Builder(IntExpression.Literal(timeoutMilliseconds.Value));
}
if (continueOnErrorStatusVariable is not null || continueOnErrorBodyVariable is not null)
{
ContinueOnErrorBehavior.Builder continueBuilder = new();
if (continueOnErrorStatusVariable is not null)
{
continueBuilder.StatusCode = PropertyPath.Create(FormatVariablePath(continueOnErrorStatusVariable));
}
if (continueOnErrorBodyVariable is not null)
{
continueBuilder.ErrorResponseBody = PropertyPath.Create(FormatVariablePath(continueOnErrorBodyVariable));
}
builder.ErrorHandling = continueBuilder;
}
return AssignParent<HttpRequestAction>(builder);
}
private sealed class MockHttpRequestHandler : Mock<IHttpRequestHandler>
{
private HttpRequestInfo? _lastRequest;
public MockHttpRequestHandler(HttpRequestResult result, Exception? throwOnSend = null)
{
this.Setup(handler => handler.SendAsync(It.IsAny<HttpRequestInfo>(), It.IsAny<CancellationToken>()))
.Returns<HttpRequestInfo, CancellationToken>((info, _) =>
{
this._lastRequest = info;
if (throwOnSend is not null)
{
throw throwOnSend;
}
return Task.FromResult(result);
});
}
public void VerifySent(Func<HttpRequestInfo, bool> predicate)
{
Assert.NotNull(this._lastRequest);
Assert.True(predicate(this._lastRequest!), "Sent HTTP request did not match expected predicate.");
}
}
}
@@ -1,15 +0,0 @@
kind: Workflow
trigger:
kind: OnConversationStart
id: my_workflow
actions:
- kind: HttpRequestAction
id: http_request
method: GET
url: =Concatenate("https://api.example.test/items/", System.LastMessageText)
headers:
Accept: application/json
response: Local.HttpResult
responseHeaders: Local.HttpHeaders
+1 -1
View File
@@ -61,7 +61,7 @@ repos:
additional_dependencies: ["bandit[toml]"]
- repo: https://github.com/astral-sh/uv-pre-commit
# uv version.
rev: 0.11.6
rev: 0.10.10
hooks:
# Update the uv lockfile
- id: uv-lock
-1
View File
@@ -69,7 +69,6 @@ python/
### Azure Integrations
- [foundry](packages/foundry/README.md) - Microsoft Foundry chat, agent, memory, and embedding integrations
- [azure-contentunderstanding](packages/azure-contentunderstanding/AGENTS.md) - Azure Content Understanding context provider
- [azure-ai-search](packages/azure-ai-search/AGENTS.md) - Azure AI Search RAG
- [azure-cosmos](packages/azure-cosmos/AGENTS.md) - Azure Cosmos DB-backed history provider
- [azurefunctions](packages/azurefunctions/AGENTS.md) - Azure Functions hosting
+1 -44
View File
@@ -7,47 +7,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased]
## [1.2.2] - 2026-04-29
### Added
- **agent-framework-azure-contentunderstanding**: New alpha package — Azure AI Content Understanding context provider that auto-analyzes file attachments (documents, images, audio, video) and injects structured results into the LLM context, with multi-document session state, configurable timeout, output filtering via `AnalysisSection`, and auto-registered `list_documents` / `get_analyzed_document` tools ([#4829](https://github.com/microsoft/agent-framework/pull/4829))
- **agent-framework-foundry-hosting**: Add hosted Durable Workflow support — propagate full conversation history to workflow agents and wire `Workflow.as_agent()` end-to-end via the foundry hosting layer ([#5531](https://github.com/microsoft/agent-framework/pull/5531))
### Changed
- **agent-framework-orchestrations**: [BREAKING] Standardize orchestration terminal outputs as `AgentResponse` so `Workflow.as_agent()` returns the final answer only; aligns sequential-approval (`with_request_info`) and concurrent (`intermediate_outputs=True`) flows on the same output contract ([#5301](https://github.com/microsoft/agent-framework/pull/5301))
- **agent-framework-core**, **agent-framework-declarative**: Preserve `Workflow.run()` shared state across calls so multi-turn `WorkflowAgent` invocations retain context, accept `list[Message]` input in the declarative start executor, and coerce `Enum` values when serializing PowerFx symbols ([#5531](https://github.com/microsoft/agent-framework/pull/5531))
- **dependencies**: Update workspace package dependencies and preserve `mcp[ws]` / `uvicorn[standard]` extras through override-dependencies in `/python` ([#5555](https://github.com/microsoft/agent-framework/pull/5555))
### Fixed
- **agent-framework-core**: Fix observability spans not being correctly nested when using streaming ([#5552](https://github.com/microsoft/agent-framework/pull/5552))
- **agent-framework-openai**: Fix `file_search` citations breaking the assistant-message history roundtrip — skip `hosted_file` content in the assistant role so the Responses API no longer rejects `input_file` ([#5557](https://github.com/microsoft/agent-framework/pull/5557))
## [1.2.1] - 2026-04-28
### Added
- **agent-framework-foundry-hosting**: Add file data type support to hosted-agent Responses, refresh `foundry-hosted-agents` samples, and add response test coverage ([#5485](https://github.com/microsoft/agent-framework/pull/5485))
- **samples**: Add `requirements.txt` and `.env.example` to the `a2a/` hosting sample for pip-based setup ([#5510](https://github.com/microsoft/agent-framework/pull/5510))
### Changed
- **dependencies**: Update `rich` requirement from `<15.0.0,>=13.7.1` to `>=13.7.1,<16.0.0` in `/python` ([#5227](https://github.com/microsoft/agent-framework/pull/5227))
- **dependencies**: Bump `prek` from `0.3.8` to `0.3.9` in `/python` ([#5228](https://github.com/microsoft/agent-framework/pull/5228))
- **dependencies**: Bump `python-multipart` from `0.0.22` to `0.0.26` in `/python` ([#5286](https://github.com/microsoft/agent-framework/pull/5286))
- **dependencies**: Bump `pyasn1` from `0.6.2` to `0.6.3` in `/python` ([#4748](https://github.com/microsoft/agent-framework/pull/4748))
- **dependencies**: Bump `pytest` from `9.0.2` to `9.0.3` in `/python/packages/ag-ui` ([#5461](https://github.com/microsoft/agent-framework/pull/5461))
- **dependencies**: Bump `pytest` from `9.0.2` to `9.0.3` in `/python/packages/devui` ([#5492](https://github.com/microsoft/agent-framework/pull/5492))
- **dependencies**: Bump `pytest` from `9.0.2` to `9.0.3` in `/python/packages/lab` ([#5470](https://github.com/microsoft/agent-framework/pull/5470))
- **dependencies**: Bump `uv` from `0.11.3` to `0.11.6` in `/python/packages/lab` ([#5469](https://github.com/microsoft/agent-framework/pull/5469))
- **dependencies**: Bump `vite` from `7.1.12` to `7.3.2` in `/python/packages/devui/frontend` ([#5127](https://github.com/microsoft/agent-framework/pull/5127))
- **dependencies**: Bump `vite` from `7.1.12` to `7.3.2` in `/python/samples/05-end-to-end/chatkit-integration/frontend` ([#5126](https://github.com/microsoft/agent-framework/pull/5126))
- **dependencies**: Bump `postcss` from `8.5.6` to `8.5.10` in `/python/packages/devui/frontend` ([#5484](https://github.com/microsoft/agent-framework/pull/5484))
- **dependencies**: Bump `postcss` from `8.5.6` to `8.5.10` in `/python/samples/05-end-to-end/chatkit-integration/frontend` ([#5491](https://github.com/microsoft/agent-framework/pull/5491))
- **dependencies**: Bump `postcss` from `8.5.6` to `8.5.12` in `/python/samples/05-end-to-end/ag_ui_workflow_handoff/frontend` ([#5527](https://github.com/microsoft/agent-framework/pull/5527))
- **dependencies**: Bump `picomatch` from `4.0.3` to `4.0.4` in `/python/packages/devui/frontend` ([#4921](https://github.com/microsoft/agent-framework/pull/4921))
- **dependencies**: Bump `picomatch` from `4.0.3` to `4.0.4` in `/python/samples/05-end-to-end/ag_ui_workflow_handoff/frontend` ([#4936](https://github.com/microsoft/agent-framework/pull/4936))
### Fixed
- **agent-framework-core**: Prevent `inner_exception` from being lost in `AgentFrameworkException` ([#5167](https://github.com/microsoft/agent-framework/pull/5167))
## [1.2.0] - 2026-04-24
### Added
@@ -1018,9 +977,7 @@ Release candidate for **agent-framework-core** and **agent-framework-azure-ai**
For more information, see the [announcement blog post](https://devblogs.microsoft.com/foundry/introducing-microsoft-agent-framework-the-open-source-engine-for-agentic-ai-apps/).
[Unreleased]: https://github.com/microsoft/agent-framework/compare/python-1.2.2...HEAD
[1.2.2]: https://github.com/microsoft/agent-framework/compare/python-1.2.1...python-1.2.2
[1.2.1]: https://github.com/microsoft/agent-framework/compare/python-1.2.0...python-1.2.1
[Unreleased]: https://github.com/microsoft/agent-framework/compare/python-1.2.0...HEAD
[1.2.0]: https://github.com/microsoft/agent-framework/compare/python-1.1.1...python-1.2.0
[1.1.1]: https://github.com/microsoft/agent-framework/compare/python-1.1.0...python-1.1.1
[1.1.0]: https://github.com/microsoft/agent-framework/compare/python-1.0.1...python-1.1.0
-1
View File
@@ -18,7 +18,6 @@ Status is grouped into these buckets:
| `agent-framework-a2a` | `python/packages/a2a` | `beta` |
| `agent-framework-ag-ui` | `python/packages/ag-ui` | `beta` |
| `agent-framework-anthropic` | `python/packages/anthropic` | `beta` |
| `agent-framework-azure-contentunderstanding` | `python/packages/azure-contentunderstanding` | `alpha` |
| `agent-framework-azure-ai-search` | `python/packages/azure-ai-search` | `beta` |
| `agent-framework-azure-cosmos` | `python/packages/azure-cosmos` | `beta` |
| `agent-framework-azurefunctions` | `python/packages/azurefunctions` | `beta` |
+2 -2
View File
@@ -4,7 +4,7 @@ description = "A2A integration for Microsoft Agent Framework."
authors = [{ name = "Microsoft", email = "af-support@microsoft.com"}]
readme = "README.md"
requires-python = ">=3.10"
version = "1.0.0b260429"
version = "1.0.0b260424"
license-files = ["LICENSE"]
urls.homepage = "https://aka.ms/agent-framework"
urls.source = "https://github.com/microsoft/agent-framework/tree/main/python"
@@ -23,7 +23,7 @@ classifiers = [
"Typing :: Typed",
]
dependencies = [
"agent-framework-core>=1.2.2,<2",
"agent-framework-core>=1.2.0,<2",
"a2a-sdk>=0.3.5,<0.3.24",
]
+3 -3
View File
@@ -1,6 +1,6 @@
[project]
name = "agent-framework-ag-ui"
version = "1.0.0b260429"
version = "1.0.0b260424"
description = "AG-UI protocol integration for Agent Framework"
readme = "README.md"
license-files = ["LICENSE"]
@@ -22,7 +22,7 @@ classifiers = [
"Typing :: Typed",
]
dependencies = [
"agent-framework-core>=1.2.2,<2",
"agent-framework-core>=1.2.0,<2",
"ag-ui-protocol>=0.1.16,<0.2",
"fastapi>=0.115.0,<0.133.1",
"uvicorn[standard]>=0.30.0,<0.42.0"
@@ -30,7 +30,7 @@ dependencies = [
[project.optional-dependencies]
dev = [
"pytest==9.0.3",
"pytest==9.0.2",
"httpx==0.28.1",
]
+2 -2
View File
@@ -4,7 +4,7 @@ description = "Anthropic integration for Microsoft Agent Framework."
authors = [{ name = "Microsoft", email = "af-support@microsoft.com"}]
readme = "README.md"
requires-python = ">=3.10"
version = "1.0.0b260429"
version = "1.0.0b260424"
license-files = ["LICENSE"]
urls.homepage = "https://aka.ms/agent-framework"
urls.source = "https://github.com/microsoft/agent-framework/tree/main/python"
@@ -23,7 +23,7 @@ classifiers = [
"Typing :: Typed",
]
dependencies = [
"agent-framework-core>=1.2.2,<2",
"agent-framework-core>=1.2.0,<2",
"anthropic>=0.80.0,<0.80.1",
]
@@ -4,7 +4,7 @@ description = "Azure AI Search integration for Microsoft Agent Framework."
authors = [{ name = "Microsoft", email = "af-support@microsoft.com"}]
readme = "README.md"
requires-python = ">=3.10"
version = "1.0.0b260429"
version = "1.0.0b260424"
license-files = ["LICENSE"]
urls.homepage = "https://aka.ms/agent-framework"
urls.source = "https://github.com/microsoft/agent-framework/tree/main/python"
@@ -23,7 +23,7 @@ classifiers = [
"Typing :: Typed",
]
dependencies = [
"agent-framework-core>=1.2.2,<2",
"agent-framework-core>=1.2.0,<2",
"azure-search-documents>=11.7.0b2,<11.7.0b3",
]
@@ -1,3 +0,0 @@
# Local-only files (not committed)
_local_only/
*_local_only*
@@ -1,71 +0,0 @@
# AGENTS.md — azure-contentunderstanding
## Package Overview
`agent-framework-azure-contentunderstanding` integrates Azure Content Understanding (CU)
into the Agent Framework as a context provider. It automatically analyzes file attachments
(documents, images, audio, video) and injects structured results into the LLM context.
## Public API
| Symbol | Type | Description |
|--------|------|-------------|
| `ContentUnderstandingContextProvider` | class | Main context provider — extends `ContextProvider` |
| `AnalysisSection` | enum | Output section selector (MARKDOWN, FIELDS, etc.) |
| `DocumentStatus` | enum | Document lifecycle state (ANALYZING, UPLOADING, READY, FAILED) |
| `FileSearchBackend` | ABC | Abstract vector store file operations interface |
| `FileSearchConfig` | dataclass | Configuration for CU + vector store RAG mode |
## Architecture
- **`_context_provider.py`** — Main provider implementation. Overrides `before_run()` to detect
file attachments, call the CU API, manage session state with multi-document tracking,
and auto-register retrieval tools for follow-up turns.
- **Analyzer auto-detection** — When `analyzer_id=None` (default), `_resolve_analyzer_id()`
selects the CU analyzer based on media type prefix: `audio/` → `prebuilt-audioSearch`,
`video/` → `prebuilt-videoSearch`, everything else → `prebuilt-documentSearch`.
- **Multi-segment output** — CU splits long video/audio into multiple scene segments
(each a separate `contents[]` entry with its own `startTimeMs`, `endTimeMs`, `markdown`,
and `fields`). `_extract_sections()` produces:
- `segments`: list of per-segment dicts, each with `markdown`, `fields`, `start_time_s`, `end_time_s`
- `markdown`: concatenated at top level with `---` separators (for file_search uploads)
- `duration_seconds`: computed from global `min(startTimeMs)` → `max(endTimeMs)`
- Metadata (`kind`, `resolution`): taken from the first segment
- **Speaker diarization (not identification)** — CU transcripts label speakers as
`<Speaker 1>`, `<Speaker 2>`, etc. CU does **not** identify speakers by name.
- **file_search RAG** — When `FileSearchConfig` is provided, CU-extracted markdown is
uploaded to an OpenAI vector store and a `file_search` tool is registered on the context
instead of injecting the full document content. This enables token-efficient retrieval
for large documents.
- **`_models.py`** — `AnalysisSection` enum, `DocumentStatus` enum, `DocumentEntry` TypedDict,
`FileSearchConfig` dataclass.
- **`_file_search.py`** — `FileSearchBackend` ABC, `OpenAIFileSearchBackend`,
`FoundryFileSearchBackend`.
## Key Patterns
- Follows the Azure AI Search context provider pattern (same lifecycle, config style).
- Uses provider-scoped `state` dict for multi-document tracking across turns.
- Auto-registers `list_documents()` tool via `context.extend_tools()`.
- Configurable timeout (`max_wait`) with `asyncio.create_task()` background fallback.
- Strips supported binary attachments from `input_messages` to prevent LLM API errors.
- Explicit `analyzer_id` always overrides auto-detection (user preference wins).
- Vector store resources are cleaned up in `close()` / `__aexit__`.
## Samples
| Sample | Description |
|--------|-------------|
| `01_document_qa.py` | Upload a PDF via URL, ask questions about it |
| `02_multi_turn_session.py` | AgentSession persistence across turns |
| `03_multimodal_chat.py` | PDF + audio + video parallel analysis |
| `04_invoice_processing.py` | Structured field extraction with `prebuilt-invoice` analyzer |
| `05_large_doc_file_search.py` | CU extraction + OpenAI vector store RAG |
| `02-devui/01-multimodal_agent/` | DevUI web UI for CU-powered chat |
| `02-devui/02-file_search_agent/` | DevUI web UI combining CU + file_search RAG |
## Running Tests
```bash
uv run poe test -P azure-contentunderstanding
```
@@ -1,21 +0,0 @@
MIT License
Copyright (c) Microsoft Corporation.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE
@@ -1,127 +0,0 @@
# Get Started with Azure Content Understanding in Microsoft Agent Framework
Please install this package via pip:
```bash
pip install agent-framework-azure-contentunderstanding --pre
```
## Azure Content Understanding Integration
### Prerequisites
Before using this package, you need an Azure Content Understanding resource:
1. An active **Azure subscription** ([create one for free](https://azure.microsoft.com/pricing/purchase-options/azure-account))
2. A **Microsoft Foundry resource** created in a [supported region](https://learn.microsoft.com/azure/ai-services/content-understanding/language-region-support)
3. **Default model deployments** configured for your resource (GPT-4.1, GPT-4.1-mini, text-embedding-3-large)
Follow the [prerequisites section](https://learn.microsoft.com/azure/ai-services/content-understanding/quickstart/use-rest-api?tabs=portal%2Cdocument&pivots=programming-language-rest#prerequisites) in the Azure Content Understanding quickstart for setup instructions.
### Introduction
The Azure Content Understanding integration provides a context provider that automatically analyzes file attachments (documents, images, audio, video) using [Azure Content Understanding](https://learn.microsoft.com/azure/ai-services/content-understanding/) and injects structured results into the LLM context.
- **Document & image analysis**: State-of-the-art OCR with markdown extraction, table preservation, and structured field extraction — handles scanned PDFs, handwritten content, and complex layouts
- **Audio & video analysis**: Transcription, speaker diarization, and per-segment summaries
- **Background processing**: Configurable timeout with async background fallback for large files
- **file_search integration**: Optional vector store upload for token-efficient RAG on large documents
> Learn more about Azure Content Understanding capabilities at [https://learn.microsoft.com/azure/ai-services/content-understanding/](https://learn.microsoft.com/azure/ai-services/content-understanding/)
### Basic Usage Example
See the [samples directory](samples/) which demonstrates:
- Single PDF upload and Q&A ([01_document_qa](samples/01-get-started/01_document_qa.py))
- Multi-turn sessions with cached results ([02_multi_turn_session](samples/01-get-started/02_multi_turn_session.py))
- PDF + audio + video parallel analysis ([03_multimodal_chat](samples/01-get-started/03_multimodal_chat.py))
- Structured field extraction with prebuilt-invoice ([04_invoice_processing](samples/01-get-started/04_invoice_processing.py))
- CU extraction + OpenAI vector store RAG ([05_large_doc_file_search](samples/01-get-started/05_large_doc_file_search.py))
- Interactive web UI with DevUI ([02-devui](samples/02-devui/))
```python
import asyncio
from agent_framework import Agent, AgentSession, Message, Content
from agent_framework.foundry import FoundryChatClient
from agent_framework.foundry import ContentUnderstandingContextProvider
from azure.identity import AzureCliCredential
credential = AzureCliCredential()
cu = ContentUnderstandingContextProvider(
endpoint="https://my-resource.cognitiveservices.azure.com/",
credential=credential,
max_wait=None, # block until CU extraction completes before sending to LLM
)
client = FoundryChatClient(
project_endpoint="https://your-project.services.ai.azure.com",
model="gpt-4.1",
credential=credential,
)
async def main():
async with cu:
agent = Agent(
client=client,
name="DocumentQA",
instructions="You are a helpful document analyst.",
context_providers=[cu],
)
session = AgentSession()
response = await agent.run(
Message(role="user", contents=[
Content.from_text("What's on this invoice?"),
Content.from_uri(
"https://raw.githubusercontent.com/Azure-Samples/"
"azure-ai-content-understanding-assets/main/document/invoice.pdf",
media_type="application/pdf",
additional_properties={"filename": "invoice.pdf"},
),
]),
session=session,
)
print(response.text)
asyncio.run(main())
```
### Supported File Types
| Category | Types |
|----------|-------|
| Documents | PDF, DOCX, XLSX, PPTX, HTML, TXT, Markdown |
| Images | JPEG, PNG, TIFF, BMP |
| Audio | WAV, MP3, M4A, FLAC, OGG |
| Video | MP4, MOV, AVI, WebM |
For the complete list of supported file types and size limits, see [Azure Content Understanding service limits](https://learn.microsoft.com/azure/ai-services/content-understanding/service-limits#input-file-limits).
### Environment Variables
The provider supports automatic endpoint resolution from environment variables.
When ``endpoint`` is not passed to the constructor, it is loaded from
``AZURE_CONTENTUNDERSTANDING_ENDPOINT``:
```python
# Endpoint auto-loaded from AZURE_CONTENTUNDERSTANDING_ENDPOINT env var
cu = ContentUnderstandingContextProvider(credential=credential)
```
Set these in your shell or in a `.env` file:
```bash
AZURE_CONTENTUNDERSTANDING_ENDPOINT=https://your-cu-resource.cognitiveservices.azure.com/
AZURE_AI_PROJECT_ENDPOINT=https://your-project.services.ai.azure.com
AZURE_OPENAI_DEPLOYMENT_NAME=gpt-4.1
```
You also need to be logged in with `az login` (for `AzureCliCredential`).
### Next steps
- Explore the [samples directory](samples/) for complete code examples
- Read the [Azure Content Understanding documentation](https://learn.microsoft.com/azure/ai-services/content-understanding/) for detailed service information
- Learn more about the [Microsoft Agent Framework](https://aka.ms/agent-framework)
@@ -1,28 +0,0 @@
# Copyright (c) Microsoft. All rights reserved.
"""Azure Content Understanding integration for Microsoft Agent Framework.
Provides a context provider that analyzes file attachments (documents, images,
audio, video) using Azure Content Understanding and injects structured results
into the LLM context.
"""
import importlib.metadata
from ._context_provider import ContentUnderstandingContextProvider
from ._file_search import FileSearchBackend
from ._models import AnalysisSection, DocumentStatus, FileSearchConfig
try:
__version__ = importlib.metadata.version(__name__)
except importlib.metadata.PackageNotFoundError:
__version__ = "0.0.0"
__all__ = [
"AnalysisSection",
"ContentUnderstandingContextProvider",
"DocumentStatus",
"FileSearchBackend",
"FileSearchConfig",
"__version__",
]
@@ -1,858 +0,0 @@
# Copyright (c) Microsoft. All rights reserved.
"""Azure Content Understanding context provider using ContextProvider.
This module provides ``ContentUnderstandingContextProvider``, built on the
:class:`ContextProvider` hooks pattern. It automatically detects file
attachments, analyzes them via the Azure Content Understanding API, and
injects structured results into the LLM context.
"""
from __future__ import annotations
import asyncio
import json
import logging
import sys
import time
from datetime import datetime, timezone
from typing import TYPE_CHECKING, Any, ClassVar, TypedDict
from agent_framework import (
AGENT_FRAMEWORK_USER_AGENT,
Content,
ContextProvider,
FunctionTool,
Message,
SessionContext,
)
from agent_framework._sessions import AgentSession
from agent_framework._settings import load_settings
from azure.ai.contentunderstanding.aio import ContentUnderstandingClient
from azure.ai.contentunderstanding.models import AnalysisInput, AnalysisResult
from azure.core.credentials import AzureKeyCredential
from azure.core.credentials_async import AsyncTokenCredential
if TYPE_CHECKING:
from agent_framework._agents import SupportsAgentRun
from ._detection import (
detect_and_strip_files,
)
from ._extraction import extract_sections, format_result
from ._models import AnalysisSection, DocumentEntry, DocumentStatus, FileSearchConfig
if sys.version_info >= (3, 11):
from typing import Self # pragma: no cover
else:
from typing_extensions import Self # pragma: no cover
logger = logging.getLogger("agent_framework.azure_contentunderstanding")
AzureCredentialTypes = AzureKeyCredential | AsyncTokenCredential
# Mapping from media type prefix to the appropriate prebuilt CU analyzer.
# Used when analyzer_id is None (auto-detect mode).
MEDIA_TYPE_ANALYZER_MAP: dict[str, str] = {
"audio/": "prebuilt-audioSearch",
"video/": "prebuilt-videoSearch",
}
DEFAULT_ANALYZER: str = "prebuilt-documentSearch"
class ContentUnderstandingSettings(TypedDict, total=False):
"""Settings for ContentUnderstandingContextProvider with auto-loading from environment.
Settings are resolved in this order: explicit keyword arguments, values from an
explicitly provided .env file, then environment variables with the prefix
``AZURE_CONTENTUNDERSTANDING_``.
Keys:
endpoint: Azure AI Foundry endpoint URL.
Can be set via environment variable ``AZURE_CONTENTUNDERSTANDING_ENDPOINT``.
"""
endpoint: str | None
class ContentUnderstandingContextProvider(ContextProvider):
"""Context provider that analyzes file attachments using Azure Content Understanding.
Automatically detects supported file attachments in the agent's input,
analyzes them via CU, and injects the structured results (markdown, fields)
into the LLM context. Supports multiple documents per session with background
processing for long-running analyses. Optionally integrates with a vector
store backend for ``file_search``-based RAG retrieval on LLM clients that
support it.
Args:
endpoint: Azure AI Foundry endpoint URL
(e.g., ``"https://<your-foundry-resource>.services.ai.azure.com/"``).
Can also be set via environment variable
``AZURE_CONTENTUNDERSTANDING_ENDPOINT``.
credential: An ``AzureKeyCredential`` for API key auth or an
``AsyncTokenCredential`` (e.g., ``DefaultAzureCredential``) for
Microsoft Entra ID auth.
analyzer_id: A prebuilt or custom CU analyzer ID. When ``None``
(default), a prebuilt analyzer is chosen automatically based on
the file's media type: ``prebuilt-documentSearch`` for documents
and images, ``prebuilt-audioSearch`` for audio, and
``prebuilt-videoSearch`` for video.
Analyzer reference: https://learn.microsoft.com/azure/ai-services/content-understanding/concepts/analyzer-reference
Prebuilt analyzers: https://learn.microsoft.com/azure/ai-services/content-understanding/concepts/prebuilt-analyzers
max_wait: Max seconds to wait for analysis before deferring to background.
``None`` waits until complete.
output_sections: Which CU output sections to pass to LLM.
Defaults to ``["markdown", "fields"]``.
file_search: Optional configuration for uploading CU-extracted markdown to
a vector store for token-efficient RAG retrieval. When provided, full
content injection is replaced by ``file_search`` tool registration.
The ``FileSearchConfig`` abstraction is backend-agnostic — use
``FileSearchConfig.from_openai()`` or ``FileSearchConfig.from_foundry()``
for supported providers, or supply a custom ``FileSearchBackend``
implementation for other vector store services.
source_id: Unique identifier for this provider instance, used for message
attribution and tool registration. Defaults to ``"azure_contentunderstanding"``.
env_file_path: Path to a ``.env`` file for loading settings.
env_file_encoding: Encoding of the ``.env`` file.
Per-file ``additional_properties`` on ``Content`` objects:
The provider reads the following keys from
``Content.additional_properties`` (passed via ``Content.from_data()``
or ``Content.from_uri()``):
``filename`` (str):
The document key used for tracking, status, and LLM references.
Without a filename, a UUID-based key is generated.
Must be unique within a session — uploading a file with a
duplicate filename will be rejected and the file will not be
analyzed.
``analyzer_id`` (str):
Per-file analyzer override. Takes priority over the provider-level
``analyzer_id``. Useful for mixing analyzers in the same turn
(e.g., ``prebuilt-invoice`` for invoices alongside
``prebuilt-documentSearch`` for general documents).
``content_range`` (str):
Subset of the input to analyze. For documents, use 1-based page
numbers (e.g., ``"1-3"`` for pages 1-3, ``"1,3,5-"`` for pages
1, 3, and 5 onward). For audio/video, use milliseconds
(e.g., ``"0-60000"`` for the first 60 seconds).
Example::
Content.from_data(
pdf_bytes,
"application/pdf",
additional_properties={
"filename": "invoice.pdf",
"analyzer_id": "prebuilt-invoice",
"content_range": "1-3",
},
)
"""
DEFAULT_SOURCE_ID: ClassVar[str] = "azure_contentunderstanding"
DEFAULT_MAX_WAIT_SECONDS: ClassVar[float] = 5.0
def __init__(
self,
*,
endpoint: str | None = None,
credential: AzureCredentialTypes | None = None,
client: ContentUnderstandingClient | None = None,
analyzer_id: str | None = None,
max_wait: float | None = DEFAULT_MAX_WAIT_SECONDS,
output_sections: list[AnalysisSection] | None = None,
file_search: FileSearchConfig | None = None,
source_id: str = DEFAULT_SOURCE_ID,
env_file_path: str | None = None,
env_file_encoding: str | None = None,
) -> None:
super().__init__(source_id)
if client is not None:
# Use the pre-built client directly — endpoint/credential are ignored.
self._client = client
self._owns_client = False
self._endpoint = ""
self._credential = None
else:
# Build a new client from endpoint + credential.
settings = load_settings(
ContentUnderstandingSettings,
env_prefix="AZURE_CONTENTUNDERSTANDING_",
required_fields=["endpoint"],
endpoint=endpoint,
env_file_path=env_file_path,
env_file_encoding=env_file_encoding,
)
resolved_endpoint: str = settings["endpoint"] # type: ignore[assignment] # validated by load_settings
if credential is None:
raise ValueError(
"Azure credential is required. Provide a 'credential' keyword argument "
"(e.g., AzureKeyCredential or AzureCliCredential), or pass a pre-built "
"'client' (ContentUnderstandingClient) instead."
)
self._endpoint = resolved_endpoint
self._credential = credential
self._client = ContentUnderstandingClient(
self._endpoint, self._credential, user_agent=AGENT_FRAMEWORK_USER_AGENT
)
self._owns_client = True
self.analyzer_id = analyzer_id
self.max_wait = max_wait
self.output_sections: list[AnalysisSection] = output_sections or ["markdown", "fields"]
self.file_search = file_search
# Global list of uploaded file IDs — used only by close() for
# best-effort cleanup. The authoritative per-session copy lives in
# state["_uploaded_file_ids"] (populated in before_run). This global
# list may contain entries from multiple sessions; that is intentional
# for cleanup.
self._all_uploaded_file_ids: list[str] = []
async def __aenter__(self) -> Self:
"""Async context manager entry."""
return self
async def __aexit__(
self,
exc_type: type[BaseException] | None,
exc_val: BaseException | None,
exc_tb: Any,
) -> None:
"""Async context manager exit — cleanup clients."""
await self.close()
async def close(self) -> None:
"""Close the underlying CU client and clean up resources.
Uses global tracking lists for best-effort cleanup across all
sessions that used this provider instance.
"""
# Clean up uploaded files; the vector store itself is caller-managed.
if self.file_search and self._all_uploaded_file_ids:
await self._cleanup_uploaded_files()
# Only close the client if we created it internally.
# When a pre-built client was passed in, the caller owns its lifecycle.
if self._owns_client:
await self._client.close()
async def before_run(
self,
*,
agent: SupportsAgentRun,
session: AgentSession,
context: SessionContext,
state: dict[str, Any],
) -> None:
"""Analyze file attachments and inject results into the LLM context.
This method is called automatically by the framework before each LLM invocation.
"""
documents: dict[str, DocumentEntry] = state.setdefault("documents", {})
# Per-session mutable state — isolated per session to prevent cross-session leakage.
# _pending_tokens stores serializable continuation tokens (not asyncio.Task objects)
# so that state can be persisted to disk/storage by the framework.
# Structure: {doc_key: {"continuation_token": <opaque Azure SDK string>,
# "analyzer_id": <CU analyzer used for this file>}}
pending_tokens: dict[str, dict[str, str]] = state.setdefault("_pending_tokens", {})
pending_uploads: list[tuple[str, DocumentEntry]] = state.setdefault("_pending_uploads", [])
# 1. Resolve pending background analyses via continuation tokens
await self._resolve_pending_tokens(pending_tokens, pending_uploads, documents, context)
# 1b. Upload any documents that completed in the background (file_search mode)
if pending_uploads:
# Use a bounded timeout so before_run() stays responsive and does not block
# indefinitely on slow vector store indexing.
upload_timeout = getattr(self, "max_wait", None)
remaining_uploads: list[tuple[str, DocumentEntry]] = []
for upload_key, upload_entry in pending_uploads:
try:
if upload_timeout is not None:
await asyncio.wait_for(
self._upload_to_vector_store(upload_key, upload_entry, state=state),
timeout=upload_timeout,
)
else:
await self._upload_to_vector_store(upload_key, upload_entry, state=state)
except asyncio.TimeoutError:
# Leave timed-out uploads pending so they can be retried on a later turn.
logger.warning(
"Timed out while uploading document '%s' to vector store; will retry later.",
upload_key,
)
remaining_uploads.append((upload_key, upload_entry))
except Exception:
# Log unexpected failures and drop the upload entry; this matches prior
# behavior where all pending uploads were cleared regardless of outcome.
logger.exception(
"Error while uploading document '%s' to vector store; dropping from pending list.",
upload_key,
)
context.extend_messages(
self.source_id,
[
Message(
role="user",
contents=[
(
f"Document '{upload_key}' was analyzed but failed to upload "
"to the vector store. The document content is not available for search."
)
],
)
],
)
state["_pending_uploads"] = remaining_uploads
pending_uploads = remaining_uploads
# 2. Detect CU-supported file attachments, strip them from input, and return for analysis
new_files = detect_and_strip_files(context)
# 3. Analyze new files using CU (track elapsed time for combined timeout)
file_start_times: dict[str, float] = {}
accepted_keys: set[str] = set() # doc_keys successfully accepted for analysis this turn
for doc_key, content_item, binary_data in new_files:
# Reject duplicate filenames — re-analyzing would orphan vector store entries
if doc_key in documents:
logger.warning("Duplicate document key '%s' — skipping (already exists in session).", doc_key)
context.extend_messages(
self.source_id,
[
Message(
role="user",
contents=[
(
f"The user tried to upload '{doc_key}', but a file with that name "
"was already uploaded earlier in this session. The new upload was rejected "
"and was not analyzed. Tell the user that a file with the same name "
"already exists and they need to rename the file before uploading again."
)
],
)
],
)
continue
file_start_times[doc_key] = time.monotonic()
doc_entry = await self._analyze_file(doc_key, content_item, binary_data, context, pending_tokens)
if doc_entry:
documents[doc_key] = doc_entry
accepted_keys.add(doc_key)
# 4. Inject content for ready documents and register tools
if documents:
self._register_tools(documents, context)
# 5. On upload turns, inject content for docs accepted this turn
for doc_key in accepted_keys:
entry = documents.get(doc_key)
if entry and entry["status"] == DocumentStatus.READY and entry["result"]:
# Upload to vector store if file_search is configured
if self.file_search:
# Combined timeout: subtract CU analysis time from max_wait
remaining: float | None = None
if self.max_wait is not None:
elapsed = time.monotonic() - file_start_times.get(doc_key, time.monotonic())
remaining = max(0.0, self.max_wait - elapsed)
uploaded = await self._upload_to_vector_store(doc_key, entry, timeout=remaining, state=state)
if uploaded:
context.extend_messages(
self.source_id,
[
Message(
role="user",
contents=[
(
f"The user just uploaded '{entry['filename']}'. It has been analyzed "
"using Azure Content Understanding and indexed in a vector store. "
f"When using file_search, include '{entry['filename']}' in your query "
"to retrieve content from this specific document."
)
],
)
],
)
elif entry.get("error"):
# Upload failed (not timeout — actual error)
context.extend_messages(
self.source_id,
[
Message(
role="user",
contents=[
(
f"Document '{entry['filename']}' was analyzed but failed to upload "
"to the vector store. The document content is not available for search."
)
],
)
],
)
else:
# Upload deferred to background (timeout)
context.extend_messages(
self.source_id,
[
Message(
role="user",
contents=[
(
f"Document '{entry['filename']}' has been analyzed and is being indexed. "
"Ask about it again in a moment."
)
],
)
],
)
else:
# Without file_search, inject full content into context
context.extend_messages(
self,
[
Message(role="user", contents=[format_result(entry["filename"], entry["result"])]),
],
)
context.extend_messages(
self.source_id,
[
Message(
role="user",
contents=[
(
f"The user just uploaded '{entry['filename']}'."
" It has been analyzed using Azure Content Understanding."
" The document content (markdown) and extracted fields"
" (JSON) are provided above."
" If the user's question is ambiguous,"
" prioritize this most recently uploaded document."
" Use specific field values and cite page numbers"
" when answering."
)
],
)
],
)
# 6. Register file_search tool (for LLM clients that support it)
if self.file_search:
context.extend_tools(
self.source_id,
[self.file_search.file_search_tool],
)
context.extend_instructions(
self.source_id,
"Tool usage guidelines:\n"
"- Use file_search ONLY when answering questions about document content.\n"
"- Use list_documents() for status queries (e.g. 'list docs', 'what's uploaded?').\n"
"- Do NOT call file_search for status queries — it wastes tokens.",
)
# ------------------------------------------------------------------
# Analyzer Resolution
# ------------------------------------------------------------------
def _resolve_analyzer_id(self, media_type: str) -> str:
"""Return the analyzer ID to use for the given media type.
When ``self.analyzer_id`` is set, it is always returned (explicit
override). Otherwise the media type prefix is matched against the
known mapping, falling back to ``prebuilt-documentSearch``.
"""
if self.analyzer_id is not None:
return self.analyzer_id
for prefix, analyzer in MEDIA_TYPE_ANALYZER_MAP.items():
if media_type.startswith(prefix):
return analyzer
return DEFAULT_ANALYZER
# ------------------------------------------------------------------
# Analysis
# ------------------------------------------------------------------
async def _analyze_file(
self,
doc_key: str,
content: Content,
binary_data: bytes | None,
context: SessionContext,
pending_tokens: dict[str, dict[str, str]] | None = None,
) -> DocumentEntry | None:
"""Analyze a single file via CU with timeout handling.
The analyzer is resolved in priority order:
1. Per-file override via ``content.additional_properties["analyzer_id"]``
2. Provider-level default via ``self.analyzer_id``
3. Auto-detect by media type (document/audio/video)
Returns:
A ``DocumentEntry`` (ready, analyzing, or failed), or ``None`` if
file data could not be extracted.
"""
media_type = content.media_type or "application/octet-stream"
filename = doc_key
# Per-file analyzer override from additional_properties
props = content.additional_properties or {}
per_file_analyzer = props.get("analyzer_id")
content_range = props.get("content_range")
resolved_analyzer = per_file_analyzer or self._resolve_analyzer_id(media_type)
t0 = time.monotonic()
try:
# Start CU analysis
if content.type == "uri" and content.uri and not content.uri.startswith("data:"):
poller = await self._client.begin_analyze(
resolved_analyzer,
inputs=[AnalysisInput(url=content.uri, content_range=content_range)],
)
elif binary_data:
poller = await self._client.begin_analyze_binary(
resolved_analyzer,
binary_input=binary_data,
content_type=media_type,
)
else:
context.extend_messages(
self.source_id,
[Message(role="user", contents=[f"Could not extract file data from '{filename}'."])],
)
return None
# Wait with timeout; defer to background polling on timeout.
try:
result = await asyncio.wait_for(poller.result(), timeout=self.max_wait)
except asyncio.TimeoutError:
# Save continuation token for resuming on next before_run().
# Continuation tokens are serializable strings, so state can
# be persisted to disk/storage without issues.
token = poller.continuation_token()
logger.info("Analysis of '%s' timed out; deferring to background via continuation token.", filename)
if pending_tokens is not None:
pending_tokens[doc_key] = {
"continuation_token": token,
"analyzer_id": resolved_analyzer,
}
context.extend_messages(
self.source_id,
[
Message(
role="user",
contents=[f"Document '{filename}' is being analyzed. Ask about it again in a moment."],
)
],
)
return DocumentEntry(
status=DocumentStatus.ANALYZING,
filename=filename,
media_type=media_type,
analyzer_id=resolved_analyzer,
analyzed_at=None,
analysis_duration_s=None,
upload_duration_s=None,
result=None,
error=None,
)
# Analysis completed within timeout
analysis_duration = round(time.monotonic() - t0, 2)
extracted = self._extract_sections(result)
logger.info("Analyzed '%s' with analyzer '%s' in %.1fs.", filename, resolved_analyzer, analysis_duration)
return DocumentEntry(
status=DocumentStatus.READY,
filename=filename,
media_type=media_type,
analyzer_id=resolved_analyzer,
analyzed_at=datetime.now(tz=timezone.utc).isoformat(),
analysis_duration_s=analysis_duration,
upload_duration_s=None,
result=extracted,
error=None,
)
except asyncio.TimeoutError:
raise
except Exception as e:
logger.warning("CU analysis error for '%s': %s", filename, e)
context.extend_messages(
self.source_id,
[Message(role="user", contents=[f"Could not analyze '{filename}': {e}"])],
)
return DocumentEntry(
status=DocumentStatus.FAILED,
filename=filename,
media_type=media_type,
analyzer_id=resolved_analyzer,
analyzed_at=datetime.now(tz=timezone.utc).isoformat(),
analysis_duration_s=round(time.monotonic() - t0, 2),
upload_duration_s=None,
result=None,
error=str(e),
)
# ------------------------------------------------------------------
# Pending Token Resolution
# ------------------------------------------------------------------
async def _resolve_pending_tokens(
self,
pending_tokens: dict[str, dict[str, str]],
pending_uploads: list[tuple[str, DocumentEntry]],
documents: dict[str, DocumentEntry],
context: SessionContext,
) -> None:
"""Resume pending CU analyses using serializable continuation tokens.
When a file's CU analysis exceeds ``max_wait``, a continuation token
(an opaque string from the Azure SDK) is saved in ``state`` instead of
an ``asyncio.Task``. This keeps state fully serializable — it can be
persisted to disk/storage by the framework.
On the next ``before_run()`` call, this method resumes each pending
operation by passing the token back to ``begin_analyze()``. If the
server-side operation has completed, the result is available
immediately; otherwise the token is kept for the next turn.
"""
if not pending_tokens:
return
logger.info("Resolving %d pending analysis token(s).", len(pending_tokens))
completed_keys: list[str] = []
for doc_key, token_info in pending_tokens.items():
entry = documents.get(doc_key)
if not entry:
completed_keys.append(doc_key)
continue
try:
poller = await self._client.begin_analyze( # type: ignore[call-overload, reportUnknownVariableType]
token_info["analyzer_id"],
continuation_token=token_info["continuation_token"], # pyright: ignore[reportCallIssue]
)
# Use wait_for to avoid blocking before_run indefinitely.
# poller.done() always returns False for resumed pollers (stale
# cached status), so we call poller.result() which polls the server.
#
# Timeout: at least 10s regardless of max_wait. The upload-turn
# max_wait can be very short (e.g. 5s) for responsiveness, but
# on resolution turns the resumed poller needs a network round-trip
# to fetch the result. If the analysis is still running after 10s,
# the token is kept and retried on the next turn.
MIN_RESOLUTION_TIMEOUT = 10.0
resolution_timeout = max(self.max_wait or MIN_RESOLUTION_TIMEOUT, MIN_RESOLUTION_TIMEOUT)
try:
result: AnalysisResult = await asyncio.wait_for(
poller.result(), # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType]
timeout=resolution_timeout,
) # pyright: ignore[reportUnknownVariableType]
except asyncio.TimeoutError:
# Still running — update token and keep for next turn
new_token: str = poller.continuation_token() # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType]
token_info["continuation_token"] = new_token
logger.info("Analysis for '%s' still running; keeping token for next turn.", doc_key)
continue
completed_keys.append(doc_key)
extracted = self._extract_sections(result) # pyright: ignore[reportUnknownArgumentType]
entry["status"] = DocumentStatus.READY
entry["analyzed_at"] = datetime.now(tz=timezone.utc).isoformat()
entry["result"] = extracted
entry["error"] = None
logger.info("Background analysis of '%s' completed.", entry["filename"])
# Inject newly ready content
if self.file_search:
pending_uploads.append((doc_key, entry))
else:
context.extend_messages(
self,
[
Message(role="user", contents=[format_result(entry["filename"], extracted)]),
],
)
context.extend_messages(
self.source_id,
[
Message(
role="user",
contents=[
f"Document '{entry['filename']}' analysis is now complete."
+ (
" The document is being indexed in the vector store and will become"
" searchable via file_search shortly."
if self.file_search
else " The content is provided above."
)
],
)
],
)
except Exception as e:
completed_keys.append(doc_key)
logger.warning("Background analysis of '%s' failed: %s", entry.get("filename", doc_key), e)
entry["status"] = DocumentStatus.FAILED
entry["analyzed_at"] = datetime.now(tz=timezone.utc).isoformat()
entry["error"] = str(e)
context.extend_messages(
self.source_id,
[Message(role="user", contents=[f"Document '{entry['filename']}' analysis failed: {e}"])],
)
for key in completed_keys:
del pending_tokens[key]
# ------------------------------------------------------------------
# Output Extraction & Formatting (delegates to _extraction module)
# ------------------------------------------------------------------
def _extract_sections(self, result: AnalysisResult) -> dict[str, object]:
return extract_sections(result, self.output_sections)
# ------------------------------------------------------------------
# Tool Registration
# ------------------------------------------------------------------
def _register_tools(
self,
documents: dict[str, DocumentEntry],
context: SessionContext,
) -> None:
"""Register document tools on the context.
Only ``list_documents`` is registered — the full document content is
already injected into conversation history on the upload turn, so a
separate retrieval tool is not needed.
"""
context.extend_tools(
self.source_id,
[self._make_list_documents_tool(documents)],
)
@staticmethod
def _make_list_documents_tool(documents: dict[str, DocumentEntry]) -> FunctionTool:
"""Create a tool that lists all tracked documents with their status."""
docs_ref = documents
def list_documents() -> str:
"""List all documents that have been uploaded and their analysis status."""
entries: list[dict[str, object]] = []
for name, entry in docs_ref.items():
entries.append({
"name": name,
"status": entry["status"],
"media_type": entry["media_type"],
"analyzed_at": entry["analyzed_at"],
"analysis_duration_s": entry["analysis_duration_s"],
"upload_duration_s": entry["upload_duration_s"],
})
return json.dumps(entries, indent=2, default=str)
return FunctionTool(
name="list_documents",
description=(
"List all documents that have been uploaded in this session "
"with their analysis status (analyzing, uploading, ready, or failed)."
),
func=list_documents,
)
# ------------------------------------------------------------------
# file_search Vector Store Integration
# ------------------------------------------------------------------
async def _upload_to_vector_store(
self,
doc_key: str,
entry: DocumentEntry,
*,
timeout: float | None = None,
state: dict[str, Any] | None = None,
) -> bool:
"""Upload CU-extracted markdown to the caller's vector store.
Delegates to the configured ``FileSearchBackend`` (OpenAI, Foundry,
or a custom implementation). The upload includes file upload **and**
vector store indexing (embedding + ingestion) — ``create_and_poll``
waits for the index to be fully ready before returning.
Args:
doc_key: Document identifier.
entry: The document entry with extracted results.
timeout: Max seconds to wait for upload + indexing. ``None`` waits
indefinitely. On timeout the upload is deferred to the
per-session ``_pending_uploads`` queue for the next
``before_run()`` call.
state: Per-session state dict for tracking uploaded file IDs and
pending uploads.
Returns:
True if the upload succeeded, False otherwise.
"""
if not self.file_search:
return False
result = entry.get("result")
if not result:
return False
# Upload the full formatted content (markdown + fields + segments),
# not just raw markdown — consistent with what non-file_search mode injects.
formatted = format_result(entry["filename"], result)
if not formatted:
return False
entry["status"] = DocumentStatus.UPLOADING
t0 = time.monotonic()
try:
upload_coro = self.file_search.backend.upload_file(
self.file_search.vector_store_id, f"{doc_key}.md", formatted.encode("utf-8")
)
file_id = await asyncio.wait_for(upload_coro, timeout=timeout)
upload_duration = round(time.monotonic() - t0, 2)
# Track in per-session state and global list (for close() cleanup)
if state is not None:
state.setdefault("_uploaded_file_ids", []).append(file_id)
self._all_uploaded_file_ids.append(file_id)
entry["status"] = DocumentStatus.READY
entry["upload_duration_s"] = upload_duration
logger.info("Uploaded '%s' to vector store in %.1fs (%s bytes).", doc_key, upload_duration, len(formatted))
return True
except asyncio.TimeoutError:
logger.info("Vector store upload for '%s' timed out; deferring to background.", doc_key)
entry["status"] = DocumentStatus.UPLOADING
if state is not None:
state.setdefault("_pending_uploads", []).append((doc_key, entry))
return False
except Exception as e:
logger.warning("Failed to upload '%s' to vector store: %s", doc_key, e)
entry["status"] = DocumentStatus.FAILED
entry["upload_duration_s"] = round(time.monotonic() - t0, 2)
entry["error"] = f"Vector store upload failed: {e}"
return False
async def _cleanup_uploaded_files(self) -> None:
"""Delete files uploaded by this provider via the configured backend.
The vector store itself is caller-managed and is not deleted here.
"""
if not self.file_search:
return
backend = self.file_search.backend
try:
for file_id in self._all_uploaded_file_ids:
await backend.delete_file(file_id)
self._all_uploaded_file_ids.clear()
except Exception as e:
logger.warning("Failed to clean up uploaded files: %s", e)
@@ -1,234 +0,0 @@
# Copyright (c) Microsoft. All rights reserved.
"""File detection utilities for Azure Content Understanding context provider.
Functions for scanning input messages, sniffing MIME types, deriving
document keys, and extracting binary data from content items.
"""
from __future__ import annotations
import base64
import logging
import mimetypes
import re
import uuid
import filetype
from agent_framework import Content, SessionContext
logger = logging.getLogger("agent_framework.azure_contentunderstanding")
# MIME types used to match against the resolved media type for routing files to CU analysis.
# The media type may be provided via Content.media_type or inferred (e.g., via sniffing or filename)
# when missing or generic (such as application/octet-stream). Only files whose resolved media type is
# in this set will be processed; others are skipped.
#
# Supported input file types:
# https://learn.microsoft.com/azure/ai-services/content-understanding/service-limits#input-file-limits
SUPPORTED_MEDIA_TYPES: frozenset[str] = frozenset({
# Documents and images
"application/pdf",
"image/jpeg",
"image/png",
"image/tiff",
"image/bmp",
"image/heif",
"image/heic",
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
"application/vnd.openxmlformats-officedocument.presentationml.presentation",
# Text
"text/plain",
"text/html",
"text/markdown",
"text/rtf",
"text/xml",
"application/xml",
"message/rfc822",
"application/vnd.ms-outlook",
# Audio
"audio/wav",
"audio/mpeg",
"audio/mp3",
"audio/mp4",
"audio/m4a",
"audio/flac",
"audio/ogg",
"audio/opus",
"audio/webm",
"audio/x-ms-wma",
"audio/aac",
"audio/amr",
"audio/3gpp",
# Video
"video/mp4",
"video/quicktime",
"video/x-msvideo",
"video/webm",
"video/x-flv",
"video/x-ms-wmv",
"video/x-ms-asf",
"video/x-matroska",
})
# Mapping from filetype's MIME output to our canonical SUPPORTED_MEDIA_TYPES values.
# filetype uses some x-prefixed variants that differ from our set.
MIME_ALIASES: dict[str, str] = {
"audio/x-wav": "audio/wav",
"audio/x-flac": "audio/flac",
"video/x-m4v": "video/mp4",
}
def detect_and_strip_files(
context: SessionContext,
) -> list[tuple[str, Content, bytes | None]]:
"""Scan input messages for supported file content and prepare for CU analysis.
Scans for type ``data`` or ``uri`` content supported by Azure Content
Understanding, strips them from messages to prevent raw binary being sent
to the LLM, and returns metadata for CU analysis.
Detected files are tracked via ``doc_key`` (derived from filename, URL,
or UUID) and their analysis status is managed in session state.
When the upstream MIME type is unreliable (``application/octet-stream``
or missing), binary content sniffing via ``filetype`` is used to
determine the real media type, with ``mimetypes.guess_type`` as a
filename-based fallback.
Returns:
List of (doc_key, content_item, binary_data) tuples for files to analyze.
"""
results: list[tuple[str, Content, bytes | None]] = []
strip_ids: set[int] = set()
for msg in context.input_messages:
for c in msg.contents:
if c.type not in ("data", "uri"):
continue
media_type = c.media_type
# Fast path: already a known supported type
if media_type and media_type in SUPPORTED_MEDIA_TYPES:
binary_data = extract_binary(c)
results.append((derive_doc_key(c), c, binary_data))
strip_ids.add(id(c))
continue
# Slow path: unreliable MIME — sniff binary content
if (not media_type) or (media_type == "application/octet-stream"):
binary_data = extract_binary(c)
resolved = sniff_media_type(binary_data, c)
if resolved and (resolved in SUPPORTED_MEDIA_TYPES):
c.media_type = resolved
results.append((derive_doc_key(c), c, binary_data))
strip_ids.add(id(c))
# Strip detected files from input so raw binary isn't sent to LLM
msg.contents = [c for c in msg.contents if id(c) not in strip_ids]
return results
def sniff_media_type(binary_data: bytes | None, content: Content) -> str | None:
"""Sniff the actual MIME type from binary data, with filename fallback.
Uses ``filetype`` (magic-bytes) first, then ``mimetypes.guess_type``
on the filename. Normalizes filetype's variant MIME values (e.g.
``audio/x-wav`` -> ``audio/wav``) via ``MIME_ALIASES``.
"""
# 1. Binary sniffing via filetype (needs only first 261 bytes)
if binary_data:
kind = filetype.guess(binary_data[:262]) # type: ignore[reportUnknownMemberType]
if kind:
mime: str = kind.mime # type: ignore[reportUnknownMemberType]
return MIME_ALIASES.get(mime, mime)
# 2. Filename extension fallback — try additional_properties first,
# then extract basename from external URL path
filename: str | None = None
if content.additional_properties:
filename = content.additional_properties.get("filename")
if not filename and content.uri and not content.uri.startswith("data:"):
# Extract basename from URL path (e.g. "https://example.com/report.pdf?v=1" -> "report.pdf")
filename = content.uri.split("?")[0].split("#")[0].rsplit("/", 1)[-1]
if filename:
guessed, _ = mimetypes.guess_type(filename) # uses file extension to guess MIME type
if guessed:
return MIME_ALIASES.get(guessed, guessed)
return None
def is_supported_content(content: Content) -> bool:
"""Check if a content item is a supported file type for CU analysis."""
if content.type not in ("data", "uri"):
return False
media_type = content.media_type
if not media_type:
return False
return media_type in SUPPORTED_MEDIA_TYPES
def sanitize_doc_key(raw: str) -> str:
"""Sanitize a document key to prevent prompt injection.
Removes control characters (newlines, tabs, etc.), collapses
whitespace, strips surrounding whitespace, and caps length at
255 characters.
"""
# Remove control characters (C0/C1 controls, including \n, \r, \t)
cleaned = re.sub(r"[\x00-\x1f\x7f-\x9f]", "", raw)
# Collapse whitespace
cleaned = " ".join(cleaned.split())
# Cap length
return cleaned[:255] if cleaned else f"doc_{uuid.uuid4().hex[:8]}"
def derive_doc_key(content: Content) -> str:
"""Derive a unique document key from content metadata.
The key is used to track documents in session state. Duplicate keys
within a session are rejected (not re-analyzed) to prevent orphaned
vector store entries.
The returned key is sanitized to prevent prompt injection via
crafted filenames (control characters removed, length capped).
Priority: filename > URL basename > generated UUID.
"""
# 1. Filename from additional_properties
if content.additional_properties:
filename = content.additional_properties.get("filename")
if filename and isinstance(filename, str):
return sanitize_doc_key(filename)
# 2. URL path basename for external URIs (e.g. "https://example.com/report.pdf" -> "report.pdf")
if content.type == "uri" and content.uri and not content.uri.startswith("data:"):
path = content.uri.split("?")[0].split("#")[0] # strip query params and fragments
# rstrip("/") handles trailing slashes (e.g. ".../files/" -> ".../files")
# rsplit("/", 1)[-1] splits from the right once to get the last path segment
basename = path.rstrip("/").rsplit("/", 1)[-1]
if basename:
return sanitize_doc_key(basename)
# 3. Fallback: generate a unique ID for anonymous uploads (no filename, no URL)
return f"doc_{uuid.uuid4().hex[:8]}"
def extract_binary(content: Content) -> bytes | None:
"""Extract binary data from a data URI content item.
Only handles ``data:`` URIs (base64-encoded). Returns ``None`` for
external URLs -- those are passed directly to CU via ``begin_analyze``.
"""
if content.uri and content.uri.startswith("data:"):
try:
_, data_part = content.uri.split(",", 1)
return base64.b64decode(data_part)
except Exception:
logger.warning("Failed to decode base64 data URI")
return None
return None
@@ -1,297 +0,0 @@
# Copyright (c) Microsoft. All rights reserved.
"""Output extraction and formatting for Azure Content Understanding results.
Converts CU ``AnalysisResult`` objects into plain Python dicts suitable
for LLM consumption, and formats them as human-readable text.
"""
from __future__ import annotations
import json
from typing import Any, cast
from azure.ai.contentunderstanding.models import AnalysisResult
from ._models import AnalysisSection
def extract_sections(
result: AnalysisResult,
output_sections: list[AnalysisSection],
) -> dict[str, object]:
"""Extract configured sections from a CU analysis result.
For single-segment results (documents, images, short audio), returns a flat
dict with ``markdown`` and ``fields`` at the top level.
For multi-segment results (e.g. video split into scenes), fields are kept
with their respective segments in a ``segments`` list so the LLM can see
which fields belong to which part of the content:
- ``segments``: list of per-segment dicts with ``markdown``, ``fields``,
``start_time_s``, and ``end_time_s``
- ``markdown``: still concatenated at top level for file_search uploads
- ``duration_seconds``: computed from the global time span
- ``kind`` / ``resolution``: taken from the first segment
"""
extracted: dict[str, object] = {}
contents = result.contents
if not contents:
return extracted
# --- Warnings from the CU service (ODataV4Format with code/message/target) ---
if result.warnings:
warnings_out: list[dict[str, str]] = []
for w in result.warnings:
entry: dict[str, str] = {}
code = getattr(w, "code", None)
if code:
entry["code"] = code
msg = getattr(w, "message", None)
entry["message"] = msg if msg else str(w)
target = getattr(w, "target", None)
if target:
entry["target"] = target
warnings_out.append(entry)
extracted["warnings"] = warnings_out
# --- Media metadata (from first segment) ---
first = contents[0]
kind = getattr(first, "kind", None)
if kind:
extracted["kind"] = kind
width = getattr(first, "width", None)
height = getattr(first, "height", None)
if width and height:
extracted["resolution"] = f"{width}x{height}"
# Compute total duration from the global time span of all segments.
global_start: int | None = None
global_end: int | None = None
for content in contents:
s = getattr(content, "start_time_ms", None)
if s is None:
s = getattr(content, "startTimeMs", None)
e = getattr(content, "end_time_ms", None)
if e is None:
e = getattr(content, "endTimeMs", None)
if s is not None:
global_start = s if global_start is None else min(global_start, s)
if e is not None:
global_end = e if global_end is None else max(global_end, e)
if global_start is not None and global_end is not None:
extracted["duration_seconds"] = round((global_end - global_start) / 1000, 1)
is_multi_segment = len(contents) > 1
# --- Single-segment: flat output (documents, images, short audio) ---
if not is_multi_segment:
if "markdown" in output_sections and contents[0].markdown:
extracted["markdown"] = contents[0].markdown
if "fields" in output_sections and contents[0].fields:
fields: dict[str, object] = {}
for name, field in contents[0].fields.items():
entry_dict: dict[str, object] = {
"type": getattr(field, "type", None),
"value": extract_field_value(field),
}
confidence = getattr(field, "confidence", None)
if confidence is not None:
entry_dict["confidence"] = confidence
fields[name] = entry_dict
if fields:
extracted["fields"] = fields
# Content-level category (e.g. from classifier analyzers)
category = getattr(contents[0], "category", None)
if category:
extracted["category"] = category
return extracted
# --- Multi-segment: per-segment output (video scenes, long audio) ---
# Each segment keeps its own markdown + fields together so the LLM can
# see which fields (e.g. Summary) belong to which part of the content.
segments_out: list[dict[str, object]] = []
md_parts: list[str] = [] # also collect for top-level concatenated markdown
for content in contents:
seg: dict[str, object] = {}
# Time range for this segment
s = getattr(content, "start_time_ms", None)
if s is None:
s = getattr(content, "startTimeMs", None)
e = getattr(content, "end_time_ms", None)
if e is None:
e = getattr(content, "endTimeMs", None)
if s is not None:
seg["start_time_s"] = round(s / 1000, 1)
if e is not None:
seg["end_time_s"] = round(e / 1000, 1)
# Per-segment markdown
if "markdown" in output_sections and content.markdown:
seg["markdown"] = content.markdown
md_parts.append(content.markdown)
# Per-segment fields
if "fields" in output_sections and content.fields:
seg_fields: dict[str, object] = {}
for name, field in content.fields.items():
seg_entry: dict[str, object] = {
"type": getattr(field, "type", None),
"value": extract_field_value(field),
}
confidence = getattr(field, "confidence", None)
if confidence is not None:
seg_entry["confidence"] = confidence
seg_fields[name] = seg_entry
if seg_fields:
seg["fields"] = seg_fields
# Per-segment category (e.g. from classifier analyzers)
category = getattr(content, "category", None)
if category:
seg["category"] = category
segments_out.append(seg)
extracted["segments"] = segments_out
# Top-level concatenated markdown (used by file_search for vector store upload)
if md_parts:
extracted["markdown"] = "\n\n---\n\n".join(md_parts)
return extracted
def extract_field_value(field: Any) -> object:
"""Extract the plain Python value from a CU ``ContentField``.
Uses the SDK's ``.value`` convenience property, which dynamically
reads the correct ``value_*`` attribute for each field type.
Object and array types are recursively flattened so that the
output contains only plain Python primitives (str, int, float,
date, dict, list) -- no SDK model objects or raw wire format
(``valueNumber``, ``spans``, ``source``, etc.).
"""
field_type = getattr(field, "type", None)
raw = getattr(field, "value", None)
# Object fields -> recursively resolve nested sub-fields
if field_type == "object" and raw is not None and isinstance(raw, dict):
return {str(k): flatten_field(v) for k, v in cast(dict[str, Any], raw).items()}
# Array fields -> list of flattened items (each with value + optional confidence)
if field_type == "array" and raw is not None and isinstance(raw, list):
return [flatten_field(item) for item in cast(list[Any], raw)]
# Scalar fields (string, number, date, etc.) -- .value returns native Python type
return raw
def flatten_field(field: Any) -> object:
"""Flatten a CU ``ContentField`` into a ``{type, value, confidence}`` dict.
Used for sub-fields inside object and array types to preserve
per-field confidence scores. Confidence is omitted when ``None``
to reduce token usage.
"""
field_type = getattr(field, "type", None)
value = extract_field_value(field)
confidence = getattr(field, "confidence", None)
result: dict[str, object] = {"type": field_type, "value": value}
if confidence is not None:
result["confidence"] = confidence
return result
def format_result(filename: str, result: dict[str, object]) -> str:
"""Format extracted CU result for LLM consumption.
For multi-segment results (video/audio with ``segments``), each segment's
markdown and fields are grouped together so the LLM can see which fields
belong to which part of the content.
"""
kind = result.get("kind")
is_video = kind == "audioVisual"
is_audio = kind == "audio"
# Header -- media-aware label
if is_video:
label = "Video analysis"
elif is_audio:
label = "Audio analysis"
else:
label = "Document analysis"
parts: list[str] = [f'{label} of "{filename}":']
# Media metadata line (duration, resolution)
meta_items: list[str] = []
duration = result.get("duration_seconds")
if duration is not None:
mins, secs = divmod(int(duration), 60) # type: ignore[call-overload]
meta_items.append(f"Duration: {mins}:{secs:02d}")
resolution = result.get("resolution")
if resolution:
meta_items.append(f"Resolution: {resolution}")
if meta_items:
parts.append(" | ".join(meta_items))
# --- Multi-segment: format each segment with its own content + fields ---
raw_segments = result.get("segments")
segments: list[dict[str, object]] = (
cast(list[dict[str, object]], raw_segments) if isinstance(raw_segments, list) else []
)
if segments:
for i, seg in enumerate(segments):
# Segment header with time range
start = seg.get("start_time_s")
end = seg.get("end_time_s")
if start is not None and end is not None:
s_min, s_sec = divmod(int(start), 60) # type: ignore[call-overload]
e_min, e_sec = divmod(int(end), 60) # type: ignore[call-overload]
parts.append(f"\n### Segment {i + 1} ({s_min}:{s_sec:02d} - {e_min}:{e_sec:02d})")
else:
parts.append(f"\n### Segment {i + 1}")
# Segment markdown
seg_md = seg.get("markdown")
if seg_md:
parts.append(f"\n```markdown\n{seg_md}\n```")
# Segment fields
seg_fields = seg.get("fields")
if isinstance(seg_fields, dict) and seg_fields:
fields_json = json.dumps(seg_fields, indent=2, default=str)
parts.append(f"\n**Fields:**\n```json\n{fields_json}\n```")
return "\n".join(parts)
# --- Single-segment: flat format ---
fields_raw = result.get("fields")
fields: dict[str, object] = cast(dict[str, object], fields_raw) if isinstance(fields_raw, dict) else {}
# For audio: promote Summary field as prose before markdown
if is_audio and fields:
summary_field = fields.get("Summary")
if isinstance(summary_field, dict):
sf = cast(dict[str, object], summary_field)
if sf.get("value"):
parts.append(f"\n## Summary\n\n{sf['value']}")
# Markdown content
markdown = result.get("markdown")
if markdown:
parts.append(f"\n## Content\n\n```markdown\n{markdown}\n```")
# Fields section
if fields:
remaining = dict(fields)
if is_audio:
remaining = {k: v for k, v in remaining.items() if k != "Summary"}
if remaining:
fields_json = json.dumps(remaining, indent=2, default=str)
parts.append(f"\n## Extracted Fields\n\n```json\n{fields_json}\n```")
return "\n".join(parts)
@@ -1,101 +0,0 @@
# Copyright (c) Microsoft. All rights reserved.
"""File search backend abstraction for vector store file operations.
Provides a unified interface for uploading CU-extracted content to
vector stores across different LLM clients. Two implementations:
- ``OpenAIFileSearchBackend`` — for ``OpenAIChatClient`` (Responses API)
- ``FoundryFileSearchBackend`` — for ``FoundryChatClient`` (Responses API via Azure)
Both share the same OpenAI-compatible vector store file API but differ
in the file upload ``purpose`` value.
Vector store creation, tool construction, and lifecycle management are
the caller's responsibility — the backend only handles file upload/delete.
"""
from __future__ import annotations
import io
from abc import ABC, abstractmethod
from typing import Any
class FileSearchBackend(ABC):
"""Abstract interface for vector store file operations.
Implementations handle the differences between OpenAI and Foundry
file upload APIs (e.g., different ``purpose`` values).
Vector store creation, deletion, and ``file_search`` tool construction
are **not** part of this interface — those are managed by the caller.
"""
@abstractmethod
async def upload_file(self, vector_store_id: str, filename: str, content: bytes) -> str:
"""Upload a file to a vector store and return the file ID."""
@abstractmethod
async def delete_file(self, file_id: str) -> None:
"""Delete a previously uploaded file by ID."""
class _OpenAICompatBackend(FileSearchBackend):
"""Shared base for OpenAI-compatible file upload backends.
Both OpenAI and Foundry use the same ``client.files.*`` and
``client.vector_stores.files.*`` API surface. Subclasses only
override the file upload ``purpose``.
"""
_FILE_PURPOSE: str # Subclasses must set this
def __init__(self, client: Any) -> None:
self._client = client
async def upload_file(self, vector_store_id: str, filename: str, content: bytes) -> str:
uploaded = await self._client.files.create(
file=(filename, io.BytesIO(content)),
purpose=self._FILE_PURPOSE,
)
# Use create_and_poll to wait for indexing to complete before returning.
# Without this, file_search queries may return no results immediately
# after upload because the vector store index isn't ready yet.
await self._client.vector_stores.files.create_and_poll(
vector_store_id=vector_store_id,
file_id=uploaded.id,
)
return uploaded.id # type: ignore[no-any-return]
async def delete_file(self, file_id: str) -> None:
await self._client.files.delete(file_id)
class OpenAIFileSearchBackend(_OpenAICompatBackend):
"""File search backend for OpenAI Responses API.
Use with ``OpenAIChatClient`` or ``AzureOpenAIResponsesClient``.
Requires an ``AsyncOpenAI`` or ``AsyncAzureOpenAI`` client.
Args:
client: An async OpenAI client (``AsyncOpenAI`` or ``AsyncAzureOpenAI``)
that supports ``client.files.*`` and ``client.vector_stores.*`` APIs.
"""
_FILE_PURPOSE = "user_data"
class FoundryFileSearchBackend(_OpenAICompatBackend):
"""File search backend for Azure AI Foundry.
Use with ``FoundryChatClient``. Requires the OpenAI-compatible client
obtained from ``FoundryChatClient.client`` (i.e.,
``project_client.get_openai_client()``).
Args:
client: The OpenAI-compatible async client from a ``FoundryChatClient``
(access via ``foundry_client.client``).
"""
_FILE_PURPOSE = "assistants"
@@ -1,115 +0,0 @@
# Copyright (c) Microsoft. All rights reserved.
from __future__ import annotations
from dataclasses import dataclass
from enum import Enum
from typing import Any, Literal, TypedDict
from ._file_search import FileSearchBackend, FoundryFileSearchBackend, OpenAIFileSearchBackend
class DocumentStatus(str, Enum):
"""Analysis lifecycle state of a tracked document."""
ANALYZING = "analyzing"
"""CU analysis is in progress (deferred to background)."""
UPLOADING = "uploading"
"""Analysis complete; vector store upload + indexing is in progress."""
READY = "ready"
"""Analysis (and upload, if applicable) completed successfully."""
FAILED = "failed"
"""Analysis or upload failed."""
AnalysisSection = Literal["markdown", "fields"]
"""Which sections of the CU output to pass to the LLM.
- ``"markdown"``: Full document text with tables as HTML, reading order preserved.
- ``"fields"``: Extracted typed fields with confidence scores (when available).
"""
class DocumentEntry(TypedDict):
"""Tracks the analysis state of a single document in session state."""
status: DocumentStatus
filename: str
media_type: str
analyzer_id: str
analyzed_at: str | None
analysis_duration_s: float | None
upload_duration_s: float | None
result: dict[str, object] | None
error: str | None
@dataclass
class FileSearchConfig:
"""Configuration for uploading CU-extracted content to an existing vector store.
When provided to ``ContentUnderstandingContextProvider``, analyzed document
markdown is automatically uploaded to the specified vector store and the
given ``file_search`` tool is registered on the context. This enables
token-efficient RAG retrieval on follow-up turns for large documents.
The caller is responsible for creating and managing the vector store and
the ``file_search`` tool. Use :meth:`from_openai` or :meth:`from_foundry`
factory methods for convenience.
Args:
backend: A ``FileSearchBackend`` that handles file upload/delete
operations for the target vector store.
vector_store_id: The ID of a pre-existing vector store to upload to.
file_search_tool: A ``file_search`` tool object created via the LLM
client's ``get_file_search_tool()`` factory method. This is
registered on the context via ``extend_tools`` so the LLM can
retrieve uploaded content.
"""
backend: FileSearchBackend
vector_store_id: str
file_search_tool: Any
@staticmethod
def from_openai(
client: Any,
*,
vector_store_id: str,
file_search_tool: Any,
) -> FileSearchConfig:
"""Create a config for OpenAI Responses API (``OpenAIChatClient``).
Args:
client: An ``AsyncOpenAI`` or ``AsyncAzureOpenAI`` client.
vector_store_id: The ID of the vector store to upload to.
file_search_tool: Tool from ``OpenAIChatClient.get_file_search_tool()``.
"""
return FileSearchConfig(
backend=OpenAIFileSearchBackend(client),
vector_store_id=vector_store_id,
file_search_tool=file_search_tool,
)
@staticmethod
def from_foundry(
client: Any,
*,
vector_store_id: str,
file_search_tool: Any,
) -> FileSearchConfig:
"""Create a config for Azure AI Foundry (``FoundryChatClient``).
Args:
client: The OpenAI-compatible client from ``FoundryChatClient.client``.
vector_store_id: The ID of the vector store to upload to.
file_search_tool: Tool from ``FoundryChatClient.get_file_search_tool()``.
"""
return FileSearchConfig(
backend=FoundryFileSearchBackend(client),
vector_store_id=vector_store_id,
file_search_tool=file_search_tool,
)
@@ -1,101 +0,0 @@
[project]
name = "agent-framework-azure-contentunderstanding"
description = "Azure Content Understanding integration for Microsoft Agent Framework."
authors = [{ name = "Microsoft", email = "af-support@microsoft.com" }]
readme = "README.md"
requires-python = ">=3.10"
version = "1.0.0a260429"
license-files = ["LICENSE"]
urls.homepage = "https://aka.ms/agent-framework"
urls.source = "https://github.com/microsoft/agent-framework/tree/main/python"
urls.release_notes = "https://github.com/microsoft/agent-framework/releases?q=tag%3Apython-1&expanded=true"
urls.issues = "https://github.com/microsoft/agent-framework/issues"
classifiers = [
"License :: OSI Approved :: MIT License",
"Development Status :: 3 - Alpha",
"Intended Audience :: Developers",
"Programming Language :: Python :: 3",
"Programming Language :: Python :: 3.10",
"Programming Language :: Python :: 3.11",
"Programming Language :: Python :: 3.12",
"Programming Language :: Python :: 3.13",
"Programming Language :: Python :: 3.14",
"Typing :: Typed",
]
dependencies = [
"agent-framework-core>=1.2.2,<2",
"agent-framework-foundry>=1.2.2,<2",
"azure-ai-contentunderstanding>=1.0.1,<1.1",
"aiohttp>=3.9,<4",
"filetype>=1.2,<2",
]
[tool.uv]
prerelease = "if-necessary-or-explicit"
environments = [
"sys_platform == 'darwin'",
"sys_platform == 'linux'",
"sys_platform == 'win32'"
]
[tool.uv-dynamic-versioning]
fallback-version = "0.0.0"
[tool.pytest.ini_options]
testpaths = 'tests'
addopts = "-ra -q -r fEX"
asyncio_mode = "auto"
asyncio_default_fixture_loop_scope = "function"
timeout = 120
markers = [
"integration: marks tests as integration tests that require external services",
]
[tool.ruff]
extend = "../../pyproject.toml"
[tool.ruff.lint.per-file-ignores]
"**/tests/**" = ["D", "INP", "TD", "ERA001", "RUF", "S"]
"samples/**" = ["D", "INP", "ERA001", "RUF", "S", "T201", "CPY"]
[tool.coverage.run]
omit = ["**/__init__.py"]
[tool.pyright]
extends = "../../pyproject.toml"
include = ["agent_framework_azure_contentunderstanding"]
exclude = ['tests']
[tool.mypy]
plugins = ['pydantic.mypy']
strict = true
python_version = "3.10"
ignore_missing_imports = true
disallow_untyped_defs = true
no_implicit_optional = true
check_untyped_defs = true
warn_return_any = true
show_error_codes = true
warn_unused_ignores = false
disallow_incomplete_defs = true
disallow_untyped_decorators = true
[tool.bandit]
targets = ["agent_framework_azure_contentunderstanding"]
exclude_dirs = ["tests"]
[tool.poe]
executor.type = "uv"
include = "../../shared_tasks.toml"
[tool.poe.tasks.mypy]
help = "Run MyPy for this package."
cmd = "mypy --config-file $POE_ROOT/pyproject.toml agent_framework_azure_contentunderstanding"
[tool.poe.tasks.test]
help = "Run the default unit test suite for this package."
cmd = 'pytest -m "not integration" --cov=agent_framework_azure_contentunderstanding --cov-report=term-missing:skip-covered tests'
[build-system]
requires = ["flit-core >= 3.11,<4.0"]
build-backend = "flit_core.buildapi"
@@ -1,117 +0,0 @@
# Copyright (c) Microsoft. All rights reserved.
# /// script
# requires-python = ">=3.10"
# dependencies = [
# "agent-framework-azure-contentunderstanding",
# "agent-framework-foundry",
# "azure-identity",
# ]
# ///
# Run with: uv run packages/azure-contentunderstanding/samples/01-get-started/01_document_qa.py
import asyncio
import os
from pathlib import Path
from agent_framework import Agent, Content, Message
from agent_framework.foundry import ContentUnderstandingContextProvider, FoundryChatClient
from azure.identity import AzureCliCredential
from dotenv import load_dotenv
load_dotenv()
"""
Document Q&A — PDF upload with CU-powered extraction
This sample demonstrates the simplest CU integration: upload a PDF and
ask questions about it. Azure Content Understanding extracts structured
markdown with table preservation — superior to LLM-only vision for
scanned PDFs, handwritten content, and complex layouts.
Environment variables:
FOUNDRY_PROJECT_ENDPOINT — Azure AI Foundry project endpoint
FOUNDRY_MODEL — Model deployment name (e.g. gpt-4.1)
AZURE_CONTENTUNDERSTANDING_ENDPOINT — CU endpoint URL
"""
# Path to a sample PDF — uses the shared sample asset if available,
# otherwise falls back to a public URL
SAMPLE_PDF_PATH = Path(__file__).resolve().parents[1] / "shared" / "sample_assets" / "invoice.pdf"
async def main() -> None:
credential = AzureCliCredential()
# Set up Azure Content Understanding context provider
cu = ContentUnderstandingContextProvider(
endpoint=os.environ["AZURE_CONTENTUNDERSTANDING_ENDPOINT"],
credential=credential,
analyzer_id="prebuilt-documentSearch", # RAG-optimized document analyzer
max_wait=None, # wait until CU analysis finishes (no background deferral)
)
# Set up the LLM client
client = FoundryChatClient(
project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"],
model=os.environ["FOUNDRY_MODEL"],
credential=credential,
)
# Create agent with CU context provider.
# The provider extracts document content via CU and injects it into the
# LLM context so the agent can answer questions about the document.
async with cu:
agent = Agent(
client=client,
name="DocumentQA",
instructions=(
"You are a helpful document analyst. Use the analyzed document "
"content and extracted fields to answer questions precisely."
),
context_providers=[cu],
)
# --- Turn 1: Upload PDF and ask a question ---
# 4. Upload PDF and ask questions
# The CU provider extracts markdown + fields from the PDF and injects
# the full content into context so the agent can answer precisely.
print("--- Upload PDF and ask questions ---")
pdf_bytes = SAMPLE_PDF_PATH.read_bytes()
response = await agent.run(
Message(
role="user",
contents=[
Content.from_text(
"What is this document about? Who is the vendor, and what is the total amount due?"
),
Content.from_data(
pdf_bytes,
"application/pdf",
# Always provide filename — used as the document key
additional_properties={"filename": SAMPLE_PDF_PATH.name},
),
],
)
)
usage = response.usage_details or {}
print(f"Agent: {response}")
print(f" [Input tokens: {usage.get('input_token_count', 'N/A')}]\n")
if __name__ == "__main__":
asyncio.run(main())
"""
Sample output:
--- Upload PDF and ask questions ---
Agent: This document is an **invoice** for services and fees billed to
**MICROSOFT CORPORATION** (Invoice **INV-100**), including line items
(e.g., Consulting Services, Document Fee, Printing Fee) and a billing summary.
- **Vendor:** **CONTOSO LTD.**
- **Total amount due:** **$610.00**
[Input tokens: 988]
"""
@@ -1,143 +0,0 @@
# Copyright (c) Microsoft. All rights reserved.
# /// script
# requires-python = ">=3.10"
# dependencies = [
# "agent-framework-azure-contentunderstanding",
# "agent-framework-foundry",
# "azure-identity",
# ]
# ///
# Run with: uv run packages/azure-contentunderstanding/samples/01-get-started/02_multi_turn_session.py
import asyncio
import os
from pathlib import Path
from agent_framework import Agent, AgentSession, Content, Message
from agent_framework.foundry import ContentUnderstandingContextProvider, FoundryChatClient
from azure.identity import AzureCliCredential
from dotenv import load_dotenv
load_dotenv()
"""
Multi-Turn Session — Cached results across turns
This sample demonstrates multi-turn document Q&A using an AgentSession.
The session persists CU analysis results and conversation history across
turns so the agent can answer follow-up questions about previously
uploaded documents without re-analyzing them.
Key concepts:
- AgentSession keeps CU state and conversation history across agent.run() calls
- Turn 1: CU analyzes the PDF and injects full content into context
- Turn 2: Unrelated question — agent answers from general knowledge
- Turn 3: Detailed question — agent uses document content from conversation
history (injected in Turn 1) to answer precisely
Environment variables:
FOUNDRY_PROJECT_ENDPOINT — Azure AI Foundry project endpoint
FOUNDRY_MODEL — Model deployment name (e.g. gpt-4.1)
AZURE_CONTENTUNDERSTANDING_ENDPOINT — CU endpoint URL
"""
SAMPLE_PDF_PATH = Path(__file__).resolve().parents[1] / "shared" / "sample_assets" / "invoice.pdf"
async def main() -> None:
# 1. Set up credentials and CU context provider
credential = AzureCliCredential()
cu = ContentUnderstandingContextProvider(
endpoint=os.environ["AZURE_CONTENTUNDERSTANDING_ENDPOINT"],
credential=credential,
analyzer_id="prebuilt-documentSearch",
max_wait=None, # wait until CU analysis finishes (no background deferral)
)
# 2. Set up the LLM client
client = FoundryChatClient(
project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"],
model=os.environ["FOUNDRY_MODEL"],
credential=credential,
)
# 3. Create agent and persistent session
async with cu:
agent = Agent(
client=client,
name="DocumentQA",
instructions=(
"You are a helpful document analyst. Use the analyzed document "
"content and extracted fields to answer questions precisely."
),
context_providers=[cu],
)
# Create a persistent session — this keeps CU state across turns
session = AgentSession()
# 4. Turn 1: Upload PDF
# CU analyzes the PDF and injects full content into context.
print("--- Turn 1: Upload PDF ---")
pdf_bytes = SAMPLE_PDF_PATH.read_bytes()
response = await agent.run(
Message(
role="user",
contents=[
Content.from_text("What is this document about?"),
Content.from_data(
pdf_bytes,
"application/pdf",
additional_properties={"filename": SAMPLE_PDF_PATH.name},
),
],
),
session=session, # <-- persist state across turns
)
usage = response.usage_details or {}
print(f"Agent: {response}")
print(f" [Input tokens: {usage.get('input_token_count', 'N/A')}]\n")
# 5. Turn 2: Unrelated question
# No document needed — agent answers from general knowledge.
print("--- Turn 2: Unrelated question ---")
response = await agent.run("What is the capital of France?", session=session)
usage = response.usage_details or {}
print(f"Agent: {response}")
print(f" [Input tokens: {usage.get('input_token_count', 'N/A')}]\n")
# 6. Turn 3: Detailed follow-up
# The agent answers from the full document content that was injected
# into conversation history in Turn 1. No re-analysis or tool call needed.
print("--- Turn 3: Detailed follow-up ---")
response = await agent.run(
"What is the shipping address on the invoice?",
session=session,
)
usage = response.usage_details or {}
print(f"Agent: {response}")
print(f" [Input tokens: {usage.get('input_token_count', 'N/A')}]\n")
if __name__ == "__main__":
asyncio.run(main())
"""
Sample output:
--- Turn 1: Upload PDF ---
Agent: This document is an **invoice** from **CONTOSO LTD.** to **MICROSOFT
CORPORATION**. Amount Due: $610.00. Invoice INV-100, dated 11/15/2019.
[Input tokens: 975]
--- Turn 2: Unrelated question ---
Agent: Paris.
[Input tokens: 1134]
--- Turn 3: Detailed follow-up ---
Agent: Shipping address (SHIP TO): Microsoft Delivery, 123 Ship St,
Redmond WA, 98052.
[Input tokens: 1155]
"""
@@ -1,186 +0,0 @@
# Copyright (c) Microsoft. All rights reserved.
# /// script
# requires-python = ">=3.10"
# dependencies = [
# "agent-framework-azure-contentunderstanding",
# "agent-framework-foundry",
# "azure-identity",
# ]
# ///
# Run with: uv run packages/azure-contentunderstanding/samples/01-get-started/03_multimodal_chat.py
import asyncio
import os
import time
from pathlib import Path
from agent_framework import Agent, AgentSession, Content, Message
from agent_framework.foundry import ContentUnderstandingContextProvider, FoundryChatClient
from azure.identity import AzureCliCredential
from dotenv import load_dotenv
load_dotenv()
"""
Multi-Modal Chat — PDF, audio, and video in a single turn
This sample demonstrates CU's multi-modal capability: upload a PDF invoice,
an audio call recording, and a video file all at once. The provider analyzes
all three in parallel using the right CU analyzer for each media type.
The provider auto-detects the media type and selects the right CU analyzer:
- PDF/images → prebuilt-documentSearch
- Audio → prebuilt-audioSearch
- Video → prebuilt-videoSearch
Environment variables:
FOUNDRY_PROJECT_ENDPOINT — Azure AI Foundry project endpoint
FOUNDRY_MODEL — Model deployment name (e.g. gpt-4.1)
AZURE_CONTENTUNDERSTANDING_ENDPOINT — CU endpoint URL
"""
# Local PDF from package assets
SAMPLE_PDF = Path(__file__).resolve().parents[1] / "shared" / "sample_assets" / "invoice.pdf"
# Public audio/video from Azure CU samples repo (raw GitHub URLs)
_CU_ASSETS = "https://raw.githubusercontent.com/Azure-Samples/azure-ai-content-understanding-assets/main"
AUDIO_URL = f"{_CU_ASSETS}/audio/callCenterRecording.mp3"
VIDEO_URL = f"{_CU_ASSETS}/videos/sdk_samples/FlightSimulator.mp4"
async def main() -> None:
# 1. Set up credentials and CU context provider
credential = AzureCliCredential()
# No analyzer_id specified — the provider auto-detects from media type:
# PDF/images → prebuilt-documentSearch
# Audio → prebuilt-audioSearch
# Video → prebuilt-videoSearch
cu = ContentUnderstandingContextProvider(
endpoint=os.environ["AZURE_CONTENTUNDERSTANDING_ENDPOINT"],
credential=credential,
max_wait=None, # wait until each analysis finishes
)
# 2. Set up the LLM client
client = FoundryChatClient(
project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"],
model=os.environ["FOUNDRY_MODEL"],
credential=credential,
)
# 3. Create agent and session
async with cu:
agent = Agent(
client=client,
name="MultiModalAgent",
instructions=(
"You are a helpful assistant that can analyze documents, audio, "
"and video files. Answer questions using the extracted content."
),
context_providers=[cu],
)
session = AgentSession()
# --- Turn 1: Upload all 3 modalities at once ---
# The provider analyzes all files in parallel using the appropriate
# CU analyzer for each media type. All results are injected into
# the same context so the agent can answer about all of them.
turn1_prompt = (
"I'm uploading three files: an invoice PDF, a call center "
"audio recording, and a flight simulator video. "
"Give a brief summary of each file."
)
print("--- Turn 1: Upload PDF + audio + video (parallel analysis) ---")
print(" (CU analysis may take a few minutes for these audio/video files...)")
print(f"User: {turn1_prompt}")
t0 = time.perf_counter()
response = await agent.run(
Message(
role="user",
contents=[
Content.from_text(turn1_prompt),
Content.from_data(
SAMPLE_PDF.read_bytes(),
"application/pdf",
additional_properties={"filename": "invoice.pdf"},
),
Content.from_uri(
AUDIO_URL,
media_type="audio/mp3",
additional_properties={"filename": "callCenterRecording.mp3"},
),
Content.from_uri(
VIDEO_URL,
media_type="video/mp4",
additional_properties={"filename": "FlightSimulator.mp4"},
),
],
),
session=session,
)
elapsed = time.perf_counter() - t0
usage = response.usage_details or {}
print(f" [Analyzed in {elapsed:.1f}s | Input tokens: {usage.get('input_token_count', 'N/A')}]")
print(f"Agent: {response}\n")
# --- Turn 2: Detail question about the PDF ---
turn2_prompt = "What are the line items and their amounts on the invoice?"
print("--- Turn 2: PDF detail ---")
print(f"User: {turn2_prompt}")
response = await agent.run(turn2_prompt, session=session)
usage = response.usage_details or {}
print(f" [Input tokens: {usage.get('input_token_count', 'N/A')}]")
print(f"Agent: {response}\n")
# --- Turn 3: Detail question about the audio ---
turn3_prompt = "What was the customer's issue in the call recording?"
print("--- Turn 3: Audio detail ---")
print(f"User: {turn3_prompt}")
response = await agent.run(turn3_prompt, session=session)
usage = response.usage_details or {}
print(f" [Input tokens: {usage.get('input_token_count', 'N/A')}]")
print(f"Agent: {response}\n")
# --- Turn 4: Detail question about the video ---
turn4_prompt = "What key scenes or actions are shown in the flight simulator video?"
print("--- Turn 4: Video detail ---")
print(f"User: {turn4_prompt}")
response = await agent.run(turn4_prompt, session=session)
usage = response.usage_details or {}
print(f" [Input tokens: {usage.get('input_token_count', 'N/A')}]")
print(f"Agent: {response}\n")
# --- Turn 5: Cross-document question ---
turn5_prompt = (
"Across all three files, which one contains financial data, "
"which one involves a customer interaction, and which one is "
"a visual demonstration?"
)
print("--- Turn 5: Cross-document question ---")
print(f"User: {turn5_prompt}")
response = await agent.run(turn5_prompt, session=session)
usage = response.usage_details or {}
print(f" [Input tokens: {usage.get('input_token_count', 'N/A')}]")
print(f"Agent: {response}\n")
if __name__ == "__main__":
asyncio.run(main())
"""
Sample output:
--- Turn 1: Upload PDF + audio + video (parallel analysis) ---
User: I'm uploading three files...
(CU analysis may take 1-2 minutes for audio/video files...)
[Analyzed in ~94s | Input tokens: ~2939]
Agent: ### invoice.pdf: An invoice from CONTOSO LTD. to MICROSOFT CORPORATION...
### callCenterRecording.mp3: A customer service call about point balance...
### FlightSimulator.mp4: A clip discussing neural text-to-speech...
--- Turn 2-5: Detail and cross-document questions ---
(Agent answers from conversation history without re-analysis)
"""
@@ -1,193 +0,0 @@
# Copyright (c) Microsoft. All rights reserved.
# /// script
# requires-python = ">=3.10"
# dependencies = [
# "agent-framework-azure-contentunderstanding",
# "agent-framework-foundry",
# "azure-identity",
# "pydantic",
# ]
# ///
# Run with: uv run packages/azure-contentunderstanding/samples/01-get-started/04_invoice_processing.py
import asyncio
import os
from pathlib import Path
from agent_framework import Agent, AgentSession, Content, Message
from agent_framework.foundry import ContentUnderstandingContextProvider, FoundryChatClient
from azure.identity import AzureCliCredential
from dotenv import load_dotenv
from pydantic import BaseModel, Field
load_dotenv()
"""
Invoice Processing — Structured output with prebuilt-invoice analyzer
This sample demonstrates CU's structured field extraction combined with
LLM structured output (Pydantic model). The prebuilt-invoice analyzer extracts
typed fields (VendorName, InvoiceTotal, DueDate, LineItems, etc.) with
confidence scores. We use output_sections=["fields"] only (no markdown needed)
since we want the LLM to produce a structured JSON response from the extracted
fields, not summarize document text.
Environment variables:
FOUNDRY_PROJECT_ENDPOINT — Azure AI Foundry project endpoint
FOUNDRY_MODEL — Model deployment name (e.g. gpt-4.1)
AZURE_CONTENTUNDERSTANDING_ENDPOINT — CU endpoint URL
"""
SAMPLE_PDF_PATH = Path(__file__).resolve().parents[1] / "shared" / "sample_assets" / "invoice.pdf"
# Structured output model — the LLM will return JSON matching this schema
# Structured output models — the LLM returns JSON matching this schema.
#
# Note: the prebuilt-invoice analyzer extracts an extensive set of fields
# (VendorName, BillingAddress, ShippingAddress, TaxDetails, PONumber, etc.).
# This sample defines a simplified schema to extract only the fields of
# interest to the caller. The LLM maps the full CU field output to this
# subset automatically.
# Learn more about prebuilt analyzers: https://learn.microsoft.com/azure/ai-services/content-understanding/concepts/prebuilt-analyzers
class LineItem(BaseModel):
description: str
quantity: float | None = None
unit_price: float | None = None
amount: float | None = None
class LowConfidenceField(BaseModel):
field_name: str
confidence: float
class InvoiceResult(BaseModel):
vendor_name: str
total_amount: float | None = None
currency: str = "USD"
due_date: str | None = None
line_items: list[LineItem] = Field(default_factory=list)
low_confidence_fields: list[LowConfidenceField] = Field(
default_factory=list,
description="Fields with confidence < 0.8, including their confidence score",
)
async def main() -> None:
# 1. Set up credentials and CU context provider
credential = AzureCliCredential()
# Default analyzer is prebuilt-documentSearch (RAG-optimized).
# Per-file override via additional_properties["analyzer_id"] lets us
# use prebuilt-invoice for structured field extraction on specific files.
#
# Only request "fields" (not "markdown") — we want the extracted typed
# fields for structured output, not the raw document text.
cu = ContentUnderstandingContextProvider(
endpoint=os.environ["AZURE_CONTENTUNDERSTANDING_ENDPOINT"],
credential=credential,
analyzer_id="prebuilt-documentSearch", # default for all files
max_wait=None, # wait until CU analysis finishes
output_sections=["fields"], # fields only — structured output doesn't need markdown
)
# 2. Set up the LLM client
client = FoundryChatClient(
project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"],
model=os.environ["FOUNDRY_MODEL"],
credential=credential,
)
# 3. Create agent and session
async with cu:
agent = Agent(
client=client,
name="InvoiceProcessor",
instructions=(
"You are an invoice processing assistant. Extract invoice data from "
"the provided CU fields (JSON with confidence scores). Return structured "
"output matching the requested schema. Flag fields with confidence < 0.8 "
"in the low_confidence_fields list."
),
context_providers=[cu],
)
session = AgentSession()
# 4. Upload an invoice PDF — uses structured output (Pydantic model)
print("--- Upload Invoice (Structured Output) ---")
pdf_bytes = SAMPLE_PDF_PATH.read_bytes()
response = await agent.run(
Message(
role="user",
contents=[
Content.from_text(
"Process this invoice. Extract the vendor name, total amount, due date, and all line items."
),
Content.from_data(
pdf_bytes,
"application/pdf",
# Per-file analyzer override: use prebuilt-invoice for
# structured field extraction (VendorName, InvoiceTotal, etc.)
# instead of the provider default (prebuilt-documentSearch).
additional_properties={
"filename": SAMPLE_PDF_PATH.name,
"analyzer_id": "prebuilt-invoice",
},
),
],
),
session=session,
options={"response_format": InvoiceResult},
)
# Parse the structured output from JSON text
try:
invoice = InvoiceResult.model_validate_json(response.text)
print(f"Vendor: {invoice.vendor_name}")
print(f"Total: {invoice.currency} {invoice.total_amount}")
print(f"Due date: {invoice.due_date}")
print(f"Line items ({len(invoice.line_items)}):")
for item in invoice.line_items:
print(f" - {item.description}: {item.amount}")
if invoice.low_confidence_fields:
print("âš  Low confidence fields:")
for f in invoice.low_confidence_fields:
print(f" - {f.field_name}: {f.confidence:.3f}")
except Exception:
print(f"Agent (raw): {response.text}\n")
# 5. Follow-up: free-text question about the invoice
print("\n--- Follow-up (Free Text) ---")
response = await agent.run(
"What is the payment term? Are there any fields with low confidence?",
session=session,
)
print(f"Agent: {response}\n")
if __name__ == "__main__":
asyncio.run(main())
"""
Sample output:
--- Upload Invoice (Structured Output) ---
Vendor: CONTOSO LTD.
Total: USD 110.0
Due date: 2019-12-15
Line items (3):
- Consulting Services: 60.0
- Document Fee: 30.0
- Printing Fee: 10.0
âš  Low confidence: VendorName, CustomerName
--- Follow-up (Free Text) ---
Agent: The payment terms are not explicitly stated on the invoice...
"""
@@ -1,166 +0,0 @@
# Copyright (c) Microsoft. All rights reserved.
# /// script
# requires-python = ">=3.10"
# dependencies = [
# "agent-framework-azure-contentunderstanding",
# "agent-framework-foundry",
# "azure-identity",
# ]
# ///
# Run with: uv run packages/azure-contentunderstanding/samples/01-get-started/05_large_doc_file_search.py
import asyncio
import os
from pathlib import Path
from agent_framework import Agent, AgentSession, Content, Message
from agent_framework.foundry import (
ContentUnderstandingContextProvider,
FileSearchConfig,
FoundryChatClient,
)
from azure.identity import AzureCliCredential
from dotenv import load_dotenv
load_dotenv()
"""
Large Document + file_search RAG — CU extraction + OpenAI vector store
For large documents (100+ pages) or long audio/video, injecting the full
CU-extracted content into the LLM context is impractical. This sample shows
how to use the built-in file_search integration: CU extracts markdown and
automatically uploads it to an OpenAI vector store for token-efficient RAG.
When ``FileSearchConfig`` is provided, the provider:
1. Extracts markdown via CU (handles scanned PDFs, audio, video)
2. Uploads the extracted markdown to a vector store
3. Registers a ``file_search`` tool on the agent context
4. Cleans up the vector store on close
Architecture:
Large PDF -> CU extracts markdown -> auto-upload to vector store -> file_search
Follow-up -> file_search retrieves top-k chunks -> LLM answers
NOTE: Requires an async OpenAI client for vector store operations.
This sample uses a single small invoice PDF for simplicity. In practice,
you can upload multiple files in the same session (each is indexed
separately in the vector store), and this pattern is most valuable for
large documents (up to 300 pages), long audio recordings, or video files
where full-context injection would exceed the LLM's context window.
CU supports PDFs up to 300 pages / 200 MB, and audio files up to 300 MB
— see the full service limits:
https://learn.microsoft.com/azure/ai-services/content-understanding/service-limits#input-file-limits
Environment variables:
FOUNDRY_PROJECT_ENDPOINT — Azure AI Foundry project endpoint
FOUNDRY_MODEL — Model deployment name (e.g. gpt-4.1)
AZURE_CONTENTUNDERSTANDING_ENDPOINT — CU endpoint URL
"""
SAMPLE_PDF_PATH = Path(__file__).resolve().parents[1] / "shared" / "sample_assets" / "invoice.pdf"
async def main() -> None:
# 1. Set up credentials and LLM client
credential = AzureCliCredential()
client = FoundryChatClient(
project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"],
model=os.environ["FOUNDRY_MODEL"],
credential=credential,
)
# 2. Get the async OpenAI client from FoundryChatClient for vector store operations
openai_client = client.client
# 3. Create vector store and file_search tool
vector_store = await openai_client.vector_stores.create(
name="cu_large_doc_demo",
expires_after={"anchor": "last_active_at", "days": 1},
)
file_search_tool = client.get_file_search_tool(vector_store_ids=[vector_store.id])
# 4. Configure CU provider with file_search integration
# When file_search is set, CU-extracted markdown is automatically uploaded
# to the vector store and the file_search tool is registered on the context.
cu = ContentUnderstandingContextProvider(
endpoint=os.environ["AZURE_CONTENTUNDERSTANDING_ENDPOINT"],
credential=credential,
analyzer_id="prebuilt-documentSearch",
max_wait=None, # wait until CU analysis + vector store upload finishes
file_search=FileSearchConfig.from_foundry(
openai_client,
vector_store_id=vector_store.id,
file_search_tool=file_search_tool,
),
)
pdf_bytes = SAMPLE_PDF_PATH.read_bytes()
# The provider handles everything: CU extraction + vector store upload + file_search tool
async with cu:
agent = Agent(
client=client,
name="LargeDocAgent",
instructions=(
"You are a document analyst. Use the file_search tool to find "
"relevant sections from the document and answer precisely. "
"Cite specific sections when answering."
),
context_providers=[cu],
)
session = AgentSession()
# Turn 1: Upload — CU extracts and uploads to vector store automatically
print("--- Turn 1: Upload document ---")
response = await agent.run(
Message(
role="user",
contents=[
Content.from_text("What are the key points in this document?"),
Content.from_data(
pdf_bytes,
"application/pdf",
additional_properties={"filename": SAMPLE_PDF_PATH.name},
),
],
),
session=session,
)
print(f"Agent: {response}\n")
# Turn 2: Follow-up — file_search retrieves relevant chunks (token efficient)
print("--- Turn 2: Follow-up (RAG) ---")
response = await agent.run(
"What numbers or financial metrics are mentioned?",
session=session,
)
print(f"Agent: {response}\n")
# Explicitly delete the vector store created for this sample
await openai_client.vector_stores.delete(vector_store.id)
print("Done. Vector store deleted.")
if __name__ == "__main__":
asyncio.run(main())
"""
Sample output:
--- Turn 1: Upload document ---
Agent: An invoice from Contoso Ltd. to Microsoft Corporation (INV-100).
Line items: Consulting Services $60, Document Fee $30, Printing Fee $10.
Subtotal $100, Sales tax $10, Total $110, Previous balance $500, Amount due $610.
--- Turn 2: Follow-up (RAG) ---
Agent: Subtotal $100.00, Sales tax $10.00, Total $110.00,
Previous unpaid balance $500.00, Amount due $610.00.
Line items: 2 hours @ $30 = $60, 3 @ $10 = $30, 10 pages @ $1 = $10.
Done. Vector store cleaned up automatically.
"""
@@ -1,33 +0,0 @@
# DevUI Multi-Modal Agent
Interactive web UI for uploading and chatting with documents, images, audio, and video using Azure Content Understanding.
## Setup
1. Set environment variables (or create a `.env` file in `python/`):
```bash
FOUNDRY_PROJECT_ENDPOINT=https://your-project.api.azureml.ms
AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME=gpt-4.1
AZURE_CONTENTUNDERSTANDING_ENDPOINT=https://your-cu-resource.cognitiveservices.azure.com/
```
2. Log in with Azure CLI:
```bash
az login
```
3. Run with DevUI:
```bash
uv run poe devui --agent packages/azure-contentunderstanding/samples/devui_multimodal_agent
```
4. Open the DevUI URL in your browser and start uploading files.
## What You Can Do
- **Upload PDFs** — including scanned/image-based PDFs that LLM vision struggles with
- **Upload images** — handwritten notes, infographics, charts
- **Upload audio** — meeting recordings, call center calls (transcription with speaker ID)
- **Upload video** — product demos, training videos (frame extraction + transcription)
- **Ask questions** across all uploaded documents
- **Check status** — "which documents are ready?" uses the auto-registered `list_documents()` tool
@@ -1,6 +0,0 @@
# Copyright (c) Microsoft. All rights reserved.
"""DevUI Multi-Modal Agent with Azure Content Understanding."""
from .agent import agent
__all__ = ["agent"]
@@ -1,66 +0,0 @@
# Copyright (c) Microsoft. All rights reserved.
"""DevUI Multi-Modal Agent — file upload + CU-powered analysis.
This agent uses Azure Content Understanding to analyze uploaded files
(PDFs, scanned documents, handwritten images, audio recordings, video)
and answer questions about them through the DevUI web interface.
Unlike the standard azure_responses_agent which sends files directly to the LLM,
this agent uses CU for structured extraction — superior for scanned PDFs,
handwritten content, audio transcription, and video analysis.
Required environment variables:
FOUNDRY_PROJECT_ENDPOINT — Azure AI Foundry project endpoint
FOUNDRY_MODEL — Model deployment name (e.g. gpt-4.1)
AZURE_CONTENTUNDERSTANDING_ENDPOINT — CU endpoint URL
Run with DevUI:
uv run poe devui --agent packages/azure-contentunderstanding/samples/devui_multimodal_agent
"""
import os
from agent_framework import Agent
from agent_framework.foundry import ContentUnderstandingContextProvider, FoundryChatClient
from azure.core.credentials import AzureKeyCredential
from azure.identity import AzureCliCredential
from dotenv import load_dotenv
load_dotenv()
# --- Auth ---
_credential = AzureCliCredential()
_cu_api_key = os.environ.get("AZURE_CONTENTUNDERSTANDING_API_KEY")
_cu_credential = AzureKeyCredential(_cu_api_key) if _cu_api_key else _credential
cu = ContentUnderstandingContextProvider(
endpoint=os.environ["AZURE_CONTENTUNDERSTANDING_ENDPOINT"],
credential=_cu_credential,
# max_wait controls how long before_run() waits for CU analysis before
# deferring to background. For interactive DevUI use, a short timeout
# (e.g. 5s) keeps the chat responsive — the agent tells the user the
# file is still being analyzed and resolves it on the next turn.
# Use max_wait=None to always wait for analysis to complete.
max_wait=5.0,
)
client = FoundryChatClient(
project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"],
model=os.environ["FOUNDRY_MODEL"],
credential=_credential,
)
agent = Agent(
client=client,
name="MultiModalDocAgent",
instructions=(
"You are a helpful document analysis assistant. "
"When a user uploads files, they are automatically analyzed using Azure Content Understanding. "
"Use list_documents() to check which documents are ready, pending, or failed "
"and to see which files are available for answering questions. "
"Tell the user if any documents are still being analyzed. "
"You can process PDFs, scanned documents, handwritten images, audio recordings, and video files. "
"When answering, cite specific content from the documents."
),
context_providers=[cu],
)
@@ -1,51 +0,0 @@
# DevUI File Search Agent
Interactive web UI for uploading and chatting with documents, images, audio, and video using Azure Content Understanding + OpenAI file_search RAG.
## How It Works
1. **Upload** any supported file (PDF, image, audio, video) via the DevUI chat
2. **CU analyzes** the file — auto-selects the right analyzer per media type
3. **Markdown extracted** by CU is uploaded to an OpenAI vector store
4. **file_search** tool is registered — LLM retrieves top-k relevant chunks
5. **Ask questions** across all uploaded documents with token-efficient RAG
## Setup
1. Set environment variables (or create a `.env` file in `python/`):
```bash
FOUNDRY_PROJECT_ENDPOINT=https://your-project.services.ai.azure.com/
AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME=gpt-4.1
AZURE_CONTENTUNDERSTANDING_ENDPOINT=https://your-cu-resource.services.ai.azure.com/
```
2. Log in with Azure CLI:
```bash
az login
```
3. Run with DevUI:
```bash
devui packages/azure-contentunderstanding/samples/devui_azure_openai_file_search_agent
```
4. Open the DevUI URL in your browser and start uploading files.
## Supported File Types
| Type | Formats | CU Analyzer (auto-detected) |
|------|---------|----------------------------|
| Documents | PDF, DOCX, XLSX, PPTX, HTML, TXT, Markdown | `prebuilt-documentSearch` |
| Images | JPEG, PNG, TIFF, BMP | `prebuilt-documentSearch` |
| Audio | WAV, MP3, FLAC, OGG, M4A | `prebuilt-audioSearch` |
| Video | MP4, MOV, AVI, WebM | `prebuilt-videoSearch` |
## vs. devui_multimodal_agent
| Feature | multimodal_agent | file_search_agent |
|---------|-----------------|-------------------|
| CU extraction | âś… Full content injected | âś… Content indexed in vector store |
| RAG | ❌ | ✅ file_search retrieves top-k chunks |
| Large docs (100+ pages) | ⚠️ May exceed context window | ✅ Token-efficient |
| Multiple large files | ⚠️ Context overflow risk | ✅ All indexed, searchable |
| Best for | Small docs, quick inspection | Large docs, multi-file Q&A |
@@ -1,6 +0,0 @@
# Copyright (c) Microsoft. All rights reserved.
"""DevUI Multi-Modal Agent with CU + file_search RAG."""
from .agent import agent
__all__ = ["agent"]
@@ -1,105 +0,0 @@
# Copyright (c) Microsoft. All rights reserved.
"""DevUI Multi-Modal Agent — CU extraction + file_search RAG.
This agent combines Azure Content Understanding with OpenAI file_search
for token-efficient RAG over large or multi-modal documents.
Upload flow:
1. CU extracts high-quality markdown (handles scanned PDFs, audio, video)
2. Extracted markdown is auto-uploaded to an OpenAI vector store
3. file_search tool is registered so the LLM retrieves top-k chunks
4. Vector store is configured to auto-expire after inactivity
This is ideal for large documents (100+ pages), long audio recordings,
or multiple files in the same conversation where full-context injection
would exceed the LLM's context window.
Analyzer auto-detection:
When no analyzer_id is specified, the provider auto-selects the
appropriate CU analyzer based on media type:
- Documents/images → prebuilt-documentSearch
- Audio → prebuilt-audioSearch
- Video → prebuilt-videoSearch
Required environment variables:
FOUNDRY_PROJECT_ENDPOINT — Azure AI Foundry project endpoint
FOUNDRY_MODEL — Model deployment name (e.g. gpt-4.1)
AZURE_CONTENTUNDERSTANDING_ENDPOINT — CU endpoint URL
Run with DevUI:
devui packages/azure-contentunderstanding/samples/devui_azure_openai_file_search_agent
"""
import os
from agent_framework import Agent
from agent_framework.foundry import (
ContentUnderstandingContextProvider,
FileSearchConfig,
FoundryChatClient,
)
from azure.ai.projects import AIProjectClient
from azure.core.credentials import AzureKeyCredential
from azure.identity import AzureCliCredential
from dotenv import load_dotenv
load_dotenv()
# --- Auth ---
_credential = AzureCliCredential()
_cu_api_key = os.environ.get("AZURE_CONTENTUNDERSTANDING_API_KEY")
_cu_credential = AzureKeyCredential(_cu_api_key) if _cu_api_key else _credential
_endpoint = os.environ["FOUNDRY_PROJECT_ENDPOINT"]
# --- LLM client + sync vector store setup ---
# DevUI loads agent modules synchronously at startup while an event loop is already
# running, so we cannot use async APIs here. A sync AIProjectClient is used for
# one-time vector store creation; runtime file uploads use client.client (async).
client = FoundryChatClient(
project_endpoint=_endpoint,
model=os.environ["FOUNDRY_MODEL"],
credential=_credential,
)
_sync_project = AIProjectClient(endpoint=_endpoint, credential=_credential) # type: ignore[arg-type]
_sync_openai = _sync_project.get_openai_client()
_vector_store = _sync_openai.vector_stores.create(
name="devui_cu_file_search",
expires_after={"anchor": "last_active_at", "days": 1},
)
_sync_openai.close()
_file_search_tool = client.get_file_search_tool(
vector_store_ids=[_vector_store.id],
max_num_results=3, # limit chunks to reduce input token usage
)
# --- CU context provider with file_search ---
# client.client is the async OpenAI client used for runtime file uploads.
# No analyzer_id → auto-selects per media type (documents, audio, video)
cu = ContentUnderstandingContextProvider(
endpoint=os.environ["AZURE_CONTENTUNDERSTANDING_ENDPOINT"],
credential=_cu_credential,
file_search=FileSearchConfig.from_foundry(
client.client, # reuse the LLM client's internal AsyncAzureOpenAI for file uploads
vector_store_id=_vector_store.id,
file_search_tool=_file_search_tool,
),
)
agent = Agent(
client=client,
name="FileSearchDocAgent",
instructions=(
"You are a helpful document analysis assistant with RAG capabilities. "
"When a user uploads files, they are automatically analyzed using Azure Content Understanding "
"and indexed in a vector store for efficient retrieval. "
"Analysis takes time (seconds for documents, longer for audio/video) — if a document "
"is still pending, let the user know and suggest they ask again shortly. "
"You can process PDFs, scanned documents, handwritten images, audio recordings, and video files. "
"Multiple files can be uploaded and queried in the same conversation. "
"When answering, cite specific content from the documents."
),
context_providers=[cu],
)
@@ -1,34 +0,0 @@
# DevUI Foundry File Search Agent
Interactive web UI for uploading and chatting with documents, images, audio, and video using Azure Content Understanding + Foundry file_search RAG.
This is the **Foundry** variant. For the Azure OpenAI Responses API variant, see `devui_azure_openai_file_search_agent`.
## How It Works
1. **Upload** any supported file (PDF, image, audio, video) via the DevUI chat
2. **CU analyzes** the file — auto-selects the right analyzer per media type
3. **Markdown extracted** by CU is uploaded to a Foundry vector store
4. **file_search** tool is registered — LLM retrieves top-k relevant chunks
5. **Ask questions** across all uploaded documents with token-efficient RAG
## Setup
1. Set environment variables (or create a `.env` file in `python/`):
```bash
FOUNDRY_PROJECT_ENDPOINT=https://your-project.services.ai.azure.com/
FOUNDRY_MODEL=gpt-4.1
AZURE_CONTENTUNDERSTANDING_ENDPOINT=https://your-cu-resource.services.ai.azure.com/
```
2. Log in with Azure CLI:
```bash
az login
```
3. Run with DevUI:
```bash
devui packages/azure-contentunderstanding/samples/devui_foundry_file_search_agent
```
4. Open the DevUI URL in your browser and start uploading files.
@@ -1 +0,0 @@
# Copyright (c) Microsoft. All rights reserved.
@@ -1,109 +0,0 @@
# Copyright (c) Microsoft. All rights reserved.
"""DevUI Multi-Modal Agent — CU extraction + file_search RAG via Azure AI Foundry.
This agent combines Azure Content Understanding with Foundry's file_search
for token-efficient RAG over large or multi-modal documents.
Upload flow:
1. CU extracts high-quality markdown (handles scanned PDFs, audio, video)
2. Extracted markdown is uploaded to a Foundry vector store
3. file_search tool is registered so the LLM retrieves top-k chunks
4. Uploaded files are cleaned up on server shutdown
This sample uses ``FoundryChatClient`` and ``FoundryFileSearchBackend``.
For the OpenAI Responses API variant, see ``devui_azure_openai_file_search_agent``.
Analyzer auto-detection:
When no analyzer_id is specified, the provider auto-selects the
appropriate CU analyzer based on media type:
- Documents/images → prebuilt-documentSearch
- Audio → prebuilt-audioSearch
- Video → prebuilt-videoSearch
Required environment variables:
FOUNDRY_PROJECT_ENDPOINT — Azure AI Foundry project endpoint
FOUNDRY_MODEL — Model deployment name (e.g. gpt-4.1)
AZURE_CONTENTUNDERSTANDING_ENDPOINT — CU endpoint URL
Run with DevUI:
devui packages/azure-contentunderstanding/samples/devui_foundry_file_search_agent
"""
import os
from agent_framework import Agent
from agent_framework.foundry import (
ContentUnderstandingContextProvider,
FileSearchConfig,
FoundryChatClient,
)
from azure.core.credentials import AzureKeyCredential
from azure.identity import AzureCliCredential
from dotenv import load_dotenv
from openai import AzureOpenAI
load_dotenv()
# --- Auth ---
# AzureCliCredential for Foundry. CU API key optional if on a different resource.
_credential = AzureCliCredential()
_cu_api_key = os.environ.get("AZURE_CONTENTUNDERSTANDING_API_KEY")
_cu_credential = AzureKeyCredential(_cu_api_key) if _cu_api_key else _credential
# --- Foundry LLM client ---
client = FoundryChatClient(
project_endpoint=os.environ.get("FOUNDRY_PROJECT_ENDPOINT", ""),
model=os.environ.get("FOUNDRY_MODEL", ""),
credential=_credential,
)
# --- Create vector store (sync client to avoid event loop conflicts in DevUI) ---
_token = _credential.get_token("https://ai.azure.com/.default").token
_sync_openai = AzureOpenAI(
azure_endpoint=os.environ.get("FOUNDRY_PROJECT_ENDPOINT", ""),
azure_ad_token=_token,
api_version="2025-04-01-preview",
)
_vector_store = _sync_openai.vector_stores.create(
name="devui_cu_foundry_file_search",
expires_after={"anchor": "last_active_at", "days": 1},
)
_sync_openai.close()
_file_search_tool = client.get_file_search_tool(
vector_store_ids=[_vector_store.id],
max_num_results=3, # limit chunks to reduce input token usage
)
# --- CU context provider with file_search ---
# No analyzer_id → auto-selects per media type (documents, audio, video)
cu = ContentUnderstandingContextProvider(
endpoint=os.environ["AZURE_CONTENTUNDERSTANDING_ENDPOINT"],
credential=_cu_credential,
# max_wait is the combined budget for CU analysis + vector store upload.
# For file_search mode, 10s gives enough time for small documents to be
# analyzed and indexed in one turn. Larger files (audio, video) will
# be deferred to background and resolved on the next turn.
max_wait=10.0,
file_search=FileSearchConfig.from_foundry(
client.client,
vector_store_id=_vector_store.id,
file_search_tool=_file_search_tool,
),
)
agent = Agent(
client=client,
name="FoundryFileSearchDocAgent",
instructions=(
"You are a helpful document analysis assistant with RAG capabilities. "
"When a user uploads files, they are automatically analyzed using Azure Content Understanding "
"and indexed in a vector store for efficient retrieval. "
"Analysis takes time (seconds for documents, longer for audio/video) — if a document "
"is still pending, let the user know and suggest they ask again shortly. "
"You can process PDFs, scanned documents, handwritten images, audio recordings, and video files. "
"Multiple files can be uploaded and queried in the same conversation. "
"When answering, cite specific content from the documents."
),
context_providers=[cu],
)
@@ -1,39 +0,0 @@
# Azure Content Understanding Samples
These samples demonstrate how to use the `agent-framework-azure-contentunderstanding` package to add document, image, audio, and video understanding to your agents.
## Prerequisites
1. Azure CLI logged in: `az login`
2. Environment variables set (or `.env` file in the `python/` directory):
```
FOUNDRY_PROJECT_ENDPOINT=https://your-project.services.ai.azure.com
FOUNDRY_MODEL=gpt-4.1
AZURE_CONTENTUNDERSTANDING_ENDPOINT=https://your-cu-resource.cognitiveservices.azure.com/
```
## Samples
### 01-get-started — Script samples (easy → advanced)
| # | Sample | Description | Run |
|---|--------|-------------|-----|
| 01 | [Document Q&A](01-get-started/01_document_qa.py) | Upload a PDF, ask questions with CU-powered extraction | `uv run samples/01-get-started/01_document_qa.py` |
| 02 | [Multi-Turn Session](01-get-started/02_multi_turn_session.py) | AgentSession persistence across turns | `uv run samples/01-get-started/02_multi_turn_session.py` |
| 03 | [Multi-Modal Chat](01-get-started/03_multimodal_chat.py) | PDF + audio + video parallel analysis | `uv run samples/01-get-started/03_multimodal_chat.py` |
| 04 | [Invoice Processing](01-get-started/04_invoice_processing.py) | Structured field extraction with prebuilt-invoice | `uv run samples/01-get-started/04_invoice_processing.py` |
| 05 | [Large Doc + file_search](01-get-started/05_large_doc_file_search.py) | CU extraction + OpenAI vector store RAG | `uv run samples/01-get-started/05_large_doc_file_search.py` |
### 02-devui — Interactive web UI samples
| # | Sample | Description | Run |
|---|--------|-------------|-----|
| 01 | [Multi-Modal Agent](02-devui/01-multimodal_agent/) | Web UI for file upload + CU-powered chat | `devui samples/02-devui/01-multimodal_agent` |
| 02a | [file_search (Azure OpenAI backend)](02-devui/02-file_search_agent/azure_openai_backend/) | DevUI with CU + Azure OpenAI vector store | `devui samples/02-devui/02-file_search_agent/azure_openai_backend` |
| 02b | [file_search (Foundry backend)](02-devui/02-file_search_agent/foundry_backend/) | DevUI with CU + Foundry vector store | `devui samples/02-devui/02-file_search_agent/foundry_backend` |
## Install (preview)
```bash
pip install --pre agent-framework-azure-contentunderstanding
```
@@ -1,106 +0,0 @@
# Copyright (c) Microsoft. All rights reserved.
from __future__ import annotations
import asyncio
import json
from pathlib import Path
from typing import Any
from unittest.mock import AsyncMock, MagicMock
import pytest
from azure.ai.contentunderstanding.models import AnalysisResult
FIXTURES_DIR = Path(__file__).parent / "fixtures"
def _load_fixture(name: str) -> dict[str, Any]:
return json.loads((FIXTURES_DIR / name).read_text()) # type: ignore[no-any-return]
@pytest.fixture
def pdf_fixture_raw() -> dict[str, Any]:
return _load_fixture("analyze_pdf_result.json")
@pytest.fixture
def pdf_analysis_result(pdf_fixture_raw: dict[str, Any]) -> AnalysisResult:
return AnalysisResult(pdf_fixture_raw)
@pytest.fixture
def audio_fixture_raw() -> dict[str, Any]:
return _load_fixture("analyze_audio_result.json")
@pytest.fixture
def audio_analysis_result(audio_fixture_raw: dict[str, Any]) -> AnalysisResult:
return AnalysisResult(audio_fixture_raw)
@pytest.fixture
def invoice_fixture_raw() -> dict[str, Any]:
return _load_fixture("analyze_invoice_result.json")
@pytest.fixture
def invoice_analysis_result(invoice_fixture_raw: dict[str, Any]) -> AnalysisResult:
return AnalysisResult(invoice_fixture_raw)
@pytest.fixture
def video_fixture_raw() -> dict[str, Any]:
return _load_fixture("analyze_video_result.json")
@pytest.fixture
def video_analysis_result(video_fixture_raw: dict[str, Any]) -> AnalysisResult:
return AnalysisResult(video_fixture_raw)
@pytest.fixture
def image_fixture_raw() -> dict[str, Any]:
return _load_fixture("analyze_image_result.json")
@pytest.fixture
def image_analysis_result(image_fixture_raw: dict[str, Any]) -> AnalysisResult:
return AnalysisResult(image_fixture_raw)
@pytest.fixture
def mock_cu_client() -> AsyncMock:
"""Create a mock ContentUnderstandingClient."""
client = AsyncMock()
client.close = AsyncMock()
return client
def make_mock_poller(result: AnalysisResult) -> AsyncMock:
"""Create a mock poller that returns the given result immediately."""
poller = AsyncMock()
poller.result = AsyncMock(return_value=result)
poller.continuation_token = MagicMock(return_value="mock_continuation_token")
poller.done = MagicMock(return_value=True)
return poller
def make_slow_poller(result: AnalysisResult, delay: float = 10.0) -> MagicMock:
"""Create a mock poller that simulates a timeout then eventually returns."""
poller = MagicMock()
async def slow_result() -> AnalysisResult:
await asyncio.sleep(delay)
return result
poller.result = slow_result
poller.continuation_token = MagicMock(return_value="mock_slow_continuation_token")
poller.done = MagicMock(return_value=False)
return poller
def make_failing_poller(error: Exception) -> AsyncMock:
"""Create a mock poller that raises an exception."""
poller = AsyncMock()
poller.result = AsyncMock(side_effect=error)
return poller
@@ -1,13 +0,0 @@
{
"id": "synthetic-audio-001",
"status": "Succeeded",
"analyzer_id": "prebuilt-audioSearch",
"api_version": "2025-05-01-preview",
"created_at": "2026-03-21T10:05:00Z",
"contents": [
{
"markdown": "## Call Center Recording\n\n**Duration:** 2 minutes 15 seconds\n**Speakers:** 2\n\n### Transcript\n\n**Speaker 1 (Agent):** Thank you for calling Contoso support. My name is Sarah. How can I help you today?\n\n**Speaker 2 (Customer):** Hi Sarah, I'm calling about my recent order number ORD-5678. It was supposed to arrive yesterday but I haven't received it.\n\n**Speaker 1 (Agent):** I'm sorry to hear that. Let me look up your order. Can you confirm your name and email address?\n\n**Speaker 2 (Customer):** Sure, it's John Smith, john.smith@example.com.\n\n**Speaker 1 (Agent):** Thank you, John. I can see your order was shipped on March 18th. It looks like there was a delay with the carrier. The updated delivery estimate is March 22nd.\n\n**Speaker 2 (Customer):** That's helpful, thank you. Is there anything I can do to track it?\n\n**Speaker 1 (Agent):** Yes, I'll send you a tracking link to your email right away. Is there anything else I can help with?\n\n**Speaker 2 (Customer):** No, that's all. Thanks for your help.\n\n**Speaker 1 (Agent):** You're welcome! Have a great day.",
"fields": {}
}
]
}
@@ -1,857 +0,0 @@
{
"analyzerId": "prebuilt-documentSearch",
"apiVersion": "2025-11-01",
"createdAt": "2026-03-21T22:44:21Z",
"stringEncoding": "codePoint",
"warnings": [],
"contents": [
{
"path": "input1",
"markdown": "# Contoso Q1 2025 Financial Summary\n\nTotal revenue for Q1 2025 was $42.7 million, an increase of 18% over Q1 2024.\nOperating expenses were $31.2 million. Net profit was $11.5 million. The largest\nrevenue segment was Cloud Services at $19.3 million, followed by Professional\nServices at $14.8 million and Product Licensing at $8.6 million. Headcount at end of\nQ1 was 1,247 employees across 8 offices worldwide.\n",
"fields": {
"Summary": {
"type": "string",
"valueString": "The document provides a financial summary for Contoso in Q1 2025, reporting total revenue of $42.7 million, an 18% increase from Q1 2024. Operating expenses were $31.2 million, resulting in a net profit of $11.5 million. The largest revenue segment was Cloud Services with $19.3 million, followed by Professional Services at $14.8 million and Product Licensing at $8.6 million. The company had 1,247 employees across 8 offices worldwide at the end of Q1.",
"spans": [
{
"offset": 37,
"length": 77
},
{
"offset": 115,
"length": 80
},
{
"offset": 196,
"length": 77
},
{
"offset": 274,
"length": 84
},
{
"offset": 359,
"length": 50
}
],
"confidence": 0.592,
"source": "D(1,212.0000,334.0000,1394.0000,334.0000,1394.0000,374.0000,212.0000,374.0000);D(1,213.0000,379.0000,1398.0000,379.0000,1398.0000,422.0000,213.0000,422.0000);D(1,212.0000,423.0000,1389.0000,423.0000,1389.0000,464.0000,212.0000,464.0000);D(1,213.0000,468.0000,1453.0000,468.0000,1453.0000,510.0000,213.0000,510.0000);D(1,213.0000,512.0000,1000.0000,512.0000,1000.0000,554.0000,213.0000,554.0000)"
}
},
"kind": "document",
"startPageNumber": 1,
"endPageNumber": 1,
"unit": "pixel",
"pages": [
{
"pageNumber": 1,
"angle": -0.0242,
"width": 1700,
"height": 2200,
"spans": [
{
"offset": 0,
"length": 410
}
],
"words": [
{
"content": "Contoso",
"span": {
"offset": 2,
"length": 7
},
"confidence": 0.99,
"source": "D(1,214,222,401,222,401,274,214,273)"
},
{
"content": "Q1",
"span": {
"offset": 10,
"length": 2
},
"confidence": 0.957,
"source": "D(1,414,222,473,222,473,275,414,274)"
},
{
"content": "2025",
"span": {
"offset": 13,
"length": 4
},
"confidence": 0.929,
"source": "D(1,494,222,607,222,607,276,494,275)"
},
{
"content": "Financial",
"span": {
"offset": 18,
"length": 9
},
"confidence": 0.975,
"source": "D(1,624,222,819,223,819,277,624,276)"
},
{
"content": "Summary",
"span": {
"offset": 28,
"length": 7
},
"confidence": 0.991,
"source": "D(1,836,223,1050,225,1050,279,836,277)"
},
{
"content": "Total",
"span": {
"offset": 37,
"length": 5
},
"confidence": 0.996,
"source": "D(1,212,335,287,334,288,374,212,373)"
},
{
"content": "revenue",
"span": {
"offset": 43,
"length": 7
},
"confidence": 0.994,
"source": "D(1,299,334,417,334,418,374,299,374)"
},
{
"content": "for",
"span": {
"offset": 51,
"length": 3
},
"confidence": 0.994,
"source": "D(1,427,334,467,334,467,374,427,374)"
},
{
"content": "Q1",
"span": {
"offset": 55,
"length": 2
},
"confidence": 0.944,
"source": "D(1,475,334,515,334,515,374,475,374)"
},
{
"content": "2025",
"span": {
"offset": 58,
"length": 4
},
"confidence": 0.876,
"source": "D(1,528,334,604,334,604,374,529,374)"
},
{
"content": "was",
"span": {
"offset": 63,
"length": 3
},
"confidence": 0.991,
"source": "D(1,613,334,672,334,672,374,613,374)"
},
{
"content": "$",
"span": {
"offset": 67,
"length": 1
},
"confidence": 0.999,
"source": "D(1,681,334,698,334,698,374,681,374)"
},
{
"content": "42.7",
"span": {
"offset": 68,
"length": 4
},
"confidence": 0.946,
"source": "D(1,700,334,765,334,765,374,700,374)"
},
{
"content": "million",
"span": {
"offset": 73,
"length": 7
},
"confidence": 0.977,
"source": "D(1,775,334,867,334,867,374,776,374)"
},
{
"content": ",",
"span": {
"offset": 80,
"length": 1
},
"confidence": 0.998,
"source": "D(1,870,334,877,334,877,374,870,374)"
},
{
"content": "an",
"span": {
"offset": 82,
"length": 2
},
"confidence": 0.998,
"source": "D(1,888,334,922,334,922,374,888,374)"
},
{
"content": "increase",
"span": {
"offset": 85,
"length": 8
},
"confidence": 0.991,
"source": "D(1,934,334,1058,335,1059,374,934,374)"
},
{
"content": "of",
"span": {
"offset": 94,
"length": 2
},
"confidence": 0.982,
"source": "D(1,1069,335,1098,335,1098,374,1069,374)"
},
{
"content": "18",
"span": {
"offset": 97,
"length": 2
},
"confidence": 0.963,
"source": "D(1,1108,335,1142,335,1142,374,1108,374)"
},
{
"content": "%",
"span": {
"offset": 99,
"length": 1
},
"confidence": 0.998,
"source": "D(1,1143,335,1171,335,1171,374,1143,374)"
},
{
"content": "over",
"span": {
"offset": 101,
"length": 4
},
"confidence": 0.946,
"source": "D(1,1181,335,1248,335,1248,374,1181,374)"
},
{
"content": "Q1",
"span": {
"offset": 106,
"length": 2
},
"confidence": 0.875,
"source": "D(1,1256,335,1295,335,1295,374,1256,374)"
},
{
"content": "2024",
"span": {
"offset": 109,
"length": 4
},
"confidence": 0.683,
"source": "D(1,1310,335,1384,335,1384,374,1310,374)"
},
{
"content": ".",
"span": {
"offset": 113,
"length": 1
},
"confidence": 0.991,
"source": "D(1,1385,335,1394,335,1394,374,1385,374)"
},
{
"content": "Operating",
"span": {
"offset": 115,
"length": 9
},
"confidence": 0.996,
"source": "D(1,213,380,358,380,358,422,213,422)"
},
{
"content": "expenses",
"span": {
"offset": 125,
"length": 8
},
"confidence": 0.997,
"source": "D(1,369,380,513,379,513,421,369,421)"
},
{
"content": "were",
"span": {
"offset": 134,
"length": 4
},
"confidence": 0.998,
"source": "D(1,521,379,595,379,595,421,521,421)"
},
{
"content": "$",
"span": {
"offset": 139,
"length": 1
},
"confidence": 0.999,
"source": "D(1,603,379,620,379,620,421,603,421)"
},
{
"content": "31.2",
"span": {
"offset": 140,
"length": 4
},
"confidence": 0.938,
"source": "D(1,623,379,686,379,686,421,623,421)"
},
{
"content": "million",
"span": {
"offset": 145,
"length": 7
},
"confidence": 0.913,
"source": "D(1,696,379,790,379,790,421,696,421)"
},
{
"content": ".",
"span": {
"offset": 152,
"length": 1
},
"confidence": 0.975,
"source": "D(1,793,379,800,379,800,421,793,421)"
},
{
"content": "Net",
"span": {
"offset": 154,
"length": 3
},
"confidence": 0.976,
"source": "D(1,811,379,862,379,862,420,811,421)"
},
{
"content": "profit",
"span": {
"offset": 158,
"length": 6
},
"confidence": 0.993,
"source": "D(1,871,379,947,379,947,420,871,420)"
},
{
"content": "was",
"span": {
"offset": 165,
"length": 3
},
"confidence": 0.997,
"source": "D(1,954,379,1012,379,1012,420,953,420)"
},
{
"content": "$",
"span": {
"offset": 169,
"length": 1
},
"confidence": 0.998,
"source": "D(1,1021,379,1039,379,1039,420,1021,420)"
},
{
"content": "11.5",
"span": {
"offset": 170,
"length": 4
},
"confidence": 0.954,
"source": "D(1,1043,379,1106,379,1106,421,1043,420)"
},
{
"content": "million",
"span": {
"offset": 175,
"length": 7
},
"confidence": 0.837,
"source": "D(1,1118,379,1208,379,1208,421,1118,421)"
},
{
"content": ".",
"span": {
"offset": 182,
"length": 1
},
"confidence": 0.978,
"source": "D(1,1210,379,1217,379,1217,421,1210,421)"
},
{
"content": "The",
"span": {
"offset": 184,
"length": 3
},
"confidence": 0.949,
"source": "D(1,1228,379,1285,379,1285,421,1228,421)"
},
{
"content": "largest",
"span": {
"offset": 188,
"length": 7
},
"confidence": 0.978,
"source": "D(1,1295,379,1398,379,1398,421,1295,421)"
},
{
"content": "revenue",
"span": {
"offset": 196,
"length": 7
},
"confidence": 0.995,
"source": "D(1,212,425,334,425,334,464,212,464)"
},
{
"content": "segment",
"span": {
"offset": 204,
"length": 7
},
"confidence": 0.996,
"source": "D(1,344,425,472,424,472,464,344,464)"
},
{
"content": "was",
"span": {
"offset": 212,
"length": 3
},
"confidence": 0.998,
"source": "D(1,480,424,541,424,541,464,480,464)"
},
{
"content": "Cloud",
"span": {
"offset": 216,
"length": 5
},
"confidence": 0.997,
"source": "D(1,550,424,636,424,637,464,551,464)"
},
{
"content": "Services",
"span": {
"offset": 222,
"length": 8
},
"confidence": 0.995,
"source": "D(1,647,424,774,424,774,464,647,464)"
},
{
"content": "at",
"span": {
"offset": 231,
"length": 2
},
"confidence": 0.996,
"source": "D(1,784,424,812,424,812,464,784,464)"
},
{
"content": "$",
"span": {
"offset": 234,
"length": 1
},
"confidence": 0.998,
"source": "D(1,820,424,837,424,837,464,820,464)"
},
{
"content": "19.3",
"span": {
"offset": 235,
"length": 4
},
"confidence": 0.879,
"source": "D(1,840,424,903,423,903,463,840,464)"
},
{
"content": "million",
"span": {
"offset": 240,
"length": 7
},
"confidence": 0.876,
"source": "D(1,915,423,1006,423,1006,463,915,463)"
},
{
"content": ",",
"span": {
"offset": 247,
"length": 1
},
"confidence": 0.999,
"source": "D(1,1008,423,1015,423,1015,463,1008,463)"
},
{
"content": "followed",
"span": {
"offset": 249,
"length": 8
},
"confidence": 0.978,
"source": "D(1,1026,423,1148,424,1148,463,1026,463)"
},
{
"content": "by",
"span": {
"offset": 258,
"length": 2
},
"confidence": 0.986,
"source": "D(1,1160,424,1194,424,1194,463,1160,463)"
},
{
"content": "Professional",
"span": {
"offset": 261,
"length": 12
},
"confidence": 0.965,
"source": "D(1,1204,424,1389,424,1389,463,1204,463)"
},
{
"content": "Services",
"span": {
"offset": 274,
"length": 8
},
"confidence": 0.991,
"source": "D(1,213,469,341,469,341,510,213,510)"
},
{
"content": "at",
"span": {
"offset": 283,
"length": 2
},
"confidence": 0.997,
"source": "D(1,352,469,380,469,380,510,352,510)"
},
{
"content": "$",
"span": {
"offset": 286,
"length": 1
},
"confidence": 0.998,
"source": "D(1,388,469,405,469,405,510,388,510)"
},
{
"content": "14.8",
"span": {
"offset": 287,
"length": 4
},
"confidence": 0.973,
"source": "D(1,410,469,472,469,472,510,410,510)"
},
{
"content": "million",
"span": {
"offset": 292,
"length": 7
},
"confidence": 0.987,
"source": "D(1,483,469,575,469,575,510,483,510)"
},
{
"content": "and",
"span": {
"offset": 300,
"length": 3
},
"confidence": 0.999,
"source": "D(1,585,469,638,469,638,510,585,510)"
},
{
"content": "Product",
"span": {
"offset": 304,
"length": 7
},
"confidence": 0.995,
"source": "D(1,652,469,765,469,765,510,652,510)"
},
{
"content": "Licensing",
"span": {
"offset": 312,
"length": 9
},
"confidence": 0.993,
"source": "D(1,777,469,914,469,914,510,777,510)"
},
{
"content": "at",
"span": {
"offset": 322,
"length": 2
},
"confidence": 0.998,
"source": "D(1,925,469,953,469,953,510,925,510)"
},
{
"content": "$",
"span": {
"offset": 325,
"length": 1
},
"confidence": 0.998,
"source": "D(1,961,469,978,469,978,510,961,510)"
},
{
"content": "8.6",
"span": {
"offset": 326,
"length": 3
},
"confidence": 0.958,
"source": "D(1,980,469,1025,469,1025,510,980,510)"
},
{
"content": "million",
"span": {
"offset": 330,
"length": 7
},
"confidence": 0.908,
"source": "D(1,1036,469,1128,468,1128,510,1036,510)"
},
{
"content": ".",
"span": {
"offset": 337,
"length": 1
},
"confidence": 0.987,
"source": "D(1,1130,468,1137,468,1137,510,1130,510)"
},
{
"content": "Headcount",
"span": {
"offset": 339,
"length": 9
},
"confidence": 0.934,
"source": "D(1,1150,468,1310,468,1310,510,1150,510)"
},
{
"content": "at",
"span": {
"offset": 349,
"length": 2
},
"confidence": 0.993,
"source": "D(1,1318,468,1348,468,1348,510,1318,510)"
},
{
"content": "end",
"span": {
"offset": 352,
"length": 3
},
"confidence": 0.947,
"source": "D(1,1355,468,1410,468,1410,510,1355,510)"
},
{
"content": "of",
"span": {
"offset": 356,
"length": 2
},
"confidence": 0.974,
"source": "D(1,1419,468,1453,468,1453,509,1419,509)"
},
{
"content": "Q1",
"span": {
"offset": 359,
"length": 2
},
"confidence": 0.931,
"source": "D(1,213,512,252,512,252,554,213,554)"
},
{
"content": "was",
"span": {
"offset": 362,
"length": 3
},
"confidence": 0.847,
"source": "D(1,267,512,326,512,326,554,267,554)"
},
{
"content": "1,247",
"span": {
"offset": 366,
"length": 5
},
"confidence": 0.523,
"source": "D(1,338,512,419,512,419,554,338,554)"
},
{
"content": "employees",
"span": {
"offset": 372,
"length": 9
},
"confidence": 0.972,
"source": "D(1,429,513,591,512,591,554,429,554)"
},
{
"content": "across",
"span": {
"offset": 382,
"length": 6
},
"confidence": 0.972,
"source": "D(1,601,512,697,512,697,554,601,554)"
},
{
"content": "8",
"span": {
"offset": 389,
"length": 1
},
"confidence": 0.946,
"source": "D(1,708,512,725,512,725,553,708,554)"
},
{
"content": "offices",
"span": {
"offset": 391,
"length": 7
},
"confidence": 0.95,
"source": "D(1,736,512,831,512,831,553,736,553)"
},
{
"content": "worldwide",
"span": {
"offset": 399,
"length": 9
},
"confidence": 0.988,
"source": "D(1,840,512,989,512,989,552,840,553)"
},
{
"content": ".",
"span": {
"offset": 408,
"length": 1
},
"confidence": 0.996,
"source": "D(1,991,512,1000,512,1000,552,991,552)"
}
],
"lines": [
{
"content": "Contoso Q1 2025 Financial Summary",
"source": "D(1,214,221,1050,225,1050,279,213,273)",
"span": {
"offset": 2,
"length": 33
}
},
{
"content": "Total revenue for Q1 2025 was $42.7 million, an increase of 18% over Q1 2024.",
"source": "D(1,212,334,1394,335,1394,374,212,374)",
"span": {
"offset": 37,
"length": 77
}
},
{
"content": "Operating expenses were $31.2 million. Net profit was $11.5 million. The largest",
"source": "D(1,213,379,1398,378,1398,421,213,422)",
"span": {
"offset": 115,
"length": 80
}
},
{
"content": "revenue segment was Cloud Services at $19.3 million, followed by Professional",
"source": "D(1,212,424,1389,423,1389,463,212,464)",
"span": {
"offset": 196,
"length": 77
}
},
{
"content": "Services at $14.8 million and Product Licensing at $8.6 million. Headcount at end of",
"source": "D(1,213,469,1453,468,1453,510,213,511)",
"span": {
"offset": 274,
"length": 84
}
},
{
"content": "Q1 was 1,247 employees across 8 offices worldwide.",
"source": "D(1,213,512,1000,512,1000,554,213,554)",
"span": {
"offset": 359,
"length": 50
}
}
]
}
],
"paragraphs": [
{
"role": "title",
"content": "Contoso Q1 2025 Financial Summary",
"source": "D(1,214,219,1050,225,1050,279,213,273)",
"span": {
"offset": 0,
"length": 35
}
},
{
"content": "Total revenue for Q1 2025 was $42.7 million, an increase of 18% over Q1 2024. Operating expenses were $31.2 million. Net profit was $11.5 million. The largest revenue segment was Cloud Services at $19.3 million, followed by Professional Services at $14.8 million and Product Licensing at $8.6 million. Headcount at end of Q1 was 1,247 employees across 8 offices worldwide.",
"source": "D(1,212,334,1453,333,1454,553,212,554)",
"span": {
"offset": 37,
"length": 372
}
}
],
"sections": [
{
"span": {
"offset": 0,
"length": 409
},
"elements": [
"/paragraphs/0",
"/paragraphs/1"
]
}
],
"analyzerId": "prebuilt-documentSearch",
"mimeType": "image/png"
}
]
}
@@ -1,114 +0,0 @@
{
"analyzerId": "prebuilt-invoice",
"apiVersion": "2025-11-01",
"createdAt": "2026-03-21T22:44:33Z",
"stringEncoding": "codePoint",
"warnings": [],
"contents": [
{
"markdown": "# Master Services Agreement\n\nClient: Alpine Industries Inc.\n\nContract Reference: MSA-2025-ALP-00847\n\nEffective Date: January 15, 2025\nPrepared for: Robert Chen, Chief Executive Officer, Alpine Industries Inc.\n\nAddress: 742 Evergreen Blvd, Denver, CO 80203\n\nThis Master Services Agreement (the 'Agreement') is entered into by and between Alpine Industries\nInc. (the 'Client') and TechServe Global Partners (the 'Provider'). This agreement governs the provision\nof managed technology services as descri",
"fields": {
"VendorName": {
"type": "string",
"valueString": "TechServe Global Partners",
"confidence": 0.71
},
"DueDate": {
"type": "date",
"valueDate": "2025-02-15",
"confidence": 0.793
},
"InvoiceDate": {
"type": "date",
"valueDate": "2025-01-15",
"confidence": 0.693
},
"InvoiceId": {
"type": "string",
"valueString": "INV-100",
"confidence": 0.489
},
"AmountDue": {
"type": "object",
"valueObject": {
"Amount": {
"type": "number",
"valueNumber": 610,
"confidence": 0.758
},
"CurrencyCode": {
"type": "string",
"valueString": "USD"
}
}
},
"SubtotalAmount": {
"type": "object",
"valueObject": {
"Amount": {
"type": "number",
"valueNumber": 100,
"confidence": 0.902
},
"CurrencyCode": {
"type": "string",
"valueString": "USD"
}
}
},
"LineItems": {
"type": "array",
"valueArray": [
{
"type": "object",
"valueObject": {
"Description": {
"type": "string",
"valueString": "Consulting Services",
"confidence": 0.664
},
"Quantity": {
"type": "number",
"valueNumber": 2,
"confidence": 0.957
},
"UnitPrice": {
"type": "object",
"valueObject": {
"Amount": {
"type": "number",
"valueNumber": 30,
"confidence": 0.956
},
"CurrencyCode": {
"type": "string",
"valueString": "USD"
}
}
}
}
},
{
"type": "object",
"valueObject": {
"Description": {
"type": "string",
"valueString": "Document Fee",
"confidence": 0.712
},
"Quantity": {
"type": "number",
"valueNumber": 3,
"confidence": 0.939
}
}
}
]
}
},
"kind": "document",
"startPageNumber": 1,
"endPageNumber": 100
}
]
}
@@ -1,23 +0,0 @@
{
"analyzerId": "prebuilt-documentSearch",
"apiVersion": "2025-11-01",
"createdAt": "2026-03-21T22:44:09Z",
"contents": [
{
"path": "input1",
"markdown": "# Contoso Q1 2025 Financial Summary\n\nTotal revenue for Q1 2025 was $42.7 million, an increase of 18% over Q1 2024.\nOperating expenses were $31.2 million. Net profit was $11.5 million. The largest\nrevenue segment was Cloud Services at $19.3 million, followed by Professional\nServices at $14.8 million and Product Licensing at $8.6 million. Headcount at end of\nQ1 was 1,247 employees across 8 offices worldwide.\n\n<!-- PageBreak -->\n\n\n# Contoso Q2 2025 Financial Summary\n\nTotal revenue for Q2 2025 was $48.1 million, an increase of 22% over Q2 2024.\nOperating expenses were $33.9 million. Net profit was $14.2 million. Cloud Services\ngrew to $22.5 million, Professional Services was $15.7 million, and Product Licensing\nwas $9.9 million. The company opened a new office in Tokyo, bringing the total to 9\noffices. Headcount grew to 1,389 employees.\n\n<!-- PageBreak -->\n\n\n## Contoso Product Roadmap 2025\n\nThree major product launches are planned for 2025: (1) Contoso CloudVault - an\nenterprise document storage solution, launching August 2025, with an expected price\nof $29.99/user/month. (2) Contoso DataPulse - a real-time analytics dashboard,\nlaunching October 2025. (3) Contoso SecureLink - a zero-trust networking product,\nlaunching December 2025. Total R&D; budget for 2025 is $18.4 million.\n\n<!-- PageBreak -->\n\n\n# Contoso Employee Satisfaction Survey Results\n\nThe annual employee satisfaction survey was completed in March 2025 with a 87%\nresponse rate. Overall satisfaction score was 4.2 out of 5.0. Work-life balance scored\n3.8/5.0. Career growth opportunities scored 3.9/5.0. Compensation satisfaction\nscored 3.6/5.0. The top requested improvement was 'more flexible remote work\noptions' cited by 62% of respondents. Employee retention rate for the trailing 12\nmonths was 91%.\n\n<!-- PageBreak -->\n\n\n## Contoso Partnership Announcements\n\nContoso announced three strategic partnerships in H1 2025: (1) A joint venture with\nMeridian Technologies for AI-powered document processing, valued at $5.2 million\nover 3 years. (2) A distribution agreement with Pacific Rim Solutions covering 12\ncountries in Asia-Pacific. (3) A technology integration partnership with NovaBridge\nSystems for unified identity management. The Chief Partnership Officer, Helena\nNakagawa, stated the partnerships are expected to generate an additional $15 million\nin revenue by 2027.\n",
"fields": {
"Summary": {
"type": "string",
"valueString": "The document provides a comprehensive overview of Contoso's key business metrics and initiatives for 2025, including financial performance for Q1 and Q2 with revenue, expenses, and profit details; a product roadmap with three major launches and R&D budget; employee satisfaction survey results highlighting scores and retention; and strategic partnership announcements expected to boost future revenue.",
"confidence": 0.46
}
},
"kind": "document",
"startPageNumber": 1,
"endPageNumber": 5,
"mimeType": "application/pdf",
"analyzerId": "prebuilt-documentSearch"
}
]
}
@@ -1,51 +0,0 @@
{
"id": "synthetic-video-001",
"status": "Succeeded",
"analyzer_id": "prebuilt-videoSearch",
"api_version": "2025-05-01-preview",
"created_at": "2026-03-21T10:15:00Z",
"contents": [
{
"kind": "audioVisual",
"startTimeMs": 1000,
"endTimeMs": 14000,
"width": 640,
"height": 480,
"markdown": "# Video: 00:01.000 => 00:14.000\n\nTranscript\n```\nWEBVTT\n\n00:01.000 --> 00:05.000\n<Speaker 1>Welcome to the Contoso Product Demo.\n\n00:05.000 --> 00:14.000\n<Speaker 1>Today we'll be showcasing our latest cloud infrastructure management tool.\n```",
"fields": {
"Summary": {
"type": "string",
"valueString": "Introduction to the Contoso Product Demo showcasing the latest cloud infrastructure management tool."
}
}
},
{
"kind": "audioVisual",
"startTimeMs": 15000,
"endTimeMs": 35000,
"width": 640,
"height": 480,
"markdown": "# Video: 00:15.000 => 00:35.000\n\nTranscript\n```\nWEBVTT\n\n00:15.000 --> 00:25.000\n<Speaker 1>As you can see on the dashboard, the system provides real-time monitoring of all deployed resources.\n\n00:25.000 --> 00:35.000\n<Speaker 1>Key features include automated scaling, cost optimization, and security compliance monitoring.\n```",
"fields": {
"Summary": {
"type": "string",
"valueString": "Dashboard walkthrough covering real-time monitoring, automated scaling, cost optimization, and security compliance."
}
}
},
{
"kind": "audioVisual",
"startTimeMs": 36000,
"endTimeMs": 42000,
"width": 640,
"height": 480,
"markdown": "# Video: 00:36.000 => 00:42.000\n\nTranscript\n```\nWEBVTT\n\n00:36.000 --> 00:42.000\n<Speaker 1>Visit contoso.com/cloud-manager to learn more and start your free trial.\n```",
"fields": {
"Summary": {
"type": "string",
"valueString": "Call to action directing viewers to contoso.com/cloud-manager for more information and a free trial."
}
}
}
]
}
File diff suppressed because it is too large Load Diff
@@ -1,312 +0,0 @@
# Copyright (c) Microsoft. All rights reserved.
"""Integration tests for ContentUnderstandingContextProvider.
These tests require a live Azure Content Understanding endpoint.
Set AZURE_CONTENTUNDERSTANDING_ENDPOINT to enable them.
To generate fixtures for unit tests, run these tests with --update-fixtures flag
and the resulting JSON files will be written to tests/cu/fixtures/.
"""
from __future__ import annotations
import json
import os
from pathlib import Path
import pytest
skip_if_cu_integration_tests_disabled = pytest.mark.skipif(
not os.environ.get("AZURE_CONTENTUNDERSTANDING_ENDPOINT"),
reason="CU integration tests disabled (AZURE_CONTENTUNDERSTANDING_ENDPOINT not set)",
)
FIXTURES_DIR = Path(__file__).parent / "fixtures"
# Shared sample asset — same PDF used by samples and integration tests
INVOICE_PDF_PATH = Path(__file__).resolve().parents[2] / "samples" / "shared" / "sample_assets" / "invoice.pdf"
@pytest.mark.flaky
@pytest.mark.integration
@skip_if_cu_integration_tests_disabled
async def test_analyze_pdf_binary() -> None:
"""Analyze a PDF via binary upload and optionally capture fixture."""
from azure.ai.contentunderstanding.aio import ContentUnderstandingClient
from azure.identity.aio import DefaultAzureCredential
endpoint = os.environ["AZURE_CONTENTUNDERSTANDING_ENDPOINT"]
analyzer_id = os.environ.get("AZURE_CONTENTUNDERSTANDING_ANALYZER_ID", "prebuilt-documentSearch")
pdf_path = INVOICE_PDF_PATH
assert pdf_path.exists(), f"Test fixture not found: {pdf_path}"
pdf_bytes = pdf_path.read_bytes()
async with DefaultAzureCredential() as credential, ContentUnderstandingClient(endpoint, credential) as client:
poller = await client.begin_analyze_binary(
analyzer_id,
binary_input=pdf_bytes,
content_type="application/pdf",
)
result = await poller.result()
assert result.contents
assert result.contents[0].markdown
assert len(result.contents[0].markdown) > 10
assert "CONTOSO LTD." in result.contents[0].markdown
# Optionally capture fixture
if os.environ.get("CU_UPDATE_FIXTURES"):
FIXTURES_DIR.mkdir(exist_ok=True)
fixture_path = FIXTURES_DIR / "analyze_pdf_result.json"
fixture_path.write_text(json.dumps(result.as_dict(), indent=2, default=str))
@pytest.mark.flaky
@pytest.mark.integration
@skip_if_cu_integration_tests_disabled
async def test_before_run_e2e() -> None:
"""End-to-end test: Content.from_data → before_run → state populated."""
from agent_framework import Content, Message, SessionContext
from agent_framework._sessions import AgentSession
from azure.identity.aio import DefaultAzureCredential
from agent_framework_azure_contentunderstanding import ContentUnderstandingContextProvider
endpoint = os.environ["AZURE_CONTENTUNDERSTANDING_ENDPOINT"]
pdf_path = INVOICE_PDF_PATH
assert pdf_path.exists(), f"Test fixture not found: {pdf_path}"
pdf_bytes = pdf_path.read_bytes()
async with DefaultAzureCredential() as credential:
cu = ContentUnderstandingContextProvider(
endpoint=endpoint,
credential=credential,
max_wait=None, # wait until analysis completes (no background deferral)
)
async with cu:
msg = Message(
role="user",
contents=[
Content.from_text("What's in this document?"),
Content.from_data(
pdf_bytes,
"application/pdf",
additional_properties={"filename": "invoice.pdf"},
),
],
)
context = SessionContext(input_messages=[msg])
state: dict[str, object] = {}
session = AgentSession()
from unittest.mock import MagicMock
await cu.before_run(agent=MagicMock(), session=session, context=context, state=state)
docs = state.get("documents", {})
assert isinstance(docs, dict)
assert "invoice.pdf" in docs
doc_entry = docs["invoice.pdf"]
assert doc_entry["status"] == "ready"
assert doc_entry["result"] is not None
assert doc_entry["result"].get("markdown")
assert len(doc_entry["result"]["markdown"]) > 10
assert "CONTOSO LTD." in doc_entry["result"]["markdown"]
# Raw GitHub URL for a public invoice PDF from the CU samples repo
_INVOICE_PDF_URL = (
"https://raw.githubusercontent.com/Azure-Samples/azure-ai-content-understanding-assets/main/document/invoice.pdf"
)
@pytest.mark.flaky
@pytest.mark.integration
@skip_if_cu_integration_tests_disabled
async def test_before_run_uri_content() -> None:
"""End-to-end test: Content.from_uri with an external URL → before_run → state populated.
Verifies that CU can analyze a file referenced by URL (not base64 data).
Uses a public invoice PDF from the Azure CU samples repository.
"""
from agent_framework import Content, Message, SessionContext
from agent_framework._sessions import AgentSession
from azure.identity.aio import DefaultAzureCredential
from agent_framework_azure_contentunderstanding import ContentUnderstandingContextProvider
endpoint = os.environ["AZURE_CONTENTUNDERSTANDING_ENDPOINT"]
async with DefaultAzureCredential() as credential:
cu = ContentUnderstandingContextProvider(
endpoint=endpoint,
credential=credential,
max_wait=None, # wait until analysis completes (no background deferral)
)
async with cu:
msg = Message(
role="user",
contents=[
Content.from_text("What's on this invoice?"),
Content.from_uri(
uri=_INVOICE_PDF_URL,
media_type="application/pdf",
additional_properties={"filename": "invoice.pdf"},
),
],
)
context = SessionContext(input_messages=[msg])
state: dict[str, object] = {}
session = AgentSession()
from unittest.mock import MagicMock
await cu.before_run(agent=MagicMock(), session=session, context=context, state=state)
docs = state.get("documents", {})
assert isinstance(docs, dict)
assert "invoice.pdf" in docs
doc_entry = docs["invoice.pdf"]
assert doc_entry["status"] == "ready"
assert doc_entry["result"] is not None
assert doc_entry["result"].get("markdown")
assert len(doc_entry["result"]["markdown"]) > 10
assert "CONTOSO LTD." in doc_entry["result"]["markdown"]
@pytest.mark.flaky
@pytest.mark.integration
@skip_if_cu_integration_tests_disabled
async def test_before_run_data_uri_content() -> None:
"""End-to-end test: Content.from_uri with a base64 data URI → before_run → state populated.
Verifies that CU can analyze a file embedded as a data URI (data:application/pdf;base64,...).
This tests the data URI path: from_uri with "data:" prefix → type="data" → begin_analyze_binary.
"""
import base64
from agent_framework import Content, Message, SessionContext
from agent_framework._sessions import AgentSession
from azure.identity.aio import DefaultAzureCredential
from agent_framework_azure_contentunderstanding import ContentUnderstandingContextProvider
endpoint = os.environ["AZURE_CONTENTUNDERSTANDING_ENDPOINT"]
pdf_path = INVOICE_PDF_PATH
assert pdf_path.exists(), f"Test fixture not found: {pdf_path}"
pdf_bytes = pdf_path.read_bytes()
b64 = base64.b64encode(pdf_bytes).decode("ascii")
data_uri = f"data:application/pdf;base64,{b64}"
async with DefaultAzureCredential() as credential:
cu = ContentUnderstandingContextProvider(
endpoint=endpoint,
credential=credential,
max_wait=None, # wait until analysis completes
)
async with cu:
msg = Message(
role="user",
contents=[
Content.from_text("What's on this invoice?"),
Content.from_uri(
uri=data_uri,
media_type="application/pdf",
additional_properties={"filename": "invoice_b64.pdf"},
),
],
)
context = SessionContext(input_messages=[msg])
state: dict[str, object] = {}
session = AgentSession()
from unittest.mock import MagicMock
await cu.before_run(agent=MagicMock(), session=session, context=context, state=state)
docs = state.get("documents", {})
assert isinstance(docs, dict)
assert "invoice_b64.pdf" in docs
doc_entry = docs["invoice_b64.pdf"]
assert doc_entry["status"] == "ready"
assert doc_entry["result"] is not None
assert doc_entry["result"].get("markdown")
assert len(doc_entry["result"]["markdown"]) > 10
assert "CONTOSO LTD." in doc_entry["result"]["markdown"]
@pytest.mark.flaky
@pytest.mark.integration
@skip_if_cu_integration_tests_disabled
async def test_before_run_background_analysis() -> None:
"""End-to-end test: max_wait timeout → background analysis → resolved on next turn.
Uses a short max_wait (0.5s) so CU analysis is deferred to background.
Then waits for analysis to complete and calls before_run again to verify
the background task resolves and the document becomes ready.
"""
import asyncio
from agent_framework import Content, Message, SessionContext
from agent_framework._sessions import AgentSession
from azure.identity.aio import DefaultAzureCredential
from agent_framework_azure_contentunderstanding import ContentUnderstandingContextProvider
endpoint = os.environ["AZURE_CONTENTUNDERSTANDING_ENDPOINT"]
async with DefaultAzureCredential() as credential:
cu = ContentUnderstandingContextProvider(
endpoint=endpoint,
credential=credential,
max_wait=0.5, # short timeout to force background deferral
)
async with cu:
# Turn 1: upload file — should time out and defer to background
msg = Message(
role="user",
contents=[
Content.from_text("What's on this invoice?"),
Content.from_uri(
uri=_INVOICE_PDF_URL,
media_type="application/pdf",
additional_properties={"filename": "invoice.pdf"},
),
],
)
context = SessionContext(input_messages=[msg])
state: dict[str, object] = {}
session = AgentSession()
from unittest.mock import MagicMock
await cu.before_run(agent=MagicMock(), session=session, context=context, state=state)
docs = state.get("documents", {})
assert isinstance(docs, dict)
assert "invoice.pdf" in docs
assert docs["invoice.pdf"]["status"] == "analyzing", (
f"Expected 'analyzing' but got '{docs['invoice.pdf']['status']}' — "
"CU responded too fast for the 0.5s timeout"
)
assert docs["invoice.pdf"]["result"] is None
# Wait for background analysis to complete
await asyncio.sleep(30)
# Turn 2: no new files — should resolve the background task
msg2 = Message(role="user", contents=[Content.from_text("Is it ready?")])
context2 = SessionContext(input_messages=[msg2])
await cu.before_run(agent=MagicMock(), session=session, context=context2, state=state)
assert docs["invoice.pdf"]["status"] == "ready"
assert docs["invoice.pdf"]["result"] is not None
assert docs["invoice.pdf"]["result"].get("markdown")
assert "CONTOSO LTD." in docs["invoice.pdf"]["result"]["markdown"]
@@ -1,67 +0,0 @@
# Copyright (c) Microsoft. All rights reserved.
from __future__ import annotations
from unittest.mock import AsyncMock
from agent_framework_azure_contentunderstanding._models import (
DocumentEntry,
DocumentStatus,
FileSearchConfig,
)
class TestDocumentEntry:
def test_construction(self) -> None:
entry: DocumentEntry = {
"status": DocumentStatus.READY,
"filename": "invoice.pdf",
"media_type": "application/pdf",
"analyzer_id": "prebuilt-documentSearch",
"analyzed_at": "2026-01-01T00:00:00+00:00",
"analysis_duration_s": 1.23,
"upload_duration_s": None,
"result": {"markdown": "# Title"},
"error": None,
}
assert entry["status"] == DocumentStatus.READY
assert entry["filename"] == "invoice.pdf"
assert entry["analyzer_id"] == "prebuilt-documentSearch"
assert entry["analysis_duration_s"] == 1.23
assert entry["upload_duration_s"] is None
def test_failed_entry(self) -> None:
entry: DocumentEntry = {
"status": DocumentStatus.FAILED,
"filename": "bad.pdf",
"media_type": "application/pdf",
"analyzer_id": "prebuilt-documentSearch",
"analyzed_at": "2026-01-01T00:00:00+00:00",
"analysis_duration_s": 0.5,
"upload_duration_s": None,
"result": None,
"error": "Service unavailable",
}
assert entry["status"] == DocumentStatus.FAILED
assert entry["error"] == "Service unavailable"
assert entry["result"] is None
class TestFileSearchConfig:
def test_required_fields(self) -> None:
backend = AsyncMock()
tool = {"type": "file_search", "vector_store_ids": ["vs_123"]}
config = FileSearchConfig(backend=backend, vector_store_id="vs_123", file_search_tool=tool)
assert config.backend is backend
assert config.vector_store_id == "vs_123"
assert config.file_search_tool is tool
def test_from_openai_factory(self) -> None:
from agent_framework_azure_contentunderstanding._file_search import OpenAIFileSearchBackend
client = AsyncMock()
tool = {"type": "file_search", "vector_store_ids": ["vs_abc"]}
config = FileSearchConfig.from_openai(client, vector_store_id="vs_abc", file_search_tool=tool)
assert isinstance(config.backend, OpenAIFileSearchBackend)
assert config.vector_store_id == "vs_abc"
assert config.file_search_tool is tool
+2 -2
View File
@@ -4,7 +4,7 @@ description = "Azure Cosmos DB history provider integration for Microsoft Agent
authors = [{ name = "Microsoft", email = "af-support@microsoft.com"}]
readme = "README.md"
requires-python = ">=3.10"
version = "1.0.0b260429"
version = "1.0.0b260424"
license-files = ["LICENSE"]
urls.homepage = "https://aka.ms/agent-framework"
urls.source = "https://github.com/microsoft/agent-framework/tree/main/python"
@@ -23,7 +23,7 @@ classifiers = [
"Typing :: Typed",
]
dependencies = [
"agent-framework-core>=1.2.2,<2",
"agent-framework-core>=1.2.0,<2",
"azure-cosmos>=4.3.0,<5",
]
@@ -4,7 +4,7 @@ description = "Azure Functions integration for Microsoft Agent Framework."
authors = [{ name = "Microsoft", email = "af-support@microsoft.com"}]
readme = "README.md"
requires-python = ">=3.10"
version = "1.0.0b260429"
version = "1.0.0b260424"
license-files = ["LICENSE"]
urls.homepage = "https://aka.ms/agent-framework"
urls.source = "https://github.com/microsoft/agent-framework/tree/main/python"
@@ -22,7 +22,7 @@ classifiers = [
"Typing :: Typed",
]
dependencies = [
"agent-framework-core>=1.2.2,<2",
"agent-framework-core>=1.2.0,<2",
"agent-framework-durabletask",
"azure-functions>=1.24.0,<2",
"azure-functions-durable>=1.3.1,<2",
+2 -2
View File
@@ -4,7 +4,7 @@ description = "Amazon Bedrock integration for Microsoft Agent Framework."
authors = [{ name = "Microsoft", email = "af-support@microsoft.com"}]
readme = "README.md"
requires-python = ">=3.10"
version = "1.0.0b260429"
version = "1.0.0b260424"
license-files = ["LICENSE"]
urls.homepage = "https://aka.ms/agent-framework"
urls.source = "https://github.com/microsoft/agent-framework/tree/main/python"
@@ -23,7 +23,7 @@ classifiers = [
"Typing :: Typed",
]
dependencies = [
"agent-framework-core>=1.2.2,<2",
"agent-framework-core>=1.2.0,<2",
"boto3>=1.35.0,<2.0.0",
"botocore>=1.35.0,<2.0.0",
]
+2 -2
View File
@@ -4,7 +4,7 @@ description = "OpenAI ChatKit integration for Microsoft Agent Framework."
authors = [{ name = "Microsoft", email = "af-support@microsoft.com"}]
readme = "README.md"
requires-python = ">=3.10"
version = "1.0.0b260429"
version = "1.0.0b260424"
license-files = ["LICENSE"]
urls.homepage = "https://aka.ms/agent-framework"
urls.source = "https://github.com/microsoft/agent-framework/tree/main/python"
@@ -22,7 +22,7 @@ classifiers = [
"Typing :: Typed",
]
dependencies = [
"agent-framework-core>=1.2.2,<2",
"agent-framework-core>=1.2.0,<2",
"openai-chatkit>=1.4.1,<2.0.0",
]
+2 -2
View File
@@ -4,7 +4,7 @@ description = "Claude Agent SDK integration for Microsoft Agent Framework."
authors = [{ name = "Microsoft", email = "af-support@microsoft.com"}]
readme = "README.md"
requires-python = ">=3.10"
version = "1.0.0b260429"
version = "1.0.0b260424"
license-files = ["LICENSE"]
urls.homepage = "https://aka.ms/agent-framework"
urls.source = "https://github.com/microsoft/agent-framework/tree/main/python"
@@ -23,7 +23,7 @@ classifiers = [
"Typing :: Typed",
]
dependencies = [
"agent-framework-core>=1.2.2,<2",
"agent-framework-core>=1.2.0,<2",
"claude-agent-sdk>=0.1.36,<0.1.49",
]
+2 -2
View File
@@ -4,7 +4,7 @@ description = "Copilot Studio integration for Microsoft Agent Framework."
authors = [{ name = "Microsoft", email = "af-support@microsoft.com"}]
readme = "README.md"
requires-python = ">=3.10"
version = "1.0.0b260429"
version = "1.0.0b260424"
license-files = ["LICENSE"]
urls.homepage = "https://aka.ms/agent-framework"
urls.source = "https://github.com/microsoft/agent-framework/tree/main/python"
@@ -23,7 +23,7 @@ classifiers = [
"Typing :: Typed",
]
dependencies = [
"agent-framework-core>=1.2.2,<2",
"agent-framework-core>=1.2.0,<2",
"microsoft-agents-copilotstudio-client>=0.3.1,<0.3.2",
]
@@ -247,7 +247,6 @@ from ._workflows._workflow_executor import (
WorkflowExecutor,
)
from .exceptions import (
AgentFrameworkException,
MiddlewareException,
UserInputRequiredException,
WorkflowCheckpointException,
@@ -281,7 +280,6 @@ __all__ = [
"AgentExecutor",
"AgentExecutorRequest",
"AgentExecutorResponse",
"AgentFrameworkException",
"AgentMiddleware",
"AgentMiddlewareLayer",
"AgentMiddlewareTypes",
@@ -79,11 +79,9 @@ def _detect_hosted_environment() -> None:
except (ModuleNotFoundError, ValueError):
return
with contextlib.suppress(ImportError, AttributeError):
from azure.ai.agentserver.core import ( # pyright: ignore[reportMissingImports]
AgentConfig, # pyright: ignore[reportUnknownVariableType]
)
from azure.ai.agentserver.core import AgentConfig # pyright: ignore[reportMissingImports]
if AgentConfig.from_env().is_hosted: # pyright: ignore[reportUnknownMemberType]
if AgentConfig.from_env().is_hosted:
_add_user_agent_prefix(_HOSTED_USER_AGENT_PREFIX)
_hosted_env_detected = True
+7 -53
View File
@@ -3,7 +3,6 @@
from __future__ import annotations
import base64
import contextlib
import json
import logging
import re
@@ -2891,7 +2890,6 @@ class ResponseStream(AsyncIterable[UpdateT], Generic[UpdateT, FinalT]):
self._inner_stream_source: ResponseStream[Any, Any] | Awaitable[ResponseStream[Any, Any]] | None = None
self._wrap_inner: bool = False
self._map_update: Callable[[Any], UpdateT | Awaitable[UpdateT]] | None = None
self._pull_context_manager_factories: list[Callable[[], contextlib.AbstractContextManager[Any]]] = []
def map(
self,
@@ -3010,18 +3008,11 @@ class ResponseStream(AsyncIterable[UpdateT], Generic[UpdateT, FinalT]):
return self
async def __anext__(self) -> UpdateT:
if self._iterator is None:
stream = await self._get_stream()
self._iterator = stream.__aiter__()
try:
with contextlib.ExitStack() as stack:
for factory in self._pull_context_manager_factories:
stack.enter_context(factory())
# Resolve the underlying stream inside the pull contexts so that any
# spans/contexts created during stream resolution (e.g. inner chat
# completion spans created on the first pull of a wrapped agent stream)
# inherit the active context (e.g. an outer agent invoke span).
if self._iterator is None:
stream = await self._get_stream()
self._iterator = stream.__aiter__()
update: UpdateT = await self._iterator.__anext__()
update: UpdateT = await self._iterator.__anext__()
except StopAsyncIteration:
self._consumed = True
await self._run_cleanup_hooks()
@@ -3047,25 +3038,9 @@ class ResponseStream(AsyncIterable[UpdateT], Generic[UpdateT, FinalT]):
update = hooked
return update
async def _resolve_stream_with_pull_contexts(self) -> AsyncIterable[UpdateT]:
"""Resolve the underlying stream while activating any registered pull context managers.
Used by ``__await__`` and ``get_final_response`` so that any spans/contexts created
during stream resolution (e.g. when the source is an Awaitable that internally
creates child telemetry spans) inherit the same active context as iterator pulls.
``__anext__`` resolves the stream inside its own ExitStack and so calls ``_get_stream``
directly.
"""
if self._stream is not None:
return await self._get_stream()
with contextlib.ExitStack() as stack:
for factory in self._pull_context_manager_factories:
stack.enter_context(factory())
return await self._get_stream()
def __await__(self) -> Any:
async def _wrap() -> ResponseStream[UpdateT, FinalT]:
await self._resolve_stream_with_pull_contexts()
await self._get_stream()
return self
return _wrap().__await__()
@@ -3089,12 +3064,10 @@ class ResponseStream(AsyncIterable[UpdateT], Generic[UpdateT, FinalT]):
"""
if self._wrap_inner:
if self._inner_stream is None:
# Use _resolve_stream_with_pull_contexts() so that any spans/contexts
# created while resolving the awaitable (e.g. inner telemetry spans)
# inherit the same active context as iterator pulls. This also handles
# Use _get_stream() to resolve the awaitable - this properly handles
# the case where _stream_source and _inner_stream_source are the same
# coroutine (e.g., from from_awaitable), avoiding double-await errors.
await self._resolve_stream_with_pull_contexts()
await self._get_stream()
if self._inner_stream is None:
raise RuntimeError("Inner stream not available")
if not self._finalized and not self._consumed:
@@ -3204,25 +3177,6 @@ class ResponseStream(AsyncIterable[UpdateT], Generic[UpdateT, FinalT]):
self._cleanup_hooks.append(hook)
return self
def with_pull_context_manager(
self,
cm_factory: Callable[[], contextlib.AbstractContextManager[Any]],
) -> ResponseStream[UpdateT, FinalT]:
"""Register a context manager factory invoked around each underlying iterator pull.
The factory is called once per ``__anext__`` and the returned context manager wraps
the await of the underlying iterator. This is useful for state that needs to be
active while the inner async work runs - for example, attaching an OpenTelemetry
span to the current context so child spans created by inner code (HTTP clients,
tool execution) are correctly parented.
Because the context manager is entered and exited within the same ``__anext__``
invocation, attach/detach style operations remain symmetric in the same async
context regardless of where the stream is iterated.
"""
self._pull_context_manager_factories.append(cm_factory)
return self
async def _run_cleanup_hooks(self) -> None:
if self._cleanup_run:
return
@@ -437,13 +437,6 @@ class WorkflowAgent(BaseAgent):
yield event
elif checkpoint_id is not None:
# Restore the prior workflow state from the checkpoint. Shared
# state (e.g. accumulated conversation history maintained by the
# workflow's executors) survives across turns because Workflow.run
# no longer wipes state per call. Callers who want to deliver a
# new user message after restore should make a second
# `workflow.run(message=...)` call - they are NOT mutually
# exclusive on the same instance, but each must be its own call.
if streaming:
async for event in self.workflow.run(
stream=True,
@@ -535,7 +528,6 @@ class WorkflowAgent(BaseAgent):
raw_representations.append(output_event)
else:
data = output_event.data
if isinstance(data, AgentResponseUpdate):
# We cannot support AgentResponseUpdate in non-streaming mode. This is because the message
# sequence cannot be guaranteed when there are streaming updates in between non-streaming
@@ -636,23 +628,16 @@ class WorkflowAgent(BaseAgent):
A list of AgentResponseUpdate objects. Empty list if the event is not relevant.
"""
if event.type == "output":
# Convert workflow output to agent response updates.
# Handle different data types appropriately.
data = event.data
executor_id = event.executor_id
if isinstance(data, AgentResponseUpdate):
# Construct a fresh AgentResponseUpdate so we don't mutate a payload
# that AgentExecutor still holds a reference to in its `updates` list.
return [
AgentResponseUpdate(
contents=list(data.contents),
role=data.role,
author_name=data.author_name or executor_id,
response_id=data.response_id,
message_id=data.message_id,
created_at=data.created_at,
raw_representation=data.raw_representation,
)
]
# Pass through AgentResponseUpdate directly (streaming from AgentExecutor)
if not data.author_name:
data.author_name = executor_id
return [data]
if isinstance(data, AgentResponse):
# Convert each message in AgentResponse to an AgentResponseUpdate
updates: list[AgentResponseUpdate] = []
@@ -156,9 +156,8 @@ class AgentExecutor(Executor):
the agent run.
- "custom": use the provided context_filter function to determine which messages to include
as context for the agent run.
context_filter: A function that takes the full conversation (list of Messages) as input and returns
a filtered list of Messages to be used as context for the agent run. This is required
if context_mode is set to "custom".
context_filter: An optional function for filtering conversation context when context_mode is set
to "custom".
"""
# Prefer provided id; else use agent.name if present; else generate deterministic prefix
exec_id = id or resolve_agent_id(agent)
@@ -278,12 +278,7 @@ class Runner:
"Please rebuild the original workflow before resuming."
)
# Restore state. Clear first so import_state (which merges) does
# not leak stale keys from a prior run on this Workflow instance.
# This matters more now that Workflow.run() no longer wipes state
# per call - the only reset point for shared state on a reused
# instance is at restore time.
self._state.clear()
# Restore state
self._state.import_state(checkpoint.state)
# Restore executor states using the restored state
await self._restore_executor_states()
@@ -299,7 +299,7 @@ class Workflow(DictConvertible):
async def _run_workflow_with_tracing(
self,
initial_executor_fn: Callable[[], Awaitable[None]] | None = None,
is_continuation: bool = False,
reset_context: bool = True,
streaming: bool = False,
function_invocation_kwargs: Mapping[str, Mapping[str, Any]] | Mapping[str, Any] | None = None,
client_kwargs: Mapping[str, Mapping[str, Any]] | Mapping[str, Any] | None = None,
@@ -310,19 +310,13 @@ class Workflow(DictConvertible):
of external callers to maintain context across different workflow runs.
Args:
initial_executor_fn: Optional function to execute initial executor.
is_continuation: True when this run is a continuation of prior
work (a checkpoint restore or a responses-only replay) rather
than a fresh new turn delivered via the start executor with
``message=...``. Continuations preserve per-run accounting
(iteration counter and run kwargs) from the prior turn;
fresh-message runs reset them. Shared workflow state is
preserved in both cases.
streaming: Whether to enable streaming mode for agents.
initial_executor_fn: Optional function to execute initial executor
reset_context: Whether to reset the context for a new run
streaming: Whether to enable streaming mode for agents
function_invocation_kwargs: Optional kwargs to store in State for function
invocations in subagents.
invocations in subagents
client_kwargs: Optional kwargs to store in State for chat client
invocations in subagents.
invocations in subagents
Yields:
WorkflowEvent: The events generated during the workflow execution.
@@ -351,26 +345,16 @@ class Workflow(DictConvertible):
in_progress = WorkflowEvent.status(WorkflowRunState.IN_PROGRESS)
yield in_progress # noqa: RUF070
# Per-run reset for fresh-message runs only. We deliberately
# do NOT clear shared workflow state (`_state.clear()`) or the
# runner context's in-flight messages (`reset_for_new_run()`)
# here - state and pending work persist across `run()` calls
# so that a `WorkflowAgent` can deliver multi-turn input on
# the same instance and have prior turns' context survive.
# Iteration counting and per-run kwargs ARE per-run though,
# so they're reset here.
if not is_continuation:
# Reset context for a new run if supported
if reset_context:
self._runner.reset_iteration_count()
self._runner.context.reset_for_new_run()
self._state.clear()
# Store run kwargs in State so executors can access them.
# Per-run kwargs semantics:
# - On a fresh message run, prior kwargs go away (set to {}
# by default, or to the new kwargs if provided). This
# prevents stale kwargs from a prior turn leaking into the
# current turn.
# - On a continuation (checkpoint restore or responses), the
# prior run's kwargs are preserved unless the caller
# explicitly provides new kwargs.
# Only overwrite when new kwargs are explicitly provided or state was
# just cleared (fresh run). On continuation (reset_context=False) with
# no new kwargs, preserve the kwargs from the original run.
if function_invocation_kwargs is not None or client_kwargs is not None:
combined_kwargs: dict[str, Any] = {}
if function_invocation_kwargs is not None:
@@ -382,12 +366,11 @@ class Workflow(DictConvertible):
client_kwargs, "client_kwargs"
)
self._state.set(WORKFLOW_RUN_KWARGS_KEY, combined_kwargs)
elif not is_continuation:
elif reset_context:
self._state.set(WORKFLOW_RUN_KWARGS_KEY, {})
self._state.commit() # Commit immediately so kwargs are available
# Set streaming mode (always set explicitly per run since
# reset_for_new_run() no longer runs to clear it).
# Set streaming mode after reset
self._runner_context.set_streaming(streaming)
# Execute initial setup if provided
@@ -602,31 +585,13 @@ class Workflow(DictConvertible):
if checkpoint_storage is not None:
self._runner.context.set_runtime_checkpoint_storage(checkpoint_storage)
# Async validation: a fresh-message run is only allowed when the
# runner context has fully drained from any prior run. If it still
# has in-flight executor messages, the prior run didn't complete -
# the caller must either resume from a checkpoint or wait for the
# prior run to drain. (Pending request_info events are intentionally
# NOT blocked here: a follow-up run with message=... is the normal
# way to deliver a response to those pending requests, e.g. via
# WorkflowAgent._process_pending_requests.)
# NOTE: _validate_run_params already enforces that ``message`` is
# mutually exclusive with both ``checkpoint_id`` and ``responses``,
# so we don't need to re-check those here.
if message is not None and await self._runner.context.has_messages():
raise RuntimeError(
"Cannot start a new run with 'message' while in-flight executor "
"messages remain from a prior run. Resume from a checkpoint "
"(checkpoint_id=...) or wait for the prior run to complete. "
"Workflows that need to recover from a mid-run failure must use "
"checkpointing; there is no in-process recovery path."
)
initial_executor_fn = self._resolve_execution_mode(message, responses, checkpoint_id, checkpoint_storage)
initial_executor_fn, reset_context = self._resolve_execution_mode(
message, responses, checkpoint_id, checkpoint_storage
)
async for event in self._run_workflow_with_tracing(
initial_executor_fn=initial_executor_fn,
is_continuation=(message is None),
reset_context=reset_context,
streaming=streaming,
function_invocation_kwargs=function_invocation_kwargs,
client_kwargs=client_kwargs,
@@ -709,8 +674,12 @@ class Workflow(DictConvertible):
responses: Mapping[str, Any] | None,
checkpoint_id: str | None,
checkpoint_storage: CheckpointStorage | None,
) -> Callable[[], Awaitable[None]]:
"""Determine the initial executor function based on parameters."""
) -> tuple[Callable[[], Awaitable[None]], bool]:
"""Determine the initial executor function and reset_context flag based on parameters.
Returns:
A tuple of (initial_executor_fn, reset_context).
"""
if responses is not None:
if checkpoint_id is not None:
# Combined: restore checkpoint then send responses
@@ -720,9 +689,13 @@ class Workflow(DictConvertible):
else:
# Send responses only (requires pending requests in workflow state)
initial_executor_fn = functools.partial(self._send_responses_internal, responses)
return initial_executor_fn
return initial_executor_fn, False
# Regular run or checkpoint restoration
return functools.partial(self._execute_with_message_or_checkpoint, message, checkpoint_id, checkpoint_storage)
initial_executor_fn = functools.partial(
self._execute_with_message_or_checkpoint, message, checkpoint_id, checkpoint_storage
)
reset_context = message is not None and checkpoint_id is None
return initial_executor_fn, reset_context
async def _restore_and_send_responses(
self,
@@ -361,7 +361,7 @@ class WorkflowExecutor(Executor):
return any(is_instance_of(message.data, input_type) for input_type in self.workflow.input_types)
@handler
async def process_workflow(self, input_data: object, ctx: WorkflowContext[Any, Any]) -> None:
async def process_workflow(self, input_data: object, ctx: WorkflowContext[Any]) -> None:
"""Execute the sub-workflow with raw input data.
This handler starts a new sub-workflow execution. When the sub-workflow
@@ -428,7 +428,7 @@ class WorkflowExecutor(Executor):
async def handle_message_wrapped_request_response(
self,
response: SubWorkflowResponseMessage,
ctx: WorkflowContext[Any, Any],
ctx: WorkflowContext[Any],
) -> None:
"""Handle response from parent for a forwarded request.
@@ -34,8 +34,7 @@ class AgentFrameworkException(Exception):
logger.log(log_level, message, exc_info=inner_exception)
if inner_exception:
super().__init__(message, inner_exception, *args) # type: ignore
else:
super().__init__(message, *args) # type: ignore
super().__init__(message, *args) # type: ignore
# region Agent Exceptions
@@ -4,7 +4,6 @@
This module lazily re-exports objects from:
- ``agent-framework-anthropic``
- ``agent-framework-azure-contentunderstanding``
- ``agent-framework-foundry``
- ``agent-framework-foundry-local``
"""
@@ -13,15 +12,7 @@ import importlib
from typing import Any
_IMPORTS: dict[str, tuple[str, str]] = {
"AnalysisSection": ("agent_framework_azure_contentunderstanding", "agent-framework-azure-contentunderstanding"),
"AnthropicFoundryClient": ("agent_framework_anthropic", "agent-framework-anthropic"),
"ContentUnderstandingContextProvider": (
"agent_framework_azure_contentunderstanding",
"agent-framework-azure-contentunderstanding",
),
"DocumentStatus": ("agent_framework_azure_contentunderstanding", "agent-framework-azure-contentunderstanding"),
"FileSearchBackend": ("agent_framework_azure_contentunderstanding", "agent-framework-azure-contentunderstanding"),
"FileSearchConfig": ("agent_framework_azure_contentunderstanding", "agent-framework-azure-contentunderstanding"),
"FoundryAgent": ("agent_framework_foundry", "agent-framework-foundry"),
"FoundryAgentOptions": ("agent_framework_foundry", "agent-framework-foundry"),
"FoundryChatClient": ("agent_framework_foundry", "agent-framework-foundry"),
@@ -4,13 +4,6 @@
# Install the relevant packages for full type support.
from agent_framework_anthropic import AnthropicFoundryClient, RawAnthropicFoundryClient
from agent_framework_azure_contentunderstanding import ( # pyright: ignore[reportMissingImports]
AnalysisSection, # pyright: ignore[reportUnknownVariableType]
ContentUnderstandingContextProvider, # pyright: ignore[reportUnknownVariableType]
DocumentStatus, # pyright: ignore[reportUnknownVariableType]
FileSearchBackend, # pyright: ignore[reportUnknownVariableType]
FileSearchConfig, # pyright: ignore[reportUnknownVariableType]
)
from agent_framework_foundry import (
FoundryAgent,
FoundryChatClient,
@@ -38,12 +31,7 @@ from agent_framework_foundry_local import (
)
__all__ = [
"AnalysisSection",
"AnthropicFoundryClient",
"ContentUnderstandingContextProvider",
"DocumentStatus",
"FileSearchBackend",
"FileSearchConfig",
"FoundryAgent",
"FoundryChatClient",
"FoundryChatOptions",
@@ -26,7 +26,6 @@ from time import perf_counter, time_ns
from typing import TYPE_CHECKING, Any, ClassVar, Final, Generic, Literal, TypedDict, cast, overload
from dotenv import load_dotenv
from opentelemetry import context as otel_context
from opentelemetry import metrics, trace
from . import __version__ as version_info
@@ -1278,8 +1277,27 @@ class ChatTelemetryLayer(Generic[OptionsCoT]):
)
if stream:
span = _start_streaming_span(attributes, OtelAttr.REQUEST_MODEL)
result_stream = cast(
ResponseStream[ChatResponseUpdate, ChatResponse[Any]],
super_get_response(
messages=messages,
stream=True,
options=opts,
compaction_strategy=compaction_strategy,
tokenizer=tokenizer,
function_invocation_kwargs=function_invocation_kwargs,
client_kwargs=merged_client_kwargs,
),
)
# Create span directly without trace.use_span() context attachment.
# Streaming spans are closed asynchronously in cleanup hooks, which run
# in a different async context than creation — using use_span() would
# cause "Failed to detach context" errors from OpenTelemetry.
operation = attributes.get(OtelAttr.OPERATION, "operation")
span_name = attributes.get(OtelAttr.REQUEST_MODEL, "unknown")
span = get_tracer().start_span(f"{operation} {span_name}")
span.set_attributes(attributes)
if OBSERVABILITY_SETTINGS.SENSITIVE_DATA_ENABLED and messages:
_capture_messages(
span=span,
@@ -1301,24 +1319,6 @@ class ChatTelemetryLayer(Generic[OptionsCoT]):
def _record_duration() -> None:
duration_state["duration"] = perf_counter() - start_time
try:
result_stream = cast(
ResponseStream[ChatResponseUpdate, ChatResponse[Any]],
super_get_response(
messages=messages,
stream=True,
options=opts,
compaction_strategy=compaction_strategy,
tokenizer=tokenizer,
function_invocation_kwargs=function_invocation_kwargs,
client_kwargs=merged_client_kwargs,
),
)
except Exception as exception:
capture_exception(span=span, exception=exception, timestamp=time_ns())
_close_span()
raise
async def _finalize_stream() -> None:
from ._types import ChatResponse
@@ -1357,18 +1357,11 @@ class ChatTelemetryLayer(Generic[OptionsCoT]):
finally:
_close_span()
# The pull context manager attaches the span around each underlying iterator pull so
# that child spans created during the pull (e.g. HTTP requests, inner tool execution)
# are parented under this chat span. Attach and detach happen in the same async
# context as the pull, avoiding cross-context cleanup issues. The weakref finalizer
# ensures the span is closed even if the stream is garbage collected without being
# consumed.
wrapped_stream: ResponseStream[ChatResponseUpdate, ChatResponse[Any]] = (
result_stream
.with_cleanup_hook(_record_duration)
.with_cleanup_hook(_finalize_stream)
.with_pull_context_manager(lambda: _activate_span(span))
)
# Register a weak reference callback to close the span if stream is garbage collected
# without being consumed. This ensures spans don't leak if users don't consume streams.
wrapped_stream: ResponseStream[ChatResponseUpdate, ChatResponse[Any]] = result_stream.with_cleanup_hook(
_record_duration
).with_cleanup_hook(_finalize_stream)
weakref.finalize(wrapped_stream, _close_span)
return wrapped_stream
@@ -1550,8 +1543,23 @@ class AgentTelemetryLayer:
inner_accumulated_usage_token = INNER_ACCUMULATED_USAGE.set({})
if stream:
span = _start_streaming_span(attributes, OtelAttr.AGENT_NAME)
try:
run_result: object = execute()
if isinstance(run_result, ResponseStream):
result_stream: ResponseStream[AgentResponseUpdate, AgentResponse[Any]] = run_result # pyright: ignore[reportUnknownVariableType]
elif isinstance(run_result, Awaitable):
result_stream = ResponseStream.from_awaitable(run_result) # type: ignore[arg-type] # pyright: ignore[reportArgumentType]
else:
raise RuntimeError("Streaming telemetry requires a ResponseStream result.")
except Exception:
INNER_RESPONSE_TELEMETRY_CAPTURED_FIELDS.reset(inner_response_telemetry_captured_fields_token)
INNER_ACCUMULATED_USAGE.reset(inner_accumulated_usage_token)
raise
operation = attributes.get(OtelAttr.OPERATION, "operation")
span_name = attributes.get(OtelAttr.AGENT_NAME, "unknown")
span = get_tracer().start_span(f"{operation} {span_name}")
span.set_attributes(attributes)
if OBSERVABILITY_SETTINGS.SENSITIVE_DATA_ENABLED and messages:
_capture_messages(
span=span,
@@ -1573,21 +1581,6 @@ class AgentTelemetryLayer:
def _record_duration() -> None:
duration_state["duration"] = perf_counter() - start_time
try:
run_result: object = execute()
if isinstance(run_result, ResponseStream):
result_stream: ResponseStream[AgentResponseUpdate, AgentResponse[Any]] = run_result # pyright: ignore[reportUnknownVariableType]
elif isinstance(run_result, Awaitable):
result_stream = ResponseStream.from_awaitable(run_result) # type: ignore[arg-type] # pyright: ignore[reportArgumentType]
else:
raise RuntimeError("Streaming telemetry requires a ResponseStream result.")
except Exception as exception:
capture_exception(span=span, exception=exception, timestamp=time_ns())
INNER_RESPONSE_TELEMETRY_CAPTURED_FIELDS.reset(inner_response_telemetry_captured_fields_token)
INNER_ACCUMULATED_USAGE.reset(inner_accumulated_usage_token)
_close_span()
raise
async def _finalize_stream() -> None:
from ._types import AgentResponse
@@ -1627,18 +1620,9 @@ class AgentTelemetryLayer:
INNER_ACCUMULATED_USAGE.reset(inner_accumulated_usage_token)
_close_span()
# The pull context manager attaches the span around each underlying iterator pull so
# that child spans created during the pull (e.g. inner chat completion spans from the
# underlying ChatTelemetryLayer) are parented under this agent invoke span. Attach and
# detach happen in the same async context as the pull, avoiding cross-context cleanup
# issues. The weakref finalizer ensures the span is closed even if the stream is
# garbage collected without being consumed.
wrapped_stream: ResponseStream[AgentResponseUpdate, AgentResponse[Any]] = (
result_stream
.with_cleanup_hook(_record_duration)
.with_cleanup_hook(_finalize_stream)
.with_pull_context_manager(lambda: _activate_span(span))
)
wrapped_stream: ResponseStream[AgentResponseUpdate, AgentResponse[Any]] = result_stream.with_cleanup_hook(
_record_duration
).with_cleanup_hook(_finalize_stream)
weakref.finalize(wrapped_stream, _close_span)
return wrapped_stream
@@ -1825,27 +1809,6 @@ def get_function_span(
)
@contextlib.contextmanager
def _activate_span(span: trace.Span) -> Generator[None]:
"""Attach ``span`` as the current span in the OpenTelemetry context.
Designed to be used as a per-pull context manager registered on a
``ResponseStream`` via ``with_pull_context_manager``: it attaches the span
before each underlying iterator pull and detaches immediately after, so
child spans created during the pull (HTTP clients, inner chat completions,
tool execution) are correctly parented under ``span``.
Because attach and detach happen within the same ``__anext__`` invocation
(and therefore the same async task / contextvars context), there is no risk
of "Failed to detach context" warnings from cross-context cleanup.
"""
token = otel_context.attach(trace.set_span_in_context(span))
try:
yield
finally:
otel_context.detach(token)
@contextlib.contextmanager
def _get_span(
attributes: dict[str, Any],
@@ -1868,29 +1831,6 @@ def _get_span(
yield current_span
def _start_streaming_span(attributes: dict[str, Any], span_name_attribute: str) -> trace.Span:
"""Start a non-current span for a streaming operation.
Unlike :func:`_get_span`, the returned span is not attached to the current
OpenTelemetry context. The caller is responsible for:
- Ending the span via cleanup hooks on the wrapped
:class:`~agent_framework._types.ResponseStream`.
- Activating the span around each iterator pull via
:func:`_activate_span` registered with ``with_pull_context_manager`` so
that child spans created during stream production inherit it as parent.
Streaming spans are closed asynchronously in cleanup hooks that run in a
different async context than creation, so attaching the span at creation
time would cause "Failed to detach context" errors from OpenTelemetry.
"""
operation = attributes.get(OtelAttr.OPERATION, "operation")
span_name = attributes.get(span_name_attribute, "unknown")
span = get_tracer().start_span(f"{operation} {span_name}")
span.set_attributes(attributes)
return span
def _get_instructions_from_options(options: Any) -> str | list[str] | None:
"""Extract instructions from options dict."""
if options is None:

Some files were not shown because too many files have changed in this diff Show More