diff --git a/dotnet/Directory.Packages.props b/dotnet/Directory.Packages.props index 16277bd502..50cc9766c3 100644 --- a/dotnet/Directory.Packages.props +++ b/dotnet/Directory.Packages.props @@ -17,9 +17,9 @@ - - - + + + @@ -31,10 +31,10 @@ - - - - + + + + @@ -43,24 +43,24 @@ - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + diff --git a/dotnet/samples/GettingStarted/AgentSample.cs b/dotnet/samples/GettingStarted/AgentSample.cs index 273749fc9a..79f9dc8b8a 100644 --- a/dotnet/samples/GettingStarted/AgentSample.cs +++ b/dotnet/samples/GettingStarted/AgentSample.cs @@ -116,7 +116,7 @@ public class AgentSample(ITestOutputHelper output) : BaseSample(output) => new(new PersistentAgentsClient(TestConfiguration.AzureAI.Endpoint, new AzureCliCredential()), options.Id!); private NewOpenAIAssistantChatClient GetOpenAIAssistantChatClient(ChatClientAgentOptions options) - => new(new(TestConfiguration.OpenAI.ApiKey), options.Id!, null); + => new(new AssistantClient(TestConfiguration.OpenAI.ApiKey), options.Id!); #endregion diff --git a/dotnet/samples/GettingStarted/Steps/Step03_ChatClientAgent_UsingCodeInterpreterTools.cs b/dotnet/samples/GettingStarted/Steps/Step03_ChatClientAgent_UsingCodeInterpreterTools.cs index ea4c3ed8da..d1f3bb4d52 100644 --- a/dotnet/samples/GettingStarted/Steps/Step03_ChatClientAgent_UsingCodeInterpreterTools.cs +++ b/dotnet/samples/GettingStarted/Steps/Step03_ChatClientAgent_UsingCodeInterpreterTools.cs @@ -21,7 +21,7 @@ public sealed class Step03_ChatClientAgent_UsingCodeInterpreterTools(ITestOutput [InlineData(ChatClientProviders.OpenAIAssistant)] public async Task RunningWithFileReferenceAsync(ChatClientProviders provider) { - var codeInterpreterTool = new NewHostedCodeInterpreterTool() + var codeInterpreterTool = new HostedCodeInterpreterTool() { Inputs = [new HostedFileContent(await UploadFileAsync("Resources/groceries.txt", provider))] }; diff --git a/dotnet/samples/GettingStarted/Steps/Step07_ChatClientAgent_UsingFileSearchTools.cs b/dotnet/samples/GettingStarted/Steps/Step07_ChatClientAgent_UsingFileSearchTools.cs index d00b1bdb64..8b26942177 100644 --- a/dotnet/samples/GettingStarted/Steps/Step07_ChatClientAgent_UsingFileSearchTools.cs +++ b/dotnet/samples/GettingStarted/Steps/Step07_ChatClientAgent_UsingFileSearchTools.cs @@ -29,7 +29,7 @@ public sealed class Step07_ChatClientAgent_UsingFileSearchTools(ITestOutputHelpe var vectorStoreId = await CreateVectorStoreAsync([fileId], provider); // Create a file search tool that can access the vector store. - var fileSearchTool = new NewHostedFileSearchTool() + var fileSearchTool = new HostedFileSearchTool() { Inputs = [new HostedVectorStoreContent(vectorStoreId)], }; diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI/HostedFileContent.cs b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI/HostedFileContent.cs deleted file mode 100644 index 3b6b7cb9a5..0000000000 --- a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI/HostedFileContent.cs +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Diagnostics; -using System.Diagnostics.CodeAnalysis; -using Microsoft.Shared.Diagnostics; - -namespace Microsoft.Extensions.AI; - -/// -/// Represents a file that is hosted by the AI service. -/// -/// -/// Unlike which contains the data for a file or blob, this class represents a file that is hosted -/// by the AI service and referenced by an identifier. Such identifiers are specific to the provider. -/// -[DebuggerDisplay("FileId = {FileId}")] -[ExcludeFromCodeCoverage] -public sealed class HostedFileContent : AIContent -{ - private string _fileId; - - /// - /// Initializes a new instance of the class. - /// - /// The ID of the hosted file. - /// is . - /// is empty or composed entirely of whitespace. - public HostedFileContent(string fileId) - { - _fileId = Throw.IfNullOrWhitespace(fileId); - } - - /// - /// Gets or sets the ID of the hosted file. - /// - /// is . - /// is empty or composed entirely of whitespace. - public string FileId - { - get => _fileId; - set => _fileId = Throw.IfNullOrWhitespace(value); - } -} diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI/HostedVectorStoreContent.cs b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI/HostedVectorStoreContent.cs deleted file mode 100644 index beb64804c5..0000000000 --- a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI/HostedVectorStoreContent.cs +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Diagnostics; -using System.Diagnostics.CodeAnalysis; -using Microsoft.Shared.Diagnostics; - -namespace Microsoft.Extensions.AI; - -/// -/// Represents a vector store that is hosted by the AI service. -/// -/// -/// Unlike which contains the data for a file or blob, this class represents a vector store that is hosted -/// by the AI service and referenced by an identifier. Such identifiers are specific to the provider. -/// -[DebuggerDisplay("VectorStoreId = {VectorStoreId}")] -[ExcludeFromCodeCoverage] -public sealed class HostedVectorStoreContent : AIContent -{ - private string? _vectorStoreId; - - /// - /// Initializes a new instance of the class. - /// - /// The ID of the hosted vector store. - /// is . - /// is empty or composed entirely of whitespace. - public HostedVectorStoreContent(string vectorStoreId) - { - _vectorStoreId = Throw.IfNullOrWhitespace(vectorStoreId); - } - - /// - /// Gets or sets the ID of the hosted vector store. - /// - /// is . - /// is empty or composed entirely of whitespace. - public string VectorStoreId - { - get => _vectorStoreId ?? string.Empty; - set => _vectorStoreId = Throw.IfNullOrWhitespace(value); - } -} diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI/NewHostedCodeInterpreterTool.cs b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI/NewHostedCodeInterpreterTool.cs deleted file mode 100644 index 7fcbcfea58..0000000000 --- a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI/NewHostedCodeInterpreterTool.cs +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Collections.Generic; -using System.Diagnostics.CodeAnalysis; - -namespace Microsoft.Extensions.AI; - -/// -/// Proposal for abstraction updates based on the common code interpreter tool properties. -/// Based on the decision, the abstraction can be updated in M.E.AI directly. -/// -[ExcludeFromCodeCoverage] -public class NewHostedCodeInterpreterTool : HostedCodeInterpreterTool -{ - /// Gets or sets a collection of to be used as input to the code interpreter tool. - /// - /// Services support different varied kinds of inputs. Most support the IDs of files that are hosted by the service, - /// represented via . Some also support binary data, represented via . - /// Unsupported inputs will be ignored by the to which the tool is passed. - /// - public IList? Inputs { get; set; } -} diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI/NewHostedFileSearchTool.cs b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI/NewHostedFileSearchTool.cs deleted file mode 100644 index f280d78fc1..0000000000 --- a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI/NewHostedFileSearchTool.cs +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -// Line removed as it is unnecessary due to ImplicitUsings being enabled. - -using System.Collections.Generic; -using System.Diagnostics.CodeAnalysis; - -namespace Microsoft.Extensions.AI; - -/// -/// Proposal for abstraction updates based on the common file search tool properties. -/// This provides a standardized interface for file search functionality across providers. -/// -[ExcludeFromCodeCoverage] -public class NewHostedFileSearchTool : AITool -{ - /// Gets or sets a collection of to be used as input to the code interpreter tool. - /// - /// Services support different varied kinds of inputs. Most support the IDs of vector stores that are hosted by the service, - /// represented via . Some also support binary data, represented via . - /// Unsupported inputs will be ignored by the to which the tool is passed. - /// - public IList? Inputs { get; set; } -} diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/Microsoft.Extensions.AI.Agents.Abstractions.csproj b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/Microsoft.Extensions.AI.Agents.Abstractions.csproj index e0ad6ff297..37c7bc871b 100644 --- a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/Microsoft.Extensions.AI.Agents.Abstractions.csproj +++ b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/Microsoft.Extensions.AI.Agents.Abstractions.csproj @@ -30,5 +30,4 @@ - diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents.AzureAI/NewPersistentAgentsChatClient.cs b/dotnet/src/Microsoft.Extensions.AI.Agents.AzureAI/NewPersistentAgentsChatClient.cs index 59ff21fbd3..943f01846d 100644 --- a/dotnet/src/Microsoft.Extensions.AI.Agents.AzureAI/NewPersistentAgentsChatClient.cs +++ b/dotnet/src/Microsoft.Extensions.AI.Agents.AzureAI/NewPersistentAgentsChatClient.cs @@ -295,7 +295,7 @@ namespace Azure.AI.Agents.Persistent BinaryData.FromBytes(JsonSerializer.SerializeToUtf8Bytes(aiFunction.JsonSchema, AgentsChatClientJsonContext.Default.JsonElement)))); break; - case NewHostedCodeInterpreterTool codeTool: + case HostedCodeInterpreterTool codeTool: toolDefinitions.Add(new CodeInterpreterToolDefinition()); if (codeTool.Inputs is { Count: > 0 }) @@ -313,7 +313,7 @@ namespace Azure.AI.Agents.Persistent } break; - case NewHostedFileSearchTool fileSearchTool: + case HostedFileSearchTool fileSearchTool: toolDefinitions.Add(new FileSearchToolDefinition()); if (fileSearchTool.Inputs is { Count: > 0 }) diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents.OpenAI/NewOpenAIAssistantChatClient.cs b/dotnet/src/Microsoft.Extensions.AI.Agents.OpenAI/NewOpenAIAssistantChatClient.cs index f6a4ef9515..cd3408df8f 100644 --- a/dotnet/src/Microsoft.Extensions.AI.Agents.OpenAI/NewOpenAIAssistantChatClient.cs +++ b/dotnet/src/Microsoft.Extensions.AI.Agents.OpenAI/NewOpenAIAssistantChatClient.cs @@ -11,7 +11,6 @@ using System.Runtime.CompilerServices; using System.Text; using System.Text.Json; using System.Text.Json.Nodes; -using System.Text.Json.Serialization; using Microsoft.Shared.Diagnostics; using OpenAI; using OpenAI.Assistants; @@ -48,7 +47,7 @@ public sealed class NewOpenAIAssistantChatClient : IChatClient /// List of tools associated with the assistant. private IReadOnlyList? _assistantTools; - /// Initializes a new instance of the class for the specified . + /// Initializes a new instance of the class for the specified . public NewOpenAIAssistantChatClient(AssistantClient assistantClient, string assistantId, string? defaultThreadId = null) { _client = Throw.IfNull(assistantClient); @@ -96,7 +95,7 @@ public sealed class NewOpenAIAssistantChatClient : IChatClient { switch (tool) { - case NewHostedCodeInterpreterTool codeTool: + case HostedCodeInterpreterTool codeTool: if (codeTool.Inputs is { Count: > 0 }) { @@ -115,7 +114,7 @@ public sealed class NewOpenAIAssistantChatClient : IChatClient break; - case NewHostedFileSearchTool fileSearchTool: + case HostedFileSearchTool fileSearchTool: // Handle file IDs for file search tool if (fileSearchTool.Inputs is { Count: > 0 }) @@ -155,7 +154,7 @@ public sealed class NewOpenAIAssistantChatClient : IChatClient _ = Throw.IfNull(messages); // Extract necessary state from messages and options. - (RunCreationOptions runOptions, List? toolResults) = await CreateRunOptionsAsync(messages, options, cancellationToken).ConfigureAwait(false); + (RunCreationOptions runOptions, ToolResources? toolResources, List? toolResults) = await CreateRunOptionsAsync(messages, options, cancellationToken).ConfigureAwait(false); // Get the thread ID. string? threadId = options?.ConversationId ?? _defaultThreadId; @@ -202,7 +201,7 @@ public sealed class NewOpenAIAssistantChatClient : IChatClient // No thread ID was provided, so create a new thread. ThreadCreationOptions threadCreationOptions = new() { - ToolResources = CreateToolResources(options) + ToolResources = toolResources, }; foreach (var message in runOptions.AdditionalMessages) @@ -268,18 +267,19 @@ public sealed class NewOpenAIAssistantChatClient : IChatClient if (ru is RequiredActionUpdate rau && rau.ToolCallId is string toolCallId && rau.FunctionName is string functionName) { - ruUpdate.Contents.Add( - new FunctionCallContent( - JsonSerializer.Serialize([ru.Value.Id, toolCallId], OpenAIJsonContext.Default.StringArray), - functionName, - JsonSerializer.Deserialize(rau.FunctionArguments, OpenAIJsonContext.Default.IDictionaryStringObject)!)); + var fcc = OpenAIClientExtensions2.ParseCallContent( + rau.FunctionArguments, + JsonSerializer.Serialize([ru.Value.Id, toolCallId], OpenAIJsonContext.Default.StringArray), + functionName); + fcc.RawRepresentation = ru; + ruUpdate.Contents.Add(fcc); } yield return ruUpdate; break; case MessageContentUpdate mcu: - yield return new(mcu.Role == MessageRole.User ? ChatRole.User : ChatRole.Assistant, mcu.Text) + ChatResponseUpdate textUpdate = new(mcu.Role == MessageRole.User ? ChatRole.User : ChatRole.Assistant, mcu.Text) { AuthorName = _assistantId, ConversationId = threadId, @@ -287,10 +287,48 @@ public sealed class NewOpenAIAssistantChatClient : IChatClient RawRepresentation = mcu, ResponseId = responseId, }; + + // Add any annotations from the text update. The OpenAI Assistants API does not support passing these back + // into the model (MessageContent.FromXx does not support providing annotations), so they end up being one way and are dropped + // on subsequent requests. + if (mcu.TextAnnotation is { } tau) + { + string? fileId = null; + string? toolName = null; + if (!string.IsNullOrWhiteSpace(tau.InputFileId)) + { + fileId = tau.InputFileId; + toolName = "file_search"; + } + else if (!string.IsNullOrWhiteSpace(tau.OutputFileId)) + { + fileId = tau.OutputFileId; + toolName = "code_interpreter"; + } + + if (fileId is not null) + { + if (textUpdate.Contents.Count == 0) + { + // Create a empty chunk of text content to hold the annotation. + textUpdate.Contents.Add(new TextContent(string.Empty)); + } + + (((TextContent)textUpdate.Contents[0]).Annotations ??= []).Add(new CitationAnnotation + { + RawRepresentation = tau, + AnnotatedRegions = [new TextSpanAnnotatedRegion { StartIndex = tau.StartIndex, EndIndex = tau.EndIndex }], + FileId = fileId, + ToolName = toolName, + }); + } + } + + yield return textUpdate; break; default: - yield return new ChatResponseUpdate + yield return new() { AuthorName = _assistantId, ConversationId = threadId, @@ -329,7 +367,7 @@ public sealed class NewOpenAIAssistantChatClient : IChatClient /// Creates the to use for the request and extracts any function result contents /// that need to be submitted as tool results. /// - private async ValueTask<(RunCreationOptions RunOptions, List? ToolResults)> CreateRunOptionsAsync( + private async ValueTask<(RunCreationOptions RunOptions, ToolResources? Resources, List? ToolResults)> CreateRunOptionsAsync( IEnumerable messages, ChatOptions? options, CancellationToken cancellationToken) { // Create the options instance to populate, either a fresh or using one the caller provides. @@ -337,6 +375,8 @@ public sealed class NewOpenAIAssistantChatClient : IChatClient options?.RawRepresentationFactory?.Invoke(this) as RunCreationOptions ?? new(); + ToolResources? resources = null; + // Populate the run options from the ChatOptions, if provided. if (options is not null) { @@ -375,51 +415,42 @@ public sealed class NewOpenAIAssistantChatClient : IChatClient runOptions.ToolsOverride.Add(ToOpenAIAssistantsFunctionToolDefinition(aiFunction, options)); break; - case NewHostedCodeInterpreterTool codeTool: - var codeInterpreterToolDefinition = new CodeInterpreterToolDefinition(); - runOptions.ToolsOverride.Add(codeInterpreterToolDefinition); + case HostedCodeInterpreterTool codeInterpreterTool: + var interpreterToolDef = ToolDefinition.CreateCodeInterpreter(); + runOptions.ToolsOverride.Add(interpreterToolDef); - if (codeTool.Inputs is { Count: > 0 }) + if (codeInterpreterTool.Inputs?.Count is > 0) { - var threadInitializationMessage = new ThreadInitializationMessage(OpenAI.Assistants.MessageRole.User, [OpenAI.Assistants.MessageContent.FromText("attachments")]); - - foreach (var input in codeTool.Inputs) + ThreadInitializationMessage? threadInitializationMessage = null; + foreach (var input in codeInterpreterTool.Inputs) { - switch (input) + if (input is HostedFileContent hostedFile) { - case HostedFileContent fileContent: - // Use the file ID from the HostedFileContent. - threadInitializationMessage.Attachments.Add(new(fileContent.FileId, [codeInterpreterToolDefinition])); - break; + threadInitializationMessage ??= new(MessageRole.User, [MessageContent.FromText("attachments")]); + threadInitializationMessage.Attachments.Add(new(hostedFile.FileId, [interpreterToolDef])); } } - runOptions.AdditionalMessages.Add(threadInitializationMessage); + if (threadInitializationMessage is not null) + { + runOptions.AdditionalMessages.Add(threadInitializationMessage); + } } break; - case NewHostedFileSearchTool fileSearchTool: - var fileSearchToolDefinition = new FileSearchToolDefinition(); - runOptions.ToolsOverride.Add(fileSearchToolDefinition); - - // Handle file IDs for file search tool - if (fileSearchTool.Inputs is { Count: > 0 }) + case HostedFileSearchTool fileSearchTool: + runOptions.ToolsOverride.Add(ToolDefinition.CreateFileSearch(fileSearchTool.MaximumResultCount)); + if (fileSearchTool.Inputs is { Count: > 0 } fileSearchInputs) { - var threadInitializationMessage = new ThreadInitializationMessage(OpenAI.Assistants.MessageRole.User, [OpenAI.Assistants.MessageContent.FromText("file search attachments")]); - - foreach (var input in fileSearchTool.Inputs) + foreach (var input in fileSearchInputs) { - switch (input) + if (input is HostedVectorStoreContent file) { - case HostedFileContent fileContent: - // Use the file ID from the HostedFileContent. - threadInitializationMessage.Attachments.Add(new(fileContent.FileId, [fileSearchToolDefinition])); - break; + (resources ??= new()).FileSearch ??= new(); + resources.FileSearch.VectorStoreIds.Add(file.VectorStoreId); } } - - runOptions.AdditionalMessages.Add(threadInitializationMessage); } break; @@ -522,6 +553,10 @@ public sealed class NewOpenAIAssistantChatClient : IChatClient { switch (content) { + case AIContent when content.RawRepresentation is MessageContent rawRep: + messageContents.Add(rawRep); + break; + case TextContent text: messageContents.Add(MessageContent.FromText(text.Text)); break; @@ -530,18 +565,9 @@ public sealed class NewOpenAIAssistantChatClient : IChatClient messageContents.Add(MessageContent.FromImageUri(image.Uri)); break; - // Assistants doesn't support data URIs. - //case DataContent image when image.HasTopLevelMediaType("image"): - // messageContents.Add(MessageContent.FromImageUri(new Uri(image.Uri))); - // break; - case FunctionResultContent result: (functionResults ??= []).Add(result); break; - - case AIContent when content.RawRepresentation is MessageContent rawRep: - messageContents.Add(rawRep); - break; } } @@ -555,7 +581,7 @@ public sealed class NewOpenAIAssistantChatClient : IChatClient runOptions.AdditionalInstructions = instructions?.ToString(); - return (runOptions, functionResults); + return (runOptions, resources, functionResults); } /// Convert instances to instances. @@ -613,6 +639,28 @@ internal static class OpenAIClientExtensions2 /// Gets a for "developer". internal static ChatRole ChatRoleDeveloper { get; } = new ChatRole("developer"); + /// Creates a new instance of parsing arguments using a specified encoding and parser. + /// The input arguments to be parsed. + /// The function call ID. + /// The function name. + /// A new instance of containing the parse result. + /// is . + /// is . + internal static FunctionCallContent ParseCallContent(string json, string callId, string name) => + FunctionCallContent.CreateFromParsedArguments(json, callId, name, + static json => JsonSerializer.Deserialize(json, OpenAIJsonContext.Default.IDictionaryStringObject)!); + + /// Creates a new instance of parsing arguments using a specified encoding and parser. + /// The input arguments to be parsed. + /// The function call ID. + /// The function name. + /// A new instance of containing the parse result. + /// is . + /// is . + internal static FunctionCallContent ParseCallContent(BinaryData utf8json, string callId, string name) => + FunctionCallContent.CreateFromParsedArguments(utf8json, callId, name, + static utf8json => JsonSerializer.Deserialize(utf8json, OpenAIJsonContext.Default.IDictionaryStringObject)!); + /// /// Gets the JSON schema transformer cache conforming to OpenAI strict / structured output restrictions per /// https://platform.openai.com/docs/guides/structured-outputs?api-mode=responses#supported-schemas. @@ -707,31 +755,4 @@ internal static class OpenAIClientExtensions2 return functionParameters; } - - /// Used to create the JSON payload for an OpenAI tool description. - internal sealed class ToolJson - { - [JsonPropertyName("type")] - public string Type { get; set; } = "object"; - - [JsonPropertyName("required")] - public HashSet Required { get; set; } = []; - - [JsonPropertyName("properties")] - public Dictionary Properties { get; set; } = []; - - [JsonPropertyName("additionalProperties")] - public bool AdditionalProperties { get; set; } - } } - -/// Source-generated JSON type information for use by all OpenAI implementations. -[JsonSourceGenerationOptions(JsonSerializerDefaults.Web, - UseStringEnumConverter = true, - DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, - WriteIndented = true)] -[JsonSerializable(typeof(OpenAIClientExtensions2.ToolJson))] -[JsonSerializable(typeof(IDictionary))] -[JsonSerializable(typeof(string[]))] -[JsonSerializable(typeof(JsonElement))] -internal sealed partial class OpenAIJsonContext : JsonSerializerContext; diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents.OpenAI/OpenAIAssistantClientExtensions.cs b/dotnet/src/Microsoft.Extensions.AI.Agents.OpenAI/OpenAIAssistantClientExtensions.cs index 71277db74e..2c84d3a8e8 100644 --- a/dotnet/src/Microsoft.Extensions.AI.Agents.OpenAI/OpenAIAssistantClientExtensions.cs +++ b/dotnet/src/Microsoft.Extensions.AI.Agents.OpenAI/OpenAIAssistantClientExtensions.cs @@ -1,5 +1,9 @@ // Copyright (c) Microsoft. All rights reserved. +using System.Text; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Text.Json.Serialization; using Microsoft.Extensions.AI; using Microsoft.Extensions.AI.Agents; using Microsoft.Extensions.Logging; @@ -20,6 +24,9 @@ namespace OpenAI; /// public static class OpenAIAssistantClientExtensions { + /// Key into AdditionalProperties used to store a strict option. + private const string StrictKey = "strictJsonSchema"; + /// /// Creates an AI agent from an using the OpenAI Assistant API. /// @@ -80,7 +87,7 @@ public static class OpenAIAssistantClientExtensions switch (tool) { case AIFunction aiFunction: - assistantOptions.Tools.Add(NewOpenAIAssistantChatClient.ToOpenAIAssistantsFunctionToolDefinition(aiFunction)); + assistantOptions.Tools.Add(ToOpenAIAssistantsFunctionToolDefinition(aiFunction)); break; case HostedCodeInterpreterTool: @@ -107,7 +114,7 @@ public static class OpenAIAssistantClientExtensions }; #pragma warning disable CA2000 // Dispose objects before losing scope - var chatClient = new NewOpenAIAssistantChatClient(client, assistantId); + var chatClient = client.AsIChatClient(assistantId); #pragma warning restore CA2000 // Dispose objects before losing scope return new ChatClientAgent(chatClient, agentOptions, loggerFactory); } @@ -172,7 +179,7 @@ public static class OpenAIAssistantClientExtensions switch (tool) { case AIFunction aiFunction: - assistantOptions.Tools.Add(NewOpenAIAssistantChatClient.ToOpenAIAssistantsFunctionToolDefinition(aiFunction)); + assistantOptions.Tools.Add(ToOpenAIAssistantsFunctionToolDefinition(aiFunction)); break; case HostedCodeInterpreterTool: @@ -199,8 +206,139 @@ public static class OpenAIAssistantClientExtensions }; #pragma warning disable CA2000 // Dispose objects before losing scope - var chatClient = new NewOpenAIAssistantChatClient(client, assistantId); + var chatClient = client.AsIChatClient(assistantId); #pragma warning restore CA2000 // Dispose objects before losing scope return new ChatClientAgent(chatClient, agentOptions, loggerFactory); } + + /// Converts an Extensions function to an OpenAI assistants function tool. + private static FunctionToolDefinition ToOpenAIAssistantsFunctionToolDefinition(AIFunction aiFunction, ChatOptions? options = null) + { + bool? strict = + HasStrict(aiFunction.AdditionalProperties) ?? + HasStrict(options?.AdditionalProperties); + + return new FunctionToolDefinition(aiFunction.Name) + { + Description = aiFunction.Description, + Parameters = ToOpenAIFunctionParameters(aiFunction, strict), + StrictParameterSchemaEnabled = strict, + }; + } + + /// Extracts from an the parameters and strictness setting for use with OpenAI's APIs. + private static BinaryData ToOpenAIFunctionParameters(AIFunction aiFunction, bool? strict) + { + // Perform any desirable transformations on the function's JSON schema, if it'll be used in a strict setting. + JsonElement jsonSchema = strict is true ? + StrictSchemaTransformCache.GetOrCreateTransformedSchema(aiFunction) : + aiFunction.JsonSchema; + + // Roundtrip the schema through the ToolJson model type to remove extra properties + // and force missing ones into existence, then return the serialized UTF8 bytes as BinaryData. + var tool = jsonSchema.Deserialize(OpenAIJsonContext.Default.ToolJson)!; + var functionParameters = BinaryData.FromBytes(JsonSerializer.SerializeToUtf8Bytes(tool, OpenAIJsonContext.Default.ToolJson)); + + return functionParameters; + } + + /// Gets whether the properties specify that strict schema handling is desired. + private static bool? HasStrict(IReadOnlyDictionary? additionalProperties) => + additionalProperties?.TryGetValue(StrictKey, out object? strictObj) is true && + strictObj is bool strictValue ? + strictValue : null; + + private static AIJsonSchemaTransformCache StrictSchemaTransformCache { get; } = new(new() + { + DisallowAdditionalProperties = true, + ConvertBooleanSchemas = true, + MoveDefaultKeywordToDescription = true, + RequireAllProperties = true, + TransformSchemaNode = (ctx, node) => + { + // Move content from common but unsupported properties to description. In particular, we focus on properties that + // the AIJsonUtilities schema generator might produce and/or that are explicitly mentioned in the OpenAI documentation. + + if (node is JsonObject schemaObj) + { + StringBuilder? additionalDescription = null; + + ReadOnlySpan unsupportedProperties = + [ + // Produced by AIJsonUtilities but not in allow list at https://platform.openai.com/docs/guides/structured-outputs#supported-properties: + "contentEncoding", "contentMediaType", "not", + + // Explicitly mentioned at https://platform.openai.com/docs/guides/structured-outputs?api-mode=responses#key-ordering as being unsupported with some models: + "minLength", "maxLength", "pattern", "format", + "minimum", "maximum", "multipleOf", + "patternProperties", + "minItems", "maxItems", + + // Explicitly mentioned at https://learn.microsoft.com/azure/ai-services/openai/how-to/structured-outputs?pivots=programming-language-csharp&tabs=python-secure%2Cdotnet-entra-id#unsupported-type-specific-keywords + // as being unsupported with Azure OpenAI: + "unevaluatedProperties", "propertyNames", "minProperties", "maxProperties", + "unevaluatedItems", "contains", "minContains", "maxContains", "uniqueItems", + ]; + + foreach (string propName in unsupportedProperties) + { + if (schemaObj[propName] is { } propNode) + { + _ = schemaObj.Remove(propName); + AppendLine(ref additionalDescription, propName, propNode); + } + } + + if (additionalDescription is not null) + { + schemaObj["description"] = schemaObj["description"] is { } descriptionNode && descriptionNode.GetValueKind() == JsonValueKind.String ? + $"{descriptionNode.GetValue()}{Environment.NewLine}{additionalDescription}" : + additionalDescription.ToString(); + } + + return node; + + static void AppendLine(ref StringBuilder? sb, string propName, JsonNode propNode) + { + sb ??= new(); + + if (sb.Length > 0) + { + _ = sb.AppendLine(); + } + + _ = sb.Append(propName).Append(": ").Append(propNode); + } + } + + return node; + }, + }); + + /// Used to create the JSON payload for an OpenAI tool description. + internal sealed class ToolJson + { + [JsonPropertyName("type")] + public string Type { get; set; } = "object"; + + [JsonPropertyName("required")] + public HashSet Required { get; set; } = []; + + [JsonPropertyName("properties")] + public Dictionary Properties { get; set; } = []; + + [JsonPropertyName("additionalProperties")] + public bool AdditionalProperties { get; set; } + } } + +/// Source-generated JSON type information for use by all OpenAI implementations. +[JsonSourceGenerationOptions(JsonSerializerDefaults.Web, + UseStringEnumConverter = true, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + WriteIndented = true)] +[JsonSerializable(typeof(OpenAIAssistantClientExtensions.ToolJson))] +[JsonSerializable(typeof(IDictionary))] +[JsonSerializable(typeof(string[]))] +[JsonSerializable(typeof(JsonElement))] +internal sealed partial class OpenAIJsonContext : JsonSerializerContext;