.NET: Add rag samples with sample TextSearchStore (#1664)

* Port store for adding text to a vector store to AF

* Fix typo.

* Change TextSearchStore to sample, and add sample to use it and do rag with a custom schema

* Add more tests and fix broken ones

* Fix merge issue

* Fix sample after merge.

* Convert TextSearchStore to use Dynamic mode to be AOT compatible.

* Add some more clarification on when to use assistant messages in rag searches.

dotnet/src/Microsoft.Agents.AI/Data/TextSearchProvider.cs

@@ -45,6 +45,7 @@ public sealed class TextSearchProvider : AIContextProvider
     private readonly AITool[] _tools;
     private readonly Queue<string> _recentMessagesText;
     private readonly TextSearchProviderOptions _options;
+    private readonly List<ChatRole> _recentMessageRolesIncluded;
 
     /// <summary>
     /// Initializes a new instance of the <see cref="TextSearchProvider"/> class.
@@ -60,6 +61,7 @@ public sealed class TextSearchProvider : AIContextProvider
         Throw.IfLessThan(this._options.RecentMessageMemoryLimit, 0);
         this._logger = loggerFactory?.CreateLogger<TextSearchProvider>();
         this._recentMessagesText = new();
+        this._recentMessageRolesIncluded = this._options.RecentMessageRolesIncluded ?? [ChatRole.User];
 
         // Create the on-demand search tool (only used if behavior is OnDemandFunctionCalling)
         this._tools =
@@ -91,6 +93,7 @@ public sealed class TextSearchProvider : AIContextProvider
         this._options = options ?? new();
         Throw.IfLessThan(this._options.RecentMessageMemoryLimit, 0);
         this._logger = loggerFactory?.CreateLogger<TextSearchProvider>();
+        this._recentMessageRolesIncluded = this._options.RecentMessageRolesIncluded ?? [ChatRole.User];
 
         List<string>? restoredMessages = null;
 
@@ -163,7 +166,7 @@ public sealed class TextSearchProvider : AIContextProvider
 
         return new AIContext
         {
-            Messages = [new ChatMessage(ChatRole.User, formatted)]
+            Messages = [new ChatMessage(ChatRole.User, formatted) { AdditionalProperties = new AdditionalPropertiesDictionary() { ["IsTextSearchProviderOutput"] = true } }]
         };
     }
 
@@ -183,7 +186,12 @@ public sealed class TextSearchProvider : AIContextProvider
 
         var messagesText = context.RequestMessages
             .Concat(context.ResponseMessages ?? [])
-            .Where(m => (m.Role == ChatRole.User || m.Role == ChatRole.Assistant) && !string.IsNullOrWhiteSpace(m.Text))
+            .Where(m =>
+                this._recentMessageRolesIncluded.Contains(m.Role) &&
+                !string.IsNullOrWhiteSpace(m.Text) &&
+                // Filter out any messages that were added by this class in InvokingAsync, since we don't want
+                // a feedback loop where previous search results are used to find new search results.
+                (m.AdditionalProperties == null || m.AdditionalProperties.TryGetValue("IsTextSearchProviderOutput", out bool isTextSearchProviderOutput) == false || !isTextSearchProviderOutput))
             .Select(m => m.Text)
             .ToList();
         if (messagesText.Count > limit)
@@ -262,15 +270,15 @@ public sealed class TextSearchProvider : AIContextProvider
         for (int i = 0; i < results.Count; i++)
         {
             var result = results[i];
-            if (!string.IsNullOrWhiteSpace(result.Name))
+            if (!string.IsNullOrWhiteSpace(result.SourceName))
             {
-                sb.AppendLine($"SourceDocName: {result.Name}");
+                sb.AppendLine($"SourceDocName: {result.SourceName}");
             }
-            if (!string.IsNullOrWhiteSpace(result.Link))
+            if (!string.IsNullOrWhiteSpace(result.SourceLink))
             {
-                sb.AppendLine($"SourceDocLink: {result.Link}");
+                sb.AppendLine($"SourceDocLink: {result.SourceLink}");
             }
-            sb.AppendLine($"Contents: {result.Value}");
+            sb.AppendLine($"Contents: {result.Text}");
             sb.AppendLine("----");
         }
         sb.AppendLine(this._options.CitationsPrompt ?? DefaultCitationsPrompt);
@@ -286,17 +294,17 @@ public sealed class TextSearchProvider : AIContextProvider
         /// <summary>
         /// Gets or sets the display name of the source document (optional).
         /// </summary>
-        public string? Name { get; set; }
+        public string? SourceName { get; set; }
 
         /// <summary>
         /// Gets or sets a link/URL to the source document (optional).
         /// </summary>
-        public string? Link { get; set; }
+        public string? SourceLink { get; set; }
 
         /// <summary>
         /// Gets or sets the textual content of the retrieved chunk.
         /// </summary>
-        public string Value { get; set; } = string.Empty;
+        public string Text { get; set; } = string.Empty;
 
         /// <summary>
         /// Gets or sets the raw representation of the search result from the data source.

dotnet/src/Microsoft.Agents.AI/Data/TextSearchProviderOptions.cs

@@ -59,6 +59,26 @@ public sealed class TextSearchProviderOptions
     /// </value>
     public int RecentMessageMemoryLimit { get; set; }
 
+    /// <summary>
+    /// Gets or sets the list of <see cref="ChatRole"/> types to filter recent messages to
+    /// when deciding which recent messages to include when constructing the search input.
+    /// </summary>
+    /// <remarks>
+    /// <para>
+    /// Depending on your scenario, you may want to use only user messages, only assistant messages,
+    /// or both. For example, if the assistant may often provide clarifying questions or if the conversation
+    /// is expected to be particularly chatty, you may want to include assistant messages in the search context as well.
+    /// </para>
+    /// <para>
+    /// Be careful when including assistant messages though, as they may skew the search results towards
+    /// information that has already been provided by the assistant, rather than focusing on the user's current needs.
+    /// </para>
+    /// </remarks>
+    /// <value>
+    /// When not specified, defaults to only <see cref="ChatRole.User"/>.
+    /// </value>
+    public List<ChatRole>? RecentMessageRolesIncluded { get; set; }
+
     /// <summary>
     /// Behavior choices for the provider.
     /// </summary>

dotnet/src/Microsoft.Agents.AI/Microsoft.Agents.AI.csproj

@@ -19,6 +19,7 @@
   <ItemGroup>
     <ProjectReference Include="..\Microsoft.Agents.AI.Abstractions\Microsoft.Agents.AI.Abstractions.csproj" />
     <PackageReference Include="Microsoft.Extensions.AI" />
+    <PackageReference Include="Microsoft.Extensions.VectorData.Abstractions" />
     <PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" />
     <PackageReference Include="System.Diagnostics.DiagnosticSource" />
   </ItemGroup>
@@ -31,6 +32,7 @@
 
   <ItemGroup>
     <InternalsVisibleTo Include="Microsoft.Agents.AI.UnitTests" />
+    <InternalsVisibleTo Include="DynamicProxyGenAssembly2, PublicKey=0024000004800000940000000602000000240000525341310004000001000100c547cac37abd99c8db225ef2f6c8a3602f3b3606cc9891605d02baa56104f4cfc0734aa39b93bf7852f7d9266654753cc297e7d2edfe0bac1cdcf9f717241550e0a7b191195b7667bb4f64bcb8e2121380fd1d9d46ad2d92d2d15605093924cceaf74c4861eff62abf69b9291ed0a340e113be11e6a7d3113e92484cf7045cc7"/>
   </ItemGroup>
 
 </Project>