From ada5b83c8087ad0d11d5d4bdd7b444c9f39c4ff3 Mon Sep 17 00:00:00 2001
From: westey <164392973+westey-m@users.noreply.github.com>
Date: Tue, 4 Nov 2025 16:24:26 +0000
Subject: [PATCH] .NET: Add rag samples with sample TextSearchStore (#1664)
* Port store for adding text to a vector store to AF
* Fix typo.
* Change TextSearchStore to sample, and add sample to use it and do rag with a custom schema
* Add more tests and fix broken ones
* Fix merge issue
* Fix sample after merge.
* Convert TextSearchStore to use Dynamic mode to be AOT compatible.
* Add some more clarification on when to use assistant messages in rag searches.
---
dotnet/Directory.Packages.props | 2 +
dotnet/agent-framework-dotnet.slnx | 5 +
.../Catalog/AgentWithTextSearchRag/Program.cs | 18 +-
.../AgentWithRAG_Step01_BasicTextRAG.csproj | 22 +
.../Program.cs | 107 +++++
.../TextSearchStore/TextSearchDocument.cs | 51 +++
.../TextSearchStore/TextSearchStore.cs | 392 ++++++++++++++++++
.../TextSearchStore/TextSearchStoreOptions.cs | 140 +++++++
.../TextSearchStoreUpsertOptions.cs | 17 +
...ithRAG_Step02_ExternalDataSourceRAG.csproj | 22 +
.../Program.cs | 134 ++++++
.../README.md | 60 +++
.../GettingStarted/AgentWithRAG/README.md | 8 +
.../Agent_Step18_TextSearchRag/Program.cs | 22 +-
.../Data/TextSearchProvider.cs | 28 +-
.../Data/TextSearchProviderOptions.cs | 20 +
.../Microsoft.Agents.AI.csproj | 2 +
.../Data/TextSearchProviderTests.cs | 71 +++-
18 files changed, 1079 insertions(+), 42 deletions(-)
create mode 100644 dotnet/samples/GettingStarted/AgentWithRAG/AgentWithRAG_Step01_BasicTextRAG/AgentWithRAG_Step01_BasicTextRAG.csproj
create mode 100644 dotnet/samples/GettingStarted/AgentWithRAG/AgentWithRAG_Step01_BasicTextRAG/Program.cs
create mode 100644 dotnet/samples/GettingStarted/AgentWithRAG/AgentWithRAG_Step01_BasicTextRAG/TextSearchStore/TextSearchDocument.cs
create mode 100644 dotnet/samples/GettingStarted/AgentWithRAG/AgentWithRAG_Step01_BasicTextRAG/TextSearchStore/TextSearchStore.cs
create mode 100644 dotnet/samples/GettingStarted/AgentWithRAG/AgentWithRAG_Step01_BasicTextRAG/TextSearchStore/TextSearchStoreOptions.cs
create mode 100644 dotnet/samples/GettingStarted/AgentWithRAG/AgentWithRAG_Step01_BasicTextRAG/TextSearchStore/TextSearchStoreUpsertOptions.cs
create mode 100644 dotnet/samples/GettingStarted/AgentWithRAG/AgentWithRAG_Step02_ExternalDataSourceRAG/AgentWithRAG_Step02_ExternalDataSourceRAG.csproj
create mode 100644 dotnet/samples/GettingStarted/AgentWithRAG/AgentWithRAG_Step02_ExternalDataSourceRAG/Program.cs
create mode 100644 dotnet/samples/GettingStarted/AgentWithRAG/AgentWithRAG_Step02_ExternalDataSourceRAG/README.md
create mode 100644 dotnet/samples/GettingStarted/AgentWithRAG/README.md
diff --git a/dotnet/Directory.Packages.props b/dotnet/Directory.Packages.props
index 08532904c4..69d3e03d31 100644
--- a/dotnet/Directory.Packages.props
+++ b/dotnet/Directory.Packages.props
@@ -65,9 +65,11 @@
+
+
diff --git a/dotnet/agent-framework-dotnet.slnx b/dotnet/agent-framework-dotnet.slnx
index 1bfb5814f9..de8aef42fc 100644
--- a/dotnet/agent-framework-dotnet.slnx
+++ b/dotnet/agent-framework-dotnet.slnx
@@ -67,6 +67,11 @@
+
+
+
+
+
diff --git a/dotnet/samples/Catalog/AgentWithTextSearchRag/Program.cs b/dotnet/samples/Catalog/AgentWithTextSearchRag/Program.cs
index 931ce50014..3e86534edd 100644
--- a/dotnet/samples/Catalog/AgentWithTextSearchRag/Program.cs
+++ b/dotnet/samples/Catalog/AgentWithTextSearchRag/Program.cs
@@ -52,9 +52,9 @@ static Task> MockSearchAsync(st
{
results.Add(new()
{
- Name = "Contoso Outdoors Return Policy",
- Link = "https://contoso.com/policies/returns",
- Value = "Customers may return any item within 30 days of delivery. Items should be unused and include original packaging. Refunds are issued to the original payment method within 5 business days of inspection."
+ SourceName = "Contoso Outdoors Return Policy",
+ SourceLink = "https://contoso.com/policies/returns",
+ Text = "Customers may return any item within 30 days of delivery. Items should be unused and include original packaging. Refunds are issued to the original payment method within 5 business days of inspection."
});
}
@@ -62,9 +62,9 @@ static Task> MockSearchAsync(st
{
results.Add(new()
{
- Name = "Contoso Outdoors Shipping Guide",
- Link = "https://contoso.com/help/shipping",
- Value = "Standard shipping is free on orders over $50 and typically arrives in 3-5 business days within the continental United States. Expedited options are available at checkout."
+ SourceName = "Contoso Outdoors Shipping Guide",
+ SourceLink = "https://contoso.com/help/shipping",
+ Text = "Standard shipping is free on orders over $50 and typically arrives in 3-5 business days within the continental United States. Expedited options are available at checkout."
});
}
@@ -72,9 +72,9 @@ static Task> MockSearchAsync(st
{
results.Add(new()
{
- Name = "TrailRunner Tent Care Instructions",
- Link = "https://contoso.com/manuals/trailrunner-tent",
- Value = "Clean the tent fabric with lukewarm water and a non-detergent soap. Allow it to air dry completely before storage and avoid prolonged UV exposure to extend the lifespan of the waterproof coating."
+ SourceName = "TrailRunner Tent Care Instructions",
+ SourceLink = "https://contoso.com/manuals/trailrunner-tent",
+ Text = "Clean the tent fabric with lukewarm water and a non-detergent soap. Allow it to air dry completely before storage and avoid prolonged UV exposure to extend the lifespan of the waterproof coating."
});
}
diff --git a/dotnet/samples/GettingStarted/AgentWithRAG/AgentWithRAG_Step01_BasicTextRAG/AgentWithRAG_Step01_BasicTextRAG.csproj b/dotnet/samples/GettingStarted/AgentWithRAG/AgentWithRAG_Step01_BasicTextRAG/AgentWithRAG_Step01_BasicTextRAG.csproj
new file mode 100644
index 0000000000..0c8a9f2dfc
--- /dev/null
+++ b/dotnet/samples/GettingStarted/AgentWithRAG/AgentWithRAG_Step01_BasicTextRAG/AgentWithRAG_Step01_BasicTextRAG.csproj
@@ -0,0 +1,22 @@
+
+
+
+ Exe
+ net9.0
+
+ enable
+ enable
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/dotnet/samples/GettingStarted/AgentWithRAG/AgentWithRAG_Step01_BasicTextRAG/Program.cs b/dotnet/samples/GettingStarted/AgentWithRAG/AgentWithRAG_Step01_BasicTextRAG/Program.cs
new file mode 100644
index 0000000000..611e11c22c
--- /dev/null
+++ b/dotnet/samples/GettingStarted/AgentWithRAG/AgentWithRAG_Step01_BasicTextRAG/Program.cs
@@ -0,0 +1,107 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+// This sample shows how to use TextSearchProvider to add retrieval augmented generation (RAG) capabilities to an AI agent.
+// The sample uses an In-Memory vector store, which can easily be replaced with any other vector store that implements the Microsoft.Extensions.VectorData abstractions.
+// The TextSearchProvider runs a search against the vector store via the TextSearchStore before each model invocation and injects the results into the model context.
+// The TextSearchStore is a sample store implementation that hardcodes a storage schema and uses the vector store to store and retrieve documents.
+
+using Azure.AI.OpenAI;
+using Azure.Identity;
+using Microsoft.Agents.AI;
+using Microsoft.Agents.AI.Data;
+using Microsoft.Agents.AI.Samples;
+using Microsoft.Extensions.AI;
+using Microsoft.Extensions.VectorData;
+using Microsoft.SemanticKernel.Connectors.InMemory;
+using OpenAI;
+
+var endpoint = Environment.GetEnvironmentVariable("AZURE_OPENAI_ENDPOINT") ?? throw new InvalidOperationException("AZURE_OPENAI_ENDPOINT is not set.");
+var deploymentName = Environment.GetEnvironmentVariable("AZURE_OPENAI_DEPLOYMENT_NAME") ?? "gpt-4o-mini";
+var embeddingDeploymentName = Environment.GetEnvironmentVariable("AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME") ?? "text-embedding-3-large";
+
+AzureOpenAIClient azureOpenAIClient = new(
+ new Uri(endpoint),
+ new AzureCliCredential());
+
+// Create an In-Memory vector store that uses the Azure OpenAI embedding model to generate embeddings.
+VectorStore vectorStore = new InMemoryVectorStore(new()
+{
+ EmbeddingGenerator = azureOpenAIClient.GetEmbeddingClient(embeddingDeploymentName).AsIEmbeddingGenerator()
+});
+
+// Create a store that defines a storage schema, and uses the vector store to store and retrieve documents.
+TextSearchStore textSearchStore = new(vectorStore, "product-and-policy-info", 3072);
+
+// Upload sample documents into the store.
+await textSearchStore.UpsertDocumentsAsync(GetSampleDocuments());
+
+// Create an adapter function that the TextSearchProvider can use to run searches against the TextSearchStore.
+Func>> SearchAdapter = async (text, ct) =>
+{
+ // Here we are limiting the search results to the single top result to demonstrate that we are accurately matching
+ // specific search results for each question, but in a real world case, more results should be used.
+ var searchResults = await textSearchStore.SearchAsync(text, 1, ct);
+ return searchResults.Select(r => new TextSearchProvider.TextSearchResult
+ {
+ SourceName = r.SourceName,
+ SourceLink = r.SourceLink,
+ Text = r.Text ?? string.Empty,
+ RawRepresentation = r
+ });
+};
+
+// Configure the options for the TextSearchProvider.
+TextSearchProviderOptions textSearchOptions = new()
+{
+ // Run the search prior to every model invocation.
+ SearchTime = TextSearchProviderOptions.TextSearchBehavior.BeforeAIInvoke,
+};
+
+// Create the AI agent with the TextSearchProvider as the AI context provider.
+AIAgent agent = azureOpenAIClient
+ .GetChatClient(deploymentName)
+ .CreateAIAgent(new ChatClientAgentOptions
+ {
+ Instructions = "You are a helpful support specialist for Contoso Outdoors. Answer questions using the provided context and cite the source document when available.",
+ AIContextProviderFactory = ctx => ctx.SerializedState.ValueKind is not System.Text.Json.JsonValueKind.Null and not System.Text.Json.JsonValueKind.Undefined
+ ? new TextSearchProvider(SearchAdapter, ctx.SerializedState, ctx.JsonSerializerOptions, textSearchOptions)
+ : new TextSearchProvider(SearchAdapter, textSearchOptions)
+ });
+
+AgentThread thread = agent.GetNewThread();
+
+Console.WriteLine(">> Asking about returns\n");
+Console.WriteLine(await agent.RunAsync("Hi! I need help understanding the return policy.", thread));
+
+Console.WriteLine("\n>> Asking about shipping\n");
+Console.WriteLine(await agent.RunAsync("How long does standard shipping usually take?", thread));
+
+Console.WriteLine("\n>> Asking about product care\n");
+Console.WriteLine(await agent.RunAsync("What is the best way to maintain the TrailRunner tent fabric?", thread));
+
+// Produces some sample search documents.
+// Each one contains a source name and link, which the agent can use to cite sources in its responses.
+static IEnumerable GetSampleDocuments()
+{
+ yield return new TextSearchDocument
+ {
+ SourceId = "return-policy-001",
+ SourceName = "Contoso Outdoors Return Policy",
+ SourceLink = "https://contoso.com/policies/returns",
+ Text = "Customers may return any item within 30 days of delivery. Items should be unused and include original packaging. Refunds are issued to the original payment method within 5 business days of inspection."
+ };
+ yield return new TextSearchDocument
+ {
+ SourceId = "shipping-guide-001",
+ SourceName = "Contoso Outdoors Shipping Guide",
+ SourceLink = "https://contoso.com/help/shipping",
+ Text = "Standard shipping is free on orders over $50 and typically arrives in 3-5 business days within the continental United States. Expedited options are available at checkout."
+ };
+ yield return new TextSearchDocument
+ {
+ SourceId = "tent-care-001",
+ SourceName = "TrailRunner Tent Care Instructions",
+ SourceLink = "https://contoso.com/manuals/trailrunner-tent",
+ Text = "Clean the tent fabric with lukewarm water and a non-detergent soap. Allow it to air dry completely before storage and avoid prolonged UV exposure to extend the lifespan of the waterproof coating."
+ };
+}
diff --git a/dotnet/samples/GettingStarted/AgentWithRAG/AgentWithRAG_Step01_BasicTextRAG/TextSearchStore/TextSearchDocument.cs b/dotnet/samples/GettingStarted/AgentWithRAG/AgentWithRAG_Step01_BasicTextRAG/TextSearchStore/TextSearchDocument.cs
new file mode 100644
index 0000000000..773d3ff6f3
--- /dev/null
+++ b/dotnet/samples/GettingStarted/AgentWithRAG/AgentWithRAG_Step01_BasicTextRAG/TextSearchStore/TextSearchDocument.cs
@@ -0,0 +1,51 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+namespace Microsoft.Agents.AI.Samples;
+
+///
+/// Represents a document that can be used for Retrieval Augmented Generation (RAG) that stores textual data.
+///
+public sealed class TextSearchDocument
+{
+ ///
+ /// Gets or sets an optional list of namespaces that the document should belong to.
+ ///
+ ///
+ /// A namespace is a logical grouping of documents, e.g. may include a group id to scope the document to a specific group of users.
+ ///
+ public IList Namespaces { get; set; } = [];
+
+ ///
+ /// Gets or sets the content as text.
+ ///
+ public string? Text { get; set; }
+
+ ///
+ /// Gets or sets an optional source ID for the document.
+ ///
+ ///
+ /// This ID should be unique within the collection that the document is stored in, and can
+ /// be used to map back to the source artifact for this document.
+ /// If updates need to be made later or the source document was deleted and this document
+ /// also needs to be deleted, this id can be used to find the document again.
+ ///
+ public string? SourceId { get; set; }
+
+ ///
+ /// Gets or sets an optional name for the source document.
+ ///
+ ///
+ /// This can be used to provide display names for citation links when the document is referenced as
+ /// part of a response to a query.
+ ///
+ public string? SourceName { get; set; }
+
+ ///
+ /// Gets or sets an optional link back to the source of the document.
+ ///
+ ///
+ /// This can be used to provide citation links when the document is referenced as
+ /// part of a response to a query.
+ ///
+ public string? SourceLink { get; set; }
+}
diff --git a/dotnet/samples/GettingStarted/AgentWithRAG/AgentWithRAG_Step01_BasicTextRAG/TextSearchStore/TextSearchStore.cs b/dotnet/samples/GettingStarted/AgentWithRAG/AgentWithRAG_Step01_BasicTextRAG/TextSearchStore/TextSearchStore.cs
new file mode 100644
index 0000000000..502c17dba1
--- /dev/null
+++ b/dotnet/samples/GettingStarted/AgentWithRAG/AgentWithRAG_Step01_BasicTextRAG/TextSearchStore/TextSearchStore.cs
@@ -0,0 +1,392 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System.Linq.Expressions;
+using System.Text.RegularExpressions;
+using Microsoft.Extensions.VectorData;
+
+namespace Microsoft.Agents.AI.Samples;
+
+///
+/// A class that allows for easy storage and retrieval of documents in a Vector Store for Retrieval Augmented Generation (RAG).
+///
+///
+///
+/// This class provides an opinionated schema for storing documents in a vector store. It is valuable for simple scenarios
+/// where you want to store text + embedding, or a reference to an external document + embedding without needing to customize the schema.
+/// If you want to control the schema yourself, use an implementation of directly instead.
+///
+///
+/// This class and its related types are currently provided as a sample implementation, but may be promoted to a first-class supported API in future releases.
+///
+///
+public sealed partial class TextSearchStore : IDisposable
+{
+#if NET
+ [GeneratedRegex(@"\p{L}+", RegexOptions.IgnoreCase, "en-US")]
+ private static partial Regex AnyLanguageWordRegex();
+
+ private static readonly Func> s_defaultWordSegmenter = text => AnyLanguageWordRegex().Matches(text).Select(x => x.Value).ToList();
+#else
+ private static readonly Regex s_anyLanguageWordRegex = new(@"\p{L}+", RegexOptions.Compiled);
+ private static Regex AnyLanguageWordRegex() => s_anyLanguageWordRegex;
+
+ private static readonly Func> s_defaultWordSegmenter = text =>
+ {
+ List words = new();
+ foreach (Match word in AnyLanguageWordRegex().Matches(text))
+ {
+ words.Add(word.Value);
+ }
+ return words;
+ };
+#endif
+
+ private readonly VectorStore _vectorStore;
+ private readonly TextSearchStoreOptions _options;
+ private readonly Func> _wordSegmenter;
+
+ private readonly VectorStoreCollection> _vectorStoreRecordCollection;
+ private readonly SemaphoreSlim _collectionInitializationLock = new(1, 1);
+ private bool _collectionInitialized;
+ private bool _disposedValue;
+
+ ///
+ /// Initializes a new instance of the class.
+ ///
+ /// The vector store to store and read the memories from.
+ /// The name of the collection in the vector store to store and read the memories from.
+ /// The number of dimensions to use for the memory embeddings.
+ /// Options to configure the behavior of this class.
+ /// Thrown if the key type provided is not supported.
+ public TextSearchStore(
+ VectorStore vectorStore,
+ string collectionName,
+ int vectorDimensions,
+ TextSearchStoreOptions? options = default)
+ {
+ // Verify
+ if (vectorStore is null)
+ {
+ throw new ArgumentNullException(nameof(vectorStore));
+ }
+
+ if (string.IsNullOrWhiteSpace(collectionName))
+ {
+ throw new ArgumentException("Collection name cannot be null or whitespace.", nameof(collectionName));
+ }
+
+ if (vectorDimensions < 1)
+ {
+ throw new ArgumentOutOfRangeException(nameof(vectorDimensions), "Vector dimensions must be greater than zero.");
+ }
+
+ if (options?.KeyType is not null && options.KeyType != typeof(string) && options.KeyType != typeof(Guid))
+ {
+ throw new NotSupportedException($"Unsupported key of type '{options.KeyType.Name}'");
+ }
+
+ if (options?.KeyType is not null && options.KeyType != typeof(string) && options?.UseSourceIdAsPrimaryKey is true)
+ {
+ throw new NotSupportedException($"The {nameof(TextSearchStoreOptions.UseSourceIdAsPrimaryKey)} option can only be used when the key type is 'string'.");
+ }
+
+ // Assign
+ this._vectorStore = vectorStore;
+ this._options = options ?? new TextSearchStoreOptions();
+ this._wordSegmenter = this._options.WordSegmenter ?? s_defaultWordSegmenter;
+
+ // Create a definition so that we can use the dimensions provided at runtime.
+ VectorStoreCollectionDefinition ragDocumentDefinition = new()
+ {
+ Properties = new List()
+ {
+ new VectorStoreKeyProperty("Key", this._options.KeyType ?? typeof(string)),
+ new VectorStoreDataProperty("Namespaces", typeof(List)) { IsIndexed = true },
+ new VectorStoreDataProperty("SourceId", typeof(string)) { IsIndexed = true },
+ new VectorStoreDataProperty("Text", typeof(string)) { IsFullTextIndexed = true },
+ new VectorStoreDataProperty("SourceName", typeof(string)),
+ new VectorStoreDataProperty("SourceLink", typeof(string)),
+ new VectorStoreVectorProperty("TextEmbedding", typeof(string), vectorDimensions),
+ }
+ };
+
+ this._vectorStoreRecordCollection = this._vectorStore.GetDynamicCollection(collectionName, ragDocumentDefinition);
+ }
+
+ ///
+ /// Upserts a batch of text chunks into the vector store.
+ ///
+ /// The text chunks to upload.
+ /// The to monitor for cancellation requests. The default is .
+ /// A task that completes when the documents have been upserted.
+ public async Task UpsertTextAsync(IEnumerable textChunks, CancellationToken cancellationToken = default)
+ {
+ if (textChunks == null)
+ {
+ throw new ArgumentNullException(nameof(textChunks));
+ }
+
+ var vectorStoreRecordCollection = await this.EnsureCollectionExistsAsync(cancellationToken).ConfigureAwait(false);
+
+ var storageDocuments = textChunks.Select(textChunk =>
+ {
+ // Without text we cannot generate a vector.
+ if (string.IsNullOrWhiteSpace(textChunk))
+ {
+ throw new ArgumentException("One of the provided text chunks is null.", nameof(textChunks));
+ }
+
+ return new Dictionary
+ {
+ { "Key", this.GenerateUniqueKey(null) },
+ { "Namespaces", new List() },
+ { "Text", textChunk },
+ { "TextEmbedding", textChunk },
+ };
+ });
+
+ await vectorStoreRecordCollection.UpsertAsync(storageDocuments, cancellationToken).ConfigureAwait(false);
+ }
+
+ ///
+ /// Upserts a batch of documents into the vector store.
+ ///
+ /// The documents to upload.
+ /// Optional options to control the upsert behavior.
+ /// The to monitor for cancellation requests. The default is .
+ /// A task that completes when the documents have been upserted.
+ public async Task UpsertDocumentsAsync(IEnumerable documents, TextSearchStoreUpsertOptions? options = null, CancellationToken cancellationToken = default)
+ {
+ if (documents is null)
+ {
+ throw new ArgumentNullException(nameof(documents));
+ }
+
+ var vectorStoreRecordCollection = await this.EnsureCollectionExistsAsync(cancellationToken).ConfigureAwait(false);
+
+ var storageDocuments = documents.Select(document =>
+ {
+ if (document is null)
+ {
+ throw new ArgumentNullException(nameof(documents), "One of the provided documents is null.");
+ }
+
+ // Without text we cannot generate a vector.
+ if (string.IsNullOrWhiteSpace(document.Text))
+ {
+ throw new ArgumentException($"The {nameof(TextSearchDocument.Text)} property must be set.", nameof(document));
+ }
+
+ // If we aren't persisting the text, we need a source id or link to refer back to the original document.
+ if (options?.DoNotPersistSourceText is true && string.IsNullOrWhiteSpace(document.SourceId) && string.IsNullOrWhiteSpace(document.SourceLink))
+ {
+ throw new ArgumentException($"Either the {nameof(TextSearchDocument.SourceId)} or {nameof(TextSearchDocument.SourceLink)} properties must be set when the {nameof(TextSearchStoreUpsertOptions.DoNotPersistSourceText)} setting is true.", nameof(document));
+ }
+
+ var key = this.GenerateUniqueKey(this._options.UseSourceIdAsPrimaryKey ?? false ? document.SourceId : null);
+
+ return new Dictionary()
+ {
+ { "Key", key },
+ { "Namespaces", document.Namespaces.ToList() },
+ { "SourceId", document.SourceId },
+ { "Text", options?.DoNotPersistSourceText is true ? null : document.Text },
+ { "SourceName", document.SourceName },
+ { "SourceLink", document.SourceLink },
+ { "TextEmbedding", document.Text },
+ };
+ });
+
+ await vectorStoreRecordCollection.UpsertAsync(storageDocuments, cancellationToken).ConfigureAwait(false);
+ }
+
+ ///
+ /// Search the database for documents similar to the provided query.
+ ///
+ /// The text query to find similar documents to.
+ /// The maximum number of results to return.
+ /// The to monitor for cancellation requests. The default is .
+ /// The search results.
+ public async Task> SearchAsync(string query, int top, CancellationToken cancellationToken = default)
+ {
+ var searchResult = await this.SearchCoreAsync(query, top, cancellationToken).ConfigureAwait(false);
+
+ return searchResult.Select(x => new TextSearchDocument()
+ {
+ Namespaces = (List)x["Namespaces"]!,
+ Text = (string?)x["Text"],
+ SourceId = (string?)x["SourceId"],
+ SourceName = (string?)x["SourceName"],
+ SourceLink = (string?)x["SourceLink"],
+ });
+ }
+
+ ///
+ /// Internal search implementation with hydration of id / link only storage.
+ ///
+ /// The text query to find similar documents to.
+ /// The maximum number of results to return.
+ /// The to monitor for cancellation requests. The default is .
+ /// The search results.
+ private async Task>> SearchCoreAsync(string query, int top, CancellationToken cancellationToken = default)
+ {
+ // Short circuit if the query is empty.
+ if (string.IsNullOrWhiteSpace(query))
+ {
+ return [];
+ }
+
+ var vectorStoreRecordCollection = await this.EnsureCollectionExistsAsync(cancellationToken).ConfigureAwait(false);
+
+ // If the user has not opted out of hybrid search, check if the vector store supports it.
+ var hybridSearchCollection = this._options.UseHybridSearch ?? true ?
+ vectorStoreRecordCollection.GetService(typeof(IKeywordHybridSearchable>)) as IKeywordHybridSearchable> :
+ null;
+
+ // Optional filter to limit the search to a specific namespace.
+ Expression, bool>>? filter = string.IsNullOrWhiteSpace(this._options.SearchNamespace) ? null : x => ((List)x["Namespaces"]!).Contains(this._options.SearchNamespace);
+
+ // Execute a hybrid search if possible, otherwise perform a regular vector search.
+ var searchResult = hybridSearchCollection is null
+ ? vectorStoreRecordCollection.SearchAsync(
+ query,
+ top,
+ options: new()
+ {
+ Filter = filter,
+ },
+ cancellationToken: cancellationToken)
+ : hybridSearchCollection.HybridSearchAsync(
+ query,
+ this._wordSegmenter(query),
+ top,
+ options: new()
+ {
+ Filter = filter,
+ },
+ cancellationToken: cancellationToken);
+
+ // Retrieve the documents from the search results.
+ List> searchResponseDocs = new();
+ await foreach (var searchResponseDoc in searchResult.WithCancellation(cancellationToken).ConfigureAwait(false))
+ {
+ searchResponseDocs.Add(searchResponseDoc.Record);
+ }
+
+ // Find any source ids and links for which the text needs to be retrieved.
+ var sourceIdsToRetrieve = searchResponseDocs
+ .Where(x => string.IsNullOrWhiteSpace((string?)x["Text"]))
+ .Select(x => new TextSearchStoreOptions.SourceRetrievalRequest((string?)x["SourceId"], (string?)x["SourceLink"]))
+ .ToList();
+
+ // If we have none, we can return early.
+ if (sourceIdsToRetrieve.Count == 0)
+ {
+ return searchResponseDocs;
+ }
+
+ if (this._options.SourceRetrievalCallback is null)
+ {
+ throw new InvalidOperationException($"The {nameof(TextSearchStoreOptions.SourceRetrievalCallback)} option must be set if retrieving documents without stored text.");
+ }
+
+ // Retrieve the source text for the documents that need it.
+ var retrievalResponses = await this._options.SourceRetrievalCallback(sourceIdsToRetrieve).ConfigureAwait(false);
+
+ if (retrievalResponses is null)
+ {
+ throw new InvalidOperationException($"The {nameof(TextSearchStoreOptions.SourceRetrievalCallback)} must return a non-null value.");
+ }
+
+ // Update the retrieved documents with the retrieved text.
+ return searchResponseDocs.GroupJoin(
+ retrievalResponses,
+ searchResponseDoc => (searchResponseDoc["SourceId"], searchResponseDoc["SourceLink"]),
+ retrievalResponse => (retrievalResponse.SourceId, retrievalResponse.SourceLink),
+ (searchResponseDoc, textRetrievalResponse) => (searchResponseDoc, textRetrievalResponse))
+ .SelectMany(
+ joinedSet => joinedSet.textRetrievalResponse.DefaultIfEmpty(),
+ (combined, textRetrievalResponse) =>
+ {
+ combined.searchResponseDoc["Text"] = textRetrievalResponse?.Text ?? combined.searchResponseDoc["Text"];
+ return combined.searchResponseDoc;
+ });
+ }
+
+ ///
+ /// Thread safe method to get the collection and ensure that it is created at least once.
+ ///
+ /// The to monitor for cancellation requests. The default is .
+ /// The created collection.
+ private async Task>> EnsureCollectionExistsAsync(CancellationToken cancellationToken)
+ {
+ // Return immediately if the collection is already created, no need to do any locking in this case.
+ if (this._collectionInitialized)
+ {
+ return this._vectorStoreRecordCollection;
+ }
+
+ // Wait on a lock to ensure that only one thread can create the collection.
+ await this._collectionInitializationLock.WaitAsync(cancellationToken).ConfigureAwait(false);
+
+ // If multiple threads waited on the lock, and the first already created the collection,
+ // we can return immediately without doing any work in subsequent threads.
+ if (this._collectionInitialized)
+ {
+ this._collectionInitializationLock.Release();
+ return this._vectorStoreRecordCollection;
+ }
+
+ // Only the winning thread should reach this point and create the collection.
+ try
+ {
+ await this._vectorStoreRecordCollection.EnsureCollectionExistsAsync(cancellationToken).ConfigureAwait(false);
+ this._collectionInitialized = true;
+ }
+ finally
+ {
+ this._collectionInitializationLock.Release();
+ }
+
+ return this._vectorStoreRecordCollection;
+ }
+
+ ///
+ /// Generates a unique key for the RAG document.
+ ///
+ /// Source id of the source document for this RAG document.
+ /// A new unique key.
+ /// Thrown if the requested key type is not supported.
+ private object GenerateUniqueKey(string? sourceId)
+ => this._options.KeyType switch
+ {
+ _ when (this._options.KeyType == null || this._options.KeyType == typeof(string)) && !string.IsNullOrWhiteSpace(sourceId) => sourceId!,
+ _ when this._options.KeyType == null || this._options.KeyType == typeof(string) => Guid.NewGuid().ToString(),
+ _ when this._options.KeyType == typeof(Guid) => Guid.NewGuid(),
+
+ _ => throw new NotSupportedException($"Unsupported key of type '{this._options.KeyType.Name}'")
+ };
+
+ ///
+ private void Dispose(bool disposing)
+ {
+ if (!this._disposedValue)
+ {
+ if (disposing)
+ {
+ this._vectorStoreRecordCollection.Dispose();
+ this._collectionInitializationLock.Dispose();
+ }
+
+ this._disposedValue = true;
+ }
+ }
+
+ ///
+ public void Dispose()
+ {
+ // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method
+ this.Dispose(disposing: true);
+ GC.SuppressFinalize(this);
+ }
+}
diff --git a/dotnet/samples/GettingStarted/AgentWithRAG/AgentWithRAG_Step01_BasicTextRAG/TextSearchStore/TextSearchStoreOptions.cs b/dotnet/samples/GettingStarted/AgentWithRAG/AgentWithRAG_Step01_BasicTextRAG/TextSearchStore/TextSearchStoreOptions.cs
new file mode 100644
index 0000000000..53da092c82
--- /dev/null
+++ b/dotnet/samples/GettingStarted/AgentWithRAG/AgentWithRAG_Step01_BasicTextRAG/TextSearchStore/TextSearchStoreOptions.cs
@@ -0,0 +1,140 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+namespace Microsoft.Agents.AI.Samples;
+
+///
+/// Contains options for the .
+///
+public sealed class TextSearchStoreOptions
+{
+ ///
+ /// Gets or sets an optional namespace to pre-filter the possible
+ /// records with when doing a vector search.
+ ///
+ public string? SearchNamespace { get; init; }
+
+ ///
+ /// Gets or sets a value indicating whether to use the source ID as the primary key for records.
+ ///
+ ///
+ ///
+ /// Using the source ID as the primary key allows for easy updates from the source for any changed
+ /// records, since those records can just be upserted again, and will overwrite the previous version
+ /// of the same record.
+ ///
+ ///
+ /// This setting can only be used when the chosen key type is a string.
+ ///
+ ///
+ ///
+ /// Defaults to false if not set.
+ ///
+ public bool? UseSourceIdAsPrimaryKey { get; init; }
+
+ ///
+ /// Gets or sets a value indicating whether to use hybrid search if it is available for the provided vector store.
+ ///
+ ///
+ /// Defaults to true if not set.
+ ///
+ public bool? UseHybridSearch { get; init; }
+
+ ///
+ /// Gets or sets a word segmenter function to split search text into separate words for the purposes of hybrid search.
+ /// This will not be used if is set to false.
+ ///
+ ///
+ /// Defaults to a simple text-character-based segmenter that splits the text by any character that is not a text character.
+ ///
+ public Func>? WordSegmenter { get; init; }
+
+ ///
+ /// Gets or sets the type of key to use for records in the text search store.
+ ///
+ ///
+ /// Make sure to pick a key type that is supported by the underlying vector store.
+ /// Note that you have to choose when using .
+ ///
+ /// Defaults to if not set. Only and is currently supported.
+ public Type? KeyType { get; init; }
+
+ ///
+ /// Gets or sets an optional callback to load the source text using the source id or source link
+ /// if the source text is not persisted in the database.
+ ///
+ ///
+ /// The response should include the source id or source link, as provided in the request,
+ /// plus the source text loaded from the source.
+ ///
+ public Func, Task>>? SourceRetrievalCallback { get; init; }
+
+ ///
+ /// Represents a request to the .
+ ///
+ public sealed class SourceRetrievalRequest
+ {
+ ///
+ /// Initializes a new instance of the class.
+ ///
+ /// The source ID of the document to retrieve.
+ /// The source link of the document to retrieve.
+ public SourceRetrievalRequest(string? sourceId, string? sourceLink)
+ {
+ this.SourceId = sourceId;
+ this.SourceLink = sourceLink;
+ }
+
+ ///
+ /// Gets or sets the source ID of the document to retrieve.
+ ///
+ public string? SourceId { get; set; }
+
+ ///
+ /// Gets or sets the source link of the document to retrieve.
+ ///
+ public string? SourceLink { get; set; }
+ }
+
+ ///
+ /// Represents a response from the .
+ ///
+ public sealed class SourceRetrievalResponse
+ {
+ ///
+ /// Initializes a new instance of the class.
+ ///
+ /// The request matching this response.
+ /// The source text that was retrieved.
+ public SourceRetrievalResponse(SourceRetrievalRequest request, string text)
+ {
+ if (request == null)
+ {
+ throw new ArgumentNullException(nameof(request));
+ }
+
+ if (text == null)
+ {
+ throw new ArgumentNullException(nameof(text));
+ }
+
+ this.SourceId = request.SourceId;
+ this.SourceLink = request.SourceLink;
+ this.Text = text;
+ }
+
+ ///
+ /// Gets or sets the source ID of the document that was retrieved.
+ ///
+ public string? SourceId { get; set; }
+
+ ///
+ /// Gets or sets the source link of the document that was retrieved.
+ ///
+ public string? SourceLink { get; set; }
+
+ ///
+ /// Gets or sets the source text of the document that was retrieved.
+ ///
+ public string Text { get; set; }
+ }
+}
diff --git a/dotnet/samples/GettingStarted/AgentWithRAG/AgentWithRAG_Step01_BasicTextRAG/TextSearchStore/TextSearchStoreUpsertOptions.cs b/dotnet/samples/GettingStarted/AgentWithRAG/AgentWithRAG_Step01_BasicTextRAG/TextSearchStore/TextSearchStoreUpsertOptions.cs
new file mode 100644
index 0000000000..127d7de01e
--- /dev/null
+++ b/dotnet/samples/GettingStarted/AgentWithRAG/AgentWithRAG_Step01_BasicTextRAG/TextSearchStore/TextSearchStoreUpsertOptions.cs
@@ -0,0 +1,17 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+namespace Microsoft.Agents.AI.Samples;
+
+///
+/// Contains options for .
+///
+public sealed class TextSearchStoreUpsertOptions
+{
+ ///
+ /// Gets or sets a value indicating whether the source text should be persisted in the database.
+ ///
+ ///
+ /// Defaults to if not set.
+ ///
+ public bool DoNotPersistSourceText { get; init; }
+}
diff --git a/dotnet/samples/GettingStarted/AgentWithRAG/AgentWithRAG_Step02_ExternalDataSourceRAG/AgentWithRAG_Step02_ExternalDataSourceRAG.csproj b/dotnet/samples/GettingStarted/AgentWithRAG/AgentWithRAG_Step02_ExternalDataSourceRAG/AgentWithRAG_Step02_ExternalDataSourceRAG.csproj
new file mode 100644
index 0000000000..56e2ad232b
--- /dev/null
+++ b/dotnet/samples/GettingStarted/AgentWithRAG/AgentWithRAG_Step02_ExternalDataSourceRAG/AgentWithRAG_Step02_ExternalDataSourceRAG.csproj
@@ -0,0 +1,22 @@
+
+
+
+ Exe
+ net9.0
+
+ enable
+ enable
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/dotnet/samples/GettingStarted/AgentWithRAG/AgentWithRAG_Step02_ExternalDataSourceRAG/Program.cs b/dotnet/samples/GettingStarted/AgentWithRAG/AgentWithRAG_Step02_ExternalDataSourceRAG/Program.cs
new file mode 100644
index 0000000000..e29bb58d04
--- /dev/null
+++ b/dotnet/samples/GettingStarted/AgentWithRAG/AgentWithRAG_Step02_ExternalDataSourceRAG/Program.cs
@@ -0,0 +1,134 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+// This sample shows how to use Qdrant to add retrieval augmented generation (RAG) capabilities to an AI agent.
+// While the sample is using Qdrant, it can easily be replaced with any other vector store that implements the Microsoft.Extensions.VectorData abstractions.
+// The TextSearchProvider runs a search against the vector store before each model invocation and injects the results into the model context.
+
+using Azure.AI.OpenAI;
+using Azure.Identity;
+using Microsoft.Agents.AI;
+using Microsoft.Agents.AI.Data;
+using Microsoft.Extensions.AI;
+using Microsoft.Extensions.VectorData;
+using Microsoft.SemanticKernel.Connectors.Qdrant;
+using OpenAI;
+using Qdrant.Client;
+
+var endpoint = Environment.GetEnvironmentVariable("AZURE_OPENAI_ENDPOINT") ?? throw new InvalidOperationException("AZURE_OPENAI_ENDPOINT is not set.");
+var deploymentName = Environment.GetEnvironmentVariable("AZURE_OPENAI_DEPLOYMENT_NAME") ?? "gpt-4o-mini";
+var embeddingDeploymentName = Environment.GetEnvironmentVariable("AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME") ?? "text-embedding-3-large";
+var afOverviewUrl = "https://github.com/MicrosoftDocs/semantic-kernel-docs/blob/main/agent-framework/overview/agent-framework-overview.md";
+var afMigrationUrl = "https://raw.githubusercontent.com/MicrosoftDocs/semantic-kernel-docs/refs/heads/main/agent-framework/migration-guide/from-semantic-kernel/index.md";
+
+AzureOpenAIClient azureOpenAIClient = new(
+ new Uri(endpoint),
+ new AzureCliCredential());
+
+// Create a Qdrant vector store that uses the Azure OpenAI embedding model to generate embeddings.
+QdrantClient client = new("localhost");
+VectorStore vectorStore = new QdrantVectorStore(client, ownsClient: true, new()
+{
+ EmbeddingGenerator = azureOpenAIClient.GetEmbeddingClient(embeddingDeploymentName).AsIEmbeddingGenerator()
+});
+
+// Create a collection and upsert some text into it.
+var documentationCollection = vectorStore.GetCollection("documentation");
+await documentationCollection.EnsureCollectionDeletedAsync(); // Clear out any data from previous runs.
+await documentationCollection.EnsureCollectionExistsAsync();
+await UploadDataFromMarkdown(afOverviewUrl, "Microsoft Agent Framework Overview", documentationCollection, 2000, 200);
+await UploadDataFromMarkdown(afMigrationUrl, "Semantic Kernel to Microsoft Agent Framework Migration Guide", documentationCollection, 2000, 200);
+
+// Create an adapter function that the TextSearchProvider can use to run searches against the collection.
+Func>> SearchAdapter = async (text, ct) =>
+{
+ List results = [];
+ await foreach (var result in documentationCollection.SearchAsync(text, 5, cancellationToken: ct))
+ {
+ results.Add(new TextSearchProvider.TextSearchResult
+ {
+ SourceName = result.Record.SourceName,
+ SourceLink = result.Record.SourceLink,
+ Text = result.Record.Text ?? string.Empty,
+ RawRepresentation = result
+ });
+ }
+ return results;
+};
+
+// Configure the options for the TextSearchProvider.
+TextSearchProviderOptions textSearchOptions = new()
+{
+ // Run the search prior to every model invocation.
+ SearchTime = TextSearchProviderOptions.TextSearchBehavior.BeforeAIInvoke,
+ // Use up to 4 recent messages when searching so that searches
+ // still produce valuable results even when the user is referring
+ // back to previous messages in their request.
+ RecentMessageMemoryLimit = 5
+};
+
+// Create the AI agent with the TextSearchProvider as the AI context provider.
+AIAgent agent = azureOpenAIClient
+ .GetChatClient(deploymentName)
+ .CreateAIAgent(new ChatClientAgentOptions
+ {
+ Instructions = "You are a helpful support specialist for the Microsoft Agent Framework. Answer questions using the provided context and cite the source document when available. Keep responses brief.",
+ AIContextProviderFactory = ctx => ctx.SerializedState.ValueKind is not System.Text.Json.JsonValueKind.Null and not System.Text.Json.JsonValueKind.Undefined
+ ? new TextSearchProvider(SearchAdapter, ctx.SerializedState, ctx.JsonSerializerOptions, textSearchOptions)
+ : new TextSearchProvider(SearchAdapter, textSearchOptions)
+ });
+
+AgentThread thread = agent.GetNewThread();
+
+Console.WriteLine(">> Asking about SK threads\n");
+Console.WriteLine(await agent.RunAsync("Hi! How do I create a thread in Semantic Kernel?", thread));
+
+// Here we are asking a very vague question when taken out of context,
+// but since we are including previous messages in our search using RecentMessageMemoryLimit
+// the RAG search should still produce useful results.
+Console.WriteLine("\n>> Asking about AF threads\n");
+Console.WriteLine(await agent.RunAsync("and in Agent Framework?", thread));
+
+Console.WriteLine("\n>> Contrasting Approaches\n");
+Console.WriteLine(await agent.RunAsync("Please contrast the two approaches", thread));
+
+Console.WriteLine("\n>> Asking about ancestry\n");
+Console.WriteLine(await agent.RunAsync("What are the predecessors to the Agent Framework?", thread));
+
+static async Task UploadDataFromMarkdown(string markdownUrl, string sourceName, VectorStoreCollection vectorStoreCollection, int chunkSize, int overlap)
+{
+ // Download the markdown from the given url.
+ using HttpClient client = new();
+ var markdown = await client.GetStringAsync(new Uri(markdownUrl));
+
+ // Chunk it into separate parts with some overlap between chunks
+ var chunks = new List();
+ for (int i = 0; i < markdown.Length; i += chunkSize)
+ {
+ var chunk = new DocumentationChunk
+ {
+ Key = Guid.NewGuid(),
+ SourceLink = markdownUrl,
+ SourceName = sourceName,
+ Text = markdown.Substring(i, Math.Min(chunkSize + overlap, markdown.Length - i))
+ };
+ chunks.Add(chunk);
+ }
+
+ // Upsert each chunk into the provided vector store.
+ await vectorStoreCollection.UpsertAsync(chunks);
+}
+
+// Data model that defines the database schema we want to use.
+internal sealed class DocumentationChunk
+{
+ [VectorStoreKey]
+ public Guid Key { get; set; }
+ [VectorStoreData]
+ public string SourceLink { get; set; } = string.Empty;
+ [VectorStoreData]
+ public string SourceName { get; set; } = string.Empty;
+ [VectorStoreData]
+ public string Text { get; set; } = string.Empty;
+ [VectorStoreVector(Dimensions: 3072)]
+ public string Embedding => this.Text;
+}
diff --git a/dotnet/samples/GettingStarted/AgentWithRAG/AgentWithRAG_Step02_ExternalDataSourceRAG/README.md b/dotnet/samples/GettingStarted/AgentWithRAG/AgentWithRAG_Step02_ExternalDataSourceRAG/README.md
new file mode 100644
index 0000000000..1817f0d8ca
--- /dev/null
+++ b/dotnet/samples/GettingStarted/AgentWithRAG/AgentWithRAG_Step02_ExternalDataSourceRAG/README.md
@@ -0,0 +1,60 @@
+# Agent Framework Retrieval Augmented Generation (RAG) with an external Vector Store with a custom schema
+
+This sample demonstrates how to create and run an agent that uses Retrieval Augmented Generation (RAG) with an external vector store.
+It also uses a custom schema for the documents stored in the vector store.
+This sample uses Qdrant for the vector store, but this can easily be swapped out for any vector store that has a Microsoft.Extensions.VectorStore implementation.
+
+## Prerequisites
+
+- .NET 8.0 SDK or later
+- Azure OpenAI service endpoint
+- Both a chat completion and embedding deployment configured in the Azure OpenAI resource
+- Azure CLI installed and authenticated (for Azure credential authentication)
+- User has the `Cognitive Services OpenAI Contributor` role for the Azure OpenAI resource.
+- An existing Qdrant instance. You can use a managed service or run a local instance using Docker, but the sample assumes the instance is running locally.
+
+**Note**: These samples use Azure OpenAI models. For more information, see [how to deploy Azure OpenAI models with Azure AI Foundry](https://learn.microsoft.com/en-us/azure/ai-foundry/how-to/deploy-models-openai).
+
+**Note**: These samples use Azure CLI credentials for authentication. Make sure you're logged in with `az login` and have access to the Azure OpenAI resource and have the `Cognitive Services OpenAI Contributor` role. For more information, see the [Azure CLI documentation](https://learn.microsoft.com/cli/azure/authenticate-azure-cli-interactively).
+
+## Running the sample from the console
+
+Set the following environment variables:
+
+```powershell
+$env:AZURE_OPENAI_ENDPOINT="https://your-resource.openai.azure.com/" # Replace with your Azure OpenAI resource endpoint
+$env:AZURE_OPENAI_DEPLOYMENT_NAME="gpt-4o-mini" # Optional, defaults to gpt-4o-mini
+$env:AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME="text-embedding-3-large" # Optional, defaults to text-embedding-3-large
+```
+
+If the variables are not set, you will be prompted for the values when running the samples.
+
+To use Qdrant in docker locally, start your Qdrant instance using the default port mappings.
+
+```powershell
+docker run -d --name qdrant -p 6333:6333 -p 6334:6334 qdrant/qdrant:latest
+```
+
+Execute the following command to build the sample:
+
+```powershell
+dotnet build
+```
+
+Execute the following command to run the sample:
+
+```powershell
+dotnet run --no-build
+```
+
+Or just build and run in one step:
+
+```powershell
+dotnet run
+```
+
+## Running the sample from Visual Studio
+
+Open the solution in Visual Studio and set the sample project as the startup project. Then, run the project using the built-in debugger or by pressing `F5`.
+
+You will be prompted for any required environment variables if they are not already set.
diff --git a/dotnet/samples/GettingStarted/AgentWithRAG/README.md b/dotnet/samples/GettingStarted/AgentWithRAG/README.md
new file mode 100644
index 0000000000..f45c2c2540
--- /dev/null
+++ b/dotnet/samples/GettingStarted/AgentWithRAG/README.md
@@ -0,0 +1,8 @@
+# Agent Framework Retrieval Augmented Generation (RAG)
+
+These samples show how to create an agent with the Agent Framework that uses Retrieval Augmented Generation (RAG) to enhance its responses with information from a knowledge base.
+
+|Sample|Description|
+|---|---|
+|[Basic Text RAG](./AgentWithRAG_Step01_BasicTextRAG/)|This sample demonstrates how to create and run a basic agent with simple text Retrieval Augmented Generation (RAG).|
+|[RAG with external Vector Store and custom schema](./AgentWithRAG_Step02_ExternalDataSourceRAG/)|This sample demonstrates how to create and run an agent that uses Retrieval Augmented Generation (RAG) with an external vector store. It also uses a custom schema for the documents stored in the vector store.|
diff --git a/dotnet/samples/GettingStarted/Agents/Agent_Step18_TextSearchRag/Program.cs b/dotnet/samples/GettingStarted/Agents/Agent_Step18_TextSearchRag/Program.cs
index 931ce50014..81a6b29152 100644
--- a/dotnet/samples/GettingStarted/Agents/Agent_Step18_TextSearchRag/Program.cs
+++ b/dotnet/samples/GettingStarted/Agents/Agent_Step18_TextSearchRag/Program.cs
@@ -28,7 +28,9 @@ AIAgent agent = new AzureOpenAIClient(
.CreateAIAgent(new ChatClientAgentOptions
{
Instructions = "You are a helpful support specialist for Contoso Outdoors. Answer questions using the provided context and cite the source document when available.",
- AIContextProviderFactory = _ => new TextSearchProvider(MockSearchAsync, textSearchOptions)
+ AIContextProviderFactory = ctx => ctx.SerializedState.ValueKind is not System.Text.Json.JsonValueKind.Null and not System.Text.Json.JsonValueKind.Undefined
+ ? new TextSearchProvider(MockSearchAsync, ctx.SerializedState, ctx.JsonSerializerOptions, textSearchOptions)
+ : new TextSearchProvider(MockSearchAsync, textSearchOptions)
});
AgentThread thread = agent.GetNewThread();
@@ -52,9 +54,9 @@ static Task> MockSearchAsync(st
{
results.Add(new()
{
- Name = "Contoso Outdoors Return Policy",
- Link = "https://contoso.com/policies/returns",
- Value = "Customers may return any item within 30 days of delivery. Items should be unused and include original packaging. Refunds are issued to the original payment method within 5 business days of inspection."
+ SourceName = "Contoso Outdoors Return Policy",
+ SourceLink = "https://contoso.com/policies/returns",
+ Text = "Customers may return any item within 30 days of delivery. Items should be unused and include original packaging. Refunds are issued to the original payment method within 5 business days of inspection."
});
}
@@ -62,9 +64,9 @@ static Task> MockSearchAsync(st
{
results.Add(new()
{
- Name = "Contoso Outdoors Shipping Guide",
- Link = "https://contoso.com/help/shipping",
- Value = "Standard shipping is free on orders over $50 and typically arrives in 3-5 business days within the continental United States. Expedited options are available at checkout."
+ SourceName = "Contoso Outdoors Shipping Guide",
+ SourceLink = "https://contoso.com/help/shipping",
+ Text = "Standard shipping is free on orders over $50 and typically arrives in 3-5 business days within the continental United States. Expedited options are available at checkout."
});
}
@@ -72,9 +74,9 @@ static Task> MockSearchAsync(st
{
results.Add(new()
{
- Name = "TrailRunner Tent Care Instructions",
- Link = "https://contoso.com/manuals/trailrunner-tent",
- Value = "Clean the tent fabric with lukewarm water and a non-detergent soap. Allow it to air dry completely before storage and avoid prolonged UV exposure to extend the lifespan of the waterproof coating."
+ SourceName = "TrailRunner Tent Care Instructions",
+ SourceLink = "https://contoso.com/manuals/trailrunner-tent",
+ Text = "Clean the tent fabric with lukewarm water and a non-detergent soap. Allow it to air dry completely before storage and avoid prolonged UV exposure to extend the lifespan of the waterproof coating."
});
}
diff --git a/dotnet/src/Microsoft.Agents.AI/Data/TextSearchProvider.cs b/dotnet/src/Microsoft.Agents.AI/Data/TextSearchProvider.cs
index c7cffd7c04..6858510312 100644
--- a/dotnet/src/Microsoft.Agents.AI/Data/TextSearchProvider.cs
+++ b/dotnet/src/Microsoft.Agents.AI/Data/TextSearchProvider.cs
@@ -45,6 +45,7 @@ public sealed class TextSearchProvider : AIContextProvider
private readonly AITool[] _tools;
private readonly Queue _recentMessagesText;
private readonly TextSearchProviderOptions _options;
+ private readonly List _recentMessageRolesIncluded;
///
/// Initializes a new instance of the class.
@@ -60,6 +61,7 @@ public sealed class TextSearchProvider : AIContextProvider
Throw.IfLessThan(this._options.RecentMessageMemoryLimit, 0);
this._logger = loggerFactory?.CreateLogger();
this._recentMessagesText = new();
+ this._recentMessageRolesIncluded = this._options.RecentMessageRolesIncluded ?? [ChatRole.User];
// Create the on-demand search tool (only used if behavior is OnDemandFunctionCalling)
this._tools =
@@ -91,6 +93,7 @@ public sealed class TextSearchProvider : AIContextProvider
this._options = options ?? new();
Throw.IfLessThan(this._options.RecentMessageMemoryLimit, 0);
this._logger = loggerFactory?.CreateLogger();
+ this._recentMessageRolesIncluded = this._options.RecentMessageRolesIncluded ?? [ChatRole.User];
List? restoredMessages = null;
@@ -163,7 +166,7 @@ public sealed class TextSearchProvider : AIContextProvider
return new AIContext
{
- Messages = [new ChatMessage(ChatRole.User, formatted)]
+ Messages = [new ChatMessage(ChatRole.User, formatted) { AdditionalProperties = new AdditionalPropertiesDictionary() { ["IsTextSearchProviderOutput"] = true } }]
};
}
@@ -183,7 +186,12 @@ public sealed class TextSearchProvider : AIContextProvider
var messagesText = context.RequestMessages
.Concat(context.ResponseMessages ?? [])
- .Where(m => (m.Role == ChatRole.User || m.Role == ChatRole.Assistant) && !string.IsNullOrWhiteSpace(m.Text))
+ .Where(m =>
+ this._recentMessageRolesIncluded.Contains(m.Role) &&
+ !string.IsNullOrWhiteSpace(m.Text) &&
+ // Filter out any messages that were added by this class in InvokingAsync, since we don't want
+ // a feedback loop where previous search results are used to find new search results.
+ (m.AdditionalProperties == null || m.AdditionalProperties.TryGetValue("IsTextSearchProviderOutput", out bool isTextSearchProviderOutput) == false || !isTextSearchProviderOutput))
.Select(m => m.Text)
.ToList();
if (messagesText.Count > limit)
@@ -262,15 +270,15 @@ public sealed class TextSearchProvider : AIContextProvider
for (int i = 0; i < results.Count; i++)
{
var result = results[i];
- if (!string.IsNullOrWhiteSpace(result.Name))
+ if (!string.IsNullOrWhiteSpace(result.SourceName))
{
- sb.AppendLine($"SourceDocName: {result.Name}");
+ sb.AppendLine($"SourceDocName: {result.SourceName}");
}
- if (!string.IsNullOrWhiteSpace(result.Link))
+ if (!string.IsNullOrWhiteSpace(result.SourceLink))
{
- sb.AppendLine($"SourceDocLink: {result.Link}");
+ sb.AppendLine($"SourceDocLink: {result.SourceLink}");
}
- sb.AppendLine($"Contents: {result.Value}");
+ sb.AppendLine($"Contents: {result.Text}");
sb.AppendLine("----");
}
sb.AppendLine(this._options.CitationsPrompt ?? DefaultCitationsPrompt);
@@ -286,17 +294,17 @@ public sealed class TextSearchProvider : AIContextProvider
///
/// Gets or sets the display name of the source document (optional).
///
- public string? Name { get; set; }
+ public string? SourceName { get; set; }
///
/// Gets or sets a link/URL to the source document (optional).
///
- public string? Link { get; set; }
+ public string? SourceLink { get; set; }
///
/// Gets or sets the textual content of the retrieved chunk.
///
- public string Value { get; set; } = string.Empty;
+ public string Text { get; set; } = string.Empty;
///
/// Gets or sets the raw representation of the search result from the data source.
diff --git a/dotnet/src/Microsoft.Agents.AI/Data/TextSearchProviderOptions.cs b/dotnet/src/Microsoft.Agents.AI/Data/TextSearchProviderOptions.cs
index e6b20ed6b2..7949d7918a 100644
--- a/dotnet/src/Microsoft.Agents.AI/Data/TextSearchProviderOptions.cs
+++ b/dotnet/src/Microsoft.Agents.AI/Data/TextSearchProviderOptions.cs
@@ -59,6 +59,26 @@ public sealed class TextSearchProviderOptions
///
public int RecentMessageMemoryLimit { get; set; }
+ ///
+ /// Gets or sets the list of types to filter recent messages to
+ /// when deciding which recent messages to include when constructing the search input.
+ ///
+ ///
+ ///
+ /// Depending on your scenario, you may want to use only user messages, only assistant messages,
+ /// or both. For example, if the assistant may often provide clarifying questions or if the conversation
+ /// is expected to be particularly chatty, you may want to include assistant messages in the search context as well.
+ ///
+ ///
+ /// Be careful when including assistant messages though, as they may skew the search results towards
+ /// information that has already been provided by the assistant, rather than focusing on the user's current needs.
+ ///
+ ///
+ ///
+ /// When not specified, defaults to only .
+ ///
+ public List? RecentMessageRolesIncluded { get; set; }
+
///
/// Behavior choices for the provider.
///
diff --git a/dotnet/src/Microsoft.Agents.AI/Microsoft.Agents.AI.csproj b/dotnet/src/Microsoft.Agents.AI/Microsoft.Agents.AI.csproj
index d95b8ea52f..a560dece67 100644
--- a/dotnet/src/Microsoft.Agents.AI/Microsoft.Agents.AI.csproj
+++ b/dotnet/src/Microsoft.Agents.AI/Microsoft.Agents.AI.csproj
@@ -19,6 +19,7 @@
+
@@ -31,6 +32,7 @@
+
diff --git a/dotnet/tests/Microsoft.Agents.AI.UnitTests/Data/TextSearchProviderTests.cs b/dotnet/tests/Microsoft.Agents.AI.UnitTests/Data/TextSearchProviderTests.cs
index 44423695eb..66abba6c8b 100644
--- a/dotnet/tests/Microsoft.Agents.AI.UnitTests/Data/TextSearchProviderTests.cs
+++ b/dotnet/tests/Microsoft.Agents.AI.UnitTests/Data/TextSearchProviderTests.cs
@@ -41,8 +41,8 @@ public sealed class TextSearchProviderTests
// Arrange
List results =
[
- new() { Name = "Doc1", Link = "http://example.com/doc1", Value = "Content of Doc1" },
- new() { Name = "Doc2", Link = "http://example.com/doc2", Value = "Content of Doc2" }
+ new() { SourceName = "Doc1", SourceLink = "http://example.com/doc1", Text = "Content of Doc1" },
+ new() { SourceName = "Doc2", SourceLink = "http://example.com/doc2", Text = "Content of Doc2" }
];
string? capturedInput = null;
@@ -158,8 +158,8 @@ public sealed class TextSearchProviderTests
// Arrange
List results =
[
- new() { Name = "Doc1", Link = "http://example.com/doc1", Value = "Content of Doc1" },
- new() { Name = "Doc2", Link = "http://example.com/doc2", Value = "Content of Doc2" }
+ new() { SourceName = "Doc1", SourceLink = "http://example.com/doc1", Text = "Content of Doc1" },
+ new() { SourceName = "Doc2", SourceLink = "http://example.com/doc2", Text = "Content of Doc2" }
];
Task> SearchDelegateAsync(string input, CancellationToken ct)
@@ -210,8 +210,8 @@ public sealed class TextSearchProviderTests
// Arrange
List results =
[
- new() { Name = "Doc1", Link = "http://example.com/doc1", Value = "Content of Doc1" },
- new() { Name = "Doc2", Link = "http://example.com/doc2", Value = "Content of Doc2" }
+ new() { SourceName = "Doc1", SourceLink = "http://example.com/doc1", Text = "Content of Doc1" },
+ new() { SourceName = "Doc2", SourceLink = "http://example.com/doc2", Text = "Content of Doc2" }
];
Task> SearchDelegateAsync(string input, CancellationToken ct)
@@ -244,8 +244,8 @@ public sealed class TextSearchProviderTests
var payload2 = new RawPayload { Id = "R2" };
List results =
[
- new() { Name = "Doc1", Value = "Content 1", RawRepresentation = payload1 },
- new() { Name = "Doc2", Value = "Content 2", RawRepresentation = payload2 }
+ new() { SourceName = "Doc1", Text = "Content 1", RawRepresentation = payload1 },
+ new() { SourceName = "Doc2", Text = "Content 2", RawRepresentation = payload2 }
];
Task> SearchDelegateAsync(string input, CancellationToken ct)
@@ -335,7 +335,8 @@ public sealed class TextSearchProviderTests
var options = new TextSearchProviderOptions
{
SearchTime = TextSearchProviderOptions.TextSearchBehavior.BeforeAIInvoke,
- RecentMessageMemoryLimit = 3
+ RecentMessageMemoryLimit = 3,
+ RecentMessageRolesIncluded = [ChatRole.User, ChatRole.Assistant]
};
string? capturedInput = null;
Task> SearchDelegateAsync(string input, CancellationToken ct)
@@ -374,7 +375,8 @@ public sealed class TextSearchProviderTests
var options = new TextSearchProviderOptions
{
SearchTime = TextSearchProviderOptions.TextSearchBehavior.BeforeAIInvoke,
- RecentMessageMemoryLimit = 5
+ RecentMessageMemoryLimit = 5,
+ RecentMessageRolesIncluded = [ChatRole.User, ChatRole.Assistant]
};
string? capturedInput = null;
Task> SearchDelegateAsync(string input, CancellationToken ct)
@@ -408,6 +410,46 @@ public sealed class TextSearchProviderTests
Assert.Equal("A\nB\nC\nD\nE\nF", capturedInput); // All retained (limit 5) + current request message.
}
+ [Fact]
+ public async Task InvokingAsync_WithRecentMessageRolesIncluded_ShouldFilterRolesAsync()
+ {
+ // Arrange
+ var options = new TextSearchProviderOptions
+ {
+ SearchTime = TextSearchProviderOptions.TextSearchBehavior.BeforeAIInvoke,
+ RecentMessageMemoryLimit = 4,
+ RecentMessageRolesIncluded = new List { ChatRole.Assistant } // Only retain assistant messages.
+ };
+ string? capturedInput = null;
+ Task> SearchDelegateAsync(string input, CancellationToken ct)
+ {
+ capturedInput = input;
+ return Task.FromResult>([]); // No results needed for this test.
+ }
+ var provider = new TextSearchProvider(SearchDelegateAsync, options);
+
+ // Populate memory with mixed roles; only Assistant messages (A1,A2) should be retained.
+ var initialMessages = new[]
+ {
+ new ChatMessage(ChatRole.User, "U1"),
+ new ChatMessage(ChatRole.Assistant, "A1"),
+ new ChatMessage(ChatRole.User, "U2"),
+ new ChatMessage(ChatRole.Assistant, "A2"),
+ };
+ await provider.InvokedAsync(new(initialMessages, null));
+
+ var invokingContext = new AIContextProvider.InvokingContext(new[]
+ {
+ new ChatMessage(ChatRole.User, "Question?") // Current request message always appended.
+ });
+
+ // Act
+ await provider.InvokingAsync(invokingContext, CancellationToken.None);
+
+ // Assert
+ Assert.Equal("A1\nA2\nQuestion?", capturedInput); // Only assistant messages from memory + current request.
+ }
+
#endregion
#region Serialization Tests
@@ -438,7 +480,8 @@ public sealed class TextSearchProviderTests
var options = new TextSearchProviderOptions
{
SearchTime = TextSearchProviderOptions.TextSearchBehavior.BeforeAIInvoke,
- RecentMessageMemoryLimit = 3
+ RecentMessageMemoryLimit = 3,
+ RecentMessageRolesIncluded = [ChatRole.User, ChatRole.Assistant]
};
var provider = new TextSearchProvider(this.NoResultSearchAsync, options);
var messages = new[]
@@ -467,7 +510,8 @@ public sealed class TextSearchProviderTests
var options = new TextSearchProviderOptions
{
SearchTime = TextSearchProviderOptions.TextSearchBehavior.BeforeAIInvoke,
- RecentMessageMemoryLimit = 4
+ RecentMessageMemoryLimit = 4,
+ RecentMessageRolesIncluded = [ChatRole.User, ChatRole.Assistant]
};
var provider = new TextSearchProvider(this.NoResultSearchAsync, options);
var messages = new[]
@@ -507,7 +551,8 @@ public sealed class TextSearchProviderTests
var initialProvider = new TextSearchProvider(this.NoResultSearchAsync, new TextSearchProviderOptions
{
SearchTime = TextSearchProviderOptions.TextSearchBehavior.BeforeAIInvoke,
- RecentMessageMemoryLimit = 5
+ RecentMessageMemoryLimit = 5,
+ RecentMessageRolesIncluded = [ChatRole.User, ChatRole.Assistant]
});
var messages = new[]
{