.NET: [BREAKING] Add consistent message filtering to all providers. (#3851)

* Add consistent message filtering to all providers.

* Remove old chat history filtering classes

* Fix merge issues

* Fix unit test

* Enforce non-nullable property

* Fix merging bug and make troubleshooting source info easier by adding tostring implementation

dotnet/samples/GettingStarted/AgentWithRAG/AgentWithRAG_Step01_BasicTextRAG/Program.cs

@@ -63,11 +63,15 @@ AIAgent agent = azureOpenAIClient
     {
         ChatOptions = new() { Instructions = "You are a helpful support specialist for Contoso Outdoors. Answer questions using the provided context and cite the source document when available." },
         AIContextProvider = new TextSearchProvider(SearchAdapter, textSearchOptions),
-        // Since we are using ChatCompletion which stores chat history locally, we can also add a message removal policy
+        // Since we are using ChatCompletion which stores chat history locally, we can also add a message filter
         // that removes messages produced by the TextSearchProvider before they are added to the chat history, so that
         // we don't bloat chat history with all the search result messages.
-        ChatHistoryProvider = new InMemoryChatHistoryProvider()
-            .WithAIContextProviderMessageRemoval(),
+        // By default the chat history provider will store all messages, except for those that came from chat history in the first place.
+        // We also want to maintain that exclusion here.
+        ChatHistoryProvider = new InMemoryChatHistoryProvider(new InMemoryChatHistoryProviderOptions
+        {
+            StorageInputMessageFilter = messages => messages.Where(m => m.GetAgentRequestMessageSourceType() != AgentRequestMessageSourceType.AIContextProvider && m.GetAgentRequestMessageSourceType() != AgentRequestMessageSourceType.ChatHistory)
+        }),
     });
 
 AgentSession session = await agent.CreateSessionAsync();

dotnet/samples/GettingStarted/Agents/Agent_Step07_3rdPartyChatHistoryStorage/Program.cs

@@ -144,9 +144,11 @@ namespace SampleApp
             var collection = this._vectorStore.GetCollection<string, ChatHistoryItem>("ChatHistory");
             await collection.EnsureCollectionExistsAsync(cancellationToken);
 
-            // Add both request and response messages to the store
+            // Add both request and response messages to the store, excluding messages that came from chat history.
             // Optionally messages produced by the AIContextProvider can also be persisted (not shown).
-            var allNewMessages = context.RequestMessages.Concat(context.ResponseMessages ?? []);
+            var allNewMessages = context.RequestMessages
+                .Where(m => m.GetAgentRequestMessageSourceType() != AgentRequestMessageSourceType.ChatHistory)
+                .Concat(context.ResponseMessages ?? []);
 
             await collection.UpsertAsync(allNewMessages.Select(x => new ChatHistoryItem()
             {

dotnet/samples/GettingStarted/Agents/Agent_Step20_AdditionalAIContext/Program.cs

@@ -44,10 +44,14 @@ AIAgent agent = new AzureOpenAIClient(
         You manage a TODO list for the user. When the user has completed one of the tasks it can be removed from the TODO list. Only provide the list of TODO items if asked.
         You remind users of upcoming calendar events when the user interacts with you.
         """ },
-        ChatHistoryProvider = new InMemoryChatHistoryProvider()
-            // Use WithAIContextProviderMessageRemoval, so that we don't store the messages from the AI context provider in the chat history.
+        ChatHistoryProvider = new InMemoryChatHistoryProvider(new InMemoryChatHistoryProviderOptions
+        {
+            // Use StorageInputMessageFilter to provide a custom filter for messages stored in chat history.
+            // By default the chat history provider will store all messages, except for those that came from chat history in the first place.
+            // In this case, we want to also exclude messages that came from AI context providers.
             // You may want to store these messages, depending on their content and your requirements.
-            .WithAIContextProviderMessageRemoval(),
+            StorageInputMessageFilter = messages => messages.Where(m => m.GetAgentRequestMessageSourceType() != AgentRequestMessageSourceType.AIContextProvider && m.GetAgentRequestMessageSourceType() != AgentRequestMessageSourceType.ChatHistory)
+        }),
         // Add an AI context provider that maintains a todo list for the agent and one that provides upcoming calendar entries.
         // Wrap these in an AI context provider that aggregates the other two.
         AIContextProvider = new AggregatingAIContextProvider([