From bda40ba0e1e2ed5aebbaefe52dce99113fd6bf44 Mon Sep 17 00:00:00 2001 From: westey <164392973+westey-m@users.noreply.github.com> Date: Thu, 21 May 2026 18:25:33 +0100 Subject: [PATCH] .NET: Add shell support to the HarnessAgent (#6005) * Add shell support to the HarnessAgent * Address PR comments * Address PR comments --- .../HarnessAgent.cs | 18 +++ .../HarnessAgentOptions.cs | 26 ++++ .../Microsoft.Agents.AI.Harness.csproj | 4 + .../DockerShellExecutor.cs | 12 +- .../LocalShellExecutor.cs | 2 +- .../ShellExecutor.cs | 15 +++ .../HarnessAgentOptionsTests.cs | 19 +++ .../HarnessAgentTests.cs | 113 ++++++++++++++++++ .../ShellEnvironmentProviderTests.cs | 7 ++ 9 files changed, 208 insertions(+), 8 deletions(-) diff --git a/dotnet/src/Microsoft.Agents.AI.Harness/HarnessAgent.cs b/dotnet/src/Microsoft.Agents.AI.Harness/HarnessAgent.cs index e8ce93a787..ef1af05513 100644 --- a/dotnet/src/Microsoft.Agents.AI.Harness/HarnessAgent.cs +++ b/dotnet/src/Microsoft.Agents.AI.Harness/HarnessAgent.cs @@ -6,6 +6,9 @@ using System.Diagnostics.CodeAnalysis; using System.IO; using System.Linq; using Microsoft.Agents.AI.Compaction; +#if NET +using Microsoft.Agents.AI.Tools.Shell; +#endif using Microsoft.Extensions.AI; using Microsoft.Shared.DiagnosticIds; using Microsoft.Shared.Diagnostics; @@ -201,6 +204,14 @@ public sealed class HarnessAgent : DelegatingAIAgent result.Tools.Add(new HostedWebSearchTool()); } +#if NET + if (options?.ShellExecutor is ShellExecutor shellExecutor) + { + result.Tools ??= []; + result.Tools.Add(shellExecutor.AsAIFunction()); + } +#endif + return result; } @@ -259,6 +270,13 @@ public sealed class HarnessAgent : DelegatingAIAgent } } +#if NET + if (options?.ShellExecutor is ShellExecutor shellExecutor) + { + providers.Add(new ShellEnvironmentProvider(shellExecutor, options.ShellEnvironmentProviderOptions)); + } +#endif + if (options?.AIContextProviders is IEnumerable userProviders) { providers.AddRange(userProviders); diff --git a/dotnet/src/Microsoft.Agents.AI.Harness/HarnessAgentOptions.cs b/dotnet/src/Microsoft.Agents.AI.Harness/HarnessAgentOptions.cs index 5291647882..46c64cfe2f 100644 --- a/dotnet/src/Microsoft.Agents.AI.Harness/HarnessAgentOptions.cs +++ b/dotnet/src/Microsoft.Agents.AI.Harness/HarnessAgentOptions.cs @@ -2,6 +2,9 @@ using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; +#if NET +using Microsoft.Agents.AI.Tools.Shell; +#endif using Microsoft.Extensions.AI; using Microsoft.Shared.DiagnosticIds; @@ -240,4 +243,27 @@ public sealed class HarnessAgentOptions /// This property is ignored when is or empty. /// public BackgroundAgentsProviderOptions? BackgroundAgentsProviderOptions { get; set; } + +#if NET + /// + /// Gets or sets the shell executor used to enable shell tool and environment probing via . + /// + /// + /// When non-null, a is automatically included in the agent's context + /// providers (injecting OS/shell/CWD information into the system prompt), and the executor's + /// is registered as a callable tool. + /// When (the default), no shell features are enabled. + /// + public ShellExecutor? ShellExecutor { get; set; } + + /// + /// Gets or sets optional configuration for the . + /// + /// + /// Use this to customize which tools are probed, the probe timeout, shell family override, + /// or the instructions formatter. + /// This property is ignored when is . + /// + public ShellEnvironmentProviderOptions? ShellEnvironmentProviderOptions { get; set; } +#endif } diff --git a/dotnet/src/Microsoft.Agents.AI.Harness/Microsoft.Agents.AI.Harness.csproj b/dotnet/src/Microsoft.Agents.AI.Harness/Microsoft.Agents.AI.Harness.csproj index b6e0dd05c9..31e24d4324 100644 --- a/dotnet/src/Microsoft.Agents.AI.Harness/Microsoft.Agents.AI.Harness.csproj +++ b/dotnet/src/Microsoft.Agents.AI.Harness/Microsoft.Agents.AI.Harness.csproj @@ -15,6 +15,10 @@ + + + + Microsoft Agent Framework Harness diff --git a/dotnet/src/Microsoft.Agents.AI.Tools.Shell/DockerShellExecutor.cs b/dotnet/src/Microsoft.Agents.AI.Tools.Shell/DockerShellExecutor.cs index ffe9891eb5..0da232c9f0 100644 --- a/dotnet/src/Microsoft.Agents.AI.Tools.Shell/DockerShellExecutor.cs +++ b/dotnet/src/Microsoft.Agents.AI.Tools.Shell/DockerShellExecutor.cs @@ -248,7 +248,7 @@ public sealed class DockerShellExecutor : ShellExecutor /// Build the AIFunction for this tool. /// /// - /// When is + /// When is /// (the default), the returned function is wrapped in /// . The caller must /// explicitly pass to opt out of approval @@ -259,14 +259,12 @@ public sealed class DockerShellExecutor : ShellExecutor /// Function name surfaced to the model. /// Function description for the model. /// - /// or (the default) - /// wraps the function in ; + /// (the default) wraps the function in + /// ; /// opts out and returns the raw function. /// - public AIFunction AsAIFunction(string name = "run_shell", string? description = null, bool? requireApproval = null) + public override AIFunction AsAIFunction(string name = "run_shell", string? description = null, bool requireApproval = true) { - var effectiveRequireApproval = requireApproval ?? true; - description ??= "Execute a single shell command inside an isolated Docker container and return its " + "stdout, stderr, and exit code. The container has no network, no host filesystem access " + @@ -292,7 +290,7 @@ public sealed class DockerShellExecutor : ShellExecutor }, new AIFunctionFactoryOptions { Name = name, Description = description }); - return effectiveRequireApproval ? new ApprovalRequiredAIFunction(fn) : fn; + return requireApproval ? new ApprovalRequiredAIFunction(fn) : fn; } /// diff --git a/dotnet/src/Microsoft.Agents.AI.Tools.Shell/LocalShellExecutor.cs b/dotnet/src/Microsoft.Agents.AI.Tools.Shell/LocalShellExecutor.cs index 074d521f18..97cc29629b 100644 --- a/dotnet/src/Microsoft.Agents.AI.Tools.Shell/LocalShellExecutor.cs +++ b/dotnet/src/Microsoft.Agents.AI.Tools.Shell/LocalShellExecutor.cs @@ -325,7 +325,7 @@ public sealed class LocalShellExecutor : ShellExecutor /// container where the tool itself is the boundary). /// /// An wrapping . - public AIFunction AsAIFunction(string name = "run_shell", string? description = null, bool requireApproval = true) + public override AIFunction AsAIFunction(string name = "run_shell", string? description = null, bool requireApproval = true) { if (!requireApproval && !this._acknowledgeUnsafe) { diff --git a/dotnet/src/Microsoft.Agents.AI.Tools.Shell/ShellExecutor.cs b/dotnet/src/Microsoft.Agents.AI.Tools.Shell/ShellExecutor.cs index 78af2bc36a..4eb8cf3868 100644 --- a/dotnet/src/Microsoft.Agents.AI.Tools.Shell/ShellExecutor.cs +++ b/dotnet/src/Microsoft.Agents.AI.Tools.Shell/ShellExecutor.cs @@ -3,6 +3,7 @@ using System; using System.Threading; using System.Threading.Tasks; +using Microsoft.Extensions.AI; namespace Microsoft.Agents.AI.Tools.Shell; @@ -65,6 +66,20 @@ public abstract class ShellExecutor : IAsyncDisposable /// Cancellation token. public abstract Task RunAsync(string command, CancellationToken cancellationToken = default); + /// + /// Build an bound to this executor, suitable for + /// registering with an agent as a callable tool. + /// + /// Function name visible to the model. + /// Function description for the model. + /// + /// When (the default), wraps the function in + /// so every invocation requires + /// explicit user approval before executing. + /// + /// An wrapping . + public abstract AIFunction AsAIFunction(string name = "run_shell", string? description = null, bool requireApproval = true); + /// public abstract ValueTask DisposeAsync(); } diff --git a/dotnet/tests/Microsoft.Agents.AI.Harness.UnitTests/HarnessAgentOptionsTests.cs b/dotnet/tests/Microsoft.Agents.AI.Harness.UnitTests/HarnessAgentOptionsTests.cs index d880300cd0..b876ecaea4 100644 --- a/dotnet/tests/Microsoft.Agents.AI.Harness.UnitTests/HarnessAgentOptionsTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.Harness.UnitTests/HarnessAgentOptionsTests.cs @@ -1,6 +1,9 @@ // Copyright (c) Microsoft. All rights reserved. using Moq; +#if NET +using Microsoft.Agents.AI.Tools.Shell; +#endif namespace Microsoft.Agents.AI.UnitTests; @@ -39,6 +42,10 @@ public class HarnessAgentOptionsTests Assert.Null(options.AgentSkillsSource); Assert.Null(options.BackgroundAgents); Assert.Null(options.BackgroundAgentsProviderOptions); +#if NET + Assert.Null(options.ShellExecutor); + Assert.Null(options.ShellEnvironmentProviderOptions); +#endif } /// @@ -56,6 +63,10 @@ public class HarnessAgentOptionsTests var skillsSource = new Mock().Object; var backgroundAgents = new AIAgent[] { new Mock().Object }; var backgroundAgentsOptions = new BackgroundAgentsProviderOptions(); +#if NET + var shellExecutor = new Mock().Object; + var shellEnvOptions = new ShellEnvironmentProviderOptions(); +#endif // Act var options = new HarnessAgentOptions @@ -83,6 +94,10 @@ public class HarnessAgentOptionsTests OpenTelemetrySourceName = "custom-source", BackgroundAgents = backgroundAgents, BackgroundAgentsProviderOptions = backgroundAgentsOptions, +#if NET + ShellExecutor = shellExecutor, + ShellEnvironmentProviderOptions = shellEnvOptions, +#endif }; // Assert @@ -111,5 +126,9 @@ public class HarnessAgentOptionsTests Assert.Equal("custom-source", options.OpenTelemetrySourceName); Assert.Same(backgroundAgents, options.BackgroundAgents); Assert.Same(backgroundAgentsOptions, options.BackgroundAgentsProviderOptions); +#if NET + Assert.Same(shellExecutor, options.ShellExecutor); + Assert.Same(shellEnvOptions, options.ShellEnvironmentProviderOptions); +#endif } } diff --git a/dotnet/tests/Microsoft.Agents.AI.Harness.UnitTests/HarnessAgentTests.cs b/dotnet/tests/Microsoft.Agents.AI.Harness.UnitTests/HarnessAgentTests.cs index 8eb61da488..4f08209bd8 100644 --- a/dotnet/tests/Microsoft.Agents.AI.Harness.UnitTests/HarnessAgentTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.Harness.UnitTests/HarnessAgentTests.cs @@ -5,6 +5,9 @@ using System.Collections.Generic; using System.Linq; using System.Threading; using System.Threading.Tasks; +#if NET +using Microsoft.Agents.AI.Tools.Shell; +#endif using Microsoft.Extensions.AI; using Moq; @@ -1347,4 +1350,114 @@ public class HarnessAgentTests } #endregion + +#if NET + #region Feature: ShellEnvironmentProvider + + /// + /// Verify that ShellEnvironmentProvider is included when ShellExecutor is provided. + /// + [Fact] + public void ShellEnvironmentProvider_IncludedWhenExecutorProvided() + { + // Arrange + var chatClient = new Mock().Object; + var executorMock = new Mock(); + executorMock.Setup(e => e.AsAIFunction(It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(AIFunctionFactory.Create(() => "test", "run_shell")); + var options = CreateAllDisabledOptions(); + options.ShellExecutor = executorMock.Object; + + // Act + var agent = new HarnessAgent(chatClient, TestMaxContextWindowTokens, TestMaxOutputTokens, options); + var innerAgent = agent.GetService(); + + // Assert + Assert.NotNull(innerAgent?.AIContextProviders); + Assert.Contains(innerAgent!.AIContextProviders!, p => p is ShellEnvironmentProvider); + } + + /// + /// Verify that ShellEnvironmentProvider is not included when ShellExecutor is null. + /// + [Fact] + public void ShellEnvironmentProvider_ExcludedWhenExecutorNull() + { + // Arrange + var chatClient = new Mock().Object; + var options = CreateAllDisabledOptions(); + options.ShellExecutor = null; + + // Act + var agent = new HarnessAgent(chatClient, TestMaxContextWindowTokens, TestMaxOutputTokens, options); + var innerAgent = agent.GetService(); + + // Assert + Assert.NotNull(innerAgent); + Assert.NotNull(innerAgent!.AIContextProviders); + Assert.DoesNotContain(innerAgent.AIContextProviders!, p => p is ShellEnvironmentProvider); + } + + /// + /// Verify that the shell tool AIFunction is added to ChatOptions.Tools when ShellExecutor is provided. + /// + [Fact] + public async Task ShellExecutor_ToolAddedToChatOptionsAsync() + { + // Arrange + ChatOptions? capturedOptions = null; + var chatClientMock = new Mock(); + chatClientMock + .Setup(c => c.GetResponseAsync(It.IsAny>(), It.IsAny(), It.IsAny())) + .Callback, ChatOptions?, CancellationToken>((_, opts, _) => capturedOptions = opts) + .ReturnsAsync(new ChatResponse(new ChatMessage(ChatRole.Assistant, "done"))); + + var executorMock = new Mock(); + executorMock.Setup(e => e.AsAIFunction(It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(AIFunctionFactory.Create(() => "shell output", "run_shell")); + + var options = CreateAllDisabledOptions(); + options.DisableWebSearch = true; + options.ShellExecutor = executorMock.Object; + + // Act + var agent = new HarnessAgent(chatClientMock.Object, TestMaxContextWindowTokens, TestMaxOutputTokens, options); + var session = await agent.CreateSessionAsync(); + await agent.RunAsync([new ChatMessage(ChatRole.User, "Hi")], session); + + // Assert — the shell tool should be present + Assert.NotNull(capturedOptions?.Tools); + Assert.Contains(capturedOptions!.Tools!, t => t is AIFunction f && f.Name == "run_shell"); + } + + /// + /// Verify that ShellEnvironmentProvider is present when ShellEnvironmentProviderOptions is also specified. + /// + [Fact] + public void ShellEnvironmentProvider_PresentWhenOptionsProvided() + { + // Arrange + var chatClient = new Mock().Object; + var executorMock = new Mock(); + executorMock.Setup(e => e.AsAIFunction(It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(AIFunctionFactory.Create(() => "test", "run_shell")); + var envOptions = new ShellEnvironmentProviderOptions + { + ProbeTools = ["git", "python"], + }; + var options = CreateAllDisabledOptions(); + options.ShellExecutor = executorMock.Object; + options.ShellEnvironmentProviderOptions = envOptions; + + // Act + var agent = new HarnessAgent(chatClient, TestMaxContextWindowTokens, TestMaxOutputTokens, options); + var innerAgent = agent.GetService(); + + // Assert — provider should exist (options wiring is validated by the provider's behavior) + Assert.NotNull(innerAgent?.AIContextProviders); + Assert.Contains(innerAgent!.AIContextProviders!, p => p is ShellEnvironmentProvider); + } + + #endregion +#endif } diff --git a/dotnet/tests/Microsoft.Agents.AI.Tools.Shell.UnitTests/ShellEnvironmentProviderTests.cs b/dotnet/tests/Microsoft.Agents.AI.Tools.Shell.UnitTests/ShellEnvironmentProviderTests.cs index c4488b17cc..caca315a52 100644 --- a/dotnet/tests/Microsoft.Agents.AI.Tools.Shell.UnitTests/ShellEnvironmentProviderTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.Tools.Shell.UnitTests/ShellEnvironmentProviderTests.cs @@ -6,6 +6,7 @@ using System.Reflection; using System.Runtime.InteropServices; using System.Threading; using System.Threading.Tasks; +using Microsoft.Extensions.AI; namespace Microsoft.Agents.AI.Tools.Shell.UnitTests; @@ -226,6 +227,8 @@ public sealed class ShellEnvironmentProviderTests public override Task InitializeAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; public override Task RunAsync(string command, CancellationToken cancellationToken = default) => Task.FromResult(this.Responses.Dequeue()); + public override AIFunction AsAIFunction(string name = "run_shell", string? description = null, bool requireApproval = true) => + throw new NotSupportedException(); public override ValueTask DisposeAsync() => default; } @@ -280,6 +283,8 @@ public sealed class ShellEnvironmentProviderTests public override Task InitializeAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; public override Task RunAsync(string command, CancellationToken cancellationToken = default) => Task.FromResult(this._factory(cancellationToken)); + public override AIFunction AsAIFunction(string name = "run_shell", string? description = null, bool requireApproval = true) => + throw new NotSupportedException(); public override ValueTask DisposeAsync() => default; } @@ -372,6 +377,8 @@ public sealed class ShellEnvironmentProviderTests this.RunCount++; return Task.FromResult(this.NextResult); } + public override AIFunction AsAIFunction(string name = "run_shell", string? description = null, bool requireApproval = true) => + throw new NotSupportedException(); public override ValueTask DisposeAsync() => default; } }