.NET: [BREAKING] Align FileAccess tools with Python; add directory discovery and recursive search (#6474)

* Align FileAccess with python and improve functionality

* Addressing PR comments

dotnet/src/Microsoft.Agents.AI/Harness/FileAccess/FileAccessProvider.cs

@@ -31,11 +31,12 @@ namespace Microsoft.Agents.AI;
 /// <para>
 /// This provider exposes the following tools to the agent:
 /// <list type="bullet">
-/// <item><description><c>SaveFile</c> — Save a file with the given name and content.</description></item>
-/// <item><description><c>ReadFile</c> — Read the content of a file by name.</description></item>
-/// <item><description><c>DeleteFile</c> — Delete a file by name.</description></item>
-/// <item><description><c>ListFiles</c> — List all file names.</description></item>
-/// <item><description><c>SearchFiles</c> — Search file contents using a regular expression pattern.</description></item>
+/// <item><description><c>file_access_save_file</c> — Save a file with the given name and content.</description></item>
+/// <item><description><c>file_access_read_file</c> — Read the content of a file by name.</description></item>
+/// <item><description><c>file_access_delete_file</c> — Delete a file by name.</description></item>
+/// <item><description><c>file_access_list_files</c> — List the direct child file names in a directory.</description></item>
+/// <item><description><c>file_access_list_subdirectories</c> — List the direct child subdirectory names in a directory.</description></item>
+/// <item><description><c>file_access_search_files</c> — Recursively search file contents using a regular expression pattern.</description></item>
 /// </list>
 /// </para>
 /// </remarks>
@@ -45,11 +46,13 @@ public sealed class FileAccessProvider : AIContextProvider
     private const string DefaultInstructions =
         """
         ## File Access
-        You have access to a shared file storage area via the `FileAccess_*` tools for reading, writing, and managing files.
+        You have access to a shared file storage area via the `file_access_*` tools for reading, writing, and managing files.
         These files persist beyond the current session and may be shared across sessions or agents.
         Use these tools to read input data provided by the user, write output artifacts, and manage any files the user has asked you to work with.
 
         - Never delete or overwrite existing files unless the user has explicitly asked you to do so.
+        - Files may be organized into subdirectories. Use `file_access_list_files` and `file_access_list_subdirectories` to explore the tree level by level,
+          or `file_access_search_files` to search file contents recursively across the whole store.
         """;
 
     private readonly AgentFileStore _fileStore;
@@ -137,30 +140,56 @@ public sealed class FileAccessProvider : AIContextProvider
     }
 
     /// <summary>
-    /// List all file names.
+    /// List the direct child file names of a directory. Omit <paramref name="directory"/> (or pass an empty string)
+    /// to list the store root. To enumerate files in a subdirectory, pass its relative path.
     /// </summary>
+    /// <param name="directory">The relative directory path to list. Omit or pass an empty string to list the store root.</param>
     /// <param name="cancellationToken">A token to cancel the operation.</param>
     /// <returns>A list of file names.</returns>
-    [Description("List all file names.")]
-    private async Task<List<string>> ListFilesAsync(CancellationToken cancellationToken = default)
+    [Description("List the direct child file names of a directory. Omit the directory (or pass an empty string) to list the root. To enumerate files in a subdirectory, pass its relative path, for example \"reports\" or \"reports/2024\".")]
+    private async Task<List<string>> ListFilesAsync(string? directory = null, CancellationToken cancellationToken = default)
     {
-        IReadOnlyList<string> fileNames = await this._fileStore.ListFilesAsync(string.Empty, cancellationToken).ConfigureAwait(false);
+        string target = string.IsNullOrWhiteSpace(directory) ? string.Empty : directory;
+        IReadOnlyList<string> fileNames = await this._fileStore.ListFilesAsync(target, cancellationToken).ConfigureAwait(false);
         return new List<string>(fileNames);
     }
 
     /// <summary>
-    /// Search file contents using a regular expression pattern (case-insensitive).
+    /// List the direct child subdirectory names of a directory. Omit <paramref name="directory"/> (or pass an empty string)
+    /// to list the store root. To enumerate subdirectories of a subdirectory, pass its relative path.
+    /// </summary>
+    /// <param name="directory">The relative directory path to list. Omit or pass an empty string to list the store root.</param>
+    /// <param name="cancellationToken">A token to cancel the operation.</param>
+    /// <returns>A list of subdirectory names.</returns>
+    [Description("List the direct child subdirectory names of a directory. Omit the directory (or pass an empty string) to list the root. To enumerate subdirectories of a subdirectory, pass its relative path, for example \"reports\" or \"reports/2024\". Use this together with file_access_list_files to explore the directory tree level by level.")]
+    private async Task<List<string>> ListSubdirectoriesAsync(string? directory = null, CancellationToken cancellationToken = default)
+    {
+        string target = string.IsNullOrWhiteSpace(directory) ? string.Empty : directory;
+        IReadOnlyList<string> directoryNames = await this._fileStore.ListDirectoriesAsync(target, cancellationToken).ConfigureAwait(false);
+        return new List<string>(directoryNames);
+    }
+
+    /// <summary>
+    /// Search the contents of all files in the store (recursively) using a regular expression pattern (case-insensitive).
     /// Optionally filter which files to search using a glob pattern.
     /// </summary>
     /// <param name="regexPattern">A regular expression pattern to match against file contents (case-insensitive).</param>
-    /// <param name="filePattern">An optional glob pattern to filter which files to search (e.g., "*.md", "research*"). Leave empty or omit to search all files.</param>
+    /// <param name="filePattern">An optional glob pattern to filter which files to search, matched against each file's path relative to the store root. Use <c>**</c> to match across subdirectories (e.g., "**/*.md"). Leave empty or omit to search all files.</param>
     /// <param name="cancellationToken">A token to cancel the operation.</param>
-    /// <returns>A list of search results with matching file names, snippets, and matching lines.</returns>
-    [Description("Search file contents using a regular expression pattern (case-insensitive). Optionally filter which files to search using a glob pattern (e.g., \"*.md\", \"research*\"). Returns matching file names, snippets, and matching lines with line numbers.")]
+    /// <returns>A list of search results whose file names are paths relative to the store root.</returns>
+    [Description(
+        """
+        Search the contents of all files in the store (recursively, across all subdirectories) using a regular expression pattern (case-insensitive).
+        Optionally filter which files to search using a glob pattern matched against each file's path relative to the store root:
+        - '*' matches within a single path segment
+        - '**' matches across subdirectories, so use \"**/*.md\" to match markdown files at any depth, or \"reports/**\" to restrict the search to the 'reports' subtree.
+        
+        Returns matching results whose file names are paths relative to the store root (usable with file_access_read_file), along with snippets and matching lines with line numbers.
+        """)]
     private async Task<List<FileSearchResult>> SearchFilesAsync(string regexPattern, string? filePattern = null, CancellationToken cancellationToken = default)
     {
         string? pattern = string.IsNullOrWhiteSpace(filePattern) ? null : filePattern;
-        IReadOnlyList<FileSearchResult> results = await this._fileStore.SearchFilesAsync(string.Empty, regexPattern, pattern, cancellationToken).ConfigureAwait(false);
+        IReadOnlyList<FileSearchResult> results = await this._fileStore.SearchFilesAsync(string.Empty, regexPattern, pattern, recursive: true, cancellationToken).ConfigureAwait(false);
         return new List<FileSearchResult>(results);
     }
 
@@ -170,11 +199,12 @@ public sealed class FileAccessProvider : AIContextProvider
 
         return
         [
-            AIFunctionFactory.Create(this.SaveFileAsync, new AIFunctionFactoryOptions { Name = "FileAccess_SaveFile", SerializerOptions = serializerOptions }),
-            AIFunctionFactory.Create(this.ReadFileAsync, new AIFunctionFactoryOptions { Name = "FileAccess_ReadFile", SerializerOptions = serializerOptions }),
-            AIFunctionFactory.Create(this.DeleteFileAsync, new AIFunctionFactoryOptions { Name = "FileAccess_DeleteFile", SerializerOptions = serializerOptions }),
-            AIFunctionFactory.Create(this.ListFilesAsync, new AIFunctionFactoryOptions { Name = "FileAccess_ListFiles", SerializerOptions = serializerOptions }),
-            AIFunctionFactory.Create(this.SearchFilesAsync, new AIFunctionFactoryOptions { Name = "FileAccess_SearchFiles", SerializerOptions = serializerOptions }),
+            AIFunctionFactory.Create(this.SaveFileAsync, new AIFunctionFactoryOptions { Name = "file_access_save_file", SerializerOptions = serializerOptions }),
+            AIFunctionFactory.Create(this.ReadFileAsync, new AIFunctionFactoryOptions { Name = "file_access_read_file", SerializerOptions = serializerOptions }),
+            AIFunctionFactory.Create(this.DeleteFileAsync, new AIFunctionFactoryOptions { Name = "file_access_delete_file", SerializerOptions = serializerOptions }),
+            AIFunctionFactory.Create(this.ListFilesAsync, new AIFunctionFactoryOptions { Name = "file_access_list_files", SerializerOptions = serializerOptions }),
+            AIFunctionFactory.Create(this.ListSubdirectoriesAsync, new AIFunctionFactoryOptions { Name = "file_access_list_subdirectories", SerializerOptions = serializerOptions }),
+            AIFunctionFactory.Create(this.SearchFilesAsync, new AIFunctionFactoryOptions { Name = "file_access_search_files", SerializerOptions = serializerOptions }),
         ];
     }
 }

dotnet/src/Microsoft.Agents.AI/Harness/FileMemory/FileMemoryProvider.cs

@@ -296,7 +296,7 @@ public sealed class FileMemoryProvider : AIContextProvider, IDisposable
     {
         FileMemoryState state = this._sessionState.GetOrInitializeState(AIAgent.CurrentRunContext?.Session);
         string? pattern = string.IsNullOrWhiteSpace(filePattern) ? null : filePattern;
-        IReadOnlyList<FileSearchResult> results = await this._fileStore.SearchFilesAsync(state.WorkingFolder, regexPattern, pattern, cancellationToken).ConfigureAwait(false);
+        IReadOnlyList<FileSearchResult> results = await this._fileStore.SearchFilesAsync(state.WorkingFolder, regexPattern, pattern, recursive: false, cancellationToken).ConfigureAwait(false);
 
         // Filter out internal files (description sidecars and memory index) so they stay hidden.
         var filtered = new List<FileSearchResult>(results.Count);

dotnet/src/Microsoft.Agents.AI/Harness/FileStore/AgentFileStore.cs

@@ -58,6 +58,14 @@ public abstract class AgentFileStore
     /// <returns>A list of file names in the specified directory (direct children only).</returns>
     public abstract Task<IReadOnlyList<string>> ListFilesAsync(string directory, CancellationToken cancellationToken = default);
 
+    /// <summary>
+    /// Lists the direct child subdirectories of a directory.
+    /// </summary>
+    /// <param name="directory">The relative path of the directory to list. Use an empty string for the root.</param>
+    /// <param name="cancellationToken">A token to cancel the operation.</param>
+    /// <returns>A list of subdirectory names in the specified directory (direct children only).</returns>
+    public abstract Task<IReadOnlyList<string>> ListDirectoriesAsync(string directory, CancellationToken cancellationToken = default);
+
     /// <summary>
     /// Checks whether a file exists.
     /// </summary>
@@ -76,12 +84,20 @@ public abstract class AgentFileStore
     /// </param>
     /// <param name="filePattern">
     /// An optional glob pattern to filter which files are searched (e.g., <c>"*.md"</c>, <c>"research*"</c>).
-    /// When <see langword="null"/>, all files in the directory are searched.
-    /// Uses standard glob syntax from <see cref="Matcher"/>.
+    /// When <see langword="null"/>, all files are searched.
+    /// Uses standard glob syntax from <see cref="Matcher"/>, matched against each file's path relative to
+    /// <paramref name="directory"/>. Use <c>**</c> to match across subdirectories (e.g., <c>"**/*.md"</c>).
+    /// </param>
+    /// <param name="recursive">
+    /// When <see langword="true"/>, all descendant files of <paramref name="directory"/> are searched.
+    /// When <see langword="false"/> (default), only the direct children of <paramref name="directory"/> are searched.
     /// </param>
     /// <param name="cancellationToken">A token to cancel the operation.</param>
-    /// <returns>A list of search results with matching file names, snippets, and matching lines.</returns>
-    public abstract Task<IReadOnlyList<FileSearchResult>> SearchFilesAsync(string directory, string regexPattern, string? filePattern = null, CancellationToken cancellationToken = default);
+    /// <returns>
+    /// A list of search results. Each result's <see cref="FileSearchResult.FileName"/> is the matching file's
+    /// path relative to <paramref name="directory"/>.
+    /// </returns>
+    public abstract Task<IReadOnlyList<FileSearchResult>> SearchFilesAsync(string directory, string regexPattern, string? filePattern = null, bool recursive = false, CancellationToken cancellationToken = default);
 
     /// <summary>
     /// Ensures a directory exists, creating it if necessary.

dotnet/src/Microsoft.Agents.AI/Harness/FileStore/FileSystemAgentFileStore.cs

@@ -142,6 +142,7 @@ public sealed class FileSystemAgentFileStore : AgentFileStore
         string directory,
         string regexPattern,
         string? filePattern = null,
+        bool recursive = false,
         CancellationToken cancellationToken = default)
     {
         string fullDir = this.ResolveSafeDirectoryPath(directory);
@@ -156,22 +157,13 @@ public sealed class FileSystemAgentFileStore : AgentFileStore
         Matcher? matcher = filePattern is not null ? StorePaths.CreateGlobMatcher(filePattern) : null;
         var results = new List<FileSearchResult>();
 
-        foreach (string filePath in Directory.GetFiles(fullDir))
+        foreach (string filePath in EnumerateFiles(fullDir, recursive))
         {
-            // Skip files that are symlinks/reparse points to prevent reading outside the root.
-            if ((File.GetAttributes(filePath) & FileAttributes.ReparsePoint) != 0)
-            {
-                continue;
-            }
+            // The file path relative to the search directory, using forward slashes.
+            string relativeName = GetRelativeStorePath(fullDir, filePath);
 
-            string? fileName = Path.GetFileName(filePath);
-            if (fileName is null)
-            {
-                continue;
-            }
-
-            // Apply the optional glob filter on the file name.
-            if (!StorePaths.MatchesGlob(fileName, matcher))
+            // Apply the optional glob filter on the relative path.
+            if (!StorePaths.MatchesGlob(relativeName, matcher))
             {
                 continue;
             }
@@ -218,7 +210,7 @@ public sealed class FileSystemAgentFileStore : AgentFileStore
             {
                 results.Add(new FileSearchResult
                 {
-                    FileName = fileName,
+                    FileName = relativeName,
                     Snippet = firstSnippet!,
                     MatchingLines = matchingLines,
                 });
@@ -228,6 +220,76 @@ public sealed class FileSystemAgentFileStore : AgentFileStore
         return results;
     }
 
+    /// <inheritdoc />
+    public override Task<IReadOnlyList<string>> ListDirectoriesAsync(string directory, CancellationToken cancellationToken = default)
+    {
+        string fullDir = this.ResolveSafeDirectoryPath(directory);
+
+        if (!Directory.Exists(fullDir))
+        {
+            return Task.FromResult<IReadOnlyList<string>>([]);
+        }
+
+        var directories = Directory.GetDirectories(fullDir)
+            .Where(d => (File.GetAttributes(d) & FileAttributes.ReparsePoint) == 0)
+            .Select(Path.GetFileName)
+            .Where(name => name is not null)
+            .ToList();
+
+        return Task.FromResult<IReadOnlyList<string>>(directories!);
+    }
+
+    /// <summary>
+    /// Enumerates the files directly under <paramref name="directory"/> (or all descendant files when
+    /// <paramref name="recursive"/> is <see langword="true"/>), skipping symlinks/reparse points for both
+    /// files and directories to prevent reading outside the root.
+    /// </summary>
+    private static IEnumerable<string> EnumerateFiles(string directory, bool recursive)
+    {
+        foreach (string filePath in Directory.EnumerateFiles(directory))
+        {
+            // Skip files that are symlinks/reparse points.
+            if ((File.GetAttributes(filePath) & FileAttributes.ReparsePoint) != 0)
+            {
+                continue;
+            }
+
+            yield return filePath;
+        }
+
+        if (!recursive)
+        {
+            yield break;
+        }
+
+        foreach (string subDir in Directory.EnumerateDirectories(directory))
+        {
+            // Skip symlinked/reparse-point directories so recursion cannot escape the root.
+            if ((File.GetAttributes(subDir) & FileAttributes.ReparsePoint) != 0)
+            {
+                continue;
+            }
+
+            foreach (string filePath in EnumerateFiles(subDir, recursive: true))
+            {
+                yield return filePath;
+            }
+        }
+    }
+
+    /// <summary>
+    /// Returns the path of <paramref name="filePath"/> relative to <paramref name="baseDirectory"/>,
+    /// normalized to forward-slash separators. Assumes <paramref name="filePath"/> resides under
+    /// <paramref name="baseDirectory"/> (as produced by <see cref="EnumerateFiles"/>).
+    /// </summary>
+    private static string GetRelativeStorePath(string baseDirectory, string filePath)
+    {
+        string baseTrimmed = baseDirectory.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar);
+        string relative = filePath.Substring(baseTrimmed.Length)
+            .TrimStart(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar);
+        return relative.Replace(Path.DirectorySeparatorChar, '/').Replace(Path.AltDirectorySeparatorChar, '/');
+    }
+
     /// <inheritdoc />
     public override Task CreateDirectoryAsync(string path, CancellationToken cancellationToken = default)
     {

dotnet/src/Microsoft.Agents.AI/Harness/FileStore/InMemoryAgentFileStore.cs

@@ -66,6 +66,43 @@ public sealed class InMemoryAgentFileStore : AgentFileStore
         return Task.FromResult<IReadOnlyList<string>>(files);
     }
 
+    /// <inheritdoc />
+    public override Task<IReadOnlyList<string>> ListDirectoriesAsync(string directory, CancellationToken cancellationToken = default)
+    {
+        string prefix = StorePaths.NormalizeRelativePath(directory, isDirectory: true);
+        if (prefix.Length > 0 && !prefix.EndsWith("/", StringComparison.Ordinal))
+        {
+            prefix += "/";
+        }
+
+        // A subdirectory is the first path segment of any key whose remainder (after the prefix)
+        // still contains a separator. Collect distinct first segments, preserving original casing.
+        var directories = new List<string>();
+        var seen = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
+        foreach (string key in this._files.Keys)
+        {
+            if (!key.StartsWith(prefix, StringComparison.OrdinalIgnoreCase))
+            {
+                continue;
+            }
+
+            string remainder = key.Substring(prefix.Length);
+            int separatorIndex = remainder.IndexOf("/", StringComparison.Ordinal);
+            if (separatorIndex <= 0)
+            {
+                continue;
+            }
+
+            string segment = remainder.Substring(0, separatorIndex);
+            if (seen.Add(segment))
+            {
+                directories.Add(segment);
+            }
+        }
+
+        return Task.FromResult<IReadOnlyList<string>>(directories);
+    }
+
     /// <inheritdoc />
     public override Task<bool> FileExistsAsync(string path, CancellationToken cancellationToken = default)
     {
@@ -74,7 +111,7 @@ public sealed class InMemoryAgentFileStore : AgentFileStore
     }
 
     /// <inheritdoc />
-    public override Task<IReadOnlyList<FileSearchResult>> SearchFilesAsync(string directory, string regexPattern, string? filePattern = null, CancellationToken cancellationToken = default)
+    public override Task<IReadOnlyList<FileSearchResult>> SearchFilesAsync(string directory, string regexPattern, string? filePattern = null, bool recursive = false, CancellationToken cancellationToken = default)
     {
         // Normalize the directory prefix for path matching.
         string prefix = StorePaths.NormalizeRelativePath(directory, isDirectory: true);
@@ -96,14 +133,16 @@ public sealed class InMemoryAgentFileStore : AgentFileStore
                 continue;
             }
 
-            // Exclude files in subdirectories (direct children only).
+            // The file path relative to the search directory.
             string relativeName = kvp.Key.Substring(prefix.Length);
-            if (relativeName.IndexOf("/", StringComparison.Ordinal) >= 0)
+
+            // When not recursive, exclude files in subdirectories (direct children only).
+            if (!recursive && relativeName.IndexOf("/", StringComparison.Ordinal) >= 0)
             {
                 continue;
             }
 
-            // Apply the optional glob filter on the file name.
+            // Apply the optional glob filter on the relative path.
             if (!StorePaths.MatchesGlob(relativeName, matcher))
             {
                 continue;