From b4d856d3b895974b072bc236c451fa0848e7c312 Mon Sep 17 00:00:00 2001 From: Adam Herring Date: Mon, 25 May 2026 11:09:20 -0700 Subject: [PATCH] fix(scan-project): preserve non-ASCII path bytes via `git ls-files -z` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `enumerateViaGit` ran `git ls-files -co --exclude-standard` (newline-separated output) and then `split('\n').map(trim)` on the result. Without `-z`, `git ls-files` C-escapes any byte outside the locale's "safe" set and wraps the path in double quotes — for example, a directory named `30. 🏗️ docs/` comes back as `"30. \360\237\217\227\357\270\217 docs/"`. Downstream consumers then can't round-trip those octal-quoted strings to real disk paths, so every file under such directories is silently dropped from the scan. This is particularly biting on Windows (where the issue surfaces even with UTF-8 locale settings) and for any project that uses emoji, accented characters, or CJK codepoints in directory names — which is increasingly common in design/spec/journal trees. The fix is to use `-z` (NUL-terminated output), the same approach git itself documents for downstream consumers (e.g. `xargs -0`). NUL-separated chunks are raw bytes, so every codepoint round-trips back to its real disk path on every platform. Split on `\0` instead of `\n`; drop the now- unnecessary `.trim()`. Verified on a real project with emoji-prefixed directory names: bare `git ls-files`: "30. \360\237\217\227\357\270\217\360\237\247\231\342\200\215..." `git ls-files -z`: 30. 🏗️🧙‍♂️🔮 BD-CCSP/01. Demo's/DEMO--... Discovered during a multi-agent scan of an Atlas Intelligence spoke repo; ~33 design-intent files in `30. 🏗️ BD-{app}/` directories were silently dropped per scan. Full report: atlas-intelligence-io/fleet-feedback#491. Co-Authored-By: Claude Opus 4.7 --- .../skills/understand/scan-project.mjs | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/understand-anything-plugin/skills/understand/scan-project.mjs b/understand-anything-plugin/skills/understand/scan-project.mjs index 553a82e..83ccc7d 100644 --- a/understand-anything-plugin/skills/understand/scan-project.mjs +++ b/understand-anything-plugin/skills/understand/scan-project.mjs @@ -464,17 +464,25 @@ function toPosix(p) { * so the ignore filter has to do more work in the fallback path. */ function enumerateViaGit(projectRoot) { - const result = spawnSync('git', ['ls-files', '-co', '--exclude-standard'], { + // -z = NUL-terminated output. Without it, `git ls-files` C-escapes non-ASCII + // bytes in path names — paths containing emoji, accented characters, CJK + // codepoints, etc. come back quoted with octal escapes (e.g. + // `"30. \360\237\217\227 BD-CCER/file.md"` for a path containing 🏗️). + // Those quoted-escaped strings then fail to round-trip back to real disk + // paths in downstream consumers, so files in such directories are silently + // dropped from the scan. The -z form emits raw bytes between NUL separators, + // preserving every codepoint as-is. This is the same approach git itself + // uses for `--null` everywhere downstream (xargs -0, etc.). + const result = spawnSync('git', ['ls-files', '-z', '-co', '--exclude-standard'], { cwd: projectRoot, encoding: 'utf-8', maxBuffer: 256 * 1024 * 1024, // 256MB — huge monorepos can produce >10MB of paths }); if (result.status !== 0 || !result.stdout) return null; - // Each line is one path, project-relative, already POSIX on all platforms - // because git emits forward slashes regardless of OS. + // Each NUL-separated chunk is one path, project-relative, already POSIX on + // all platforms because git emits forward slashes regardless of OS. return result.stdout - .split('\n') - .map(s => s.trim()) + .split('\0') .filter(Boolean) .map(toPosix); }