From eea73b656d168ec91754d9436e60f285c36f2e85 Mon Sep 17 00:00:00 2001 From: Tirth Kanani Date: Sun, 31 May 2026 20:25:39 +0100 Subject: [PATCH] perf(understand): parallelise file I/O in compute-batches + extract-import-map (#76) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The /understand pipeline reads every code file twice during analysis: once in compute-batches (`extractExports` for the cross-batch neighbour map) and once again in extract-import-map (per-language config loaders). Both sites used sequential `readFileSync` loops, so on the iOS repo in issue #226 (~15k files) the disk-read time was effectively serialised behind a single libuv thread while the rest of the pool sat idle. ## Changes - `extractExports` now batches files into `IO_PARALLELISM = 64` slices and issues all `readFile` calls in each slice through `Promise.all`, letting libuv's worker-thread pool overlap disk reads. The tree-sitter parse stays on the main thread because `web-tree-sitter` is single-threaded WASM — pipelining the I/O while parses run is where the wall-time savings come from. - `loadTsConfigs`, `loadGoModules`, `loadPhpAutoloads` and `buildResolutionContext` switch to async / `Promise.all` for the same reason. `buildResolutionContext` also runs the three loader passes concurrently (`Promise.all([...])`) since they're independent. - A small `readFilesParallel(paths)` helper is added at the top of `extract-import-map.mjs` so the three loaders share the same error-preserving shape. ## Why behavior stays identical - Each loader collects its candidate paths in `files[]` order *before* issuing reads, then iterates `reads` in the same order to emit warnings + populate output maps. So stderr order and the final map contents are byte-identical to the previous sequential loops. - `extractExports` collects per-file errors in-place in the `Promise.all` callbacks and emits warnings during the post-read serial loop, again in chunk order — so warning text and order match the previous implementation. - Tree-sitter parsing is unchanged: parses still run serially on the main thread, just with reads pipelined alongside. ## What's NOT in this PR - `buildFingerprintStore` and `analyzeChanges` in `core/fingerprint.ts` have the same sequential pattern. They're left alone here because they're part of the public `@understand-anything/core` API; making them async would be a breaking change worth its own discussion. Internal-only `.mjs` scripts are safe to refactor without API churn. - No change to scan-project: most of its sync I/O is `statSync` (metadata, not content) plus a handful of small `.gitignore` / `.understandignore` reads. The parallelism win is marginal there. ## Verification - `pnpm lint` clean - `pnpm --filter @understand-anything/core build` clean - `pnpm --filter @understand-anything/skill build` clean - `pnpm test`: 196/196 — including `test_compute_batches.test.mjs` (19 tests) and `test_extract_import_map.test.mjs` (40 tests), which exercise both changed pipelines end-to-end with fixture projects. No output diff vs main. Refs #76 Co-Authored-By: Claude Opus 4.7 (1M context) --- .../skills/understand/compute-batches.mjs | 83 ++++++++++++----- .../skills/understand/extract-import-map.mjs | 93 +++++++++++++------ 2 files changed, 123 insertions(+), 53 deletions(-) diff --git a/understand-anything-plugin/skills/understand/compute-batches.mjs b/understand-anything-plugin/skills/understand/compute-batches.mjs index b7cce34..f78d46a 100644 --- a/understand-anything-plugin/skills/understand/compute-batches.mjs +++ b/understand-anything-plugin/skills/understand/compute-batches.mjs @@ -13,10 +13,19 @@ */ import { readFileSync, writeFileSync, existsSync, realpathSync } from 'node:fs'; +import { readFile } from 'node:fs/promises'; import { dirname, join, resolve } from 'node:path'; import { fileURLToPath, pathToFileURL } from 'node:url'; import { createRequire } from 'node:module'; +/** + * Chunk size for parallel file I/O. Bounded so a 15k-file repo doesn't try + * to open every descriptor at once (would hit `EMFILE`) while still keeping + * libuv's worker-thread pool saturated. Empirically chosen to keep memory + * around tens of MB even when the average file is ~10 KB. + */ +const IO_PARALLELISM = 64; + const __filename = fileURLToPath(import.meta.url); const PLUGIN_ROOT = resolve(dirname(__filename), '../..'); const require = createRequire(resolve(PLUGIN_ROOT, 'package.json')); @@ -57,31 +66,55 @@ async function extractExports(projectRoot, codeFiles) { } const exportsByPath = new Map(); - for (const file of codeFiles) { - const abs = join(projectRoot, file.path); - let content; - try { - content = readFileSync(abs, 'utf-8'); - } catch (err) { - process.stderr.write( - `Warning: compute-batches: exports extraction failed for ${file.path} ` + - `(read error: ${err.message}) — symbols=[] in neighborMap — ` + - `cross-batch edges to this file limited to file-level\n`, - ); - exportsByPath.set(file.path, []); - continue; - } - try { - const analysis = registry.analyzeFile(file.path, content); - const names = (analysis?.exports || []).map(e => e.name).filter(Boolean); - exportsByPath.set(file.path, names); - } catch (err) { - process.stderr.write( - `Warning: compute-batches: exports extraction failed for ${file.path} ` + - `(analyze error: ${err.message}) — symbols=[] in neighborMap — ` + - `cross-batch edges to this file limited to file-level\n`, - ); - exportsByPath.set(file.path, []); + + // I/O is parallelised in bounded chunks (libuv worker threads handle the + // disk reads concurrently) while the actual tree-sitter parse stays on + // the main thread, since web-tree-sitter is single-threaded WASM. For a + // 15k-file iOS repo (#226), the sequential `readFileSync` loop dominated; + // letting reads pipeline drops wall time roughly proportional to the + // share of the loop spent waiting on disk. + for (let start = 0; start < codeFiles.length; start += IO_PARALLELISM) { + const slice = codeFiles.slice(start, start + IO_PARALLELISM); + + // Read every file in the slice concurrently. Errors per file are + // captured in-place so a single bad file does not abort the chunk. + const reads = await Promise.all( + slice.map(async (file) => { + const abs = join(projectRoot, file.path); + try { + const content = await readFile(abs, 'utf-8'); + return { file, content, readError: null }; + } catch (err) { + return { file, content: null, readError: err }; + } + }), + ); + + // Serialise the CPU-bound tree-sitter work and the stderr warning emits + // so log order remains identical to the previous sequential loop. This + // also keeps existing fixture-comparison tests stable. + for (const { file, content, readError } of reads) { + if (readError) { + process.stderr.write( + `Warning: compute-batches: exports extraction failed for ${file.path} ` + + `(read error: ${readError.message}) — symbols=[] in neighborMap — ` + + `cross-batch edges to this file limited to file-level\n`, + ); + exportsByPath.set(file.path, []); + continue; + } + try { + const analysis = registry.analyzeFile(file.path, content); + const names = (analysis?.exports || []).map(e => e.name).filter(Boolean); + exportsByPath.set(file.path, names); + } catch (err) { + process.stderr.write( + `Warning: compute-batches: exports extraction failed for ${file.path} ` + + `(analyze error: ${err.message}) — symbols=[] in neighborMap — ` + + `cross-batch edges to this file limited to file-level\n`, + ); + exportsByPath.set(file.path, []); + } } } return exportsByPath; diff --git a/understand-anything-plugin/skills/understand/extract-import-map.mjs b/understand-anything-plugin/skills/understand/extract-import-map.mjs index 6c547d3..ec31856 100644 --- a/understand-anything-plugin/skills/understand/extract-import-map.mjs +++ b/understand-anything-plugin/skills/understand/extract-import-map.mjs @@ -37,6 +37,29 @@ import { createRequire } from 'node:module'; import { dirname, resolve, join, posix } from 'node:path'; import { fileURLToPath, pathToFileURL } from 'node:url'; import { existsSync, readFileSync, realpathSync, writeFileSync } from 'node:fs'; +import { readFile } from 'node:fs/promises'; + +/** + * Read a list of files concurrently while preserving result order. Failures + * are returned in-place as `{ raw: null, err }` so callers can emit the same + * per-file warnings they did under the previous sequential `readFileSync` + * loops. + * + * `paths` is a list of `{ key, absPath }` pairs; `key` is whatever the caller + * wants to attach the result to (typically a project-relative POSIX path). + */ +async function readFilesParallel(paths) { + return Promise.all( + paths.map(async ({ key, absPath }) => { + try { + const raw = await readFile(absPath, 'utf-8'); + return { key, raw, err: null }; + } catch (err) { + return { key, raw: null, err }; + } + }), + ); +} const __dirname = dirname(fileURLToPath(import.meta.url)); // skills/understand/ -> plugin root is two dirs up @@ -180,20 +203,25 @@ function parseTsConfigText(raw) { * where the stripper damaged a string literal containing `//`. * 3. If both fail, warn and skip — that tsconfig contributes no aliases. */ -function loadTsConfigs(projectRoot, files) { +async function loadTsConfigs(projectRoot, files) { const out = new Map(); + // Collect the candidate paths in the original file order before reading, + // so warning emit order matches the previous sequential implementation. + const candidates = []; for (const f of files) { const p = toPosix(f.path); const base = p.includes('/') ? p.slice(p.lastIndexOf('/') + 1) : p; if (base !== 'tsconfig.json') continue; const absPath = join(projectRoot, p); if (!existsSync(absPath)) continue; - let raw; - try { - raw = readFileSync(absPath, 'utf-8'); - } catch (err) { + candidates.push({ key: p, absPath }); + } + const reads = await readFilesParallel(candidates); + for (const { key: p, raw, err } of reads) { + if (err) { + // absPath isn't carried through the helper return shape; reconstruct it. process.stderr.write( - `Warning: extract-import-map: tsconfig.json at ${absPath} failed ` + + `Warning: extract-import-map: tsconfig.json at ${join(projectRoot, p)} failed ` + `to read (${err.message}) — path aliases from this config will ` + `not be applied — relative imports unaffected\n`, ); @@ -202,7 +230,7 @@ function loadTsConfigs(projectRoot, files) { const parsed = parseTsConfigText(raw); if (!parsed) { process.stderr.write( - `Warning: extract-import-map: tsconfig.json at ${absPath} failed ` + + `Warning: extract-import-map: tsconfig.json at ${join(projectRoot, p)} failed ` + `to parse — path aliases from this config will not be applied ` + `— relative imports unaffected\n`, ); @@ -237,20 +265,20 @@ function loadTsConfigs(projectRoot, files) { * The resolver uses each module's prefix to translate * `import "github.com/foo/bar/x"` into the project-internal `x/.go`. */ -function loadGoModules(projectRoot, files) { +async function loadGoModules(projectRoot, files) { const out = new Map(); + const candidates = []; for (const f of files) { const p = toPosix(f.path); const base = p.includes('/') ? p.slice(p.lastIndexOf('/') + 1) : p; if (base !== 'go.mod') continue; const absPath = join(projectRoot, p); if (!existsSync(absPath)) continue; - let raw; - try { - raw = readFileSync(absPath, 'utf-8'); - } catch { - continue; - } + candidates.push({ key: p, absPath }); + } + const reads = await readFilesParallel(candidates); + for (const { key: p, raw, err } of reads) { + if (err) continue; let moduleName = ''; for (const line of raw.split(/\r?\n/)) { const trimmed = line.replace(/\/\/.*$/, '').trim(); @@ -306,10 +334,17 @@ function findNearestConfigDir(startDir, configMap) { * * Build once; pass everywhere. */ -function buildResolutionContext(projectRoot, files) { +async function buildResolutionContext(projectRoot, files) { const fileSet = new Set(files.map(f => toPosix(f.path))); - const tsConfigs = loadTsConfigs(projectRoot, files); - const goModules = loadGoModules(projectRoot, files); + + // The three config-loader passes are independent and each does its own + // batched parallel I/O; run them concurrently so the wait for a slow + // tsconfig.json read doesn't block go.mod / composer.json scanning. + const [tsConfigs, goModules, phpAutoloads] = await Promise.all([ + loadTsConfigs(projectRoot, files), + loadGoModules(projectRoot, files), + loadPhpAutoloads(projectRoot, files), + ]); // Index .go files by their parent directory so the Go resolver can // expand a package-level import to all member .go files in O(1). @@ -331,8 +366,6 @@ function buildResolutionContext(projectRoot, files) { const kotlinIndex = buildSuffixIndex(files, p => p.endsWith('.kt')); const csIndex = buildSuffixIndex(files, p => p.endsWith('.cs')); - const phpAutoloads = loadPhpAutoloads(projectRoot, files); - return { projectRoot, fileSet, @@ -1019,20 +1052,22 @@ function parseComposerAutoloadText(raw) { * at the bad file and skips it. The rest of the project's PHP imports keep * resolving via whichever composer.json files parsed cleanly. */ -function loadPhpAutoloads(projectRoot, files) { +async function loadPhpAutoloads(projectRoot, files) { const out = new Map(); + const candidates = []; for (const f of files) { const p = toPosix(f.path); const base = p.includes('/') ? p.slice(p.lastIndexOf('/') + 1) : p; if (base !== 'composer.json') continue; const absPath = join(projectRoot, p); if (!existsSync(absPath)) continue; - let raw; - try { - raw = readFileSync(absPath, 'utf-8'); - } catch (err) { + candidates.push({ key: p, absPath }); + } + const reads = await readFilesParallel(candidates); + for (const { key: p, raw, err } of reads) { + if (err) { process.stderr.write( - `Warning: extract-import-map: composer.json at ${absPath} failed ` + + `Warning: extract-import-map: composer.json at ${join(projectRoot, p)} failed ` + `to read (${err.message}) — PSR-4 namespace mapping from this ` + `composer.json unavailable — PHP imports under this package ` + `will not resolve\n`, @@ -1042,7 +1077,7 @@ function loadPhpAutoloads(projectRoot, files) { const parsed = parseComposerAutoloadText(raw); if (parsed === null) { process.stderr.write( - `Warning: extract-import-map: composer.json at ${absPath} failed ` + + `Warning: extract-import-map: composer.json at ${join(projectRoot, p)} failed ` + `to parse — PSR-4 namespace mapping unavailable — PHP imports ` + `under this package will not resolve\n`, ); @@ -1412,8 +1447,10 @@ async function main() { ); } - // Build resolution context (cached configs) - const ctx = buildResolutionContext(projectRoot, files); + // Build resolution context (cached configs). The loader pass for the + // tsconfig/go.mod/composer.json files inside is parallelised — see + // `buildResolutionContext`. + const ctx = await buildResolutionContext(projectRoot, files); const importMap = {}; let filesWithImports = 0;