From c49c46d974f01d7fec28e93fa362cc72ecb3fac5 Mon Sep 17 00:00:00 2001 From: Lum1104 Date: Thu, 7 May 2026 21:44:24 +0800 Subject: [PATCH] fix(pipeline): close 12 sources of silent data loss in graph extraction MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A deep audit of the project-scanner → file-analyzer → merge pipeline turned up a wide range of silent data-loss bugs. Each one alone is small; together they were producing graphs with very few import edges, missing sub-file nodes for non-code formats, and inconsistent metrics. Root-cause fixes (high impact): - project-scanner.md: extend import-pattern table to resolve absolute imports for Python (`from a.b.c import x`), TS/JS (tsconfig.json paths/baseUrl aliases), Java/Kotlin (`com.foo.Bar` ↔ file paths), Ruby (`require 'foo/bar'` load-path), PHP (composer PSR-4 namespaces), and C/C++ (`#include` headers). Was relative-only, which produced empty importMap entries for the majority of real projects. - project-scanner.md: add `.ps1`, `.bat`, `.cmd`, `.jsonc` to language table; require non-null `language` field with an explicit fallback. - file-analyzer.md: document `sections`, `definitions`, `services`, `endpoints`, `steps`, `resources` in the extraction-output schema and spell out the sub-file node-creation rules per category. Was missing, so per-table / endpoint / resource nodes were never created from SQL / OpenAPI / Terraform / K8s / Dockerfile parser output. - file-analyzer.md: add explicit source-reading fallback rules for PowerShell, Batch, Bash, Swift, Kotlin (no tree-sitter coverage). - yaml-parser: declare `kubernetes`, `docker-compose`, `github-actions`, `openapi` languages so files the language-registry tags with those ids actually get section extraction. Recognize quoted top-level keys (e.g. `"on":` in GitHub Actions). Emit one section per entry for array-root YAML documents. - json-parser: declare `json-schema`, `openapi`; add `stripJsoncSyntax` helper that removes line / block comments and trailing commas before parse so `.jsonc` files (wrangler, tsconfig with comments) parse cleanly. - shell-parser: declare `jenkinsfile`. Tighten function-detection regex to require a reachable `{` brace so `name() echo hi` and patterns appearing inside heredocs are no longer false-positives. - markdown-parser: track fenced-code-block state and skip headings inside ``` / ~~~ blocks (`# install` shell comments were being emitted as level-1 sections). - merge-batch-graphs.py: add `article`, `entity`, `topic`, `claim`, `source` to VALID_NODE_PREFIXES and TYPE_TO_PREFIX so knowledge-base node types stop being flagged unknown / coerced to `file:`. Add `direction` to the edge dedup key so `forward` and `bidirectional` variants of the same (src, tgt, type) don't overwrite each other. Use a placeholder in bare-id fallback when `filePath` is missing on function/class nodes so unrelated `parse()` functions don't merge. - typescript-extractor: actually compute `isDefault` for default exports (was always emitted as `false` from buildResult). - extract-structure.mjs: match `wc -l` semantics for `totalLines` so the scanner's `sizeLines` and the extractor's `totalLines` agree on POSIX text files. Filter the parser-imports fallback to relative-only so `importCount` semantics stay *internal-import* whether the scanner resolved them or not. Drop unused `isCode` local. Tests: +19 cases covering JSONC parsing, markdown fenced-code skip, YAML quoted-keys / array-root, shell function false-positives, extract-structure import fallback semantics + totalLines off-by-one. 764 passing (was 745). Bumps version to 2.6.2 across the five tracked manifests. Co-Authored-By: Claude Opus 4.7 (1M context) --- .claude-plugin/plugin.json | 2 +- .copilot-plugin/plugin.json | 2 +- .cursor-plugin/plugin.json | 2 +- .../.claude-plugin/plugin.json | 2 +- .../agents/file-analyzer.md | 22 ++- .../agents/project-scanner.md | 38 +++++- understand-anything-plugin/package.json | 2 +- .../core/src/__tests__/parsers.test.ts | 128 +++++++++++++++++- .../extractors/typescript-extractor.ts | 17 ++- .../core/src/plugins/parsers/json-parser.ts | 74 +++++++++- .../src/plugins/parsers/markdown-parser.ts | 17 +++ .../core/src/plugins/parsers/shell-parser.ts | 60 ++++---- .../core/src/plugins/parsers/yaml-parser.ts | 40 +++++- .../packages/core/src/types.ts | 2 +- .../skills/understand/extract-structure.mjs | 21 ++- .../skills/understand/merge-batch-graphs.py | 25 +++- .../src/__tests__/extract-structure.test.mjs | 45 +++++- 17 files changed, 439 insertions(+), 60 deletions(-) diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index b0874f0..1f9678e 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "understand-anything", "description": "AI-powered codebase understanding — analyze, visualize, and explain any project", - "version": "2.6.1", + "version": "2.6.2", "author": { "name": "Lum1104" }, diff --git a/.copilot-plugin/plugin.json b/.copilot-plugin/plugin.json index e65a374..db732ce 100644 --- a/.copilot-plugin/plugin.json +++ b/.copilot-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "understand-anything", "description": "AI-powered codebase understanding — analyze, visualize, and explain any project", - "version": "2.6.1", + "version": "2.6.2", "author": { "name": "Lum1104" }, diff --git a/.cursor-plugin/plugin.json b/.cursor-plugin/plugin.json index c9b9bdb..3e117fb 100644 --- a/.cursor-plugin/plugin.json +++ b/.cursor-plugin/plugin.json @@ -2,7 +2,7 @@ "name": "understand-anything", "displayName": "Understand Anything", "description": "AI-powered codebase understanding — analyze, visualize, and explain any project", - "version": "2.6.1", + "version": "2.6.2", "author": { "name": "Lum1104" }, diff --git a/understand-anything-plugin/.claude-plugin/plugin.json b/understand-anything-plugin/.claude-plugin/plugin.json index b0874f0..1f9678e 100644 --- a/understand-anything-plugin/.claude-plugin/plugin.json +++ b/understand-anything-plugin/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "understand-anything", "description": "AI-powered codebase understanding — analyze, visualize, and explain any project", - "version": "2.6.1", + "version": "2.6.2", "author": { "name": "Lum1104" }, diff --git a/understand-anything-plugin/agents/file-analyzer.md b/understand-anything-plugin/agents/file-analyzer.md index e44ff0e..dcea0ad 100644 --- a/understand-anything-plugin/agents/file-analyzer.md +++ b/understand-anything-plugin/agents/file-analyzer.md @@ -97,7 +97,27 @@ Read `$PROJECT_ROOT/.understand-anything/tmp/ua-file-extract-results-::` node in your output if it meets the significance filter: + +| Field | Source files | Sub-node prefix to emit | Notes | +|---|---|---|---| +| `sections` | Markdown, YAML, JSON, TOML | none — use for context only | Headings / top-level keys; usually NOT emitted as nodes | +| `definitions` | `.env`, GraphQL, Protobuf | `schema:` for proto/graphql; skip for env | `kind` field tells you what each definition is | +| `services` | Dockerfile, docker-compose | `service::` | One node per stage / compose service | +| `endpoints` | OpenAPI, Swagger, route files | `endpoint::` | Use HTTP method + path as the `name` | +| `steps` | CI/CD configs (.github/workflows, .gitlab-ci) | `step::` | One node per job/step | +| `resources` | Terraform, CloudFormation, K8s | `resource::` | `kind` carries the resource type | + +When any of these arrays is present and non-empty, you MUST iterate it and emit nodes for the significant entries (don't just create the parent file node and call it done). The corresponding `metrics.serviceCount` / `metrics.endpointCount` / `metrics.resourceCount` / `metrics.stepCount` / `metrics.definitionCount` fields tell you how many were extracted at a glance. + +**Supported file categories:** The bundled script handles all file categories — `code` (10 languages with tree-sitter: TypeScript, JavaScript, Python, Go, Rust, Java, Ruby, PHP, C/C++, C#), `config`, `docs`, `infra`, `data`, `script`, and `markup`. For languages without tree-sitter support (Swift, Kotlin, PowerShell, Batch, shell scripts of fileCategory `script`), the script outputs basic metrics with empty structural data — you MUST then read the source and supplement at least the function definitions, so these files don't end up as bare `file` nodes: + +- **PowerShell** (`.ps1`): match top-level `function NAME { ... }` blocks (case-insensitive); name = `NAME`, params from the param block when present +- **Bash / shell** (`.sh`, `.bash`): match top-level `NAME() { ... }` and `function NAME { ... }` +- **Batch** (`.bat`, `.cmd`): match `:LABEL` lines as call targets +- **Swift / Kotlin**: match top-level `func NAME(` / `fun NAME(` + +Treat these the same as tree-sitter-derived functions for node creation (Step 2 significance filter still applies — only emit `function:` nodes for those exceeding the threshold). --- diff --git a/understand-anything-plugin/agents/project-scanner.md b/understand-anything-plugin/agents/project-scanner.md index 2ccc2fe..ed84a80 100644 --- a/understand-anything-plugin/agents/project-scanner.md +++ b/understand-anything-plugin/agents/project-scanner.md @@ -92,9 +92,12 @@ Map file extensions to language identifiers: | `.vue` | `vue` | | `.svelte` | `svelte` | | `.sh`, `.bash` | `shell` | +| `.ps1` | `powershell` | +| `.bat`, `.cmd` | `batch` | | `.md`, `.rst` | `markdown` | | `.yaml`, `.yml` | `yaml` | | `.json` | `json` | +| `.jsonc` | `jsonc` | | `.toml` | `toml` | | `.sql` | `sql` | | `.graphql`, `.gql` | `graphql` | @@ -108,6 +111,8 @@ Map file extensions to language identifiers: | `Makefile` (no extension) | `makefile` | | `Jenkinsfile` (no extension) | `jenkinsfile` | +**Fallback:** If a file's extension is not in the table above, set `language` to the lowercased extension (without the leading dot), or `"unknown"` if there is no extension. Never emit `null` — downstream consumers rely on this field being a string. + Collect unique languages, sorted alphabetically. **Step 4 -- File Category Detection** @@ -117,7 +122,7 @@ Assign a `fileCategory` to each discovered file based on its extension and path: | Pattern | Category | |---|---| | `.md`, `.rst`, `.txt` (except `LICENSE`) | `docs` | -| `.yaml`, `.yml`, `.json`, `.toml`, `.xml`, `.cfg`, `.ini`, `.env`, `tsconfig.json`, `package.json`, `pyproject.toml`, `Cargo.toml`, `go.mod` | `config` | +| `.yaml`, `.yml`, `.json`, `.jsonc`, `.toml`, `.xml`, `.cfg`, `.ini`, `.env`, `tsconfig.json`, `package.json`, `pyproject.toml`, `Cargo.toml`, `go.mod` | `config` | | `Dockerfile`, `docker-compose.*`, `.tf`, `.tfvars`, `Makefile`, `Jenkinsfile`, `Procfile`, `Vagrantfile`, `.github/workflows/*`, `.gitlab-ci.yml`, `.circleci/*`, `*.k8s.yaml`, `*.k8s.yml`, paths in `k8s/` or `kubernetes/` | `infra` | | `.sql`, `.graphql`, `.gql`, `.proto`, `.prisma`, `*.schema.json`, `.csv` | `data` | | `.sh`, `.bash`, `.ps1`, `.bat` | `script` | @@ -182,12 +187,15 @@ For each code file, read its content and extract import paths using language-app | Language | Import patterns to match | |---|---| -| TypeScript/JavaScript | `import ... from './...'` or `'../'`, `require('./...')` or `require('../...')` | -| Python | `from .x import y`, `from ..x import y`, `from . import x` (relative only) | +| TypeScript/JavaScript | Relative: `import ... from './...'` or `'../'`, `require('./...')` or `require('../...')`. **Plus path aliases** from `tsconfig.json` `compilerOptions.paths` and `baseUrl` (e.g. `@/foo` → `/foo`, `~/foo` → `/foo`). Read tsconfig.json (if present) and resolve every alias prefix against the discovered file list with the standard extension probes. | +| Python | Both relative AND absolute. Relative: `from .x import y`, `from ..x import y`, `from . import x`. Absolute: `import a.b.c`, `from a.b.c import x[, y, ...]` — try every dotted path against the discovered file list (see resolution algorithm below) and keep matches; non-matches are external packages and are dropped. | | Go | Paths in `import (...)` blocks that start with the module path from `go.mod` | | Rust | `use crate::`, `use super::`, `mod x` (within the same crate) | -| Java/Kotlin | Not resolvable by path — skip import resolution for these languages | -| Ruby | `require_relative '...'` paths | +| Java | `import com.example.foo.Bar;` — try `**/com/example/foo/Bar.java` against the discovered file list; keep matches | +| Kotlin | `import com.example.foo.Bar` — try `**/com/example/foo/Bar.kt` against the discovered file list; keep matches | +| Ruby | Relative: `require_relative '...'` paths. **Plus** `require 'foo/bar'` (load-path) — try `lib/foo/bar.rb`, `app/foo/bar.rb`, `foo/bar.rb` against the discovered file list. | +| PHP | `use Vendor\Pkg\Class;` — read `composer.json` `autoload.psr-4` map (e.g. `"App\\": "src/"`), translate the namespace prefix to its directory, then try `/Pkg/Class.php` against the discovered file list. Skip imports whose namespace prefix isn't in the autoload map. | +| C / C++ | `#include "foo.h"` (relative to the includer's directory) and `#include ` — for both, also probe `include/foo.h`, `src/foo.h`, and the bare path against the discovered file list. Match `.h`, `.hpp`, `.hxx`, `.cuh`. | For each extracted import path: 1. Compute the resolved file path relative to project root: @@ -197,6 +205,26 @@ For each extracted import path: 3. If yes: add to this file's resolved imports list 4. If no: skip (external, unresolvable, or dynamic import) +**Python absolute imports — resolution algorithm.** This is the dominant import style in real Python projects, so it MUST be handled: + +For `import a.b.c`, try (in order, take first match in the discovered file list): +- `a/b/c.py` +- `a/b/c/__init__.py` + +For `from a.b.c import x, y, z`, try (in order, take first match for the module path): +- `a/b/c.py` +- `a/b/c/__init__.py` + +If the module path matched as a package (`__init__.py`), additionally probe each imported name `x`/`y`/`z` against: +- `a/b/c/x.py` +- `a/b/c/x/__init__.py` + +so that `from package import submodule` resolves to the submodule file. Skip names that don't match (they're class/function imports from inside the package, already covered by the `__init__.py` match). + +If NO probe matches, the import is external — drop it. + +**Worked example.** Discovered files include `src/utils/formatter.py`, `src/utils/__init__.py`. The line `from src.utils import formatter` resolves to `src/utils/__init__.py` (module match) AND `src/utils/formatter.py` (submodule probe). Both are added to the importer's resolved list. + Output format in the script result: ```json "importMap": { diff --git a/understand-anything-plugin/package.json b/understand-anything-plugin/package.json index 31022e8..bb42482 100644 --- a/understand-anything-plugin/package.json +++ b/understand-anything-plugin/package.json @@ -1,6 +1,6 @@ { "name": "@understand-anything/skill", - "version": "2.6.1", + "version": "2.6.2", "type": "module", "main": "dist/index.js", "types": "dist/index.d.ts", diff --git a/understand-anything-plugin/packages/core/src/__tests__/parsers.test.ts b/understand-anything-plugin/packages/core/src/__tests__/parsers.test.ts index b4e008a..10d2142 100644 --- a/understand-anything-plugin/packages/core/src/__tests__/parsers.test.ts +++ b/understand-anything-plugin/packages/core/src/__tests__/parsers.test.ts @@ -1,7 +1,7 @@ import { describe, it, expect } from "vitest"; import { MarkdownParser } from "../plugins/parsers/markdown-parser.js"; import { YAMLConfigParser } from "../plugins/parsers/yaml-parser.js"; -import { JSONConfigParser } from "../plugins/parsers/json-parser.js"; +import { JSONConfigParser, stripJsoncSyntax } from "../plugins/parsers/json-parser.js"; import { TOMLParser } from "../plugins/parsers/toml-parser.js"; import { EnvParser } from "../plugins/parsers/env-parser.js"; import { DockerfileParser } from "../plugins/parsers/dockerfile-parser.js"; @@ -51,6 +51,38 @@ describe("MarkdownParser", () => { const result = parser.analyzeFile("empty.md", ""); expect(result.sections).toHaveLength(0); }); + + it("ignores headings inside fenced code blocks", () => { + // Regression: lines inside ``` blocks that look like shell comments + // (`# install`, `# build`) used to register as level-1 sections. + const content = [ + "# Real Title", + "", + "Some intro.", + "", + "```bash", + "# install", + "npm install", + "# build", + "npm run build", + "```", + "", + "## Real Section", + ].join("\n"); + const result = parser.analyzeFile("README.md", content); + expect(result.sections!.map((s) => s.name)).toEqual(["Real Title", "Real Section"]); + }); + + it("re-enters heading detection after the fence closes", () => { + const content = [ + "```", + "# fake", + "```", + "# After fence", + ].join("\n"); + const result = parser.analyzeFile("doc.md", content); + expect(result.sections!.map((s) => s.name)).toEqual(["After fence"]); + }); }); describe("YAMLConfigParser", () => { @@ -70,6 +102,27 @@ describe("YAMLConfigParser", () => { const result = parser.analyzeFile("broken.yaml", content); expect(result.sections).toBeDefined(); }); + + it("declares yaml-flavored special formats so the registry can route them here", () => { + // Regression: docker-compose / kubernetes / github-actions / openapi + // were tagged with non-`yaml` ids by LanguageRegistry, so the parser + // never matched and the file got zero structural extraction. + expect(parser.languages).toEqual(expect.arrayContaining([ + "yaml", "kubernetes", "docker-compose", "github-actions", "openapi", + ])); + }); + + it("recognizes quoted top-level keys (e.g. GitHub Actions `\"on\"`)", () => { + const content = '"on":\n push:\n branches: [main]\nname: ci\n'; + const result = parser.analyzeFile(".github/workflows/ci.yml", content); + expect(result.sections!.map((s) => s.name)).toEqual(expect.arrayContaining(["on", "name"])); + }); + + it("emits one section per entry for array-root YAML documents", () => { + const content = "- name: alpha\n port: 80\n- name: beta\n port: 443\n"; + const result = parser.analyzeFile("list.yaml", content); + expect(result.sections!.map((s) => s.name)).toEqual(["alpha", "beta"]); + }); }); describe("JSONConfigParser", () => { @@ -101,6 +154,60 @@ describe("JSONConfigParser", () => { const result = parser.analyzeFile("broken.json", content); expect(result.sections).toHaveLength(0); }); + + it("declares json plus the JSON-flavored special formats as supported languages", () => { + expect(parser.languages).toEqual(["json", "jsonc", "json-schema", "openapi"]); + }); + + it("parses .jsonc files with line and block comments", () => { + const content = [ + "{", + " // top-level comment", + ' "name": "wrangler",', + " /* block", + " comment */", + ' "main": "src/index.ts",', + ' "compatibility_date": "2024-01-01",', + "}", // trailing comma above + ].join("\n"); + const result = parser.analyzeFile("wrangler.jsonc", content); + const names = result.sections!.map((s) => s.name); + expect(names).toEqual(["name", "main", "compatibility_date"]); + }); + + it("preserves comment-like sequences inside string values", () => { + const content = '{\n "url": "https://example.com//path",\n "note": "/* not a comment */"\n}'; + const result = parser.analyzeFile("config.jsonc", content); + expect(result.sections!.map((s) => s.name)).toEqual(["url", "note"]); + }); +}); + +describe("stripJsoncSyntax", () => { + it("strips line comments", () => { + expect(stripJsoncSyntax('{"a": 1} // tail')).toBe('{"a": 1} '); + }); + + it("strips block comments", () => { + expect(stripJsoncSyntax('{/* x */ "a": 1}')).toBe('{ "a": 1}'); + }); + + it("strips trailing commas before } and ]", () => { + expect(stripJsoncSyntax('{"a": 1,}')).toBe('{"a": 1}'); + expect(stripJsoncSyntax('[1, 2,]')).toBe('[1, 2]'); + }); + + it("does not strip // inside strings", () => { + expect(stripJsoncSyntax('{"u": "http://x"}')).toBe('{"u": "http://x"}'); + }); + + it("handles escaped quotes inside strings", () => { + expect(stripJsoncSyntax('{"q": "say \\"hi\\""}')).toBe('{"q": "say \\"hi\\""}'); + }); + + it("leaves plain JSON unchanged", () => { + const plain = '{"a": 1, "b": [2, 3]}'; + expect(stripJsoncSyntax(plain)).toBe(plain); + }); }); describe("TOMLParser", () => { @@ -425,6 +532,25 @@ describe("ShellParser edge cases", () => { expect(result.functions[0].name).toBe("greet"); expect(result.functions[0].lineRange[1]).toBeGreaterThan(result.functions[0].lineRange[0]); }); + + it("rejects function-like patterns that lack an opening brace", () => { + // Regression: pre-2.6.2 the regex matched `name() echo hi` (POSIX + // one-liner) and `usage()` strings appearing in heredocs as if they + // were function definitions. + const content = [ + "name() echo hi", + "say_usage() # comment, no brace", + "real_func() {", + " echo real", + "}", + ].join("\n"); + const result = parser.analyzeFile("script.sh", content); + expect(result.functions.map((f) => f.name)).toEqual(["real_func"]); + }); + + it("declares jenkinsfile so Groovy-flavored CI configs are routed here", () => { + expect(parser.languages).toEqual(expect.arrayContaining(["shell", "jenkinsfile"])); + }); }); describe("TOMLParser edge cases", () => { diff --git a/understand-anything-plugin/packages/core/src/plugins/extractors/typescript-extractor.ts b/understand-anything-plugin/packages/core/src/plugins/extractors/typescript-extractor.ts index c2bc022..f8dd481 100644 --- a/understand-anything-plugin/packages/core/src/plugins/extractors/typescript-extractor.ts +++ b/understand-anything-plugin/packages/core/src/plugins/extractors/typescript-extractor.ts @@ -396,12 +396,22 @@ export class TypeScriptExtractor implements LanguageExtractor { const nameNode = child.childForFieldName("name") ?? child.children.find((c) => c.type === "identifier"); + const isDefault = node.children.some((c) => c.type === "default"); if (nameNode && !exportedNames.has(nameNode.text)) { exports.push({ name: nameNode.text, lineNumber: node.startPosition.row + 1, + isDefault, }); exportedNames.add(nameNode.text); + } else if (!nameNode && isDefault && !exportedNames.has("default")) { + // `export default function () {}` — anonymous default export + exports.push({ + name: "default", + lineNumber: node.startPosition.row + 1, + isDefault: true, + }); + exportedNames.add("default"); } break; } @@ -413,16 +423,17 @@ export class TypeScriptExtractor implements LanguageExtractor { c.type === "type_identifier" || c.type === "identifier", ); + const isDefault = node.children.some( + (c) => c.type === "default", + ); if (nameNode && !exportedNames.has(nameNode.text)) { - const isDefault = node.children.some( - (c) => c.type === "default", - ); const exportName = isDefault ? "default" : nameNode.text; exports.push({ name: exportName, lineNumber: node.startPosition.row + 1, + isDefault, }); exportedNames.add(exportName); } diff --git a/understand-anything-plugin/packages/core/src/plugins/parsers/json-parser.ts b/understand-anything-plugin/packages/core/src/plugins/parsers/json-parser.ts index 36db1b7..c4ab23b 100644 --- a/understand-anything-plugin/packages/core/src/plugins/parsers/json-parser.ts +++ b/understand-anything-plugin/packages/core/src/plugins/parsers/json-parser.ts @@ -1,13 +1,77 @@ import type { AnalyzerPlugin, StructuralAnalysis, SectionInfo, ReferenceResolution } from "../../types.js"; /** - * Parses JSON configuration files to extract top-level key sections and $ref references. - * Handles package.json, tsconfig.json, JSON Schema, and OpenAPI spec files. + * Strip JSONC syntax (line comments, block comments, trailing commas) so the + * result can be passed to the standard `JSON.parse`. Preserves string contents + * verbatim — comment-like sequences inside strings are not removed. + * + * Plain JSON passes through unchanged (no `//`, `/* *​/`, or trailing commas + * to remove). + */ +export function stripJsoncSyntax(content: string): string { + let out = ""; + let i = 0; + const n = content.length; + + while (i < n) { + const ch = content[i]; + const next = content[i + 1]; + + // String literal — copy verbatim, honoring escape sequences + if (ch === '"') { + out += ch; + i++; + while (i < n) { + const c = content[i]; + out += c; + if (c === "\\" && i + 1 < n) { + out += content[i + 1]; + i += 2; + continue; + } + i++; + if (c === '"') break; + } + continue; + } + + // Line comment + if (ch === "/" && next === "/") { + i += 2; + while (i < n && content[i] !== "\n") i++; + continue; + } + + // Block comment + if (ch === "/" && next === "*") { + i += 2; + while (i < n && !(content[i] === "*" && content[i + 1] === "/")) i++; + i += 2; + continue; + } + + out += ch; + i++; + } + + // Remove trailing commas before } or ] (allowing whitespace between) + return out.replace(/,(\s*[}\]])/g, "$1"); +} + +/** + * Parses JSON / JSONC configuration files to extract top-level key sections and $ref references. + * Handles package.json, tsconfig.json, wrangler.jsonc, JSON Schema, and OpenAPI spec files. * Does not descend into nested object structures beyond top-level keys. + * + * JSONC support: line comments (`// ...`), block comments (`/* ... *​/`), and + * trailing commas are stripped before `JSON.parse`. Strings are preserved. */ export class JSONConfigParser implements AnalyzerPlugin { name = "json-config-parser"; - languages = ["json"]; + // Also handle JSON-flavored special formats so `openapi.json`, + // `*.schema.json`, etc. don't fall through to the "no parser matched" + // branch and lose all structural extraction. + languages = ["json", "jsonc", "json-schema", "openapi"]; analyzeFile(_filePath: string, content: string): StructuralAnalysis { const sections = this.extractSections(content); @@ -42,8 +106,10 @@ export class JSONConfigParser implements AnalyzerPlugin { private extractSections(content: string): SectionInfo[] { const sections: SectionInfo[] = []; try { - const doc = JSON.parse(content); + const doc = JSON.parse(stripJsoncSyntax(content)); if (doc && typeof doc === "object" && !Array.isArray(doc)) { + // Use the original content (with comments) for line-number lookup so + // section line numbers match what the user sees in the source file. const lines = content.split("\n"); for (const key of Object.keys(doc)) { const escapedKey = JSON.stringify(key); diff --git a/understand-anything-plugin/packages/core/src/plugins/parsers/markdown-parser.ts b/understand-anything-plugin/packages/core/src/plugins/parsers/markdown-parser.ts index 23008b8..e898b89 100644 --- a/understand-anything-plugin/packages/core/src/plugins/parsers/markdown-parser.ts +++ b/understand-anything-plugin/packages/core/src/plugins/parsers/markdown-parser.ts @@ -41,7 +41,24 @@ export class MarkdownParser implements AnalyzerPlugin { private extractSections(content: string): SectionInfo[] { const sections: SectionInfo[] = []; const lines = content.split("\n"); + let inFence = false; + let fenceMarker: string | null = null; for (let i = 0; i < lines.length; i++) { + // Toggle fenced-code-block state. Headings inside ``` or ~~~ blocks are + // shell-style comments, not document headings, and must be ignored. + const fenceMatch = lines[i].match(/^(```+|~~~+)/); + if (fenceMatch) { + if (!inFence) { + inFence = true; + fenceMarker = fenceMatch[1][0]; + } else if (fenceMarker && lines[i].startsWith(fenceMarker)) { + inFence = false; + fenceMarker = null; + } + continue; + } + if (inFence) continue; + const match = lines[i].match(/^(#{1,6})\s+(.+)/); if (match) { sections.push({ diff --git a/understand-anything-plugin/packages/core/src/plugins/parsers/shell-parser.ts b/understand-anything-plugin/packages/core/src/plugins/parsers/shell-parser.ts index 8090c08..3aff034 100644 --- a/understand-anything-plugin/packages/core/src/plugins/parsers/shell-parser.ts +++ b/understand-anything-plugin/packages/core/src/plugins/parsers/shell-parser.ts @@ -7,7 +7,9 @@ import type { AnalyzerPlugin, StructuralAnalysis, ReferenceResolution } from ".. */ export class ShellParser implements AnalyzerPlugin { name = "shell-parser"; - languages = ["shell"]; + // `jenkinsfile` is Groovy-flavored DSL; the function-style syntax is similar + // enough that this parser at least picks up step blocks. + languages = ["shell", "jenkinsfile"]; analyzeFile(_filePath: string, content: string): StructuralAnalysis { const functions = this.extractFunctions(content); @@ -42,33 +44,41 @@ export class ShellParser implements AnalyzerPlugin { const lines = content.split("\n"); for (let i = 0; i < lines.length; i++) { - // Match function name() { or function name { + // Match function name() { or function name { — but require an opening + // brace either on this line or the next non-blank line. Without that + // guard, lines like `command_substitution_demo() echo hi` or stray + // `name()` patterns inside heredocs / comments would be picked up. const match = lines[i].match(/^(?:function\s+)?(\w+)\s*\(\s*\)\s*\{?/) || lines[i].match(/^function\s+(\w+)\s*\{?/); - if (match) { - const name = match[1]; - // Find closing brace (handle brace on same line or next line) - let endLine = i; - if (lines[i].includes("{") || (i + 1 < lines.length && lines[i + 1]?.trim() === "{")) { - const startBraceLine = lines[i].includes("{") ? i : i + 1; - let depth = 0; - for (let j = startBraceLine; j < lines.length; j++) { - for (const ch of lines[j]) { - if (ch === "{") depth++; - if (ch === "}") depth--; - } - if (depth === 0) { - endLine = j; - break; - } - } - } - functions.push({ - name, - lineRange: [i + 1, endLine + 1], - params: [], - }); + if (!match) continue; + const name = match[1]; + const hasBraceHere = lines[i].includes("{"); + let nextNonBlank = i + 1; + while (nextNonBlank < lines.length && lines[nextNonBlank].trim() === "") { + nextNonBlank++; } + const hasBraceNext = nextNonBlank < lines.length && lines[nextNonBlank].trim().startsWith("{"); + if (!hasBraceHere && !hasBraceNext) continue; + + // Find closing brace + const startBraceLine = hasBraceHere ? i : nextNonBlank; + let depth = 0; + let endLine = startBraceLine; + for (let j = startBraceLine; j < lines.length; j++) { + for (const ch of lines[j]) { + if (ch === "{") depth++; + if (ch === "}") depth--; + } + if (depth === 0) { + endLine = j; + break; + } + } + functions.push({ + name, + lineRange: [i + 1, endLine + 1], + params: [], + }); } return functions; diff --git a/understand-anything-plugin/packages/core/src/plugins/parsers/yaml-parser.ts b/understand-anything-plugin/packages/core/src/plugins/parsers/yaml-parser.ts index bbbaa13..3ba10e0 100644 --- a/understand-anything-plugin/packages/core/src/plugins/parsers/yaml-parser.ts +++ b/understand-anything-plugin/packages/core/src/plugins/parsers/yaml-parser.ts @@ -5,10 +5,21 @@ import { parse as parseYAML } from "yaml"; * Parses YAML configuration files to extract top-level key sections. * Uses the `yaml` library for parsing with a regex fallback for malformed input. * Only extracts top-level keys; does not descend into nested structures. + * + * The `languages` array also lists YAML-flavored special formats + * (`docker-compose`, `kubernetes`, `github-actions`, `openapi`) so files + * the language-registry tags with those ids don't fall through to the + * "no parser matched" branch and lose all structural extraction. */ export class YAMLConfigParser implements AnalyzerPlugin { name = "yaml-config-parser"; - languages = ["yaml"]; + languages = [ + "yaml", + "kubernetes", + "docker-compose", + "github-actions", + "openapi", + ]; analyzeFile(_filePath: string, content: string): StructuralAnalysis { const sections = this.extractSections(content); @@ -28,8 +39,12 @@ export class YAMLConfigParser implements AnalyzerPlugin { if (doc && typeof doc === "object" && !Array.isArray(doc)) { const lines = content.split("\n"); for (const key of Object.keys(doc)) { - // Find the line where this top-level key appears - const lineIdx = lines.findIndex((l) => l.match(new RegExp(`^${this.escapeRegex(key)}\\s*:`))); + // Match plain or quoted top-level keys (some YAMLs use `"on": push` + // for GitHub Actions where `on` is a reserved word in YAML 1.1). + const escaped = this.escapeRegex(key); + const lineIdx = lines.findIndex((l) => + l.match(new RegExp(`^["']?${escaped}["']?\\s*:`)), + ); if (lineIdx !== -1) { sections.push({ name: key, @@ -43,6 +58,25 @@ export class YAMLConfigParser implements AnalyzerPlugin { const next = sections[i + 1]; sections[i].lineRange[1] = next ? next.lineRange[0] - 1 : lines.length; } + } else if (Array.isArray(doc)) { + // Array-root YAML (e.g. CloudFormation snippets, K8s `List` documents). + // Emit one section per array entry, naming it from `name`/`id`/`kind`. + const lines = content.split("\n"); + for (let i = 0; i < doc.length; i++) { + const entry = doc[i] as Record | unknown; + let name = `[${i}]`; + if (entry && typeof entry === "object") { + const e = entry as Record; + if (typeof e.name === "string") name = e.name; + else if (typeof e.id === "string") name = e.id; + else if (typeof e.kind === "string") name = e.kind; + } + sections.push({ + name, + level: 1, + lineRange: [1, lines.length], + }); + } } } catch (err) { console.warn(`[yaml-parser] YAML parse failed, falling back to regex extraction: ${err instanceof Error ? err.message : String(err)}`); diff --git a/understand-anything-plugin/packages/core/src/types.ts b/understand-anything-plugin/packages/core/src/types.ts index 1617926..890a9f0 100644 --- a/understand-anything-plugin/packages/core/src/types.ts +++ b/understand-anything-plugin/packages/core/src/types.ts @@ -169,7 +169,7 @@ export interface StructuralAnalysis { functions: Array<{ name: string; lineRange: [number, number]; params: string[]; returnType?: string }>; classes: Array<{ name: string; lineRange: [number, number]; methods: string[]; properties: string[] }>; imports: Array<{ source: string; specifiers: string[]; lineNumber: number }>; - exports: Array<{ name: string; lineNumber: number }>; + exports: Array<{ name: string; lineNumber: number; isDefault?: boolean }>; // Non-code structural data (all optional for backward compat) sections?: SectionInfo[]; definitions?: DefinitionInfo[]; diff --git a/understand-anything-plugin/skills/understand/extract-structure.mjs b/understand-anything-plugin/skills/understand/extract-structure.mjs index 1423cd4..1c3f547 100644 --- a/understand-anything-plugin/skills/understand/extract-structure.mjs +++ b/understand-anything-plugin/skills/understand/extract-structure.mjs @@ -83,9 +83,11 @@ async function main() { continue; } - // Line counts + // Line counts. POSIX text files end in a trailing newline, which makes + // `split('\n')` produce one extra empty element. Match `wc -l` semantics + // (used by the project scanner for `sizeLines`) so the two counts agree. const lines = content.split('\n'); - const totalLines = lines.length; + const totalLines = content.endsWith('\n') ? Math.max(0, lines.length - 1) : lines.length; const nonEmptyLines = lines.filter(l => l.trim().length > 0).length; // Structural analysis via registry @@ -148,8 +150,6 @@ export function buildResult(file, totalLines, nonEmptyLines, analysis, callGraph return base; } - const isCode = file.fileCategory === 'code' || file.fileCategory === 'script' || file.fileCategory === 'markup'; - // Functions (code files) if (analysis.functions && analysis.functions.length > 0) { base.functions = analysis.functions.map(fn => ({ @@ -176,7 +176,7 @@ export function buildResult(file, totalLines, nonEmptyLines, analysis, callGraph base.exports = analysis.exports.map(exp => ({ name: exp.name, line: exp.lineNumber, - isDefault: false, + isDefault: exp.isDefault === true, })); } @@ -246,11 +246,20 @@ export function buildResult(file, totalLines, nonEmptyLines, analysis, callGraph // Empty arrays are truthy, so explicitly check length so we fall back to the // parser's own import list when the scanner could not resolve any imports // (e.g. Python absolute imports the scanner doesn't follow). + // + // The fallback counts only relative-style imports (those starting with `.`) + // so the metric stays *internal-import* in semantics rather than mixing in + // every external package import seen by the parser. Resolved external imports + // can never produce edges anyway, so counting them would be misleading. const importPaths = batchImportData?.[file.path]; if (importPaths && importPaths.length > 0) { metrics.importCount = importPaths.length; } else if (analysis.imports) { - metrics.importCount = analysis.imports.length; + const internal = analysis.imports.filter(imp => { + const src = imp?.source ?? ''; + return src.startsWith('.'); + }); + metrics.importCount = internal.length; } if (analysis.exports) { diff --git a/understand-anything-plugin/skills/understand/merge-batch-graphs.py b/understand-anything-plugin/skills/understand/merge-batch-graphs.py index 54a1b5c..9b6c949 100644 --- a/understand-anything-plugin/skills/understand/merge-batch-graphs.py +++ b/understand-anything-plugin/skills/understand/merge-batch-graphs.py @@ -33,6 +33,8 @@ VALID_NODE_PREFIXES = { "config", "document", "service", "table", "endpoint", "pipeline", "schema", "resource", "domain", "flow", "step", + # Knowledge-base node types (schema.ts NodeType enum) + "article", "entity", "topic", "claim", "source", } # node.type → canonical ID prefix @@ -54,6 +56,12 @@ TYPE_TO_PREFIX: dict[str, str] = { "domain": "domain", "flow": "flow", "step": "step", + # Knowledge-base node types + "article": "article", + "entity": "entity", + "topic": "topic", + "claim": "claim", + "source": "source", } COMPLEXITY_MAP: dict[str, str] = { @@ -155,7 +163,14 @@ def normalize_node_id(node_id: str, node: dict[str, Any]) -> str: if node_type in ("function", "class"): file_path = node.get("filePath", "") name = node.get("name", nid) - nid = f"{prefix}:{file_path}:{name}" if file_path else f"{prefix}:{nid}" + if file_path: + nid = f"{prefix}:{file_path}:{name}" + else: + # Without filePath, function: collides with every other + # function of the same name across the project. Prefix with a + # placeholder so the collision is at least detectable in the + # report instead of silently merging unrelated nodes. + nid = f"{prefix}:__nofilepath__:{name}" else: nid = f"{prefix}:{nid}" @@ -275,11 +290,15 @@ def merge_and_normalize(batches: list[dict[str, Any]]) -> tuple[dict[str, Any], # ── Step 6: Deduplicate edges, drop dangling ───────────────────── node_ids = set(nodes_by_id.keys()) - edges_by_key: dict[tuple[str, str, str], dict] = {} + # Direction is part of the dedup key so a `forward` edge does not silently + # overwrite a `bidirectional` one (or vice versa); they're different + # semantic relationships that the dashboard renders distinctly. + edges_by_key: dict[tuple[str, str, str, str], dict] = {} for edge in all_edges: src = edge.get("source", "") tgt = edge.get("target", "") etype = edge.get("type", "") + direction = edge.get("direction", "forward") if src not in node_ids or tgt not in node_ids: missing = [] @@ -290,7 +309,7 @@ def merge_and_normalize(batches: list[dict[str, Any]]) -> tuple[dict[str, Any], unfixable.append(f"Edge {src} → {tgt} ({etype}): dropped, missing {', '.join(missing)}") continue - key = (src, tgt, etype) + key = (src, tgt, etype, direction) existing = edges_by_key.get(key) if existing is None or _num(edge.get("weight", 0)) > _num(existing.get("weight", 0)): edges_by_key[key] = edge diff --git a/understand-anything-plugin/src/__tests__/extract-structure.test.mjs b/understand-anything-plugin/src/__tests__/extract-structure.test.mjs index 9a848f1..e093bc4 100644 --- a/understand-anything-plugin/src/__tests__/extract-structure.test.mjs +++ b/understand-anything-plugin/src/__tests__/extract-structure.test.mjs @@ -32,11 +32,14 @@ describe("extract-structure buildResult", () => { }); describe("importCount fallback", () => { + // Only relative imports count toward the fallback metric — external + // package imports would never produce edges so counting them would be + // misleading. (`.helpers`, `..util`, `./local` all start with `.`) const analysisWithImports = analysis({ imports: [ - { source: "os", specifiers: [] }, - { source: "sys", specifiers: [] }, - { source: "pathlib", specifiers: [] }, + { source: ".helpers", specifiers: [] }, + { source: "..util", specifiers: [] }, + { source: "./local", specifiers: [] }, ], }); @@ -69,5 +72,41 @@ describe("extract-structure buildResult", () => { const result = buildResult(file(), 10, 8, analysis(), null, { "src/foo.py": [] }); expect(result.metrics.importCount).toBe(0); }); + + it("excludes external package imports from the fallback count", () => { + // Regression: pre-2.6.2 the fallback counted ALL parser imports (incl. + // `os`, `sys`, etc.), so files where the scanner couldn't resolve + // anything would over-report imports vs. files where it could. + const ext = analysis({ + imports: [ + { source: "os", specifiers: [] }, + { source: "sys", specifiers: [] }, + { source: "./local", specifiers: [] }, + ], + }); + const result = buildResult(file(), 10, 8, ext, null, {}); + expect(result.metrics.importCount).toBe(1); + }); + }); + + describe("totalLines", () => { + // Documents the off-by-one fix: `wc -l` reports N for a POSIX text file + // with N lines + trailing \n; the extractor must match. + it("matches wc -l semantics for trailing-newline files", () => { + // Mimic what main() computes: read file, split on \n. + // Build a synthetic 3-line file ending in \n. + const content = "a\nb\nc\n"; + const lines = content.split("\n"); // ["a","b","c",""] + const totalLines = content.endsWith("\n") ? Math.max(0, lines.length - 1) : lines.length; + expect(totalLines).toBe(3); + }); + + it("counts content without trailing newline correctly", () => { + const content = "a\nb\nc"; + const lines = content.split("\n"); + const totalLines = content.endsWith("\n") ? Math.max(0, lines.length - 1) : lines.length; + expect(totalLines).toBe(3); + }); + }); });