From 44e1fee31bfb44c7b25bd5dbbe4d5f83b2bafcec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?d=20=F0=9F=94=B9?= <258577966+voidborne-d@users.noreply.github.com> Date: Mon, 4 May 2026 11:49:20 +0800 Subject: [PATCH 1/2] fix(dashboard): O(N+K) per-layer aggregations, kill quadratic Array.includes (#102) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three hot paths in the dashboard ran `layer.nodeIds.includes(node.id)`, which is O(K) per check. Combined with their enclosing loops they collectively spent quadratic time per render of the overview / per filter recompute / per node-selection event. On the 4.8 MB knowledge graph reported in #102, the overview render alone took ~470 ms of synchronous main-thread work before ELK / React Flow ran — long enough for the page to register as unresponsive. Fix: precompute two indexes once when a graph is loaded. - `nodesById: Map` - `nodeIdToLayerId: Map` (first layer wins, matching prior `findNodeLayer` semantics) Both live in `useDashboardStore` and are rebuilt by `setGraph`. The three call sites: 1. `useOverviewGraph` (GraphView.tsx) — per-layer complexity aggregation moved into a new `computeLayerStats(layer, nodesById)` helper that walks `layer.nodeIds` instead of filtering all `graph.nodes`. Search match counts now read straight from `nodeIdToLayerId` instead of rebuilding a layer index on every searchResults change. 2. `filterNodes` (utils/filters.ts) — takes `nodeIdToLayerId` instead of `Layer[]`; the layer-membership check is one Map.get() per node. Updated `ExportMenu.tsx` caller to pass the store-level index. 3. `findNodeLayer` (store.ts) — replaced with `nodeIdToLayerId.get()` at the four call sites. `navigateTourToLayer` helper updated to take the index rather than the whole graph. Behavior is preserved exactly: - "First layer wins" semantics for nodes that appear in multiple layers (#102 schema doesn't forbid this). - 30 % aggregate-complexity threshold pinned by tests. - Layer filter that excludes layer-less orphans, but ungated when no layers are selected. Verified locally: Bench (`scripts/benchmark-aggregations.mjs`, node 22): 100 layers × 200 nodes (#102 shape): 475 ms → 2 ms (232× faster) 50 layers × 200 nodes: 116 ms → 0.6 ms (190× faster) 30 layers × 100 nodes: 12 ms → 0.2 ms (63× faster) Tests: `pnpm --filter @understand-anything/dashboard test` 24 → 41 pass (+17 new tests across `layerStats.test.ts` and `filters.test.ts`, including a #102 perf-regression guard at 100 layers × 100 nodes < 50 ms). `pnpm --filter @understand-anything/core test` — 654 / 654 pass. `pnpm --filter @understand-anything/dashboard exec tsc -b` — clean. `pnpm --filter @understand-anything/dashboard build` — clean. Pre-existing on master and not from this branch: `pnpm lint` errors out with "eslint: command not found" — `eslint` isn't installed by any package and the root `lint` script is bare `eslint .`. Out of scope here. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../scripts/benchmark-aggregations.mjs | 99 ++++++++++ .../dashboard/src/components/ExportMenu.tsx | 3 +- .../dashboard/src/components/GraphView.tsx | 34 ++-- .../packages/dashboard/src/store.ts | 67 +++++-- .../src/utils/__tests__/filters.test.ts | 175 ++++++++++++++++++ .../src/utils/__tests__/layerStats.test.ts | 118 ++++++++++++ .../packages/dashboard/src/utils/filters.ts | 20 +- .../dashboard/src/utils/layerStats.ts | 39 ++++ 8 files changed, 505 insertions(+), 50 deletions(-) create mode 100644 understand-anything-plugin/packages/dashboard/scripts/benchmark-aggregations.mjs create mode 100644 understand-anything-plugin/packages/dashboard/src/utils/__tests__/filters.test.ts create mode 100644 understand-anything-plugin/packages/dashboard/src/utils/__tests__/layerStats.test.ts create mode 100644 understand-anything-plugin/packages/dashboard/src/utils/layerStats.ts diff --git a/understand-anything-plugin/packages/dashboard/scripts/benchmark-aggregations.mjs b/understand-anything-plugin/packages/dashboard/scripts/benchmark-aggregations.mjs new file mode 100644 index 0000000..0c3848f --- /dev/null +++ b/understand-anything-plugin/packages/dashboard/scripts/benchmark-aggregations.mjs @@ -0,0 +1,99 @@ +// Per-layer aggregation perf benchmark. +// +// Mirrors the BEFORE shape (graph.nodes.filter(n => layer.nodeIds.includes(n.id)) +// per layer) and the AFTER shape (single nodesById Map + iterate layer.nodeIds) +// from `useOverviewGraph` in `src/components/GraphView.tsx`. Issue #102 reported +// a 4.8 MB knowledge graph that froze the dashboard on overview render — the +// quadratic Array.includes pass was the dominant synchronous cost. +// +// We can't import the dashboard helper directly (Vite-bundled, no +// per-module dist), so the new shape is reproduced here in lockstep with +// `src/utils/layerStats.ts::computeLayerStats`. +// +// Usage: +// node understand-anything-plugin/packages/dashboard/scripts/benchmark-aggregations.mjs + +import { performance } from "node:perf_hooks"; + +function makeGraph(layerCount, nodesPerLayer) { + const nodes = []; + const layers = []; + for (let li = 0; li < layerCount; li++) { + const ids = []; + for (let ni = 0; ni < nodesPerLayer; ni++) { + const id = `n-${li}-${ni}`; + const complexity = ["simple", "moderate", "complex"][(li + ni) % 3]; + nodes.push({ id, complexity }); + ids.push(id); + } + layers.push({ id: `L${li}`, nodeIds: ids }); + } + return { nodes, layers }; +} + +// --- BEFORE: O(N × K × L) per overview render ---------------------------- +function aggregateBefore(graph) { + const out = []; + for (const layer of graph.layers) { + const memberNodes = graph.nodes.filter((n) => layer.nodeIds.includes(n.id)); + const c = { simple: 0, moderate: 0, complex: 0 }; + for (const n of memberNodes) c[n.complexity]++; + const aggregate = + c.complex > memberNodes.length * 0.3 + ? "complex" + : c.moderate > memberNodes.length * 0.3 + ? "moderate" + : "simple"; + out.push({ id: layer.id, aggregateComplexity: aggregate }); + } + return out; +} + +// --- AFTER: O(N + Σ K_i) per overview render ---------------------------- +function aggregateAfter(graph, nodesById) { + const out = []; + for (const layer of graph.layers) { + const c = { simple: 0, moderate: 0, complex: 0 }; + let resolved = 0; + for (const nid of layer.nodeIds) { + const node = nodesById.get(nid); + if (!node) continue; + resolved++; + c[node.complexity]++; + } + const aggregate = + c.complex > resolved * 0.3 + ? "complex" + : c.moderate > resolved * 0.3 + ? "moderate" + : "simple"; + out.push({ id: layer.id, aggregateComplexity: aggregate }); + } + return out; +} + +function bench(label, layerCount, nodesPerLayer) { + const graph = makeGraph(layerCount, nodesPerLayer); + const nodesById = new Map(graph.nodes.map((n) => [n.id, n])); + + const t0 = performance.now(); + const before = aggregateBefore(graph); + const t1 = performance.now(); + const after = aggregateAfter(graph, nodesById); + const t2 = performance.now(); + + const beforeMs = t1 - t0; + const afterMs = t2 - t1; + const speedup = afterMs > 0 ? beforeMs / afterMs : Infinity; + const parity = JSON.stringify(before) === JSON.stringify(after); + console.log( + `${label} (${layerCount} layers × ${nodesPerLayer} nodes = ${graph.nodes.length} total): ` + + `BEFORE ${beforeMs.toFixed(1)}ms | AFTER ${afterMs.toFixed(1)}ms | ` + + `${speedup.toFixed(1)}× faster | parity ${parity}`, + ); +} + +bench("small", 10, 50); +bench("medium", 30, 100); +bench("large", 50, 200); +bench("issue#102 shape", 100, 200); diff --git a/understand-anything-plugin/packages/dashboard/src/components/ExportMenu.tsx b/understand-anything-plugin/packages/dashboard/src/components/ExportMenu.tsx index 25c7423..fbaa1c3 100644 --- a/understand-anything-plugin/packages/dashboard/src/components/ExportMenu.tsx +++ b/understand-anything-plugin/packages/dashboard/src/components/ExportMenu.tsx @@ -20,6 +20,7 @@ function downloadBlob(blob: Blob, filename: string) { export default function ExportMenu() { const graph = useDashboardStore((s) => s.graph); + const nodeIdToLayerId = useDashboardStore((s) => s.nodeIdToLayerId); const filters = useDashboardStore((s) => s.filters); const exportMenuOpen = useDashboardStore((s) => s.exportMenuOpen); const toggleExportMenu = useDashboardStore((s) => s.toggleExportMenu); @@ -187,7 +188,7 @@ export default function ExportMenu() { ? graph.nodes.filter((n) => !subFileTypes.has(n.type)) : graph.nodes; - filteredGraphNodes = filterNodes(filteredGraphNodes, graph.layers ?? [], filters); + filteredGraphNodes = filterNodes(filteredGraphNodes, nodeIdToLayerId, filters); const filteredNodeIds = new Set(filteredGraphNodes.map((n) => n.id)); let filteredGraphEdges = graph.edges.filter( diff --git a/understand-anything-plugin/packages/dashboard/src/components/GraphView.tsx b/understand-anything-plugin/packages/dashboard/src/components/GraphView.tsx index d6367af..0caaf98 100644 --- a/understand-anything-plugin/packages/dashboard/src/components/GraphView.tsx +++ b/understand-anything-plugin/packages/dashboard/src/components/GraphView.tsx @@ -51,6 +51,7 @@ import { } from "../utils/edgeAggregation"; import { deriveContainers } from "../utils/containers"; import type { DerivedContainer } from "../utils/containers"; +import { computeLayerStats } from "../utils/layerStats"; const nodeTypes = { custom: CustomNode, @@ -139,6 +140,8 @@ function SelectedNodeFitView() { function useOverviewGraph() { const graph = useDashboardStore((s) => s.graph); + const nodesById = useDashboardStore((s) => s.nodesById); + const nodeIdToLayerId = useDashboardStore((s) => s.nodeIdToLayerId); const searchResults = useDashboardStore((s) => s.searchResults); const drillIntoLayer = useDashboardStore((s) => s.drillIntoLayer); @@ -154,36 +157,25 @@ function useOverviewGraph() { return null; } - // Build search match counts per layer + // Build search match counts per layer using the precomputed + // nodeIdToLayerId index. Reusing the store-level index avoids an extra + // O(N) pass when search results change frequently. const searchMatchByLayer = new Map(); if (searchResults.length > 0) { - const nodeToLayer = new Map(); - for (const layer of layers) { - for (const nid of layer.nodeIds) { - nodeToLayer.set(nid, layer.id); - } - } for (const result of searchResults) { - const lid = nodeToLayer.get(result.nodeId); + const lid = nodeIdToLayerId.get(result.nodeId); if (lid) { searchMatchByLayer.set(lid, (searchMatchByLayer.get(lid) ?? 0) + 1); } } } - // Create cluster nodes + // Create cluster nodes. Per-layer aggregation goes through + // `computeLayerStats`, which iterates `layer.nodeIds` against the + // `nodesById` index — O(K) per layer instead of the previous + // O(N) Array.filter that ran `layer.nodeIds.includes(n.id)` (#102). const clusterNodes: LayerClusterFlowNode[] = layers.map((layer, i) => { - const memberNodes = graph.nodes.filter((n) => layer.nodeIds.includes(n.id)); - const complexCounts = { simple: 0, moderate: 0, complex: 0 }; - for (const n of memberNodes) { - complexCounts[n.complexity]++; - } - const aggregateComplexity = - complexCounts.complex > memberNodes.length * 0.3 - ? "complex" - : complexCounts.moderate > memberNodes.length * 0.3 - ? "moderate" - : "simple"; + const { aggregateComplexity } = computeLayerStats(layer, nodesById); return { id: layer.id, @@ -222,7 +214,7 @@ function useOverviewGraph() { } return { clusterNodes, flowEdges, dims }; - }, [graph, searchResults, drillIntoLayer]); + }, [graph, nodesById, nodeIdToLayerId, searchResults, drillIntoLayer]); const [overview, setOverview] = useState<{ nodes: Node[]; edges: Edge[] }>({ nodes: [], diff --git a/understand-anything-plugin/packages/dashboard/src/store.ts b/understand-anything-plugin/packages/dashboard/src/store.ts index ec833d4..564e0bd 100644 --- a/understand-anything-plugin/packages/dashboard/src/store.ts +++ b/understand-anything-plugin/packages/dashboard/src/store.ts @@ -3,6 +3,7 @@ import { SearchEngine } from "@understand-anything/core/search"; import type { SearchResult } from "@understand-anything/core/search"; import type { GraphIssue } from "@understand-anything/core/schema"; import type { + GraphNode, KnowledgeGraph, TourStep, } from "@understand-anything/core/types"; @@ -49,12 +50,29 @@ const DEFAULT_FILTERS: FilterState = { /** Categories used for node type filter toggles. Single source of truth for NodeCategory. */ export type NodeCategory = "code" | "config" | "docs" | "infra" | "data" | "domain" | "knowledge"; -/** Find which layer a node belongs to. Returns layerId or null. */ -function findNodeLayer(graph: KnowledgeGraph, nodeId: string): string | null { +/** + * Build the (id → node) and (id → layerId) lookup maps that the rest of + * the dashboard reads via store selectors. Centralised so `setGraph` and + * any future graph-replacement path stay in sync. + * + * `nodeIdToLayerId` preserves the prior `findNodeLayer` "first matching + * layer wins" semantics — if a node id appears in multiple layers (rare + * but legal in the schema), the first occurrence in `graph.layers` order + * is the one we map to. + */ +function buildGraphIndexes(graph: KnowledgeGraph): { + nodesById: Map; + nodeIdToLayerId: Map; +} { + const nodesById = new Map(); + for (const node of graph.nodes) nodesById.set(node.id, node); + const nodeIdToLayerId = new Map(); for (const layer of graph.layers) { - if (layer.nodeIds.includes(nodeId)) return layer.id; + for (const nid of layer.nodeIds) { + if (!nodeIdToLayerId.has(nid)) nodeIdToLayerId.set(nid, layer.id); + } } - return null; + return { nodesById, nodeIdToLayerId }; } /** Maximum number of entries in the sidebar navigation history. */ @@ -62,6 +80,10 @@ const MAX_HISTORY = 50; interface DashboardStore { graph: KnowledgeGraph | null; + /** id → node lookup, rebuilt by setGraph. Empty before any graph loads. */ + nodesById: Map; + /** id → layer id, rebuilt by setGraph. Empty before any graph loads. */ + nodeIdToLayerId: Map; selectedNodeId: string | null; searchQuery: string; searchResults: SearchResult[]; @@ -189,11 +211,11 @@ function getSortedTour(graph: KnowledgeGraph): TourStep[] { /** Navigate tour step to the correct layer for the first highlighted node. */ function navigateTourToLayer( - graph: KnowledgeGraph, + nodeIdToLayerId: Map, nodeIds: string[], ): Partial { if (nodeIds.length === 0) return {}; - const layerId = findNodeLayer(graph, nodeIds[0]); + const layerId = nodeIdToLayerId.get(nodeIds[0]); if (layerId) { return { navigationLevel: "layer-detail" as const, @@ -205,6 +227,8 @@ function navigateTourToLayer( export const useDashboardStore = create()((set, get) => ({ graph: null, + nodesById: new Map(), + nodeIdToLayerId: new Map(), selectedNodeId: null, searchQuery: "", searchResults: [], @@ -259,8 +283,11 @@ export const useDashboardStore = create()((set, get) => ({ const { viewMode, domainGraph, activeDomainId } = get(); // Preserve domain view if a domain graph is already loaded const keepDomainView = viewMode === "domain" && domainGraph !== null; + const { nodesById, nodeIdToLayerId } = buildGraphIndexes(graph); set({ graph, + nodesById, + nodeIdToLayerId, searchEngine, searchResults, navigationLevel: "overview", @@ -296,9 +323,9 @@ export const useDashboardStore = create()((set, get) => ({ }, navigateToNodeInLayer: (nodeId) => { - const { graph, selectedNodeId, nodeHistory } = get(); + const { graph, selectedNodeId, nodeHistory, nodeIdToLayerId } = get(); if (!graph) return; - const layerId = findNodeLayer(graph, nodeId); + const layerId = nodeIdToLayerId.get(nodeId) ?? null; const newHistory = selectedNodeId && nodeId !== selectedNodeId ? [...nodeHistory, selectedNodeId].slice(-MAX_HISTORY) @@ -323,11 +350,11 @@ export const useDashboardStore = create()((set, get) => ({ }, navigateToHistoryIndex: (index) => { - const { nodeHistory, graph } = get(); + const { nodeHistory, graph, nodeIdToLayerId } = get(); if (!graph || index < 0 || index >= nodeHistory.length) return; const targetId = nodeHistory[index]; const newHistory = nodeHistory.slice(0, index); - const layerId = findNodeLayer(graph, targetId); + const layerId = nodeIdToLayerId.get(targetId) ?? null; set({ selectedNodeId: targetId, nodeHistory: newHistory, @@ -336,11 +363,11 @@ export const useDashboardStore = create()((set, get) => ({ }, goBackNode: () => { - const { nodeHistory, graph } = get(); + const { nodeHistory, graph, nodeIdToLayerId } = get(); if (nodeHistory.length === 0 || !graph) return; const prevNodeId = nodeHistory[nodeHistory.length - 1]; const newHistory = nodeHistory.slice(0, -1); - const layerId = findNodeLayer(graph, prevNodeId); + const layerId = nodeIdToLayerId.get(prevNodeId) ?? null; if (layerId) { set({ navigationLevel: "layer-detail", @@ -483,10 +510,10 @@ export const useDashboardStore = create()((set, get) => ({ }, startTour: () => { - const { graph } = get(); + const { graph, nodeIdToLayerId } = get(); if (!graph || !graph.tour || graph.tour.length === 0) return; const sorted = getSortedTour(graph); - const layerNav = navigateTourToLayer(graph, sorted[0].nodeIds); + const layerNav = navigateTourToLayer(nodeIdToLayerId, sorted[0].nodeIds); set({ tourActive: true, currentTourStep: 0, @@ -504,11 +531,11 @@ export const useDashboardStore = create()((set, get) => ({ }), setTourStep: (step) => { - const { graph } = get(); + const { graph, nodeIdToLayerId } = get(); if (!graph || !graph.tour || graph.tour.length === 0) return; const sorted = getSortedTour(graph); if (step < 0 || step >= sorted.length) return; - const layerNav = navigateTourToLayer(graph, sorted[step].nodeIds); + const layerNav = navigateTourToLayer(nodeIdToLayerId, sorted[step].nodeIds); set({ currentTourStep: step, tourHighlightedNodeIds: sorted[step].nodeIds, @@ -517,12 +544,12 @@ export const useDashboardStore = create()((set, get) => ({ }, nextTourStep: () => { - const { graph, currentTourStep } = get(); + const { graph, currentTourStep, nodeIdToLayerId } = get(); if (!graph || !graph.tour || graph.tour.length === 0) return; const sorted = getSortedTour(graph); if (currentTourStep < sorted.length - 1) { const next = currentTourStep + 1; - const layerNav = navigateTourToLayer(graph, sorted[next].nodeIds); + const layerNav = navigateTourToLayer(nodeIdToLayerId, sorted[next].nodeIds); set({ currentTourStep: next, tourHighlightedNodeIds: sorted[next].nodeIds, @@ -532,12 +559,12 @@ export const useDashboardStore = create()((set, get) => ({ }, prevTourStep: () => { - const { graph, currentTourStep } = get(); + const { graph, currentTourStep, nodeIdToLayerId } = get(); if (!graph || !graph.tour || graph.tour.length === 0) return; if (currentTourStep > 0) { const sorted = getSortedTour(graph); const prev = currentTourStep - 1; - const layerNav = navigateTourToLayer(graph, sorted[prev].nodeIds); + const layerNav = navigateTourToLayer(nodeIdToLayerId, sorted[prev].nodeIds); set({ currentTourStep: prev, tourHighlightedNodeIds: sorted[prev].nodeIds, diff --git a/understand-anything-plugin/packages/dashboard/src/utils/__tests__/filters.test.ts b/understand-anything-plugin/packages/dashboard/src/utils/__tests__/filters.test.ts new file mode 100644 index 0000000..15c4abf --- /dev/null +++ b/understand-anything-plugin/packages/dashboard/src/utils/__tests__/filters.test.ts @@ -0,0 +1,175 @@ +import { describe, it, expect } from "vitest"; +import { filterNodes, filterEdges } from "../filters"; +import type { + GraphNode, + GraphEdge, + Layer, +} from "@understand-anything/core/types"; +import type { + FilterState, + NodeType, + Complexity, + EdgeCategory, +} from "../../store"; +import { + ALL_NODE_TYPES, + ALL_COMPLEXITIES, + ALL_EDGE_CATEGORIES, +} from "../../store"; + +function node( + id: string, + type: NodeType = "file", + complexity: Complexity = "simple", +): GraphNode { + return { + id, + type, + name: id, + summary: "", + complexity, + tags: [], + } as GraphNode; +} + +function edge(source: string, target: string, type = "imports"): GraphEdge { + return { source, target, type } as GraphEdge; +} + +function defaultFilters(overrides: Partial = {}): FilterState { + return { + nodeTypes: new Set(ALL_NODE_TYPES), + complexities: new Set(ALL_COMPLEXITIES), + layerIds: new Set(), + edgeCategories: new Set(ALL_EDGE_CATEGORIES), + ...overrides, + }; +} + +function indexLayers(layers: Layer[]): Map { + const m = new Map(); + for (const l of layers) { + for (const nid of l.nodeIds) { + if (!m.has(nid)) m.set(nid, l.id); + } + } + return m; +} + +describe("filterNodes", () => { + it("returns all nodes when no filters narrow the set", () => { + const nodes = [node("a"), node("b"), node("c")]; + const out = filterNodes(nodes, new Map(), defaultFilters()); + expect(out).toHaveLength(3); + }); + + it("filters by node type", () => { + const nodes = [node("a", "file"), node("b", "function"), node("c", "class")]; + const filters = defaultFilters({ nodeTypes: new Set(["file"]) }); + const out = filterNodes(nodes, new Map(), filters); + expect(out.map((n) => n.id)).toEqual(["a"]); + }); + + it("filters by complexity", () => { + const nodes = [ + node("a", "file", "simple"), + node("b", "file", "moderate"), + node("c", "file", "complex"), + ]; + const filters = defaultFilters({ complexities: new Set(["complex"]) }); + const out = filterNodes(nodes, new Map(), filters); + expect(out.map((n) => n.id)).toEqual(["c"]); + }); + + it("keeps a node only when its layer is selected", () => { + const nodes = [node("a"), node("b"), node("c")]; + const idx = indexLayers([ + { id: "L1", name: "L1", description: "", nodeIds: ["a", "b"] }, + { id: "L2", name: "L2", description: "", nodeIds: ["c"] }, + ]); + const filters = defaultFilters({ layerIds: new Set(["L1"]) }); + const out = filterNodes(nodes, idx, filters); + expect(out.map((n) => n.id).sort()).toEqual(["a", "b"]); + }); + + it("drops nodes that aren't in any layer when a layer filter is active", () => { + const nodes = [node("a"), node("orphan")]; + const idx = indexLayers([ + { id: "L1", name: "L1", description: "", nodeIds: ["a"] }, + ]); + const filters = defaultFilters({ layerIds: new Set(["L1"]) }); + const out = filterNodes(nodes, idx, filters); + expect(out.map((n) => n.id)).toEqual(["a"]); + }); + + it("ignores layer filter when no layers are selected (parity with prior shape)", () => { + const nodes = [node("a"), node("orphan")]; + // idx maps "a"; "orphan" isn't in any layer. With layer filter empty, + // the orphan must still pass through. + const idx = indexLayers([ + { id: "L1", name: "L1", description: "", nodeIds: ["a"] }, + ]); + const out = filterNodes(nodes, idx, defaultFilters()); + expect(out.map((n) => n.id).sort()).toEqual(["a", "orphan"]); + }); + + it("scales linearly: 10k nodes × 100 layers under 50ms (#102 regression guard)", () => { + // The pre-fix path was O(N × L × K) — `layers.some(layer => filters.layerIds.has(layer.id) && layer.nodeIds.includes(node.id))`. + // For a 10k-node / 100-layer graph with half the layers selected, that + // measured ~50ms locally on node 22 just for the filter step. + const nodes: GraphNode[] = []; + const layers: Layer[] = []; + for (let li = 0; li < 100; li++) { + const nodeIds: string[] = []; + for (let ni = 0; ni < 100; ni++) { + const id = `n-${li}-${ni}`; + nodes.push(node(id)); + nodeIds.push(id); + } + layers.push({ id: `L${li}`, name: `L${li}`, description: "", nodeIds }); + } + const idx = indexLayers(layers); + const selected = new Set(layers.slice(0, 50).map((l) => l.id)); + const filters = defaultFilters({ layerIds: selected }); + + const t0 = performance.now(); + const out = filterNodes(nodes, idx, filters); + const elapsedMs = performance.now() - t0; + + expect(out.length).toBe(50 * 100); + expect(elapsedMs).toBeLessThan(50); + }); +}); + +describe("filterEdges", () => { + it("keeps only edges whose endpoints are visible", () => { + const edges = [edge("a", "b"), edge("a", "missing"), edge("c", "b")]; + const visible = new Set(["a", "b"]); + const out = filterEdges(edges, visible, defaultFilters()); + expect(out).toEqual([edge("a", "b")]); + }); + + it("filters by edge category", () => { + const edges = [ + edge("a", "b", "imports"), // structural + edge("a", "b", "calls"), // behavioral + edge("a", "b", "reads_from"), // data-flow + ]; + const visible = new Set(["a", "b"]); + const filters = defaultFilters({ + edgeCategories: new Set(["structural"]), + }); + const out = filterEdges(edges, visible, filters); + expect(out.map((e) => e.type)).toEqual(["imports"]); + }); + + it("passes through edges with unknown types (no category match)", () => { + // getEdgeCategory returns null for unknown types, which short-circuits + // the category filter — pinning current behavior so a future refactor + // doesn't accidentally start dropping unknown edges. + const edges = [edge("a", "b", "future-edge-type")]; + const visible = new Set(["a", "b"]); + const out = filterEdges(edges, visible, defaultFilters()); + expect(out).toHaveLength(1); + }); +}); diff --git a/understand-anything-plugin/packages/dashboard/src/utils/__tests__/layerStats.test.ts b/understand-anything-plugin/packages/dashboard/src/utils/__tests__/layerStats.test.ts new file mode 100644 index 0000000..b47e767 --- /dev/null +++ b/understand-anything-plugin/packages/dashboard/src/utils/__tests__/layerStats.test.ts @@ -0,0 +1,118 @@ +import { describe, it, expect } from "vitest"; +import { computeLayerStats } from "../layerStats"; +import type { GraphNode, Layer } from "@understand-anything/core/types"; + +function node( + id: string, + complexity: GraphNode["complexity"] = "simple", +): GraphNode { + return { + id, + type: "file", + name: id, + summary: "", + complexity, + tags: [], + } as GraphNode; +} + +function layer(id: string, nodeIds: string[]): Layer { + return { + id, + name: id, + description: "", + nodeIds, + }; +} + +function indexById(nodes: GraphNode[]): Map { + return new Map(nodes.map((n) => [n.id, n])); +} + +describe("computeLayerStats", () => { + it("counts only nodes that resolve in nodesById", () => { + const nodes = [node("a", "simple"), node("b", "moderate"), node("c", "complex")]; + const l = layer("L", ["a", "b", "c", "ghost"]); + const stats = computeLayerStats(l, indexById(nodes)); + expect(stats.resolvedCount).toBe(3); + }); + + it("returns 'simple' when no complexity passes the 30% threshold", () => { + // 1 complex out of 4 = 25% — under threshold. + const nodes = [ + node("a", "simple"), + node("b", "simple"), + node("c", "simple"), + node("d", "complex"), + ]; + const stats = computeLayerStats(layer("L", ["a", "b", "c", "d"]), indexById(nodes)); + expect(stats.aggregateComplexity).toBe("simple"); + }); + + it("returns 'complex' when complex count strictly exceeds 30%", () => { + // 4 complex out of 10 = 40% — over threshold. + const nodes = Array.from({ length: 10 }, (_, i) => + node(`n${i}`, i < 4 ? "complex" : "simple"), + ); + const stats = computeLayerStats( + layer("L", nodes.map((n) => n.id)), + indexById(nodes), + ); + expect(stats.aggregateComplexity).toBe("complex"); + }); + + it("prefers 'complex' over 'moderate' when both clear the threshold", () => { + // 4 complex + 4 moderate out of 10 — complex wins via the order of checks + // in the prior implementation; this test pins that behavior. + const nodes = Array.from({ length: 10 }, (_, i) => + node(`n${i}`, i < 4 ? "complex" : i < 8 ? "moderate" : "simple"), + ); + const stats = computeLayerStats( + layer("L", nodes.map((n) => n.id)), + indexById(nodes), + ); + expect(stats.aggregateComplexity).toBe("complex"); + }); + + it("returns 'moderate' when only the moderate count clears the threshold", () => { + const nodes = Array.from({ length: 10 }, (_, i) => + node(`n${i}`, i < 4 ? "moderate" : "simple"), + ); + const stats = computeLayerStats( + layer("L", nodes.map((n) => n.id)), + indexById(nodes), + ); + expect(stats.aggregateComplexity).toBe("moderate"); + }); + + it("treats an empty layer as 'simple' with resolvedCount 0", () => { + const stats = computeLayerStats(layer("L", []), indexById([])); + expect(stats.resolvedCount).toBe(0); + expect(stats.aggregateComplexity).toBe("simple"); + }); + + it("aggregates a 100-layer / 100-nodes-per-layer graph in under 50ms (#102 regression guard)", () => { + // The pre-fix path ran graph.nodes.filter((n) => layer.nodeIds.includes(n.id)) + // per layer — O(N × K × L) — and locally took ~150ms for this shape under + // node 22. The new path is O(N + Σ K_i). Loose budget so CI variance + // doesn't flake; the pre-fix path would blow past it by 2-10×. + const nodes: GraphNode[] = []; + const layers: Layer[] = []; + for (let li = 0; li < 100; li++) { + const ids: string[] = []; + for (let ni = 0; ni < 100; ni++) { + const id = `n-${li}-${ni}`; + nodes.push(node(id, ((li + ni) % 3 === 0 ? "complex" : (li + ni) % 3 === 1 ? "moderate" : "simple"))); + ids.push(id); + } + layers.push(layer(`L${li}`, ids)); + } + const byId = indexById(nodes); + + const t0 = performance.now(); + for (const l of layers) computeLayerStats(l, byId); + const elapsedMs = performance.now() - t0; + + expect(elapsedMs).toBeLessThan(50); + }); +}); diff --git a/understand-anything-plugin/packages/dashboard/src/utils/filters.ts b/understand-anything-plugin/packages/dashboard/src/utils/filters.ts index 6e49010..d0d699d 100644 --- a/understand-anything-plugin/packages/dashboard/src/utils/filters.ts +++ b/understand-anything-plugin/packages/dashboard/src/utils/filters.ts @@ -1,15 +1,21 @@ -import type { GraphNode, GraphEdge, Layer } from "@understand-anything/core/types"; +import type { GraphNode, GraphEdge } from "@understand-anything/core/types"; import type { FilterState, NodeType, Complexity, EdgeCategory } from "../store"; import { EDGE_CATEGORY_MAP } from "../store"; /** - * Filter nodes based on active filters + * Filter nodes based on active filters. + * + * Pass `nodeIdToLayerId` from the store (precomputed once on `setGraph`) + * so the layer-membership check is O(1) per node. The previous shape took + * `Layer[]` and ran `layer.nodeIds.includes(node.id)` per node-per-layer, + * which was O(N × L × K) and dominated export time on large graphs (#102). */ export function filterNodes( nodes: GraphNode[], - layers: Layer[], + nodeIdToLayerId: Map, filters: FilterState, ): GraphNode[] { + const hasLayerFilter = filters.layerIds.size > 0; return nodes.filter((node) => { // Filter by node type if (!filters.nodeTypes.has(node.type as NodeType)) { @@ -22,11 +28,9 @@ export function filterNodes( } // Filter by layer (if any layers are selected) - if (filters.layerIds.size > 0) { - const nodeInSelectedLayer = layers.some( - (layer) => filters.layerIds.has(layer.id) && layer.nodeIds.includes(node.id) - ); - if (!nodeInSelectedLayer) { + if (hasLayerFilter) { + const layerId = nodeIdToLayerId.get(node.id); + if (!layerId || !filters.layerIds.has(layerId)) { return false; } } diff --git a/understand-anything-plugin/packages/dashboard/src/utils/layerStats.ts b/understand-anything-plugin/packages/dashboard/src/utils/layerStats.ts new file mode 100644 index 0000000..2172bca --- /dev/null +++ b/understand-anything-plugin/packages/dashboard/src/utils/layerStats.ts @@ -0,0 +1,39 @@ +import type { GraphNode, Layer } from "@understand-anything/core/types"; + +export type Complexity = "simple" | "moderate" | "complex"; + +export interface LayerStats { + /** Number of layer.nodeIds that resolve to a node in the graph. */ + resolvedCount: number; + /** Aggregate label for the cluster card; matches the prior 30% threshold. */ + aggregateComplexity: Complexity; +} + +/** + * O(layer.nodeIds.length) summary of a layer's complexity composition. + * + * Replaces the prior `graph.nodes.filter((n) => layer.nodeIds.includes(n.id))` + * pass in `useOverviewGraph`, which was O(N × K) per layer and went + * super-linear once a project had a few thousand nodes spread across many + * layers (#102: 4.8 MB graph froze on overview render). + */ +export function computeLayerStats( + layer: Layer, + nodesById: Map, +): LayerStats { + const counts: Record = { simple: 0, moderate: 0, complex: 0 }; + let resolved = 0; + for (const nid of layer.nodeIds) { + const node = nodesById.get(nid); + if (!node) continue; + resolved++; + counts[node.complexity]++; + } + const aggregateComplexity: Complexity = + counts.complex > resolved * 0.3 + ? "complex" + : counts.moderate > resolved * 0.3 + ? "moderate" + : "simple"; + return { resolvedCount: resolved, aggregateComplexity }; +} From 988533a550bf945a6dcd80c32c092384bf0f5669 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?d=20=F0=9F=94=B9?= <258577966+voidborne-d@users.noreply.github.com> Date: Mon, 4 May 2026 17:16:42 +0800 Subject: [PATCH 2/2] fix(dashboard): preserve any-layer-wins membership for filterNodes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reviewer @Lum1104 (PR #112) caught a silent semantic regression: the new `filterNodes` reads layer membership through `nodeIdToLayerId.get(node.id)`, which is first-wins. The pre-#112 path was any-layer-wins — `layers.some(layer => filters.layerIds.has(layer.id) && layer.nodeIds.includes(node.id))`. For a node X listed in both L1 and L2 with only L2 selected, the old code kept X; the new code dropped it. The schema permits multi-layer membership, so this was a behavior change, not a bug fix. Fix: keep two distinct indexes in the store. Both are rebuilt once on `setGraph`, so the O(1)-per-node performance win from #112 is preserved. - `nodeIdToLayerId: Map` — first-matching-layer wins. Drives navigation (drillIntoLayer / tour step → layer / sidebar history) where one canonical layer is the right answer. Unchanged. - `nodeIdToLayerIds: Map>` — every layer the node belongs to. Drives `filterNodes` membership checks. Restores any-layer-wins exactly. `filterNodes` now iterates the (small) layer-id set per node looking for intersection with `filters.layerIds`. ExportMenu reads `nodeIdToLayerIds` from the store. Verified locally: - Added `filters.test.ts` regression: node in (L1, L2) with only L2 selected must pass. Failed against the first-wins implementation; passes now. - `pnpm --filter @understand-anything/dashboard test` — 42 / 42 pass (was 41; +1 multi-layer regression test; perf-guard at 100 layers × 100 nodes still <50 ms). - `pnpm --filter @understand-anything/dashboard exec tsc --noEmit` — clean. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../dashboard/src/components/ExportMenu.tsx | 4 +-- .../packages/dashboard/src/store.ts | 36 +++++++++++++++---- .../src/utils/__tests__/filters.test.ts | 27 ++++++++++++-- .../packages/dashboard/src/utils/filters.ts | 22 +++++++++--- 4 files changed, 72 insertions(+), 17 deletions(-) diff --git a/understand-anything-plugin/packages/dashboard/src/components/ExportMenu.tsx b/understand-anything-plugin/packages/dashboard/src/components/ExportMenu.tsx index fbaa1c3..e3ed709 100644 --- a/understand-anything-plugin/packages/dashboard/src/components/ExportMenu.tsx +++ b/understand-anything-plugin/packages/dashboard/src/components/ExportMenu.tsx @@ -20,7 +20,7 @@ function downloadBlob(blob: Blob, filename: string) { export default function ExportMenu() { const graph = useDashboardStore((s) => s.graph); - const nodeIdToLayerId = useDashboardStore((s) => s.nodeIdToLayerId); + const nodeIdToLayerIds = useDashboardStore((s) => s.nodeIdToLayerIds); const filters = useDashboardStore((s) => s.filters); const exportMenuOpen = useDashboardStore((s) => s.exportMenuOpen); const toggleExportMenu = useDashboardStore((s) => s.toggleExportMenu); @@ -188,7 +188,7 @@ export default function ExportMenu() { ? graph.nodes.filter((n) => !subFileTypes.has(n.type)) : graph.nodes; - filteredGraphNodes = filterNodes(filteredGraphNodes, nodeIdToLayerId, filters); + filteredGraphNodes = filterNodes(filteredGraphNodes, nodeIdToLayerIds, filters); const filteredNodeIds = new Set(filteredGraphNodes.map((n) => n.id)); let filteredGraphEdges = graph.edges.filter( diff --git a/understand-anything-plugin/packages/dashboard/src/store.ts b/understand-anything-plugin/packages/dashboard/src/store.ts index 564e0bd..b635d66 100644 --- a/understand-anything-plugin/packages/dashboard/src/store.ts +++ b/understand-anything-plugin/packages/dashboard/src/store.ts @@ -55,24 +55,42 @@ export type NodeCategory = "code" | "config" | "docs" | "infra" | "data" | "doma * the dashboard reads via store selectors. Centralised so `setGraph` and * any future graph-replacement path stay in sync. * - * `nodeIdToLayerId` preserves the prior `findNodeLayer` "first matching - * layer wins" semantics — if a node id appears in multiple layers (rare - * but legal in the schema), the first occurrence in `graph.layers` order - * is the one we map to. + * Two layer indexes, intentionally distinct: + * + * - `nodeIdToLayerId` preserves the prior `findNodeLayer` "first matching + * layer wins" semantics — if a node id appears in multiple layers + * (rare but legal in the schema), the first occurrence in `graph.layers` + * order is the one we map to. Drives navigation (drillIntoLayer, tour + * step → layer, sidebar history) where a single canonical layer is the + * right answer. + * + * - `nodeIdToLayerIds` records *every* layer a node belongs to. Drives + * membership queries (filterNodes) where the prior `Layer[] + + * layer.nodeIds.includes` shape was any-layer-wins — a node in L1 and + * L2 with only L2 selected must still pass. Collapsing to first-wins + * for filtering would be a silent regression. */ function buildGraphIndexes(graph: KnowledgeGraph): { nodesById: Map; nodeIdToLayerId: Map; + nodeIdToLayerIds: Map>; } { const nodesById = new Map(); for (const node of graph.nodes) nodesById.set(node.id, node); const nodeIdToLayerId = new Map(); + const nodeIdToLayerIds = new Map>(); for (const layer of graph.layers) { for (const nid of layer.nodeIds) { if (!nodeIdToLayerId.has(nid)) nodeIdToLayerId.set(nid, layer.id); + let set = nodeIdToLayerIds.get(nid); + if (!set) { + set = new Set(); + nodeIdToLayerIds.set(nid, set); + } + set.add(layer.id); } } - return { nodesById, nodeIdToLayerId }; + return { nodesById, nodeIdToLayerId, nodeIdToLayerIds }; } /** Maximum number of entries in the sidebar navigation history. */ @@ -82,8 +100,10 @@ interface DashboardStore { graph: KnowledgeGraph | null; /** id → node lookup, rebuilt by setGraph. Empty before any graph loads. */ nodesById: Map; - /** id → layer id, rebuilt by setGraph. Empty before any graph loads. */ + /** id → layer id (first-matching-layer wins), rebuilt by setGraph. Empty before any graph loads. */ nodeIdToLayerId: Map; + /** id → set of every layer the node belongs to, rebuilt by setGraph. Empty before any graph loads. */ + nodeIdToLayerIds: Map>; selectedNodeId: string | null; searchQuery: string; searchResults: SearchResult[]; @@ -229,6 +249,7 @@ export const useDashboardStore = create()((set, get) => ({ graph: null, nodesById: new Map(), nodeIdToLayerId: new Map(), + nodeIdToLayerIds: new Map>(), selectedNodeId: null, searchQuery: "", searchResults: [], @@ -283,11 +304,12 @@ export const useDashboardStore = create()((set, get) => ({ const { viewMode, domainGraph, activeDomainId } = get(); // Preserve domain view if a domain graph is already loaded const keepDomainView = viewMode === "domain" && domainGraph !== null; - const { nodesById, nodeIdToLayerId } = buildGraphIndexes(graph); + const { nodesById, nodeIdToLayerId, nodeIdToLayerIds } = buildGraphIndexes(graph); set({ graph, nodesById, nodeIdToLayerId, + nodeIdToLayerIds, searchEngine, searchResults, navigationLevel: "overview", diff --git a/understand-anything-plugin/packages/dashboard/src/utils/__tests__/filters.test.ts b/understand-anything-plugin/packages/dashboard/src/utils/__tests__/filters.test.ts index 15c4abf..189b223 100644 --- a/understand-anything-plugin/packages/dashboard/src/utils/__tests__/filters.test.ts +++ b/understand-anything-plugin/packages/dashboard/src/utils/__tests__/filters.test.ts @@ -46,11 +46,16 @@ function defaultFilters(overrides: Partial = {}): FilterState { }; } -function indexLayers(layers: Layer[]): Map { - const m = new Map(); +function indexLayers(layers: Layer[]): Map> { + const m = new Map>(); for (const l of layers) { for (const nid of l.nodeIds) { - if (!m.has(nid)) m.set(nid, l.id); + let set = m.get(nid); + if (!set) { + set = new Set(); + m.set(nid, set); + } + set.add(l.id); } } return m; @@ -102,6 +107,22 @@ describe("filterNodes", () => { expect(out.map((n) => n.id)).toEqual(["a"]); }); + it("keeps a multi-layer node when any of its layers is selected (any-layer-wins)", () => { + // Regression for the silent first-wins behavior change in #112: a node + // X listed in both L1 (declared first) and L2, with only L2 selected, + // must still pass — matching the prior `layers.some(...)` shape. The + // first-wins `nodeIdToLayerId` index that drives navigation would + // have dropped X here. + const nodes = [node("x"), node("y")]; + const idx = indexLayers([ + { id: "L1", name: "L1", description: "", nodeIds: ["x"] }, + { id: "L2", name: "L2", description: "", nodeIds: ["x", "y"] }, + ]); + const filters = defaultFilters({ layerIds: new Set(["L2"]) }); + const out = filterNodes(nodes, idx, filters); + expect(out.map((n) => n.id).sort()).toEqual(["x", "y"]); + }); + it("ignores layer filter when no layers are selected (parity with prior shape)", () => { const nodes = [node("a"), node("orphan")]; // idx maps "a"; "orphan" isn't in any layer. With layer filter empty, diff --git a/understand-anything-plugin/packages/dashboard/src/utils/filters.ts b/understand-anything-plugin/packages/dashboard/src/utils/filters.ts index d0d699d..eef3cc2 100644 --- a/understand-anything-plugin/packages/dashboard/src/utils/filters.ts +++ b/understand-anything-plugin/packages/dashboard/src/utils/filters.ts @@ -5,14 +5,20 @@ import { EDGE_CATEGORY_MAP } from "../store"; /** * Filter nodes based on active filters. * - * Pass `nodeIdToLayerId` from the store (precomputed once on `setGraph`) + * Pass `nodeIdToLayerIds` from the store (precomputed once on `setGraph`) * so the layer-membership check is O(1) per node. The previous shape took * `Layer[]` and ran `layer.nodeIds.includes(node.id)` per node-per-layer, * which was O(N × L × K) and dominated export time on large graphs (#102). + * + * Membership semantics are any-layer-wins, matching the prior shape: a + * node in L1 and L2 with only L2 selected passes. The store's other + * index, `nodeIdToLayerId`, is first-wins and is for navigation, not + * filtering — using it here would silently drop multi-layer nodes whose + * first declared layer isn't selected. */ export function filterNodes( nodes: GraphNode[], - nodeIdToLayerId: Map, + nodeIdToLayerIds: Map>, filters: FilterState, ): GraphNode[] { const hasLayerFilter = filters.layerIds.size > 0; @@ -29,10 +35,16 @@ export function filterNodes( // Filter by layer (if any layers are selected) if (hasLayerFilter) { - const layerId = nodeIdToLayerId.get(node.id); - if (!layerId || !filters.layerIds.has(layerId)) { - return false; + const layerIds = nodeIdToLayerIds.get(node.id); + if (!layerIds) return false; + let inSelected = false; + for (const lid of layerIds) { + if (filters.layerIds.has(lid)) { + inSelected = true; + break; + } } + if (!inSelected) return false; } return true;