From 7bfcba677b89bd20a1d00c3326985840acccb0a7 Mon Sep 17 00:00:00 2001 From: AnnulusLabs Date: Sun, 22 Mar 2026 09:26:05 -0600 Subject: [PATCH] fix: move dagre layout to Web Worker for large graphs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes #14 — dashboard becomes unusable with large knowledge graphs (2,700+ nodes). - Web Worker for dagre layout: graphs above 200 nodes compute layout off the main thread via a dedicated Web Worker, keeping the UI responsive - Request ID correlation: concurrent layout calls are routed by incrementing request ID, preventing race conditions where results cross-wire - Worker error handling: onerror rejects pending promises; GraphView catches errors and clears the loading spinner - Cancellation cleanup: effect cleanup clears layouting state to prevent stuck spinners when the graph switches below the async threshold - Topology/visual split: dagre only re-runs when graph structure changes; node selection, tour highlights, and search results are applied as a cheap O(n) overlay without triggering relayout - React.memo on CustomNode: prevents O(n) re-renders when selecting individual nodes in large graphs - Zero overhead for small graphs: below 200 nodes, the original synchronous layout path is used unchanged Co-Authored-By: Claude Opus 4.6 (1M context) --- .../dashboard/src/components/CustomNode.tsx | 6 +- .../dashboard/src/components/GraphView.tsx | 477 +++++++++++------- .../packages/dashboard/src/utils/layout.ts | 80 +++ .../dashboard/src/utils/layout.worker.ts | 47 ++ 4 files changed, 431 insertions(+), 179 deletions(-) create mode 100644 understand-anything-plugin/packages/dashboard/src/utils/layout.worker.ts diff --git a/understand-anything-plugin/packages/dashboard/src/components/CustomNode.tsx b/understand-anything-plugin/packages/dashboard/src/components/CustomNode.tsx index cf88c98..295f71a 100644 --- a/understand-anything-plugin/packages/dashboard/src/components/CustomNode.tsx +++ b/understand-anything-plugin/packages/dashboard/src/components/CustomNode.tsx @@ -1,3 +1,4 @@ +import { memo } from "react"; import { Handle, Position } from "@xyflow/react"; import type { NodeProps, Node } from "@xyflow/react"; @@ -40,7 +41,7 @@ export interface CustomNodeData extends Record { export type CustomFlowNode = Node; -export default function CustomNode({ +function CustomNodeComponent({ id, data, }: NodeProps) { @@ -122,3 +123,6 @@ export default function CustomNode({ ); } + +const CustomNode = memo(CustomNodeComponent); +export default CustomNode; diff --git a/understand-anything-plugin/packages/dashboard/src/components/GraphView.tsx b/understand-anything-plugin/packages/dashboard/src/components/GraphView.tsx index 2c8c92a..3c30f19 100644 --- a/understand-anything-plugin/packages/dashboard/src/components/GraphView.tsx +++ b/understand-anything-plugin/packages/dashboard/src/components/GraphView.tsx @@ -1,4 +1,4 @@ -import { useCallback, useEffect, useMemo, useRef } from "react"; +import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { ReactFlow, ReactFlowProvider, @@ -16,11 +16,16 @@ import "@xyflow/react/dist/style.css"; import CustomNode from "./CustomNode"; import type { CustomFlowNode } from "./CustomNode"; import { useDashboardStore } from "../store"; -import { applyDagreLayout, NODE_WIDTH, NODE_HEIGHT } from "../utils/layout"; -// Layer colors are hardcoded to gold-tinted values in the group node styles +import { applyDagreLayout, applyDagreLayoutAsync, NODE_WIDTH, NODE_HEIGHT } from "../utils/layout"; const LAYER_PADDING = 40; +/** + * Node count above which layout runs in a Web Worker + * to avoid blocking the main thread. + */ +const ASYNC_LAYOUT_THRESHOLD = 200; + const nodeTypes = { custom: CustomNode }; /** @@ -84,6 +89,214 @@ function SelectedNodeFitView() { return null; } +/** + * Build topology-only flow data: nodes and edges without visual-only state + * (selection, tour highlights, search results). This output drives dagre + * layout and should only recompute when the graph structure changes. + */ +function buildTopologyData( + graph: NonNullable["graph"]>, + persona: string, + diffMode: boolean, + changedNodeIds: Set, + affectedNodeIds: Set, + handleNodeSelect: (nodeId: string) => void, +) { + const filteredGraphNodes = + persona === "non-technical" + ? graph.nodes.filter( + (n) => + n.type === "concept" || n.type === "module" || n.type === "file", + ) + : graph.nodes; + + const filteredNodeIds = new Set(filteredGraphNodes.map((n) => n.id)); + const filteredGraphEdges = + persona === "non-technical" + ? graph.edges.filter( + (e) => filteredNodeIds.has(e.source) && filteredNodeIds.has(e.target), + ) + : graph.edges; + + const flowNodes: CustomFlowNode[] = filteredGraphNodes.map((node) => ({ + id: node.id, + type: "custom" as const, + position: { x: 0, y: 0 }, + data: { + label: node.name ?? node.filePath?.split("/").pop() ?? node.id, + nodeType: node.type, + summary: node.summary, + complexity: node.complexity, + isHighlighted: false, + searchScore: undefined, + isSelected: false, + isTourHighlighted: false, + isDiffChanged: diffMode && changedNodeIds.has(node.id), + isDiffAffected: diffMode && affectedNodeIds.has(node.id), + isDiffFaded: diffMode && !changedNodeIds.has(node.id) && !affectedNodeIds.has(node.id), + onNodeClick: handleNodeSelect, + }, + })); + + const diffNodeIds = diffMode ? new Set([...changedNodeIds, ...affectedNodeIds]) : new Set(); + const flowEdges: Edge[] = filteredGraphEdges.map((edge, i) => { + const sourceInDiff = diffMode && diffNodeIds.has(edge.source); + const targetInDiff = diffMode && diffNodeIds.has(edge.target); + const isImpacted = diffMode && (sourceInDiff || targetInDiff); + + return { + id: `e-${i}`, + source: edge.source, + target: edge.target, + label: edge.type, + animated: edge.type === "calls" || isImpacted, + style: isImpacted + ? { + stroke: sourceInDiff && targetInDiff + ? "rgba(224, 82, 82, 0.7)" + : "rgba(212, 160, 48, 0.5)", + strokeWidth: 2.5, + } + : diffMode + ? { stroke: "rgba(212,165,116,0.08)", strokeWidth: 1 } + : { stroke: "rgba(212,165,116,0.3)", strokeWidth: 1.5 }, + labelStyle: diffMode && !isImpacted + ? { fill: "rgba(163,151,135,0.3)", fontSize: 10 } + : { fill: "#a39787", fontSize: 10 }, + }; + }); + + return { flowNodes, flowEdges }; +} + +/** + * Lightweight overlay of visual-only state onto already-positioned nodes. + * This is O(n) object spreads — cheap even for thousands of nodes — and + * avoids triggering a dagre relayout when selection/highlight/search changes. + */ +function applyVisualState( + nodes: (CustomFlowNode | Node)[], + selectedNodeId: string | null, + tourHighlightedNodeIds: string[], + searchResults: Array<{ nodeId: string; score: number }>, +): (CustomFlowNode | Node)[] { + const searchMap = new Map(searchResults.map((r) => [r.nodeId, r.score])); + const tourSet = new Set(tourHighlightedNodeIds); + + return nodes.map((node) => { + // Skip group nodes (layer containers) — they have no CustomNodeData + if (node.type === "group") return node; + + const searchScore = searchMap.get(node.id); + const isHighlighted = searchScore !== undefined; + const isSelected = selectedNodeId === node.id; + const isTourHighlighted = tourSet.has(node.id); + + const data = node.data as CustomFlowNode["data"]; + + // Skip creating a new object if nothing visual changed + if ( + data.isHighlighted === isHighlighted && + data.searchScore === searchScore && + data.isSelected === isSelected && + data.isTourHighlighted === isTourHighlighted + ) { + return node; + } + + return { + ...node, + data: { + ...data, + isHighlighted, + searchScore, + isSelected, + isTourHighlighted, + }, + }; + }); +} + +function applyLayerGroups( + laidNodes: CustomFlowNode[], + edges: Edge[], + layers: Array<{ id: string; name: string; nodeIds: string[] }>, + showLayers: boolean, +): { initialNodes: (CustomFlowNode | Node)[]; initialEdges: Edge[] } { + if (!showLayers || layers.length === 0) { + return { initialNodes: laidNodes, initialEdges: edges }; + } + + const nodeToLayer = new Map(); + for (const layer of layers) { + for (const nodeId of layer.nodeIds) { + nodeToLayer.set(nodeId, layer.id); + } + } + + const groupNodes: Node[] = []; + const adjustedNodes: (CustomFlowNode | Node)[] = []; + + for (const layer of layers) { + const memberNodes = laidNodes.filter((n) => layer.nodeIds.includes(n.id)); + if (memberNodes.length === 0) continue; + + let minX = Infinity, minY = Infinity, maxX = -Infinity, maxY = -Infinity; + for (const node of memberNodes) { + minX = Math.min(minX, node.position.x); + minY = Math.min(minY, node.position.y); + maxX = Math.max(maxX, node.position.x + NODE_WIDTH); + maxY = Math.max(maxY, node.position.y + NODE_HEIGHT); + } + + const groupX = minX - LAYER_PADDING; + const groupY = minY - LAYER_PADDING - 24; + const groupWidth = maxX - minX + LAYER_PADDING * 2; + const groupHeight = maxY - minY + LAYER_PADDING * 2 + 24; + + groupNodes.push({ + id: layer.id, + type: "group", + position: { x: groupX, y: groupY }, + data: { label: layer.name }, + style: { + width: groupWidth, + height: groupHeight, + backgroundColor: "rgba(212,165,116,0.05)", + borderRadius: 12, + border: "2px dashed rgba(212,165,116,0.25)", + padding: 8, + fontSize: 13, + fontWeight: 600, + color: "#d4a574", + }, + }); + + for (const node of memberNodes) { + adjustedNodes.push({ + ...node, + parentId: layer.id, + extent: "parent" as const, + position: { + x: node.position.x - groupX, + y: node.position.y - groupY, + }, + }); + } + } + + for (const node of laidNodes) { + if (!nodeToLayer.has(node.id)) { + adjustedNodes.push(node); + } + } + + return { + initialNodes: [...groupNodes, ...adjustedNodes], + initialEdges: edges, + }; +} + function GraphViewInner() { const graph = useDashboardStore((s) => s.graph); const selectedNodeId = useDashboardStore((s) => s.selectedNodeId); @@ -97,6 +310,8 @@ function GraphViewInner() { const changedNodeIds = useDashboardStore((s) => s.changedNodeIds); const affectedNodeIds = useDashboardStore((s) => s.affectedNodeIds); + const [layouting, setLayouting] = useState(false); + const handleNodeSelect = useCallback( (nodeId: string) => { selectNode(nodeId); @@ -105,190 +320,86 @@ function GraphViewInner() { [selectNode, openCodeViewer], ); - const { initialNodes, initialEdges } = useMemo(() => { - if (!graph) - return { - initialNodes: [] as (CustomFlowNode | Node)[], - initialEdges: [] as Edge[], - }; + // ── Topology memo: only recomputes when graph structure changes ── + // Does NOT depend on selectedNodeId, tourHighlightedNodeIds, or searchResults. + const { topoNodes, topoEdges, needsAsyncLayout } = useMemo(() => { + if (!graph) { + return { topoNodes: [] as CustomFlowNode[], topoEdges: [] as Edge[], needsAsyncLayout: false }; + } + const { flowNodes, flowEdges } = buildTopologyData( + graph, persona, diffMode, changedNodeIds, affectedNodeIds, + handleNodeSelect, + ); + return { topoNodes: flowNodes, topoEdges: flowEdges, needsAsyncLayout: flowNodes.length > ASYNC_LAYOUT_THRESHOLD }; + }, [graph, persona, handleNodeSelect, diffMode, changedNodeIds, affectedNodeIds]); - // Filter nodes and edges based on persona - const filteredGraphNodes = - persona === "non-technical" - ? graph.nodes.filter( - (n) => - n.type === "concept" || n.type === "module" || n.type === "file", - ) - : graph.nodes; - - const filteredNodeIds = new Set(filteredGraphNodes.map((n) => n.id)); - const filteredGraphEdges = - persona === "non-technical" - ? graph.edges.filter( - (e) => filteredNodeIds.has(e.source) && filteredNodeIds.has(e.target), - ) - : graph.edges; - - const flowNodes: CustomFlowNode[] = filteredGraphNodes.map((node) => { - const matchResult = searchResults.find((r) => r.nodeId === node.id); - return { - id: node.id, - type: "custom" as const, - position: { x: 0, y: 0 }, - data: { - label: node.name ?? node.filePath?.split("/").pop() ?? node.id, - nodeType: node.type, - summary: node.summary, - complexity: node.complexity, - isHighlighted: !!matchResult, - searchScore: matchResult?.score, - isSelected: selectedNodeId === node.id, - isTourHighlighted: tourHighlightedNodeIds.includes(node.id), - isDiffChanged: diffMode && changedNodeIds.has(node.id), - isDiffAffected: diffMode && affectedNodeIds.has(node.id), - isDiffFaded: diffMode && !changedNodeIds.has(node.id) && !affectedNodeIds.has(node.id), - onNodeClick: handleNodeSelect, - }, - }; - }); - - const diffNodeIds = diffMode ? new Set([...changedNodeIds, ...affectedNodeIds]) : new Set(); - const flowEdges: Edge[] = filteredGraphEdges.map((edge, i) => { - const sourceInDiff = diffMode && diffNodeIds.has(edge.source); - const targetInDiff = diffMode && diffNodeIds.has(edge.target); - const isImpacted = diffMode && (sourceInDiff || targetInDiff); - - return { - id: `e-${i}`, - source: edge.source, - target: edge.target, - label: edge.type, - animated: edge.type === "calls" || isImpacted, - style: isImpacted - ? { - stroke: sourceInDiff && targetInDiff - ? "rgba(224, 82, 82, 0.7)" - : "rgba(212, 160, 48, 0.5)", - strokeWidth: 2.5, - } - : diffMode - ? { stroke: "rgba(212,165,116,0.08)", strokeWidth: 1 } - : { stroke: "rgba(212,165,116,0.3)", strokeWidth: 1.5 }, - labelStyle: diffMode && !isImpacted - ? { fill: "rgba(163,151,135,0.3)", fontSize: 10 } - : { fill: "#a39787", fontSize: 10 }, - }; - }); - - // Run dagre layout on all nodes (without groups) - const laid = applyDagreLayout(flowNodes, flowEdges); - const laidNodes = laid.nodes as CustomFlowNode[]; + // ── Laid-out nodes from the last completed layout pass ── + // Stored in a ref so layout results persist across visual-state changes. + const laidOutRef = useRef<{ initialNodes: (CustomFlowNode | Node)[]; initialEdges: Edge[] } | null>(null); + // ── Sync layout: for small graphs, run dagre on the main thread ── + const syncResult = useMemo(() => { + if (!graph || needsAsyncLayout || topoNodes.length === 0) return null; + const laid = applyDagreLayout(topoNodes, topoEdges); const layers = graph.layers ?? []; - if (!showLayers || layers.length === 0) { - return { initialNodes: laidNodes, initialEdges: laid.edges }; - } + return applyLayerGroups(laid.nodes as CustomFlowNode[], laid.edges, layers, showLayers); + }, [graph, topoNodes, topoEdges, needsAsyncLayout, showLayers]); - // Build a map of nodeId -> layer for quick lookup - const nodeToLayer = new Map(); - for (const layer of layers) { - for (const nodeId of layer.nodeIds) { - nodeToLayer.set(nodeId, layer.id); - } - } + // Keep laidOutRef in sync with sync layout results + if (syncResult) { + laidOutRef.current = syncResult; + } - // Create group nodes and adjust member positions - const groupNodes: Node[] = []; - const adjustedNodes: (CustomFlowNode | Node)[] = []; + // ── Visual memo: cheap overlay of selection/highlight/search state ── + const visualNodes = useMemo(() => { + const base = laidOutRef.current; + if (!base) return [] as (CustomFlowNode | Node)[]; + return applyVisualState(base.initialNodes, selectedNodeId, tourHighlightedNodeIds, searchResults); + }, [laidOutRef.current, selectedNodeId, tourHighlightedNodeIds, searchResults]); - for (let layerIdx = 0; layerIdx < layers.length; layerIdx++) { - const layer = layers[layerIdx]; - const memberNodes = laidNodes.filter((n) => - layer.nodeIds.includes(n.id), - ); - - if (memberNodes.length === 0) continue; - - // Compute bounding box around member nodes - let minX = Infinity; - let minY = Infinity; - let maxX = -Infinity; - let maxY = -Infinity; - - for (const node of memberNodes) { - const x = node.position.x; - const y = node.position.y; - minX = Math.min(minX, x); - minY = Math.min(minY, y); - maxX = Math.max(maxX, x + NODE_WIDTH); - maxY = Math.max(maxY, y + NODE_HEIGHT); - } - - // Group node position = top-left with padding - const groupX = minX - LAYER_PADDING; - const groupY = minY - LAYER_PADDING - 24; // extra space for label - const groupWidth = maxX - minX + LAYER_PADDING * 2; - const groupHeight = maxY - minY + LAYER_PADDING * 2 + 24; - - // Create the group node - groupNodes.push({ - id: layer.id, - type: "group", - position: { x: groupX, y: groupY }, - data: { label: layer.name }, - style: { - width: groupWidth, - height: groupHeight, - backgroundColor: "rgba(212,165,116,0.05)", - borderRadius: 12, - border: `2px dashed rgba(212,165,116,0.25)`, - padding: 8, - fontSize: 13, - fontWeight: 600, - color: "#d4a574", - }, - }); - - // Adjust member node positions to be relative to the group - for (const node of memberNodes) { - adjustedNodes.push({ - ...node, - parentId: layer.id, - extent: "parent" as const, - position: { - x: node.position.x - groupX, - y: node.position.y - groupY, - }, - }); - } - } - - // Add nodes that are not in any layer (keep original positions) - for (const node of laidNodes) { - if (!nodeToLayer.has(node.id)) { - adjustedNodes.push(node); - } - } - - // Group nodes must come before their children in the array - const allNodes: (CustomFlowNode | Node)[] = [ - ...groupNodes, - ...adjustedNodes, - ]; - - return { initialNodes: allNodes, initialEdges: laid.edges }; - }, [graph, searchResults, selectedNodeId, showLayers, tourHighlightedNodeIds, persona, handleNodeSelect, diffMode, changedNodeIds, affectedNodeIds]); - - const [nodes, setNodes, onNodesChange] = useNodesState(initialNodes); - const [edges, setEdges, onEdgesChange] = useEdgesState(initialEdges); + const [nodes, setNodes, onNodesChange] = useNodesState(visualNodes); + const [edges, setEdges, onEdgesChange] = useEdgesState(laidOutRef.current?.initialEdges ?? []); + // ── Push sync layout + visual state to ReactFlow ── useEffect(() => { - setNodes(initialNodes); - }, [initialNodes, setNodes]); + if (syncResult) { + const withVisual = applyVisualState(syncResult.initialNodes, selectedNodeId, tourHighlightedNodeIds, searchResults); + setNodes(withVisual); + setEdges(syncResult.initialEdges); + } + }, [syncResult, selectedNodeId, tourHighlightedNodeIds, searchResults, setNodes, setEdges]); + // ── Push visual-only changes (no relayout) ── useEffect(() => { - setEdges(initialEdges); - }, [initialEdges, setEdges]); + if (laidOutRef.current && !layouting) { + const withVisual = applyVisualState(laidOutRef.current.initialNodes, selectedNodeId, tourHighlightedNodeIds, searchResults); + setNodes(withVisual); + } + }, [selectedNodeId, tourHighlightedNodeIds, searchResults, setNodes, layouting]); + + // ── Async layout: for large graphs, run dagre in a Web Worker ── + useEffect(() => { + if (!graph || !needsAsyncLayout || topoNodes.length === 0) return; + + let cancelled = false; + setLayouting(true); + + applyDagreLayoutAsync(topoNodes, topoEdges).then((laid) => { + if (cancelled) return; + const layers = graph.layers ?? []; + const result = applyLayerGroups(laid.nodes as CustomFlowNode[], laid.edges, layers, showLayers); + laidOutRef.current = result; + const withVisual = applyVisualState(result.initialNodes, selectedNodeId, tourHighlightedNodeIds, searchResults); + setNodes(withVisual); + setEdges(result.initialEdges); + setLayouting(false); + }).catch(() => { + if (cancelled) return; + setLayouting(false); + }); + + return () => { cancelled = true; setLayouting(false); }; + }, [graph, topoNodes, topoEdges, needsAsyncLayout, showLayers, setNodes, setEdges]); const onNodeClick = useCallback( (_: React.MouseEvent, node: { id: string }) => { @@ -314,7 +425,17 @@ function GraphViewInner() { } return ( -
+
+ {layouting && ( +
+
+
+

+ Laying out {topoNodes.length.toLocaleString()} nodes... +

+
+
+ )} void; + reject: (reason?: unknown) => void; + } +>(); + +function getWorker(): Worker { + if (!_worker) { + _worker = new Worker( + new URL("./layout.worker.ts", import.meta.url), + { type: "module" }, + ); + + _worker.onmessage = (e: MessageEvent) => { + const { requestId, positions } = e.data; + const entry = _pending.get(requestId); + _pending.delete(requestId); + + // S1: Discard stale results — only honour the latest request. + if (!entry || requestId !== _latestRequestId) return; + + const layoutedNodes = entry.nodes.map((node) => ({ + ...node, + position: positions[node.id] ?? { x: 0, y: 0 }, + })); + + entry.resolve({ nodes: layoutedNodes, edges: entry.edges }); + }; + + _worker.onerror = (err: ErrorEvent) => { + for (const [, entry] of _pending) { + entry.reject(err); + } + _pending.clear(); + }; + } + return _worker; +} + +/** + * Async dagre layout via Web Worker — used for large graphs. + * Keeps the main thread responsive while dagre computes positions. + * + * Uses request-ID correlation so concurrent calls never cross-wire, + * and only the latest request's result is honoured (stale ones are discarded). + */ +export function applyDagreLayoutAsync( + nodes: Node[], + edges: Edge[], + direction: "TB" | "LR" = "TB", +): Promise<{ nodes: Node[]; edges: Edge[] }> { + return new Promise((resolve, reject) => { + const worker = getWorker(); + const requestId = _nextRequestId++; + _latestRequestId = requestId; + + _pending.set(requestId, { nodes, edges, resolve, reject }); + + const msg: LayoutMessage = { + requestId, + nodes: nodes.map((n) => ({ id: n.id, width: NODE_WIDTH, height: NODE_HEIGHT })), + edges: edges.map((e) => ({ source: e.source, target: e.target })), + direction, + }; + + worker.postMessage(msg); + }); +} diff --git a/understand-anything-plugin/packages/dashboard/src/utils/layout.worker.ts b/understand-anything-plugin/packages/dashboard/src/utils/layout.worker.ts new file mode 100644 index 0000000..4f466f3 --- /dev/null +++ b/understand-anything-plugin/packages/dashboard/src/utils/layout.worker.ts @@ -0,0 +1,47 @@ +import dagre from "@dagrejs/dagre"; + +export interface LayoutMessage { + requestId: number; + nodes: Array<{ id: string; width: number; height: number }>; + edges: Array<{ source: string; target: string }>; + direction: "TB" | "LR"; +} + +export interface LayoutResult { + requestId: number; + positions: Record; +} + +self.onmessage = (e: MessageEvent) => { + const { requestId, nodes, edges, direction } = e.data; + + const g = new dagre.graphlib.Graph(); + g.setDefaultEdgeLabel(() => ({})); + g.setGraph({ + rankdir: direction, + nodesep: 60, + ranksep: 80, + marginx: 20, + marginy: 20, + }); + + for (const node of nodes) { + g.setNode(node.id, { width: node.width, height: node.height }); + } + + for (const edge of edges) { + g.setEdge(edge.source, edge.target); + } + + dagre.layout(g); + + const positions: Record = {}; + for (const node of nodes) { + const pos = g.node(node.id); + positions[node.id] = pos + ? { x: pos.x - node.width / 2, y: pos.y - node.height / 2 } + : { x: 0, y: 0 }; + } + + self.postMessage({ requestId, positions } satisfies LayoutResult); +};