diff --git a/packages/coding-agent/src/utils/image-resize-core.ts b/packages/coding-agent/src/utils/image-resize-core.ts new file mode 100644 index 000000000..e60820c7b --- /dev/null +++ b/packages/coding-agent/src/utils/image-resize-core.ts @@ -0,0 +1,164 @@ +import { applyExifOrientation } from "./exif-orientation.ts"; +import { loadPhoton } from "./photon.ts"; + +export interface ImageResizeOptions { + maxWidth?: number; // Default: 2000 + maxHeight?: number; // Default: 2000 + maxBytes?: number; // Default: 4.5MB of base64 payload (below Anthropic's 5MB limit) + jpegQuality?: number; // Default: 80 +} + +export interface ResizedImage { + data: string; // base64 + mimeType: string; + originalWidth: number; + originalHeight: number; + width: number; + height: number; + wasResized: boolean; +} + +// 4.5MB of base64 payload. Provides headroom below Anthropic's 5MB limit. +const DEFAULT_MAX_BYTES = 4.5 * 1024 * 1024; + +const DEFAULT_OPTIONS: Required = { + maxWidth: 2000, + maxHeight: 2000, + maxBytes: DEFAULT_MAX_BYTES, + jpegQuality: 80, +}; + +interface EncodedCandidate { + data: string; + encodedSize: number; + mimeType: string; +} + +function encodeCandidate(buffer: Uint8Array, mimeType: string): EncodedCandidate { + const data = Buffer.from(buffer).toString("base64"); + return { + data, + encodedSize: Buffer.byteLength(data, "utf-8"), + mimeType, + }; +} + +/** + * Resize an image to fit within the specified max dimensions and encoded file size. + * Returns null if the image cannot be resized below maxBytes. + * + * Uses Photon (Rust/WASM) for image processing. If Photon is not available, + * returns null. + * + * Strategy for staying under maxBytes: + * 1. First resize to maxWidth/maxHeight + * 2. Try both PNG and JPEG formats, pick the smaller one + * 3. If still too large, try JPEG with decreasing quality + * 4. If still too large, progressively reduce dimensions until 1x1 + */ +export async function resizeImageInProcess( + inputBytes: Uint8Array, + mimeType: string, + options?: ImageResizeOptions, +): Promise { + const opts = { ...DEFAULT_OPTIONS, ...options }; + const inputBase64Size = Math.ceil(inputBytes.byteLength / 3) * 4; + + const photon = await loadPhoton(); + if (!photon) { + return null; + } + + let image: ReturnType | undefined; + try { + const rawImage = photon.PhotonImage.new_from_byteslice(inputBytes); + image = applyExifOrientation(photon, rawImage, inputBytes); + if (image !== rawImage) rawImage.free(); + + const originalWidth = image.get_width(); + const originalHeight = image.get_height(); + const format = mimeType.split("/")[1] ?? "png"; + + // Check if already within all limits (dimensions AND encoded size) + if (originalWidth <= opts.maxWidth && originalHeight <= opts.maxHeight && inputBase64Size < opts.maxBytes) { + return { + data: Buffer.from(inputBytes).toString("base64"), + mimeType: mimeType || `image/${format}`, + originalWidth, + originalHeight, + width: originalWidth, + height: originalHeight, + wasResized: false, + }; + } + + // Calculate initial dimensions respecting max limits + let targetWidth = originalWidth; + let targetHeight = originalHeight; + + if (targetWidth > opts.maxWidth) { + targetHeight = Math.round((targetHeight * opts.maxWidth) / targetWidth); + targetWidth = opts.maxWidth; + } + if (targetHeight > opts.maxHeight) { + targetWidth = Math.round((targetWidth * opts.maxHeight) / targetHeight); + targetHeight = opts.maxHeight; + } + + function tryEncodings(width: number, height: number, jpegQualities: number[]): EncodedCandidate[] { + const resized = photon!.resize(image!, width, height, photon!.SamplingFilter.Lanczos3); + + try { + const candidates: EncodedCandidate[] = [encodeCandidate(resized.get_bytes(), "image/png")]; + for (const quality of jpegQualities) { + candidates.push(encodeCandidate(resized.get_bytes_jpeg(quality), "image/jpeg")); + } + return candidates; + } finally { + resized.free(); + } + } + + const qualitySteps = Array.from(new Set([opts.jpegQuality, 85, 70, 55, 40])); + let currentWidth = targetWidth; + let currentHeight = targetHeight; + + while (true) { + const candidates = tryEncodings(currentWidth, currentHeight, qualitySteps); + for (const candidate of candidates) { + if (candidate.encodedSize < opts.maxBytes) { + return { + data: candidate.data, + mimeType: candidate.mimeType, + originalWidth, + originalHeight, + width: currentWidth, + height: currentHeight, + wasResized: true, + }; + } + } + + if (currentWidth === 1 && currentHeight === 1) { + break; + } + + const nextWidth = currentWidth === 1 ? 1 : Math.max(1, Math.floor(currentWidth * 0.75)); + const nextHeight = currentHeight === 1 ? 1 : Math.max(1, Math.floor(currentHeight * 0.75)); + if (nextWidth === currentWidth && nextHeight === currentHeight) { + break; + } + + currentWidth = nextWidth; + currentHeight = nextHeight; + } + + return null; + } catch { + return null; + } finally { + if (image) { + image.free(); + } + } +} diff --git a/packages/coding-agent/src/utils/image-resize-worker.ts b/packages/coding-agent/src/utils/image-resize-worker.ts new file mode 100644 index 000000000..ee881b3d9 --- /dev/null +++ b/packages/coding-agent/src/utils/image-resize-worker.ts @@ -0,0 +1,42 @@ +import { parentPort } from "node:worker_threads"; +import { type ImageResizeOptions, type ResizedImage, resizeImageInProcess } from "./image-resize-core.ts"; + +interface ResizeImageWorkerRequest { + inputBytes: Uint8Array; + mimeType: string; + options?: ImageResizeOptions; +} + +interface ResizeImageWorkerResponse { + result?: ResizedImage | null; + error?: string; +} + +function isResizeImageWorkerRequest(value: unknown): value is ResizeImageWorkerRequest { + if (!value || typeof value !== "object") return false; + const record = value as Record; + return record.inputBytes instanceof Uint8Array && typeof record.mimeType === "string"; +} + +const port = parentPort; +if (!port) { + throw new Error("image resize worker requires parentPort"); +} + +port.once("message", (message: unknown) => { + void (async () => { + try { + if (!isResizeImageWorkerRequest(message)) { + throw new Error("Invalid image resize worker request"); + } + const result = await resizeImageInProcess(message.inputBytes, message.mimeType, message.options); + const response: ResizeImageWorkerResponse = { result }; + port.postMessage(response); + } catch (error) { + const response: ResizeImageWorkerResponse = { + error: error instanceof Error ? error.message : String(error), + }; + port.postMessage(response); + } + })(); +}); diff --git a/packages/coding-agent/src/utils/image-resize.ts b/packages/coding-agent/src/utils/image-resize.ts index 1f2425a88..29a7bae0f 100644 --- a/packages/coding-agent/src/utils/image-resize.ts +++ b/packages/coding-agent/src/utils/image-resize.ts @@ -1,166 +1,96 @@ -import { applyExifOrientation } from "./exif-orientation.ts"; -import { loadPhoton } from "./photon.ts"; +import { Worker } from "node:worker_threads"; +import type { ImageResizeOptions, ResizedImage } from "./image-resize-core.ts"; -export interface ImageResizeOptions { - maxWidth?: number; // Default: 2000 - maxHeight?: number; // Default: 2000 - maxBytes?: number; // Default: 4.5MB of base64 payload (below Anthropic's 5MB limit) - jpegQuality?: number; // Default: 80 +export type { ImageResizeOptions, ResizedImage } from "./image-resize-core.ts"; + +interface ResizeImageWorkerResponse { + result?: ResizedImage | null; + error?: string; } -export interface ResizedImage { - data: string; // base64 - mimeType: string; - originalWidth: number; - originalHeight: number; - width: number; - height: number; - wasResized: boolean; +function toTransferableBytes(input: Uint8Array): Uint8Array { + // Transfer detaches the buffer, so transfer a worker-owned copy and leave the + // caller's bytes intact. + return new Uint8Array(input); } -// 4.5MB of base64 payload. Provides headroom below Anthropic's 5MB limit. -const DEFAULT_MAX_BYTES = 4.5 * 1024 * 1024; - -const DEFAULT_OPTIONS: Required = { - maxWidth: 2000, - maxHeight: 2000, - maxBytes: DEFAULT_MAX_BYTES, - jpegQuality: 80, -}; - -interface EncodedCandidate { - data: string; - encodedSize: number; - mimeType: string; +function isResizeImageWorkerResponse(value: unknown): value is ResizeImageWorkerResponse { + return value !== null && typeof value === "object"; } -function encodeCandidate(buffer: Uint8Array, mimeType: string): EncodedCandidate { - const data = Buffer.from(buffer).toString("base64"); - return { - data, - encodedSize: Buffer.byteLength(data, "utf-8"), - mimeType, - }; +function createResizeWorker(): Worker { + const isTypeScriptRuntime = import.meta.url.endsWith(".ts"); + const workerUrl = new URL( + isTypeScriptRuntime ? "./image-resize-worker.ts" : "./image-resize-worker.js", + import.meta.url, + ); + return new Worker(workerUrl, isTypeScriptRuntime ? { execArgv: ["--import", "tsx"] } : undefined); +} + +async function resizeImageInWorker( + inputBytes: Uint8Array, + mimeType: string, + options?: ImageResizeOptions, +): Promise { + const worker = createResizeWorker(); + try { + const inputBytesForWorker = toTransferableBytes(inputBytes); + return await new Promise((resolve, reject) => { + let settled = false; + const settle = (result: ResizedImage | null): void => { + if (settled) return; + settled = true; + resolve(result); + }; + const fail = (error: Error): void => { + if (settled) return; + settled = true; + reject(error); + }; + + worker.once("message", (message: unknown) => { + if (!isResizeImageWorkerResponse(message)) { + fail(new Error("Invalid image resize worker response")); + return; + } + if (message.error) { + fail(new Error(message.error)); + return; + } + settle(message.result ?? null); + }); + worker.once("error", fail); + worker.once("exit", (code) => { + if (!settled) { + fail(new Error(`Image resize worker exited with code ${code}`)); + } + }); + worker.postMessage( + { + inputBytes: inputBytesForWorker, + mimeType, + options, + }, + [inputBytesForWorker.buffer], + ); + }); + } finally { + void worker.terminate().catch(() => undefined); + } } /** * Resize an image to fit within the specified max dimensions and encoded file size. - * Returns null if the image cannot be resized below maxBytes. - * - * Uses Photon (Rust/WASM) for image processing. If Photon is not available, - * returns null. - * - * Strategy for staying under maxBytes: - * 1. First resize to maxWidth/maxHeight - * 2. Try both PNG and JPEG formats, pick the smaller one - * 3. If still too large, try JPEG with decreasing quality - * 4. If still too large, progressively reduce dimensions until 1x1 + * Runs Photon in a worker thread so WASM decoding, resizing, and encoding do not + * block the TUI event loop. Worker failures are propagated instead of retried on + * the main thread. */ export async function resizeImage( inputBytes: Uint8Array, mimeType: string, options?: ImageResizeOptions, ): Promise { - const opts = { ...DEFAULT_OPTIONS, ...options }; - const inputBase64Size = Math.ceil(inputBytes.byteLength / 3) * 4; - - const photon = await loadPhoton(); - if (!photon) { - return null; - } - - let image: ReturnType | undefined; - try { - const rawImage = photon.PhotonImage.new_from_byteslice(inputBytes); - image = applyExifOrientation(photon, rawImage, inputBytes); - if (image !== rawImage) rawImage.free(); - - const originalWidth = image.get_width(); - const originalHeight = image.get_height(); - const format = mimeType.split("/")[1] ?? "png"; - - // Check if already within all limits (dimensions AND encoded size) - if (originalWidth <= opts.maxWidth && originalHeight <= opts.maxHeight && inputBase64Size < opts.maxBytes) { - return { - data: Buffer.from(inputBytes).toString("base64"), - mimeType: mimeType || `image/${format}`, - originalWidth, - originalHeight, - width: originalWidth, - height: originalHeight, - wasResized: false, - }; - } - - // Calculate initial dimensions respecting max limits - let targetWidth = originalWidth; - let targetHeight = originalHeight; - - if (targetWidth > opts.maxWidth) { - targetHeight = Math.round((targetHeight * opts.maxWidth) / targetWidth); - targetWidth = opts.maxWidth; - } - if (targetHeight > opts.maxHeight) { - targetWidth = Math.round((targetWidth * opts.maxHeight) / targetHeight); - targetHeight = opts.maxHeight; - } - - function tryEncodings(width: number, height: number, jpegQualities: number[]): EncodedCandidate[] { - const resized = photon!.resize(image!, width, height, photon!.SamplingFilter.Lanczos3); - - try { - const candidates: EncodedCandidate[] = [encodeCandidate(resized.get_bytes(), "image/png")]; - for (const quality of jpegQualities) { - candidates.push(encodeCandidate(resized.get_bytes_jpeg(quality), "image/jpeg")); - } - return candidates; - } finally { - resized.free(); - } - } - - const qualitySteps = Array.from(new Set([opts.jpegQuality, 85, 70, 55, 40])); - let currentWidth = targetWidth; - let currentHeight = targetHeight; - - while (true) { - const candidates = tryEncodings(currentWidth, currentHeight, qualitySteps); - for (const candidate of candidates) { - if (candidate.encodedSize < opts.maxBytes) { - return { - data: candidate.data, - mimeType: candidate.mimeType, - originalWidth, - originalHeight, - width: currentWidth, - height: currentHeight, - wasResized: true, - }; - } - } - - if (currentWidth === 1 && currentHeight === 1) { - break; - } - - const nextWidth = currentWidth === 1 ? 1 : Math.max(1, Math.floor(currentWidth * 0.75)); - const nextHeight = currentHeight === 1 ? 1 : Math.max(1, Math.floor(currentHeight * 0.75)); - if (nextWidth === currentWidth && nextHeight === currentHeight) { - break; - } - - currentWidth = nextWidth; - currentHeight = nextHeight; - } - - return null; - } catch { - return null; - } finally { - if (image) { - image.free(); - } - } + return resizeImageInWorker(inputBytes, mimeType, options); } /** diff --git a/packages/coding-agent/test/image-processing.test.ts b/packages/coding-agent/test/image-processing.test.ts index fe403d12c..3e4c46c9b 100644 --- a/packages/coding-agent/test/image-processing.test.ts +++ b/packages/coding-agent/test/image-processing.test.ts @@ -50,6 +50,22 @@ describe("convertToPng", () => { }); describe("resizeImage", () => { + it("should keep caller input bytes intact", async () => { + const input = new Uint8Array(imageBytes(TINY_PNG)); + const originalByteLength = input.byteLength; + const originalFirstByte = input[0]; + + const result = await resizeImage(input, "image/png", { + maxWidth: 100, + maxHeight: 100, + maxBytes: 1024 * 1024, + }); + + expect(result).not.toBeNull(); + expect(input.byteLength).toBe(originalByteLength); + expect(input[0]).toBe(originalFirstByte); + }); + it("should return original image if within limits", async () => { const result = await resizeImage(imageBytes(TINY_PNG), "image/png", { maxWidth: 100, diff --git a/packages/tui/src/components/loader.ts b/packages/tui/src/components/loader.ts index 9bce7293f..4a93eb08f 100644 --- a/packages/tui/src/components/loader.ts +++ b/packages/tui/src/components/loader.ts @@ -34,10 +34,10 @@ export class Loader extends Text { ) { super("", 1, 0); this.ui = ui; - this.setIndicator(indicator); this.spinnerColorFn = spinnerColorFn; this.messageColorFn = messageColorFn; this.message = message; + this.setIndicator(indicator); } render(width: number): string[] {