diff --git a/packages/coding-agent/src/cli/file-processor.ts b/packages/coding-agent/src/cli/file-processor.ts index c9ab54950..61523196f 100644 --- a/packages/coding-agent/src/cli/file-processor.ts +++ b/packages/coding-agent/src/cli/file-processor.ts @@ -50,13 +50,12 @@ export async function processFileArguments(fileArgs: string[], options?: Process if (mimeType) { // Handle image file const content = await readFile(absolutePath); - const base64Content = content.toString("base64"); let attachment: ImageContent; let dimensionNote: string | undefined; if (autoResizeImages) { - const resized = await resizeImage({ type: "image", data: base64Content, mimeType }); + const resized = await resizeImage(content, mimeType); if (!resized) { text += `[Image omitted: could not be resized below the inline image size limit.]\n`; continue; @@ -71,7 +70,7 @@ export async function processFileArguments(fileArgs: string[], options?: Process attachment = { type: "image", mimeType, - data: base64Content, + data: content.toString("base64"), }; } diff --git a/packages/coding-agent/src/core/tools/read.ts b/packages/coding-agent/src/core/tools/read.ts index 0fd210073..70115f53e 100644 --- a/packages/coding-agent/src/core/tools/read.ts +++ b/packages/coding-agent/src/core/tools/read.ts @@ -250,10 +250,9 @@ export function createReadToolDefinition( if (mimeType) { // Read image as binary. const buffer = await ops.readFile(absolutePath); - const base64 = buffer.toString("base64"); if (autoResizeImages) { // Resize image if needed before sending it back to the model. - const resized = await resizeImage({ type: "image", data: base64, mimeType }); + const resized = await resizeImage(buffer, mimeType); if (!resized) { let textNote = `Read image file [${mimeType}]\n[Image omitted: could not be resized below the inline image size limit.]`; if (nonVisionImageNote) textNote += `\n${nonVisionImageNote}`; @@ -273,7 +272,7 @@ export function createReadToolDefinition( if (nonVisionImageNote) textNote += `\n${nonVisionImageNote}`; content = [ { type: "text", text: textNote }, - { type: "image", data: base64, mimeType }, + { type: "image", data: buffer.toString("base64"), mimeType }, ]; } } else { diff --git a/packages/coding-agent/src/utils/image-resize.ts b/packages/coding-agent/src/utils/image-resize.ts index 84b4ac331..bfd864234 100644 --- a/packages/coding-agent/src/utils/image-resize.ts +++ b/packages/coding-agent/src/utils/image-resize.ts @@ -1,4 +1,3 @@ -import type { ImageContent } from "@earendil-works/pi-ai"; import { applyExifOrientation } from "./exif-orientation.js"; import { loadPhoton } from "./photon.js"; @@ -57,10 +56,13 @@ function encodeCandidate(buffer: Uint8Array, mimeType: string): EncodedCandidate * 3. If still too large, try JPEG with decreasing quality * 4. If still too large, progressively reduce dimensions until 1x1 */ -export async function resizeImage(img: ImageContent, options?: ImageResizeOptions): Promise { +export async function resizeImage( + inputBytes: Uint8Array, + mimeType: string, + options?: ImageResizeOptions, +): Promise { const opts = { ...DEFAULT_OPTIONS, ...options }; - const inputBuffer = Buffer.from(img.data, "base64"); - const inputBase64Size = Buffer.byteLength(img.data, "utf-8"); + const inputBase64Size = Math.ceil(inputBytes.byteLength / 3) * 4; const photon = await loadPhoton(); if (!photon) { @@ -69,20 +71,19 @@ export async function resizeImage(img: ImageContent, options?: ImageResizeOption let image: ReturnType | undefined; try { - const inputBytes = new Uint8Array(inputBuffer); const rawImage = photon.PhotonImage.new_from_byteslice(inputBytes); image = applyExifOrientation(photon, rawImage, inputBytes); if (image !== rawImage) rawImage.free(); const originalWidth = image.get_width(); const originalHeight = image.get_height(); - const format = img.mimeType?.split("/")[1] ?? "png"; + const format = mimeType.split("/")[1] ?? "png"; // Check if already within all limits (dimensions AND encoded size) if (originalWidth <= opts.maxWidth && originalHeight <= opts.maxHeight && inputBase64Size < opts.maxBytes) { return { - data: img.data, - mimeType: img.mimeType ?? `image/${format}`, + data: Buffer.from(inputBytes).toString("base64"), + mimeType: mimeType || `image/${format}`, originalWidth, originalHeight, width: originalWidth, diff --git a/packages/coding-agent/test/image-processing.test.ts b/packages/coding-agent/test/image-processing.test.ts index 4dba6f2d9..fe403d12c 100644 --- a/packages/coding-agent/test/image-processing.test.ts +++ b/packages/coding-agent/test/image-processing.test.ts @@ -22,6 +22,10 @@ const MEDIUM_PNG_100x100 = const LARGE_PNG_200x200 = "iVBORw0KGgoAAAANSUhEUgAAAMgAAADIEAIAAAByquWKAAAAIGNIUk0AAHomAACAhAAA+gAAAIDoAAB1MAAA6mAAADqYAAAXcJy6UTwAAAAGYktHRP///////wlY99wAAAAHdElNRQfqAQ4AMzqU03BFAAAEEElEQVR42u3dvWqUURSF4XXMRAyIhSAWQvwnWtjaWusFx9bW1kKD/wELESxEiBjDjJdwysOC57mCXW32+zGcGbtdACpsdi+T8Xz1GABzm91xMl6sHgNgblzcT/Y+rh4DYG6cv0/2H64eA2BOEgI1fHQHaoyzbXIwVo8BMCcJgRqSEKgxft9Lrn5aPQbA3Ph1klw7Wj0GwJwkBGpYWECN8fMiub63egyAORcWUMPCAmqMH3eTG59XjwEwN76/S24+Wj0GwJwkBGpYWECN8e1fcmuzegyAORcWUMPCAmqM09vJ4dfVYwDMjS9vkzuPV48BMCcJgRqeSAZqjA/nyYP91WMAzElCoIYkBGqMk8Pk6HT1GABzm+3L5JIkBApIQqCGj+5AjfHmb/Lk8uoxAOYkIVBDEgI1LCygxmZ7nFyShEABFxZQw8ICaozXf5KnV1aPATDnwgJqWFhADQsLqOFnDUANFxZQw8ICaoxXZ8mzg9VjAMy5sIAaFhZQw3tYQA1/QgHUkIRADUkI1JCEQA1JCNSQhEANFxZQwzcsoIYkBGpIQqCGJARqSEKghiQEalhYQA1/QgHUcGEBNSwsoIYkBGq4sIAaFhZQw8ICaviGBdRwYQE1LCyghiQEariwgBoWFlDDe1hADS+OAjUkIVBDEgI1JCFQQxICNSQhUMOFBdTwDQuoIQmBGpIQqCEJgRqSEKghCYEaFhZQw4ujQA0XFlDDwgJqSEKghgsLqGFhATUsLKCGb1hADRcWUMPCAmpIQqCGCwuoYWEBNbyHBdTw4ihQQxICNSQhUEMSAjUkIVBDEgI1XFhADd+wgBqSEKghCYEakhCoIQmBGpIQqGFhATW8OArUcGEBNSwsoIYkBGq4sIAaFhZQw8ICaviGBdRwYQE1LCyghiQEariwgBoWFlDDe1hADS+OAjUkIVBDEgI1JCFQQxICNSQhUMOFBdTwDQuoIQmBGpIQqCEJgRqSEKghCYEa49lB8ups9RgAc14cBWpIQqCGhQXUkIRADRcWUMPCAmqMp1eS139WjwEw5xsWUEMSAjUsLKCGJARquLCAGhYWUGM8uZy8+bt6DIA5L44CNSQhUMMTyUCNcXSanByuHgNgThICNSQhUGM82E8+nK8eA2DOzxqAGpIQqDHuPE6+vF09BsDcOPyanN5ePQbAnCQEavgdFlBj3Nok3/6tHgNgzoujQA1JCNQYNx8l39+tHgNgbtz4nPy4u3oMgDlJCNSwsIAa4/pe8vNi9RgAc37WANSQhECNce0o+XWyegyAuXH1U/L73uoxAOYkIVDDwgJqjIORnG1XjwEw508ogBqSEKgx9h8m5+9XjwEwN/Y+Jhf3V48BMCcJgRpjPE+2x6vHAJgbSbLbrR4DYO4/GqiSgXN+ksgAAAAldEVYdGRhdGU6Y3JlYXRlADIwMjYtMDEtMTRUMDA6NTE6NTcrMDA6MDDpysx4AAAAJXRFWHRkYXRlOm1vZGlmeQAyMDI2LTAxLTE0VDAwOjUxOjU3KzAwOjAwmJd0xAAAACh0RVh0ZGF0ZTp0aW1lc3RhbXAAMjAyNi0wMS0xNFQwMDo1MTo1NyswMDowMM+CVRsAAAAASUVORK5CYII="; +function imageBytes(base64Data: string): Uint8Array { + return Buffer.from(base64Data, "base64"); +} + describe("convertToPng", () => { it("should return original data for PNG input", async () => { const result = await convertToPng(TINY_PNG, "image/png"); @@ -47,10 +51,11 @@ describe("convertToPng", () => { describe("resizeImage", () => { it("should return original image if within limits", async () => { - const result = await resizeImage( - { type: "image", data: TINY_PNG, mimeType: "image/png" }, - { maxWidth: 100, maxHeight: 100, maxBytes: 1024 * 1024 }, - ); + const result = await resizeImage(imageBytes(TINY_PNG), "image/png", { + maxWidth: 100, + maxHeight: 100, + maxBytes: 1024 * 1024, + }); expect(result).not.toBeNull(); expect(result!.wasResized).toBe(false); @@ -62,10 +67,11 @@ describe("resizeImage", () => { }); it("should resize image exceeding dimension limits", async () => { - const result = await resizeImage( - { type: "image", data: MEDIUM_PNG_100x100, mimeType: "image/png" }, - { maxWidth: 50, maxHeight: 50, maxBytes: 1024 * 1024 }, - ); + const result = await resizeImage(imageBytes(MEDIUM_PNG_100x100), "image/png", { + maxWidth: 50, + maxHeight: 50, + maxBytes: 1024 * 1024, + }); expect(result).not.toBeNull(); expect(result!.wasResized).toBe(true); @@ -80,10 +86,11 @@ describe("resizeImage", () => { const originalSize = originalBuffer.length; // Set maxBytes to less than the original encoded image size - const result = await resizeImage( - { type: "image", data: LARGE_PNG_200x200, mimeType: "image/png" }, - { maxWidth: 2000, maxHeight: 2000, maxBytes: Math.floor(LARGE_PNG_200x200.length * 0.9) }, - ); + const result = await resizeImage(imageBytes(LARGE_PNG_200x200), "image/png", { + maxWidth: 2000, + maxHeight: 2000, + maxBytes: Math.floor(LARGE_PNG_200x200.length * 0.9), + }); // Should have tried to reduce size expect(result).not.toBeNull(); @@ -93,19 +100,21 @@ describe("resizeImage", () => { }); it("should return null when image cannot be resized below maxBytes", async () => { - const result = await resizeImage( - { type: "image", data: LARGE_PNG_200x200, mimeType: "image/png" }, - { maxWidth: 2000, maxHeight: 2000, maxBytes: 1 }, - ); + const result = await resizeImage(imageBytes(LARGE_PNG_200x200), "image/png", { + maxWidth: 2000, + maxHeight: 2000, + maxBytes: 1, + }); expect(result).toBeNull(); }); it("should handle JPEG input", async () => { - const result = await resizeImage( - { type: "image", data: TINY_JPEG, mimeType: "image/jpeg" }, - { maxWidth: 100, maxHeight: 100, maxBytes: 1024 * 1024 }, - ); + const result = await resizeImage(imageBytes(TINY_JPEG), "image/jpeg", { + maxWidth: 100, + maxHeight: 100, + maxBytes: 1024 * 1024, + }); expect(result).not.toBeNull(); expect(result!.wasResized).toBe(false);