refactor(coding-agent): resize images from bytes

packages/coding-agent/src/cli/file-processor.ts

@@ -50,13 +50,12 @@ export async function processFileArguments(fileArgs: string[], options?: Process
 		if (mimeType) {
 			// Handle image file
 			const content = await readFile(absolutePath);
-			const base64Content = content.toString("base64");
 
 			let attachment: ImageContent;
 			let dimensionNote: string | undefined;
 
 			if (autoResizeImages) {
-				const resized = await resizeImage({ type: "image", data: base64Content, mimeType });
+				const resized = await resizeImage(content, mimeType);
 				if (!resized) {
 					text += `<file name="${absolutePath}">[Image omitted: could not be resized below the inline image size limit.]</file>\n`;
 					continue;
@@ -71,7 +70,7 @@ export async function processFileArguments(fileArgs: string[], options?: Process
 				attachment = {
 					type: "image",
 					mimeType,
-					data: base64Content,
+					data: content.toString("base64"),
 				};
 			}
 

packages/coding-agent/src/core/tools/read.ts

@@ -250,10 +250,9 @@ export function createReadToolDefinition(
 							if (mimeType) {
 								// Read image as binary.
 								const buffer = await ops.readFile(absolutePath);
-								const base64 = buffer.toString("base64");
 								if (autoResizeImages) {
 									// Resize image if needed before sending it back to the model.
-									const resized = await resizeImage({ type: "image", data: base64, mimeType });
+									const resized = await resizeImage(buffer, mimeType);
 									if (!resized) {
 										let textNote = `Read image file [${mimeType}]\n[Image omitted: could not be resized below the inline image size limit.]`;
 										if (nonVisionImageNote) textNote += `\n${nonVisionImageNote}`;
@@ -273,7 +272,7 @@ export function createReadToolDefinition(
 									if (nonVisionImageNote) textNote += `\n${nonVisionImageNote}`;
 									content = [
 										{ type: "text", text: textNote },
-										{ type: "image", data: base64, mimeType },
+										{ type: "image", data: buffer.toString("base64"), mimeType },
 									];
 								}
 							} else {

packages/coding-agent/src/utils/image-resize.ts

@@ -1,4 +1,3 @@
-import type { ImageContent } from "@earendil-works/pi-ai";
 import { applyExifOrientation } from "./exif-orientation.js";
 import { loadPhoton } from "./photon.js";
 
@@ -57,10 +56,13 @@ function encodeCandidate(buffer: Uint8Array, mimeType: string): EncodedCandidate
  * 3. If still too large, try JPEG with decreasing quality
  * 4. If still too large, progressively reduce dimensions until 1x1
  */
-export async function resizeImage(img: ImageContent, options?: ImageResizeOptions): Promise<ResizedImage | null> {
+export async function resizeImage(
+	inputBytes: Uint8Array,
+	mimeType: string,
+	options?: ImageResizeOptions,
+): Promise<ResizedImage | null> {
 	const opts = { ...DEFAULT_OPTIONS, ...options };
-	const inputBuffer = Buffer.from(img.data, "base64");
-	const inputBase64Size = Buffer.byteLength(img.data, "utf-8");
+	const inputBase64Size = Math.ceil(inputBytes.byteLength / 3) * 4;
 
 	const photon = await loadPhoton();
 	if (!photon) {
@@ -69,20 +71,19 @@ export async function resizeImage(img: ImageContent, options?: ImageResizeOption
 
 	let image: ReturnType<typeof photon.PhotonImage.new_from_byteslice> | undefined;
 	try {
-		const inputBytes = new Uint8Array(inputBuffer);
 		const rawImage = photon.PhotonImage.new_from_byteslice(inputBytes);
 		image = applyExifOrientation(photon, rawImage, inputBytes);
 		if (image !== rawImage) rawImage.free();
 
 		const originalWidth = image.get_width();
 		const originalHeight = image.get_height();
-		const format = img.mimeType?.split("/")[1] ?? "png";
+		const format = mimeType.split("/")[1] ?? "png";
 
 		// Check if already within all limits (dimensions AND encoded size)
 		if (originalWidth <= opts.maxWidth && originalHeight <= opts.maxHeight && inputBase64Size < opts.maxBytes) {
 			return {
-				data: img.data,
-				mimeType: img.mimeType ?? `image/${format}`,
+				data: Buffer.from(inputBytes).toString("base64"),
+				mimeType: mimeType || `image/${format}`,
 				originalWidth,
 				originalHeight,
 				width: originalWidth,

packages/coding-agent/test/image-processing.test.ts

@@ -22,6 +22,10 @@ const MEDIUM_PNG_100x100 =
 const LARGE_PNG_200x200 =
 	"iVBORw0KGgoAAAANSUhEUgAAAMgAAADIEAIAAAByquWKAAAAIGNIUk0AAHomAACAhAAA+gAAAIDoAAB1MAAA6mAAADqYAAAXcJy6UTwAAAAGYktHRP///////wlY99wAAAAHdElNRQfqAQ4AMzqU03BFAAAEEElEQVR42u3dvWqUURSF4XXMRAyIhSAWQvwnWtjaWusFx9bW1kKD/wELESxEiBjDjJdwysOC57mCXW32+zGcGbtdACpsdi+T8Xz1GABzm91xMl6sHgNgblzcT/Y+rh4DYG6cv0/2H64eA2BOEgI1fHQHaoyzbXIwVo8BMCcJgRqSEKgxft9Lrn5aPQbA3Ph1klw7Wj0GwJwkBGpYWECN8fMiub63egyAORcWUMPCAmqMH3eTG59XjwEwN76/S24+Wj0GwJwkBGpYWECN8e1fcmuzegyAORcWUMPCAmqM09vJ4dfVYwDMjS9vkzuPV48BMCcJgRqeSAZqjA/nyYP91WMAzElCoIYkBGqMk8Pk6HT1GABzm+3L5JIkBApIQqCGj+5AjfHmb/Lk8uoxAOYkIVBDEgI1LCygxmZ7nFyShEABFxZQw8ICaozXf5KnV1aPATDnwgJqWFhADQsLqOFnDUANFxZQw8ICaoxXZ8mzg9VjAMy5sIAaFhZQw3tYQA1/QgHUkIRADUkI1JCEQA1JCNSQhEANFxZQwzcsoIYkBGpIQqCGJARqSEKghiQEalhYQA1/QgHUcGEBNSwsoIYkBGq4sIAaFhZQw8ICaviGBdRwYQE1LCyghiQEariwgBoWFlDDe1hADS+OAjUkIVBDEgI1JCFQQxICNSQhUMOFBdTwDQuoIQmBGpIQqCEJgRqSEKghCYEaFhZQw4ujQA0XFlDDwgJqSEKghgsLqGFhATUsLKCGb1hADRcWUMPCAmpIQqCGCwuoYWEBNbyHBdTw4ihQQxICNSQhUEMSAjUkIVBDEgI1XFhADd+wgBqSEKghCYEakhCoIQmBGpIQqGFhATW8OArUcGEBNSwsoIYkBGq4sIAaFhZQw8ICaviGBdRwYQE1LCyghiQEariwgBoWFlDDe1hADS+OAjUkIVBDEgI1JCFQQxICNSQhUMOFBdTwDQuoIQmBGpIQqCEJgRqSEKghCYEa49lB8ups9RgAc14cBWpIQqCGhQXUkIRADRcWUMPCAmqMp1eS139WjwEw5xsWUEMSAjUsLKCGJARquLCAGhYWUGM8uZy8+bt6DIA5L44CNSQhUMMTyUCNcXSanByuHgNgThICNSQhUGM82E8+nK8eA2DOzxqAGpIQqDHuPE6+vF09BsDcOPyanN5ePQbAnCQEavgdFlBj3Nok3/6tHgNgzoujQA1JCNQYNx8l39+tHgNgbtz4nPy4u3oMgDlJCNSwsIAa4/pe8vNi9RgAc37WANSQhECNce0o+XWyegyAuXH1U/L73uoxAOYkIVDDwgJqjIORnG1XjwEw508ogBqSEKgx9h8m5+9XjwEwN/Y+Jhf3V48BMCcJgRpjPE+2x6vHAJgbSbLbrR4DYO4/GqiSgXN+ksgAAAAldEVYdGRhdGU6Y3JlYXRlADIwMjYtMDEtMTRUMDA6NTE6NTcrMDA6MDDpysx4AAAAJXRFWHRkYXRlOm1vZGlmeQAyMDI2LTAxLTE0VDAwOjUxOjU3KzAwOjAwmJd0xAAAACh0RVh0ZGF0ZTp0aW1lc3RhbXAAMjAyNi0wMS0xNFQwMDo1MTo1NyswMDowMM+CVRsAAAAASUVORK5CYII=";
 
+function imageBytes(base64Data: string): Uint8Array {
+	return Buffer.from(base64Data, "base64");
+}
+
 describe("convertToPng", () => {
 	it("should return original data for PNG input", async () => {
 		const result = await convertToPng(TINY_PNG, "image/png");
@@ -47,10 +51,11 @@ describe("convertToPng", () => {
 
 describe("resizeImage", () => {
 	it("should return original image if within limits", async () => {
-		const result = await resizeImage(
-			{ type: "image", data: TINY_PNG, mimeType: "image/png" },
-			{ maxWidth: 100, maxHeight: 100, maxBytes: 1024 * 1024 },
-		);
+		const result = await resizeImage(imageBytes(TINY_PNG), "image/png", {
+			maxWidth: 100,
+			maxHeight: 100,
+			maxBytes: 1024 * 1024,
+		});
 
 		expect(result).not.toBeNull();
 		expect(result!.wasResized).toBe(false);
@@ -62,10 +67,11 @@ describe("resizeImage", () => {
 	});
 
 	it("should resize image exceeding dimension limits", async () => {
-		const result = await resizeImage(
-			{ type: "image", data: MEDIUM_PNG_100x100, mimeType: "image/png" },
-			{ maxWidth: 50, maxHeight: 50, maxBytes: 1024 * 1024 },
-		);
+		const result = await resizeImage(imageBytes(MEDIUM_PNG_100x100), "image/png", {
+			maxWidth: 50,
+			maxHeight: 50,
+			maxBytes: 1024 * 1024,
+		});
 
 		expect(result).not.toBeNull();
 		expect(result!.wasResized).toBe(true);
@@ -80,10 +86,11 @@ describe("resizeImage", () => {
 		const originalSize = originalBuffer.length;
 
 		// Set maxBytes to less than the original encoded image size
-		const result = await resizeImage(
-			{ type: "image", data: LARGE_PNG_200x200, mimeType: "image/png" },
-			{ maxWidth: 2000, maxHeight: 2000, maxBytes: Math.floor(LARGE_PNG_200x200.length * 0.9) },
-		);
+		const result = await resizeImage(imageBytes(LARGE_PNG_200x200), "image/png", {
+			maxWidth: 2000,
+			maxHeight: 2000,
+			maxBytes: Math.floor(LARGE_PNG_200x200.length * 0.9),
+		});
 
 		// Should have tried to reduce size
 		expect(result).not.toBeNull();
@@ -93,19 +100,21 @@ describe("resizeImage", () => {
 	});
 
 	it("should return null when image cannot be resized below maxBytes", async () => {
-		const result = await resizeImage(
-			{ type: "image", data: LARGE_PNG_200x200, mimeType: "image/png" },
-			{ maxWidth: 2000, maxHeight: 2000, maxBytes: 1 },
-		);
+		const result = await resizeImage(imageBytes(LARGE_PNG_200x200), "image/png", {
+			maxWidth: 2000,
+			maxHeight: 2000,
+			maxBytes: 1,
+		});
 
 		expect(result).toBeNull();
 	});
 
 	it("should handle JPEG input", async () => {
-		const result = await resizeImage(
-			{ type: "image", data: TINY_JPEG, mimeType: "image/jpeg" },
-			{ maxWidth: 100, maxHeight: 100, maxBytes: 1024 * 1024 },
-		);
+		const result = await resizeImage(imageBytes(TINY_JPEG), "image/jpeg", {
+			maxWidth: 100,
+			maxHeight: 100,
+			maxBytes: 1024 * 1024,
+		});
 
 		expect(result).not.toBeNull();
 		expect(result!.wasResized).toBe(false);