Files
pi/packages/coding-agent/src/tools/read.ts
T
Mario Zechner 84dcab219b Add image support in tool results across all providers
Tool results now use content blocks and can include both text and images.
All providers (Anthropic, Google, OpenAI Completions, OpenAI Responses)
correctly pass images from tool results to LLMs.

- Update ToolResultMessage type to use content blocks
- Add placeholder text for image-only tool results in Google/Anthropic
- OpenAI providers send tool result + follow-up user message with images
- Fix Anthropic JSON parsing for empty tool arguments
- Add comprehensive tests for image-only and text+image tool results
- Update README with tool result content blocks API
2025-11-12 10:45:56 +01:00

137 lines
3.4 KiB
TypeScript

import * as os from "node:os";
import type { AgentTool, ImageContent, TextContent } from "@mariozechner/pi-ai";
import { Type } from "@sinclair/typebox";
import { constants } from "fs";
import { access, readFile } from "fs/promises";
import { extname, resolve as resolvePath } from "path";
/**
* Expand ~ to home directory
*/
function expandPath(filePath: string): string {
if (filePath === "~") {
return os.homedir();
}
if (filePath.startsWith("~/")) {
return os.homedir() + filePath.slice(1);
}
return filePath;
}
/**
* Map of file extensions to MIME types for common image formats
*/
const IMAGE_MIME_TYPES: Record<string, string> = {
".jpg": "image/jpeg",
".jpeg": "image/jpeg",
".png": "image/png",
".gif": "image/gif",
".webp": "image/webp",
".bmp": "image/bmp",
".svg": "image/svg+xml",
};
/**
* Check if a file is an image based on its extension
*/
function isImageFile(filePath: string): string | null {
const ext = extname(filePath).toLowerCase();
return IMAGE_MIME_TYPES[ext] || null;
}
const readSchema = Type.Object({
path: Type.String({ description: "Path to the file to read (relative or absolute)" }),
});
export const readTool: AgentTool<typeof readSchema> = {
name: "read",
label: "read",
description:
"Read the contents of a file. Supports text files and images (jpg, png, gif, webp, bmp, svg). Images are sent as attachments to the model.",
parameters: readSchema,
execute: async (_toolCallId: string, { path }: { path: string }, signal?: AbortSignal) => {
const absolutePath = resolvePath(expandPath(path));
const mimeType = isImageFile(absolutePath);
return new Promise<{ content: (TextContent | ImageContent)[]; details: undefined }>((resolve, reject) => {
// Check if already aborted
if (signal?.aborted) {
reject(new Error("Operation aborted"));
return;
}
let aborted = false;
// Set up abort handler
const onAbort = () => {
aborted = true;
reject(new Error("Operation aborted"));
};
if (signal) {
signal.addEventListener("abort", onAbort, { once: true });
}
// Perform the read operation
(async () => {
try {
// Check if file exists
try {
await access(absolutePath, constants.R_OK);
} catch {
if (signal) {
signal.removeEventListener("abort", onAbort);
}
reject(new Error(`File not found: ${path}`));
return;
}
// Check if aborted before reading
if (aborted) {
return;
}
// Read the file based on type
let content: (TextContent | ImageContent)[];
if (mimeType) {
// Read as image (binary)
const buffer = await readFile(absolutePath);
const base64 = buffer.toString("base64");
content = [
{ type: "text", text: `Read image file: ${path}` },
{ type: "image", data: base64, mimeType },
];
} else {
// Read as text
const textContent = await readFile(absolutePath, "utf-8");
content = [{ type: "text", text: textContent }];
}
// Check if aborted after reading
if (aborted) {
return;
}
// Clean up abort handler
if (signal) {
signal.removeEventListener("abort", onAbort);
}
resolve({ content, details: undefined });
} catch (error: any) {
// Clean up abort handler
if (signal) {
signal.removeEventListener("abort", onAbort);
}
if (!aborted) {
reject(error);
}
}
})();
});
},
};