Files
pi/packages/coding-agent/src/tui/tui-renderer.ts
T
Mario Zechner 84dcab219b Add image support in tool results across all providers
Tool results now use content blocks and can include both text and images.
All providers (Anthropic, Google, OpenAI Completions, OpenAI Responses)
correctly pass images from tool results to LLMs.

- Update ToolResultMessage type to use content blocks
- Add placeholder text for image-only tool results in Google/Anthropic
- OpenAI providers send tool result + follow-up user message with images
- Fix Anthropic JSON parsing for empty tool arguments
- Add comprehensive tests for image-only and text+image tool results
- Update README with tool result content blocks API
2025-11-12 10:45:56 +01:00

530 lines
16 KiB
TypeScript

import type { Agent, AgentEvent, AgentState } from "@mariozechner/pi-agent";
import type { AssistantMessage, Message } from "@mariozechner/pi-ai";
import type { SlashCommand } from "@mariozechner/pi-tui";
import {
CombinedAutocompleteProvider,
Container,
Loader,
ProcessTerminal,
Spacer,
Text,
TUI,
} from "@mariozechner/pi-tui";
import chalk from "chalk";
import type { SessionManager } from "../session-manager.js";
import { AssistantMessageComponent } from "./assistant-message.js";
import { CustomEditor } from "./custom-editor.js";
import { FooterComponent } from "./footer.js";
import { ModelSelectorComponent } from "./model-selector.js";
import { ThinkingSelectorComponent } from "./thinking-selector.js";
import { ToolExecutionComponent } from "./tool-execution.js";
import { UserMessageComponent } from "./user-message.js";
/**
* TUI renderer for the coding agent
*/
export class TuiRenderer {
private ui: TUI;
private chatContainer: Container;
private statusContainer: Container;
private editor: CustomEditor;
private editorContainer: Container; // Container to swap between editor and selector
private footer: FooterComponent;
private agent: Agent;
private sessionManager: SessionManager;
private version: string;
private isInitialized = false;
private onInputCallback?: (text: string) => void;
private loadingAnimation: Loader | null = null;
private onInterruptCallback?: () => void;
private lastSigintTime = 0;
// Streaming message tracking
private streamingComponent: AssistantMessageComponent | null = null;
// Tool execution tracking: toolCallId -> component
private pendingTools = new Map<string, ToolExecutionComponent>();
// Thinking level selector
private thinkingSelector: ThinkingSelectorComponent | null = null;
// Model selector
private modelSelector: ModelSelectorComponent | null = null;
// Track if this is the first user message (to skip spacer)
private isFirstUserMessage = true;
constructor(agent: Agent, sessionManager: SessionManager, version: string) {
this.agent = agent;
this.sessionManager = sessionManager;
this.version = version;
this.ui = new TUI(new ProcessTerminal());
this.chatContainer = new Container();
this.statusContainer = new Container();
this.editor = new CustomEditor();
this.editorContainer = new Container(); // Container to hold editor or selector
this.editorContainer.addChild(this.editor); // Start with editor
this.footer = new FooterComponent(agent.state);
// Define slash commands
const thinkingCommand: SlashCommand = {
name: "thinking",
description: "Select reasoning level (opens selector UI)",
};
const modelCommand: SlashCommand = {
name: "model",
description: "Select model (opens selector UI)",
};
// Setup autocomplete for file paths and slash commands
const autocompleteProvider = new CombinedAutocompleteProvider([thinkingCommand, modelCommand], process.cwd());
this.editor.setAutocompleteProvider(autocompleteProvider);
}
async init(): Promise<void> {
if (this.isInitialized) return;
// Add header with logo and instructions
const logo = chalk.bold.cyan("pi") + chalk.dim(` v${this.version}`);
const instructions =
chalk.dim("esc") +
chalk.gray(" to interrupt") +
"\n" +
chalk.dim("ctrl+c") +
chalk.gray(" to clear") +
"\n" +
chalk.dim("ctrl+c twice") +
chalk.gray(" to exit") +
"\n" +
chalk.dim("ctrl+k") +
chalk.gray(" to delete line") +
"\n" +
chalk.dim("/") +
chalk.gray(" for commands") +
"\n" +
chalk.dim("drop files") +
chalk.gray(" to attach");
const header = new Text(logo + "\n" + instructions, 1, 0);
// Setup UI layout
this.ui.addChild(new Spacer(1));
this.ui.addChild(header);
this.ui.addChild(new Spacer(1));
this.ui.addChild(this.chatContainer);
this.ui.addChild(this.statusContainer);
this.ui.addChild(new Spacer(1));
this.ui.addChild(this.editorContainer); // Use container that can hold editor or selector
this.ui.addChild(this.footer);
this.ui.setFocus(this.editor);
// Set up custom key handlers on the editor
this.editor.onEscape = () => {
// Intercept Escape key when processing
if (this.loadingAnimation && this.onInterruptCallback) {
this.onInterruptCallback();
}
};
this.editor.onCtrlC = () => {
this.handleCtrlC();
};
// Handle editor submission
this.editor.onSubmit = (text: string) => {
text = text.trim();
if (!text) return;
// Check for /thinking command
if (text === "/thinking") {
// Show thinking level selector
this.showThinkingSelector();
this.editor.setText("");
return;
}
// Check for /model command
if (text === "/model") {
// Show model selector
this.showModelSelector();
this.editor.setText("");
return;
}
if (this.onInputCallback) {
this.onInputCallback(text);
}
};
// Start the UI
this.ui.start();
this.isInitialized = true;
}
async handleEvent(event: AgentEvent, state: AgentState): Promise<void> {
if (!this.isInitialized) {
await this.init();
}
// Update footer with current stats
this.footer.updateState(state);
switch (event.type) {
case "agent_start":
// Show loading animation
this.editor.disableSubmit = true;
// Stop old loader before clearing
if (this.loadingAnimation) {
this.loadingAnimation.stop();
}
this.statusContainer.clear();
this.loadingAnimation = new Loader(this.ui, "Working... (esc to interrupt)");
this.statusContainer.addChild(this.loadingAnimation);
this.ui.requestRender();
break;
case "message_start":
if (event.message.role === "user") {
// Show user message immediately and clear editor
this.addMessageToChat(event.message);
this.editor.setText("");
this.ui.requestRender();
} else if (event.message.role === "assistant") {
// Create assistant component for streaming
this.streamingComponent = new AssistantMessageComponent();
this.chatContainer.addChild(this.streamingComponent);
this.streamingComponent.updateContent(event.message as AssistantMessage);
this.ui.requestRender();
}
break;
case "message_update":
// Update streaming component
if (this.streamingComponent && event.message.role === "assistant") {
const assistantMsg = event.message as AssistantMessage;
this.streamingComponent.updateContent(assistantMsg);
// Create tool execution components as soon as we see tool calls
for (const content of assistantMsg.content) {
if (content.type === "toolCall") {
// Only create if we haven't created it yet
if (!this.pendingTools.has(content.id)) {
this.chatContainer.addChild(new Text("", 0, 0));
const component = new ToolExecutionComponent(content.name, content.arguments);
this.chatContainer.addChild(component);
this.pendingTools.set(content.id, component);
} else {
// Update existing component with latest arguments as they stream
const component = this.pendingTools.get(content.id);
if (component) {
component.updateArgs(content.arguments);
}
}
}
}
this.ui.requestRender();
}
break;
case "message_end":
// Skip user messages (already shown in message_start)
if (event.message.role === "user") {
break;
}
if (this.streamingComponent && event.message.role === "assistant") {
const assistantMsg = event.message as AssistantMessage;
// Update streaming component with final message (includes stopReason)
this.streamingComponent.updateContent(assistantMsg);
// If message was aborted or errored, mark all pending tool components as failed
if (assistantMsg.stopReason === "aborted" || assistantMsg.stopReason === "error") {
const errorMessage =
assistantMsg.stopReason === "aborted" ? "Operation aborted" : assistantMsg.errorMessage || "Error";
for (const [toolCallId, component] of this.pendingTools.entries()) {
component.updateResult({
content: [{ type: "text", text: errorMessage }],
isError: true,
});
}
this.pendingTools.clear();
}
// Keep the streaming component - it's now the final assistant message
this.streamingComponent = null;
}
this.ui.requestRender();
break;
case "tool_execution_start": {
// Component should already exist from message_update, but create if missing
if (!this.pendingTools.has(event.toolCallId)) {
const component = new ToolExecutionComponent(event.toolName, event.args);
this.chatContainer.addChild(component);
this.pendingTools.set(event.toolCallId, component);
this.ui.requestRender();
}
break;
}
case "tool_execution_end": {
// Update the existing tool component with the result
const component = this.pendingTools.get(event.toolCallId);
if (component) {
// Update the component with the result
const content =
typeof event.result === "string"
? [{ type: "text" as const, text: event.result }]
: event.result.content;
component.updateResult({
content,
isError: event.isError,
});
this.pendingTools.delete(event.toolCallId);
this.ui.requestRender();
}
break;
}
case "agent_end":
// Stop loading animation
if (this.loadingAnimation) {
this.loadingAnimation.stop();
this.loadingAnimation = null;
this.statusContainer.clear();
}
if (this.streamingComponent) {
this.chatContainer.removeChild(this.streamingComponent);
this.streamingComponent = null;
}
this.pendingTools.clear();
this.editor.disableSubmit = false;
this.ui.requestRender();
break;
}
}
private addMessageToChat(message: Message): void {
if (message.role === "user") {
const userMsg = message as any;
// Extract text content from content blocks
const textBlocks = userMsg.content.filter((c: any) => c.type === "text");
const textContent = textBlocks.map((c: any) => c.text).join("");
if (textContent) {
const userComponent = new UserMessageComponent(textContent, this.isFirstUserMessage);
this.chatContainer.addChild(userComponent);
this.isFirstUserMessage = false;
}
} else if (message.role === "assistant") {
const assistantMsg = message as AssistantMessage;
// Add assistant message component
const assistantComponent = new AssistantMessageComponent(assistantMsg);
this.chatContainer.addChild(assistantComponent);
}
// Note: tool calls and results are now handled via tool_execution_start/end events
}
renderInitialMessages(state: AgentState): void {
// Render all existing messages (for --continue mode)
// Reset first user message flag for initial render
this.isFirstUserMessage = true;
// Render messages
for (let i = 0; i < state.messages.length; i++) {
const message = state.messages[i];
if (message.role === "user") {
const userMsg = message as any;
const textBlocks = userMsg.content.filter((c: any) => c.type === "text");
const textContent = textBlocks.map((c: any) => c.text).join("");
if (textContent) {
const userComponent = new UserMessageComponent(textContent, this.isFirstUserMessage);
this.chatContainer.addChild(userComponent);
this.isFirstUserMessage = false;
}
} else if (message.role === "assistant") {
const assistantMsg = message as AssistantMessage;
const assistantComponent = new AssistantMessageComponent(assistantMsg);
this.chatContainer.addChild(assistantComponent);
// Create tool execution components for any tool calls
for (const content of assistantMsg.content) {
if (content.type === "toolCall") {
const component = new ToolExecutionComponent(content.name, content.arguments);
this.chatContainer.addChild(component);
// If message was aborted/errored, immediately mark tool as failed
if (assistantMsg.stopReason === "aborted" || assistantMsg.stopReason === "error") {
const errorMessage =
assistantMsg.stopReason === "aborted"
? "Operation aborted"
: assistantMsg.errorMessage || "Error";
component.updateResult({
content: [{ type: "text", text: errorMessage }],
isError: true,
});
} else {
// Store in map so we can update with results later
this.pendingTools.set(content.id, component);
}
}
}
} else if (message.role === "toolResult") {
// Update existing tool execution component with results
const toolResultMsg = message as any;
const component = this.pendingTools.get(toolResultMsg.toolCallId);
if (component) {
component.updateResult({
content: toolResultMsg.content,
isError: toolResultMsg.isError,
});
// Remove from pending map since it's complete
this.pendingTools.delete(toolResultMsg.toolCallId);
}
}
}
// Clear pending tools after rendering initial messages
this.pendingTools.clear();
this.ui.requestRender();
}
async getUserInput(): Promise<string> {
return new Promise((resolve) => {
this.onInputCallback = (text: string) => {
this.onInputCallback = undefined;
resolve(text);
};
});
}
setInterruptCallback(callback: () => void): void {
this.onInterruptCallback = callback;
}
private handleCtrlC(): void {
// Handle Ctrl+C double-press logic
const now = Date.now();
const timeSinceLastCtrlC = now - this.lastSigintTime;
if (timeSinceLastCtrlC < 500) {
// Second Ctrl+C within 500ms - exit
this.stop();
process.exit(0);
} else {
// First Ctrl+C - clear the editor
this.clearEditor();
this.lastSigintTime = now;
}
}
clearEditor(): void {
this.editor.setText("");
this.statusContainer.clear();
this.ui.requestRender();
}
showError(errorMessage: string): void {
// Show error message in the chat
this.chatContainer.addChild(new Spacer(1));
this.chatContainer.addChild(new Text(chalk.red(`Error: ${errorMessage}`), 1, 0));
this.ui.requestRender();
}
private showThinkingSelector(): void {
// Create thinking selector with current level
this.thinkingSelector = new ThinkingSelectorComponent(
this.agent.state.thinkingLevel,
(level) => {
// Apply the selected thinking level
this.agent.setThinkingLevel(level);
// Save thinking level change to session
this.sessionManager.saveThinkingLevelChange(level);
// Show confirmation message with proper spacing
this.chatContainer.addChild(new Spacer(1));
const confirmText = new Text(chalk.dim(`Thinking level: ${level}`), 1, 0);
this.chatContainer.addChild(confirmText);
// Hide selector and show editor again
this.hideThinkingSelector();
this.ui.requestRender();
},
() => {
// Just hide the selector
this.hideThinkingSelector();
this.ui.requestRender();
},
);
// Replace editor with selector
this.editorContainer.clear();
this.editorContainer.addChild(this.thinkingSelector);
this.ui.setFocus(this.thinkingSelector.getSelectList());
this.ui.requestRender();
}
private hideThinkingSelector(): void {
// Replace selector with editor in the container
this.editorContainer.clear();
this.editorContainer.addChild(this.editor);
this.thinkingSelector = null;
this.ui.setFocus(this.editor);
}
private showModelSelector(): void {
// Create model selector with current model
this.modelSelector = new ModelSelectorComponent(
this.agent.state.model,
(model) => {
// Apply the selected model
this.agent.setModel(model);
// Save model change to session
this.sessionManager.saveModelChange(`${model.provider}/${model.id}`);
// Show confirmation message with proper spacing
this.chatContainer.addChild(new Spacer(1));
const confirmText = new Text(chalk.dim(`Model: ${model.id}`), 1, 0);
this.chatContainer.addChild(confirmText);
// Hide selector and show editor again
this.hideModelSelector();
this.ui.requestRender();
},
() => {
// Just hide the selector
this.hideModelSelector();
this.ui.requestRender();
},
);
// Replace editor with selector
this.editorContainer.clear();
this.editorContainer.addChild(this.modelSelector);
this.ui.setFocus(this.modelSelector);
this.ui.requestRender();
}
private hideModelSelector(): void {
// Replace selector with editor in the container
this.editorContainer.clear();
this.editorContainer.addChild(this.editor);
this.modelSelector = null;
this.ui.setFocus(this.editor);
}
stop(): void {
if (this.loadingAnimation) {
this.loadingAnimation.stop();
this.loadingAnimation = null;
}
if (this.isInitialized) {
this.ui.stop();
this.isInitialized = false;
}
}
}