From 229e6d0347b0a8d9aed5a10761a0c6a155f54fe8 Mon Sep 17 00:00:00 2001 From: Won Park Date: Wed, 4 Mar 2026 16:54:38 -0800 Subject: [PATCH] image-gen-event/client_processing (#13512) enabling client-side to process with image-generation capabilities (setting app-server) --- .../schema/json/EventMsg.json | 142 ++++++++++++++++++ .../schema/json/ServerNotification.json | 34 +++++ .../codex_app_server_protocol.schemas.json | 122 +++++++++++++++ .../codex_app_server_protocol.v2.schemas.json | 122 +++++++++++++++ .../json/v2/ItemCompletedNotification.json | 34 +++++ .../json/v2/ItemStartedNotification.json | 34 +++++ .../schema/json/v2/ReviewStartResponse.json | 34 +++++ .../schema/json/v2/ThreadForkResponse.json | 34 +++++ .../schema/json/v2/ThreadListResponse.json | 34 +++++ .../json/v2/ThreadMetadataUpdateResponse.json | 34 +++++ .../schema/json/v2/ThreadReadResponse.json | 34 +++++ .../schema/json/v2/ThreadResumeResponse.json | 34 +++++ .../json/v2/ThreadRollbackResponse.json | 34 +++++ .../schema/json/v2/ThreadStartResponse.json | 34 +++++ .../json/v2/ThreadStartedNotification.json | 34 +++++ .../json/v2/ThreadUnarchiveResponse.json | 34 +++++ .../json/v2/TurnCompletedNotification.json | 34 +++++ .../schema/json/v2/TurnStartResponse.json | 34 +++++ .../json/v2/TurnStartedNotification.json | 34 +++++ .../schema/typescript/EventMsg.ts | 4 +- .../typescript/ImageGenerationBeginEvent.ts | 5 + .../typescript/ImageGenerationEndEvent.ts | 5 + .../schema/typescript/ImageGenerationItem.ts | 5 + .../schema/typescript/TurnItem.ts | 3 +- .../schema/typescript/index.ts | 3 + .../schema/typescript/v2/ThreadItem.ts | 2 +- .../src/protocol/thread_history.rs | 26 ++++ .../app-server-protocol/src/protocol/v2.rs | 15 ++ codex-rs/core/src/codex.rs | 2 + codex-rs/core/src/event_mapping.rs | 13 ++ codex-rs/core/src/rollout/policy.rs | 4 +- codex-rs/core/src/stream_events_utils.rs | 11 +- codex-rs/core/tests/suite/items.rs | 46 ++++++ .../src/event_processor_with_human_output.rs | 16 ++ codex-rs/mcp-server/src/codex_tool_runner.rs | 2 + codex-rs/protocol/src/items.rs | 25 +++ codex-rs/protocol/src/protocol.rs | 72 +++++++++ codex-rs/tui/src/chatwidget.rs | 18 +++ ...mage_generation_call_history_snapshot.snap | 7 + codex-rs/tui/src/chatwidget/tests.rs | 21 +++ codex-rs/tui/src/history_cell.rs | 15 ++ 41 files changed, 1244 insertions(+), 6 deletions(-) create mode 100644 codex-rs/app-server-protocol/schema/typescript/ImageGenerationBeginEvent.ts create mode 100644 codex-rs/app-server-protocol/schema/typescript/ImageGenerationEndEvent.ts create mode 100644 codex-rs/app-server-protocol/schema/typescript/ImageGenerationItem.ts create mode 100644 codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__image_generation_call_history_snapshot.snap diff --git a/codex-rs/app-server-protocol/schema/json/EventMsg.json b/codex-rs/app-server-protocol/schema/json/EventMsg.json index 526d852a2..39af1ac96 100644 --- a/codex-rs/app-server-protocol/schema/json/EventMsg.json +++ b/codex-rs/app-server-protocol/schema/json/EventMsg.json @@ -1387,6 +1387,60 @@ "title": "WebSearchEndEventMsg", "type": "object" }, + { + "properties": { + "call_id": { + "type": "string" + }, + "type": { + "enum": [ + "image_generation_begin" + ], + "title": "ImageGenerationBeginEventMsgType", + "type": "string" + } + }, + "required": [ + "call_id", + "type" + ], + "title": "ImageGenerationBeginEventMsg", + "type": "object" + }, + { + "properties": { + "call_id": { + "type": "string" + }, + "result": { + "type": "string" + }, + "revised_prompt": { + "type": [ + "string", + "null" + ] + }, + "status": { + "type": "string" + }, + "type": { + "enum": [ + "image_generation_end" + ], + "title": "ImageGenerationEndEventMsgType", + "type": "string" + } + }, + "required": [ + "call_id", + "result", + "status", + "type" + ], + "title": "ImageGenerationEndEventMsg", + "type": "object" + }, { "description": "Notification that the server is about to execute a command.", "properties": { @@ -6001,6 +6055,40 @@ "title": "WebSearchTurnItem", "type": "object" }, + { + "properties": { + "id": { + "type": "string" + }, + "result": { + "type": "string" + }, + "revised_prompt": { + "type": [ + "string", + "null" + ] + }, + "status": { + "type": "string" + }, + "type": { + "enum": [ + "ImageGeneration" + ], + "title": "ImageGenerationTurnItemType", + "type": "string" + } + }, + "required": [ + "id", + "result", + "status", + "type" + ], + "title": "ImageGenerationTurnItem", + "type": "object" + }, { "properties": { "id": { @@ -7092,6 +7180,60 @@ "title": "WebSearchEndEventMsg", "type": "object" }, + { + "properties": { + "call_id": { + "type": "string" + }, + "type": { + "enum": [ + "image_generation_begin" + ], + "title": "ImageGenerationBeginEventMsgType", + "type": "string" + } + }, + "required": [ + "call_id", + "type" + ], + "title": "ImageGenerationBeginEventMsg", + "type": "object" + }, + { + "properties": { + "call_id": { + "type": "string" + }, + "result": { + "type": "string" + }, + "revised_prompt": { + "type": [ + "string", + "null" + ] + }, + "status": { + "type": "string" + }, + "type": { + "enum": [ + "image_generation_end" + ], + "title": "ImageGenerationEndEventMsgType", + "type": "string" + } + }, + "required": [ + "call_id", + "result", + "status", + "type" + ], + "title": "ImageGenerationEndEventMsg", + "type": "object" + }, { "description": "Notification that the server is about to execute a command.", "properties": { diff --git a/codex-rs/app-server-protocol/schema/json/ServerNotification.json b/codex-rs/app-server-protocol/schema/json/ServerNotification.json index 61d15286a..b6d193231 100644 --- a/codex-rs/app-server-protocol/schema/json/ServerNotification.json +++ b/codex-rs/app-server-protocol/schema/json/ServerNotification.json @@ -2251,6 +2251,40 @@ "title": "ImageViewThreadItem", "type": "object" }, + { + "properties": { + "id": { + "type": "string" + }, + "result": { + "type": "string" + }, + "revisedPrompt": { + "type": [ + "string", + "null" + ] + }, + "status": { + "type": "string" + }, + "type": { + "enum": [ + "imageGeneration" + ], + "title": "ImageGenerationThreadItemType", + "type": "string" + } + }, + "required": [ + "id", + "result", + "status", + "type" + ], + "title": "ImageGenerationThreadItem", + "type": "object" + }, { "properties": { "id": { diff --git a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json index 0c51006bd..ff9f8d7d3 100644 --- a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json +++ b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json @@ -2618,6 +2618,60 @@ "title": "WebSearchEndEventMsg", "type": "object" }, + { + "properties": { + "call_id": { + "type": "string" + }, + "type": { + "enum": [ + "image_generation_begin" + ], + "title": "ImageGenerationBeginEventMsgType", + "type": "string" + } + }, + "required": [ + "call_id", + "type" + ], + "title": "ImageGenerationBeginEventMsg", + "type": "object" + }, + { + "properties": { + "call_id": { + "type": "string" + }, + "result": { + "type": "string" + }, + "revised_prompt": { + "type": [ + "string", + "null" + ] + }, + "status": { + "type": "string" + }, + "type": { + "enum": [ + "image_generation_end" + ], + "title": "ImageGenerationEndEventMsgType", + "type": "string" + } + }, + "required": [ + "call_id", + "result", + "status", + "type" + ], + "title": "ImageGenerationEndEventMsg", + "type": "object" + }, { "description": "Notification that the server is about to execute a command.", "properties": { @@ -7377,6 +7431,40 @@ "title": "WebSearchTurnItem", "type": "object" }, + { + "properties": { + "id": { + "type": "string" + }, + "result": { + "type": "string" + }, + "revised_prompt": { + "type": [ + "string", + "null" + ] + }, + "status": { + "type": "string" + }, + "type": { + "enum": [ + "ImageGeneration" + ], + "title": "ImageGenerationTurnItemType", + "type": "string" + } + }, + "required": [ + "id", + "result", + "status", + "type" + ], + "title": "ImageGenerationTurnItem", + "type": "object" + }, { "properties": { "id": { @@ -13595,6 +13683,40 @@ "title": "ImageViewThreadItem", "type": "object" }, + { + "properties": { + "id": { + "type": "string" + }, + "result": { + "type": "string" + }, + "revisedPrompt": { + "type": [ + "string", + "null" + ] + }, + "status": { + "type": "string" + }, + "type": { + "enum": [ + "imageGeneration" + ], + "title": "ImageGenerationThreadItemType", + "type": "string" + } + }, + "required": [ + "id", + "result", + "status", + "type" + ], + "title": "ImageGenerationThreadItem", + "type": "object" + }, { "properties": { "id": { diff --git a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json index 64ae44807..da523ea97 100644 --- a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json +++ b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json @@ -4155,6 +4155,60 @@ "title": "WebSearchEndEventMsg", "type": "object" }, + { + "properties": { + "call_id": { + "type": "string" + }, + "type": { + "enum": [ + "image_generation_begin" + ], + "title": "ImageGenerationBeginEventMsgType", + "type": "string" + } + }, + "required": [ + "call_id", + "type" + ], + "title": "ImageGenerationBeginEventMsg", + "type": "object" + }, + { + "properties": { + "call_id": { + "type": "string" + }, + "result": { + "type": "string" + }, + "revised_prompt": { + "type": [ + "string", + "null" + ] + }, + "status": { + "type": "string" + }, + "type": { + "enum": [ + "image_generation_end" + ], + "title": "ImageGenerationEndEventMsgType", + "type": "string" + } + }, + "required": [ + "call_id", + "result", + "status", + "type" + ], + "title": "ImageGenerationEndEventMsg", + "type": "object" + }, { "description": "Notification that the server is about to execute a command.", "properties": { @@ -12258,6 +12312,40 @@ "title": "ImageViewThreadItem", "type": "object" }, + { + "properties": { + "id": { + "type": "string" + }, + "result": { + "type": "string" + }, + "revisedPrompt": { + "type": [ + "string", + "null" + ] + }, + "status": { + "type": "string" + }, + "type": { + "enum": [ + "imageGeneration" + ], + "title": "ImageGenerationThreadItemType", + "type": "string" + } + }, + "required": [ + "id", + "result", + "status", + "type" + ], + "title": "ImageGenerationThreadItem", + "type": "object" + }, { "properties": { "id": { @@ -13745,6 +13833,40 @@ "title": "WebSearchTurnItem", "type": "object" }, + { + "properties": { + "id": { + "type": "string" + }, + "result": { + "type": "string" + }, + "revised_prompt": { + "type": [ + "string", + "null" + ] + }, + "status": { + "type": "string" + }, + "type": { + "enum": [ + "ImageGeneration" + ], + "title": "ImageGenerationTurnItemType", + "type": "string" + } + }, + "required": [ + "id", + "result", + "status", + "type" + ], + "title": "ImageGenerationTurnItem", + "type": "object" + }, { "properties": { "id": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ItemCompletedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/ItemCompletedNotification.json index c8824f560..8f5fcc367 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ItemCompletedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ItemCompletedNotification.json @@ -863,6 +863,40 @@ "title": "ImageViewThreadItem", "type": "object" }, + { + "properties": { + "id": { + "type": "string" + }, + "result": { + "type": "string" + }, + "revisedPrompt": { + "type": [ + "string", + "null" + ] + }, + "status": { + "type": "string" + }, + "type": { + "enum": [ + "imageGeneration" + ], + "title": "ImageGenerationThreadItemType", + "type": "string" + } + }, + "required": [ + "id", + "result", + "status", + "type" + ], + "title": "ImageGenerationThreadItem", + "type": "object" + }, { "properties": { "id": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ItemStartedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/ItemStartedNotification.json index 04dbc0738..0108ff5d7 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ItemStartedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ItemStartedNotification.json @@ -863,6 +863,40 @@ "title": "ImageViewThreadItem", "type": "object" }, + { + "properties": { + "id": { + "type": "string" + }, + "result": { + "type": "string" + }, + "revisedPrompt": { + "type": [ + "string", + "null" + ] + }, + "status": { + "type": "string" + }, + "type": { + "enum": [ + "imageGeneration" + ], + "title": "ImageGenerationThreadItemType", + "type": "string" + } + }, + "required": [ + "id", + "result", + "status", + "type" + ], + "title": "ImageGenerationThreadItem", + "type": "object" + }, { "properties": { "id": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json index 61449aa78..3c5d11cad 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json @@ -977,6 +977,40 @@ "title": "ImageViewThreadItem", "type": "object" }, + { + "properties": { + "id": { + "type": "string" + }, + "result": { + "type": "string" + }, + "revisedPrompt": { + "type": [ + "string", + "null" + ] + }, + "status": { + "type": "string" + }, + "type": { + "enum": [ + "imageGeneration" + ], + "title": "ImageGenerationThreadItemType", + "type": "string" + } + }, + "required": [ + "id", + "result", + "status", + "type" + ], + "title": "ImageGenerationThreadItem", + "type": "object" + }, { "properties": { "id": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json index 896546347..fec4e20e4 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json @@ -1453,6 +1453,40 @@ "title": "ImageViewThreadItem", "type": "object" }, + { + "properties": { + "id": { + "type": "string" + }, + "result": { + "type": "string" + }, + "revisedPrompt": { + "type": [ + "string", + "null" + ] + }, + "status": { + "type": "string" + }, + "type": { + "enum": [ + "imageGeneration" + ], + "title": "ImageGenerationThreadItemType", + "type": "string" + } + }, + "required": [ + "id", + "result", + "status", + "type" + ], + "title": "ImageGenerationThreadItem", + "type": "object" + }, { "properties": { "id": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json index 2cf3b940f..3965739d1 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json @@ -1215,6 +1215,40 @@ "title": "ImageViewThreadItem", "type": "object" }, + { + "properties": { + "id": { + "type": "string" + }, + "result": { + "type": "string" + }, + "revisedPrompt": { + "type": [ + "string", + "null" + ] + }, + "status": { + "type": "string" + }, + "type": { + "enum": [ + "imageGeneration" + ], + "title": "ImageGenerationThreadItemType", + "type": "string" + } + }, + "required": [ + "id", + "result", + "status", + "type" + ], + "title": "ImageGenerationThreadItem", + "type": "object" + }, { "properties": { "id": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json index 800010ee2..a74ee5d63 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json @@ -1215,6 +1215,40 @@ "title": "ImageViewThreadItem", "type": "object" }, + { + "properties": { + "id": { + "type": "string" + }, + "result": { + "type": "string" + }, + "revisedPrompt": { + "type": [ + "string", + "null" + ] + }, + "status": { + "type": "string" + }, + "type": { + "enum": [ + "imageGeneration" + ], + "title": "ImageGenerationThreadItemType", + "type": "string" + } + }, + "required": [ + "id", + "result", + "status", + "type" + ], + "title": "ImageGenerationThreadItem", + "type": "object" + }, { "properties": { "id": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json index b2398692f..f00275398 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json @@ -1215,6 +1215,40 @@ "title": "ImageViewThreadItem", "type": "object" }, + { + "properties": { + "id": { + "type": "string" + }, + "result": { + "type": "string" + }, + "revisedPrompt": { + "type": [ + "string", + "null" + ] + }, + "status": { + "type": "string" + }, + "type": { + "enum": [ + "imageGeneration" + ], + "title": "ImageGenerationThreadItemType", + "type": "string" + } + }, + "required": [ + "id", + "result", + "status", + "type" + ], + "title": "ImageGenerationThreadItem", + "type": "object" + }, { "properties": { "id": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json index e71203605..e8addc110 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json @@ -1453,6 +1453,40 @@ "title": "ImageViewThreadItem", "type": "object" }, + { + "properties": { + "id": { + "type": "string" + }, + "result": { + "type": "string" + }, + "revisedPrompt": { + "type": [ + "string", + "null" + ] + }, + "status": { + "type": "string" + }, + "type": { + "enum": [ + "imageGeneration" + ], + "title": "ImageGenerationThreadItemType", + "type": "string" + } + }, + "required": [ + "id", + "result", + "status", + "type" + ], + "title": "ImageGenerationThreadItem", + "type": "object" + }, { "properties": { "id": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadRollbackResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadRollbackResponse.json index 4ac54973d..e95b2d850 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadRollbackResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadRollbackResponse.json @@ -1215,6 +1215,40 @@ "title": "ImageViewThreadItem", "type": "object" }, + { + "properties": { + "id": { + "type": "string" + }, + "result": { + "type": "string" + }, + "revisedPrompt": { + "type": [ + "string", + "null" + ] + }, + "status": { + "type": "string" + }, + "type": { + "enum": [ + "imageGeneration" + ], + "title": "ImageGenerationThreadItemType", + "type": "string" + } + }, + "required": [ + "id", + "result", + "status", + "type" + ], + "title": "ImageGenerationThreadItem", + "type": "object" + }, { "properties": { "id": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json index 57098610c..52d9e2951 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json @@ -1453,6 +1453,40 @@ "title": "ImageViewThreadItem", "type": "object" }, + { + "properties": { + "id": { + "type": "string" + }, + "result": { + "type": "string" + }, + "revisedPrompt": { + "type": [ + "string", + "null" + ] + }, + "status": { + "type": "string" + }, + "type": { + "enum": [ + "imageGeneration" + ], + "title": "ImageGenerationThreadItemType", + "type": "string" + } + }, + "required": [ + "id", + "result", + "status", + "type" + ], + "title": "ImageGenerationThreadItem", + "type": "object" + }, { "properties": { "id": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json index c46d4d7bb..698793bbd 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json @@ -1215,6 +1215,40 @@ "title": "ImageViewThreadItem", "type": "object" }, + { + "properties": { + "id": { + "type": "string" + }, + "result": { + "type": "string" + }, + "revisedPrompt": { + "type": [ + "string", + "null" + ] + }, + "status": { + "type": "string" + }, + "type": { + "enum": [ + "imageGeneration" + ], + "title": "ImageGenerationThreadItemType", + "type": "string" + } + }, + "required": [ + "id", + "result", + "status", + "type" + ], + "title": "ImageGenerationThreadItem", + "type": "object" + }, { "properties": { "id": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json index eabd61402..30d1e2841 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json @@ -1215,6 +1215,40 @@ "title": "ImageViewThreadItem", "type": "object" }, + { + "properties": { + "id": { + "type": "string" + }, + "result": { + "type": "string" + }, + "revisedPrompt": { + "type": [ + "string", + "null" + ] + }, + "status": { + "type": "string" + }, + "type": { + "enum": [ + "imageGeneration" + ], + "title": "ImageGenerationThreadItemType", + "type": "string" + } + }, + "required": [ + "id", + "result", + "status", + "type" + ], + "title": "ImageGenerationThreadItem", + "type": "object" + }, { "properties": { "id": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json index 84bb60c8e..89a9e580d 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json @@ -977,6 +977,40 @@ "title": "ImageViewThreadItem", "type": "object" }, + { + "properties": { + "id": { + "type": "string" + }, + "result": { + "type": "string" + }, + "revisedPrompt": { + "type": [ + "string", + "null" + ] + }, + "status": { + "type": "string" + }, + "type": { + "enum": [ + "imageGeneration" + ], + "title": "ImageGenerationThreadItemType", + "type": "string" + } + }, + "required": [ + "id", + "result", + "status", + "type" + ], + "title": "ImageGenerationThreadItem", + "type": "object" + }, { "properties": { "id": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json b/codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json index 863814e1b..dbe082c3b 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json @@ -977,6 +977,40 @@ "title": "ImageViewThreadItem", "type": "object" }, + { + "properties": { + "id": { + "type": "string" + }, + "result": { + "type": "string" + }, + "revisedPrompt": { + "type": [ + "string", + "null" + ] + }, + "status": { + "type": "string" + }, + "type": { + "enum": [ + "imageGeneration" + ], + "title": "ImageGenerationThreadItemType", + "type": "string" + } + }, + "required": [ + "id", + "result", + "status", + "type" + ], + "title": "ImageGenerationThreadItem", + "type": "object" + }, { "properties": { "id": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json index aba5d973a..07323f202 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json @@ -977,6 +977,40 @@ "title": "ImageViewThreadItem", "type": "object" }, + { + "properties": { + "id": { + "type": "string" + }, + "result": { + "type": "string" + }, + "revisedPrompt": { + "type": [ + "string", + "null" + ] + }, + "status": { + "type": "string" + }, + "type": { + "enum": [ + "imageGeneration" + ], + "title": "ImageGenerationThreadItemType", + "type": "string" + } + }, + "required": [ + "id", + "result", + "status", + "type" + ], + "title": "ImageGenerationThreadItem", + "type": "object" + }, { "properties": { "id": { diff --git a/codex-rs/app-server-protocol/schema/typescript/EventMsg.ts b/codex-rs/app-server-protocol/schema/typescript/EventMsg.ts index 5ccbe1e69..658a97ef1 100644 --- a/codex-rs/app-server-protocol/schema/typescript/EventMsg.ts +++ b/codex-rs/app-server-protocol/schema/typescript/EventMsg.ts @@ -33,6 +33,8 @@ import type { ExecCommandEndEvent } from "./ExecCommandEndEvent"; import type { ExecCommandOutputDeltaEvent } from "./ExecCommandOutputDeltaEvent"; import type { ExitedReviewModeEvent } from "./ExitedReviewModeEvent"; import type { GetHistoryEntryResponseEvent } from "./GetHistoryEntryResponseEvent"; +import type { ImageGenerationBeginEvent } from "./ImageGenerationBeginEvent"; +import type { ImageGenerationEndEvent } from "./ImageGenerationEndEvent"; import type { ItemCompletedEvent } from "./ItemCompletedEvent"; import type { ItemStartedEvent } from "./ItemStartedEvent"; import type { ListCustomPromptsResponseEvent } from "./ListCustomPromptsResponseEvent"; @@ -79,4 +81,4 @@ import type { WebSearchEndEvent } from "./WebSearchEndEvent"; * Response event from the agent * NOTE: Make sure none of these values have optional types, as it will mess up the extension code-gen. */ -export type EventMsg = { "type": "error" } & ErrorEvent | { "type": "warning" } & WarningEvent | { "type": "realtime_conversation_started" } & RealtimeConversationStartedEvent | { "type": "realtime_conversation_realtime" } & RealtimeConversationRealtimeEvent | { "type": "realtime_conversation_closed" } & RealtimeConversationClosedEvent | { "type": "model_reroute" } & ModelRerouteEvent | { "type": "context_compacted" } & ContextCompactedEvent | { "type": "thread_rolled_back" } & ThreadRolledBackEvent | { "type": "task_started" } & TurnStartedEvent | { "type": "task_complete" } & TurnCompleteEvent | { "type": "token_count" } & TokenCountEvent | { "type": "agent_message" } & AgentMessageEvent | { "type": "user_message" } & UserMessageEvent | { "type": "agent_message_delta" } & AgentMessageDeltaEvent | { "type": "agent_reasoning" } & AgentReasoningEvent | { "type": "agent_reasoning_delta" } & AgentReasoningDeltaEvent | { "type": "agent_reasoning_raw_content" } & AgentReasoningRawContentEvent | { "type": "agent_reasoning_raw_content_delta" } & AgentReasoningRawContentDeltaEvent | { "type": "agent_reasoning_section_break" } & AgentReasoningSectionBreakEvent | { "type": "session_configured" } & SessionConfiguredEvent | { "type": "thread_name_updated" } & ThreadNameUpdatedEvent | { "type": "mcp_startup_update" } & McpStartupUpdateEvent | { "type": "mcp_startup_complete" } & McpStartupCompleteEvent | { "type": "mcp_tool_call_begin" } & McpToolCallBeginEvent | { "type": "mcp_tool_call_end" } & McpToolCallEndEvent | { "type": "web_search_begin" } & WebSearchBeginEvent | { "type": "web_search_end" } & WebSearchEndEvent | { "type": "exec_command_begin" } & ExecCommandBeginEvent | { "type": "exec_command_output_delta" } & ExecCommandOutputDeltaEvent | { "type": "terminal_interaction" } & TerminalInteractionEvent | { "type": "exec_command_end" } & ExecCommandEndEvent | { "type": "view_image_tool_call" } & ViewImageToolCallEvent | { "type": "exec_approval_request" } & ExecApprovalRequestEvent | { "type": "request_user_input" } & RequestUserInputEvent | { "type": "dynamic_tool_call_request" } & DynamicToolCallRequest | { "type": "dynamic_tool_call_response" } & DynamicToolCallResponseEvent | { "type": "elicitation_request" } & ElicitationRequestEvent | { "type": "apply_patch_approval_request" } & ApplyPatchApprovalRequestEvent | { "type": "deprecation_notice" } & DeprecationNoticeEvent | { "type": "background_event" } & BackgroundEventEvent | { "type": "undo_started" } & UndoStartedEvent | { "type": "undo_completed" } & UndoCompletedEvent | { "type": "stream_error" } & StreamErrorEvent | { "type": "patch_apply_begin" } & PatchApplyBeginEvent | { "type": "patch_apply_end" } & PatchApplyEndEvent | { "type": "turn_diff" } & TurnDiffEvent | { "type": "get_history_entry_response" } & GetHistoryEntryResponseEvent | { "type": "mcp_list_tools_response" } & McpListToolsResponseEvent | { "type": "list_custom_prompts_response" } & ListCustomPromptsResponseEvent | { "type": "list_skills_response" } & ListSkillsResponseEvent | { "type": "list_remote_skills_response" } & ListRemoteSkillsResponseEvent | { "type": "remote_skill_downloaded" } & RemoteSkillDownloadedEvent | { "type": "skills_update_available" } | { "type": "plan_update" } & UpdatePlanArgs | { "type": "turn_aborted" } & TurnAbortedEvent | { "type": "shutdown_complete" } | { "type": "entered_review_mode" } & ReviewRequest | { "type": "exited_review_mode" } & ExitedReviewModeEvent | { "type": "raw_response_item" } & RawResponseItemEvent | { "type": "item_started" } & ItemStartedEvent | { "type": "item_completed" } & ItemCompletedEvent | { "type": "agent_message_content_delta" } & AgentMessageContentDeltaEvent | { "type": "plan_delta" } & PlanDeltaEvent | { "type": "reasoning_content_delta" } & ReasoningContentDeltaEvent | { "type": "reasoning_raw_content_delta" } & ReasoningRawContentDeltaEvent | { "type": "collab_agent_spawn_begin" } & CollabAgentSpawnBeginEvent | { "type": "collab_agent_spawn_end" } & CollabAgentSpawnEndEvent | { "type": "collab_agent_interaction_begin" } & CollabAgentInteractionBeginEvent | { "type": "collab_agent_interaction_end" } & CollabAgentInteractionEndEvent | { "type": "collab_waiting_begin" } & CollabWaitingBeginEvent | { "type": "collab_waiting_end" } & CollabWaitingEndEvent | { "type": "collab_close_begin" } & CollabCloseBeginEvent | { "type": "collab_close_end" } & CollabCloseEndEvent | { "type": "collab_resume_begin" } & CollabResumeBeginEvent | { "type": "collab_resume_end" } & CollabResumeEndEvent; +export type EventMsg = { "type": "error" } & ErrorEvent | { "type": "warning" } & WarningEvent | { "type": "realtime_conversation_started" } & RealtimeConversationStartedEvent | { "type": "realtime_conversation_realtime" } & RealtimeConversationRealtimeEvent | { "type": "realtime_conversation_closed" } & RealtimeConversationClosedEvent | { "type": "model_reroute" } & ModelRerouteEvent | { "type": "context_compacted" } & ContextCompactedEvent | { "type": "thread_rolled_back" } & ThreadRolledBackEvent | { "type": "task_started" } & TurnStartedEvent | { "type": "task_complete" } & TurnCompleteEvent | { "type": "token_count" } & TokenCountEvent | { "type": "agent_message" } & AgentMessageEvent | { "type": "user_message" } & UserMessageEvent | { "type": "agent_message_delta" } & AgentMessageDeltaEvent | { "type": "agent_reasoning" } & AgentReasoningEvent | { "type": "agent_reasoning_delta" } & AgentReasoningDeltaEvent | { "type": "agent_reasoning_raw_content" } & AgentReasoningRawContentEvent | { "type": "agent_reasoning_raw_content_delta" } & AgentReasoningRawContentDeltaEvent | { "type": "agent_reasoning_section_break" } & AgentReasoningSectionBreakEvent | { "type": "session_configured" } & SessionConfiguredEvent | { "type": "thread_name_updated" } & ThreadNameUpdatedEvent | { "type": "mcp_startup_update" } & McpStartupUpdateEvent | { "type": "mcp_startup_complete" } & McpStartupCompleteEvent | { "type": "mcp_tool_call_begin" } & McpToolCallBeginEvent | { "type": "mcp_tool_call_end" } & McpToolCallEndEvent | { "type": "web_search_begin" } & WebSearchBeginEvent | { "type": "web_search_end" } & WebSearchEndEvent | { "type": "image_generation_begin" } & ImageGenerationBeginEvent | { "type": "image_generation_end" } & ImageGenerationEndEvent | { "type": "exec_command_begin" } & ExecCommandBeginEvent | { "type": "exec_command_output_delta" } & ExecCommandOutputDeltaEvent | { "type": "terminal_interaction" } & TerminalInteractionEvent | { "type": "exec_command_end" } & ExecCommandEndEvent | { "type": "view_image_tool_call" } & ViewImageToolCallEvent | { "type": "exec_approval_request" } & ExecApprovalRequestEvent | { "type": "request_user_input" } & RequestUserInputEvent | { "type": "dynamic_tool_call_request" } & DynamicToolCallRequest | { "type": "dynamic_tool_call_response" } & DynamicToolCallResponseEvent | { "type": "elicitation_request" } & ElicitationRequestEvent | { "type": "apply_patch_approval_request" } & ApplyPatchApprovalRequestEvent | { "type": "deprecation_notice" } & DeprecationNoticeEvent | { "type": "background_event" } & BackgroundEventEvent | { "type": "undo_started" } & UndoStartedEvent | { "type": "undo_completed" } & UndoCompletedEvent | { "type": "stream_error" } & StreamErrorEvent | { "type": "patch_apply_begin" } & PatchApplyBeginEvent | { "type": "patch_apply_end" } & PatchApplyEndEvent | { "type": "turn_diff" } & TurnDiffEvent | { "type": "get_history_entry_response" } & GetHistoryEntryResponseEvent | { "type": "mcp_list_tools_response" } & McpListToolsResponseEvent | { "type": "list_custom_prompts_response" } & ListCustomPromptsResponseEvent | { "type": "list_skills_response" } & ListSkillsResponseEvent | { "type": "list_remote_skills_response" } & ListRemoteSkillsResponseEvent | { "type": "remote_skill_downloaded" } & RemoteSkillDownloadedEvent | { "type": "skills_update_available" } | { "type": "plan_update" } & UpdatePlanArgs | { "type": "turn_aborted" } & TurnAbortedEvent | { "type": "shutdown_complete" } | { "type": "entered_review_mode" } & ReviewRequest | { "type": "exited_review_mode" } & ExitedReviewModeEvent | { "type": "raw_response_item" } & RawResponseItemEvent | { "type": "item_started" } & ItemStartedEvent | { "type": "item_completed" } & ItemCompletedEvent | { "type": "agent_message_content_delta" } & AgentMessageContentDeltaEvent | { "type": "plan_delta" } & PlanDeltaEvent | { "type": "reasoning_content_delta" } & ReasoningContentDeltaEvent | { "type": "reasoning_raw_content_delta" } & ReasoningRawContentDeltaEvent | { "type": "collab_agent_spawn_begin" } & CollabAgentSpawnBeginEvent | { "type": "collab_agent_spawn_end" } & CollabAgentSpawnEndEvent | { "type": "collab_agent_interaction_begin" } & CollabAgentInteractionBeginEvent | { "type": "collab_agent_interaction_end" } & CollabAgentInteractionEndEvent | { "type": "collab_waiting_begin" } & CollabWaitingBeginEvent | { "type": "collab_waiting_end" } & CollabWaitingEndEvent | { "type": "collab_close_begin" } & CollabCloseBeginEvent | { "type": "collab_close_end" } & CollabCloseEndEvent | { "type": "collab_resume_begin" } & CollabResumeBeginEvent | { "type": "collab_resume_end" } & CollabResumeEndEvent; diff --git a/codex-rs/app-server-protocol/schema/typescript/ImageGenerationBeginEvent.ts b/codex-rs/app-server-protocol/schema/typescript/ImageGenerationBeginEvent.ts new file mode 100644 index 000000000..3e424dbd0 --- /dev/null +++ b/codex-rs/app-server-protocol/schema/typescript/ImageGenerationBeginEvent.ts @@ -0,0 +1,5 @@ +// GENERATED CODE! DO NOT MODIFY BY HAND! + +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +export type ImageGenerationBeginEvent = { call_id: string, }; diff --git a/codex-rs/app-server-protocol/schema/typescript/ImageGenerationEndEvent.ts b/codex-rs/app-server-protocol/schema/typescript/ImageGenerationEndEvent.ts new file mode 100644 index 000000000..edc8c5e16 --- /dev/null +++ b/codex-rs/app-server-protocol/schema/typescript/ImageGenerationEndEvent.ts @@ -0,0 +1,5 @@ +// GENERATED CODE! DO NOT MODIFY BY HAND! + +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +export type ImageGenerationEndEvent = { call_id: string, status: string, revised_prompt?: string, result: string, }; diff --git a/codex-rs/app-server-protocol/schema/typescript/ImageGenerationItem.ts b/codex-rs/app-server-protocol/schema/typescript/ImageGenerationItem.ts new file mode 100644 index 000000000..46032bff7 --- /dev/null +++ b/codex-rs/app-server-protocol/schema/typescript/ImageGenerationItem.ts @@ -0,0 +1,5 @@ +// GENERATED CODE! DO NOT MODIFY BY HAND! + +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +export type ImageGenerationItem = { id: string, status: string, revised_prompt?: string, result: string, }; diff --git a/codex-rs/app-server-protocol/schema/typescript/TurnItem.ts b/codex-rs/app-server-protocol/schema/typescript/TurnItem.ts index 0f2ea12a2..965fb1848 100644 --- a/codex-rs/app-server-protocol/schema/typescript/TurnItem.ts +++ b/codex-rs/app-server-protocol/schema/typescript/TurnItem.ts @@ -3,9 +3,10 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. import type { AgentMessageItem } from "./AgentMessageItem"; import type { ContextCompactionItem } from "./ContextCompactionItem"; +import type { ImageGenerationItem } from "./ImageGenerationItem"; import type { PlanItem } from "./PlanItem"; import type { ReasoningItem } from "./ReasoningItem"; import type { UserMessageItem } from "./UserMessageItem"; import type { WebSearchItem } from "./WebSearchItem"; -export type TurnItem = { "type": "UserMessage" } & UserMessageItem | { "type": "AgentMessage" } & AgentMessageItem | { "type": "Plan" } & PlanItem | { "type": "Reasoning" } & ReasoningItem | { "type": "WebSearch" } & WebSearchItem | { "type": "ContextCompaction" } & ContextCompactionItem; +export type TurnItem = { "type": "UserMessage" } & UserMessageItem | { "type": "AgentMessage" } & AgentMessageItem | { "type": "Plan" } & PlanItem | { "type": "Reasoning" } & ReasoningItem | { "type": "WebSearch" } & WebSearchItem | { "type": "ImageGeneration" } & ImageGenerationItem | { "type": "ContextCompaction" } & ContextCompactionItem; diff --git a/codex-rs/app-server-protocol/schema/typescript/index.ts b/codex-rs/app-server-protocol/schema/typescript/index.ts index faad506df..fd0d333e4 100644 --- a/codex-rs/app-server-protocol/schema/typescript/index.ts +++ b/codex-rs/app-server-protocol/schema/typescript/index.ts @@ -84,6 +84,9 @@ export type { GitDiffToRemoteResponse } from "./GitDiffToRemoteResponse"; export type { GitSha } from "./GitSha"; export type { HistoryEntry } from "./HistoryEntry"; export type { ImageDetail } from "./ImageDetail"; +export type { ImageGenerationBeginEvent } from "./ImageGenerationBeginEvent"; +export type { ImageGenerationEndEvent } from "./ImageGenerationEndEvent"; +export type { ImageGenerationItem } from "./ImageGenerationItem"; export type { InitializeCapabilities } from "./InitializeCapabilities"; export type { InitializeParams } from "./InitializeParams"; export type { InitializeResponse } from "./InitializeResponse"; diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/ThreadItem.ts b/codex-rs/app-server-protocol/schema/typescript/v2/ThreadItem.ts index d937d53b4..488c07e54 100644 --- a/codex-rs/app-server-protocol/schema/typescript/v2/ThreadItem.ts +++ b/codex-rs/app-server-protocol/schema/typescript/v2/ThreadItem.ts @@ -85,4 +85,4 @@ prompt: string | null, /** * Last known status of the target agents, when available. */ -agentsStates: { [key in string]?: CollabAgentState }, } | { "type": "webSearch", id: string, query: string, action: WebSearchAction | null, } | { "type": "imageView", id: string, path: string, } | { "type": "enteredReviewMode", id: string, review: string, } | { "type": "exitedReviewMode", id: string, review: string, } | { "type": "contextCompaction", id: string, }; +agentsStates: { [key in string]?: CollabAgentState }, } | { "type": "webSearch", id: string, query: string, action: WebSearchAction | null, } | { "type": "imageView", id: string, path: string, } | { "type": "imageGeneration", id: string, status: string, revisedPrompt: string | null, result: string, } | { "type": "enteredReviewMode", id: string, review: string, } | { "type": "exitedReviewMode", id: string, review: string, } | { "type": "contextCompaction", id: string, }; diff --git a/codex-rs/app-server-protocol/src/protocol/thread_history.rs b/codex-rs/app-server-protocol/src/protocol/thread_history.rs index 111bab84c..1ad159129 100644 --- a/codex-rs/app-server-protocol/src/protocol/thread_history.rs +++ b/codex-rs/app-server-protocol/src/protocol/thread_history.rs @@ -30,6 +30,8 @@ use codex_protocol::protocol::ErrorEvent; use codex_protocol::protocol::EventMsg; use codex_protocol::protocol::ExecCommandBeginEvent; use codex_protocol::protocol::ExecCommandEndEvent; +use codex_protocol::protocol::ImageGenerationBeginEvent; +use codex_protocol::protocol::ImageGenerationEndEvent; use codex_protocol::protocol::ItemCompletedEvent; use codex_protocol::protocol::ItemStartedEvent; use codex_protocol::protocol::McpToolCallBeginEvent; @@ -141,6 +143,8 @@ impl ThreadHistoryBuilder { EventMsg::McpToolCallBegin(payload) => self.handle_mcp_tool_call_begin(payload), EventMsg::McpToolCallEnd(payload) => self.handle_mcp_tool_call_end(payload), EventMsg::ViewImageToolCall(payload) => self.handle_view_image_tool_call(payload), + EventMsg::ImageGenerationBegin(payload) => self.handle_image_generation_begin(payload), + EventMsg::ImageGenerationEnd(payload) => self.handle_image_generation_end(payload), EventMsg::CollabAgentSpawnBegin(payload) => { self.handle_collab_agent_spawn_begin(payload) } @@ -269,6 +273,7 @@ impl ThreadHistoryBuilder { | codex_protocol::items::TurnItem::AgentMessage(_) | codex_protocol::items::TurnItem::Reasoning(_) | codex_protocol::items::TurnItem::WebSearch(_) + | codex_protocol::items::TurnItem::ImageGeneration(_) | codex_protocol::items::TurnItem::ContextCompaction(_) => {} } } @@ -288,6 +293,7 @@ impl ThreadHistoryBuilder { | codex_protocol::items::TurnItem::AgentMessage(_) | codex_protocol::items::TurnItem::Reasoning(_) | codex_protocol::items::TurnItem::WebSearch(_) + | codex_protocol::items::TurnItem::ImageGeneration(_) | codex_protocol::items::TurnItem::ContextCompaction(_) => {} } } @@ -516,6 +522,26 @@ impl ThreadHistoryBuilder { self.upsert_item_in_current_turn(item); } + fn handle_image_generation_begin(&mut self, payload: &ImageGenerationBeginEvent) { + let item = ThreadItem::ImageGeneration { + id: payload.call_id.clone(), + status: String::new(), + revised_prompt: None, + result: String::new(), + }; + self.upsert_item_in_current_turn(item); + } + + fn handle_image_generation_end(&mut self, payload: &ImageGenerationEndEvent) { + let item = ThreadItem::ImageGeneration { + id: payload.call_id.clone(), + status: payload.status.clone(), + revised_prompt: payload.revised_prompt.clone(), + result: payload.result.clone(), + }; + self.upsert_item_in_current_turn(item); + } + fn handle_collab_agent_spawn_begin( &mut self, payload: &codex_protocol::protocol::CollabAgentSpawnBeginEvent, diff --git a/codex-rs/app-server-protocol/src/protocol/v2.rs b/codex-rs/app-server-protocol/src/protocol/v2.rs index cc7cbf44a..83499ae88 100644 --- a/codex-rs/app-server-protocol/src/protocol/v2.rs +++ b/codex-rs/app-server-protocol/src/protocol/v2.rs @@ -3347,6 +3347,14 @@ pub enum ThreadItem { ImageView { id: String, path: String }, #[serde(rename_all = "camelCase")] #[ts(rename_all = "camelCase")] + ImageGeneration { + id: String, + status: String, + revised_prompt: Option, + result: String, + }, + #[serde(rename_all = "camelCase")] + #[ts(rename_all = "camelCase")] EnteredReviewMode { id: String, review: String }, #[serde(rename_all = "camelCase")] #[ts(rename_all = "camelCase")] @@ -3370,6 +3378,7 @@ impl ThreadItem { | ThreadItem::CollabAgentToolCall { id, .. } | ThreadItem::WebSearch { id, .. } | ThreadItem::ImageView { id, .. } + | ThreadItem::ImageGeneration { id, .. } | ThreadItem::EnteredReviewMode { id, .. } | ThreadItem::ExitedReviewMode { id, .. } | ThreadItem::ContextCompaction { id, .. } => id, @@ -3449,6 +3458,12 @@ impl From for ThreadItem { query: search.query, action: Some(WebSearchAction::from(search.action)), }, + CoreTurnItem::ImageGeneration(image) => ThreadItem::ImageGeneration { + id: image.id, + status: image.status, + revised_prompt: image.revised_prompt, + result: image.result, + }, CoreTurnItem::ContextCompaction(compaction) => { ThreadItem::ContextCompaction { id: compaction.id } } diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index 5ba2d7f63..0d04086bd 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -5935,6 +5935,8 @@ fn realtime_text_for_event(msg: &EventMsg) -> Option { | EventMsg::PatchApplyBegin(_) | EventMsg::PatchApplyEnd(_) | EventMsg::ViewImageToolCall(_) + | EventMsg::ImageGenerationBegin(_) + | EventMsg::ImageGenerationEnd(_) | EventMsg::ExecApprovalRequest(_) | EventMsg::RequestUserInput(_) | EventMsg::DynamicToolCallRequest(_) diff --git a/codex-rs/core/src/event_mapping.rs b/codex-rs/core/src/event_mapping.rs index afffdb0f1..a5c09b1ef 100644 --- a/codex-rs/core/src/event_mapping.rs +++ b/codex-rs/core/src/event_mapping.rs @@ -142,6 +142,19 @@ pub fn parse_turn_item(item: &ResponseItem) -> Option { action, })) } + ResponseItem::ImageGenerationCall { + id, + status, + revised_prompt, + result, + } => Some(TurnItem::ImageGeneration( + codex_protocol::items::ImageGenerationItem { + id: id.clone(), + status: status.clone(), + revised_prompt: revised_prompt.clone(), + result: result.clone(), + }, + )), _ => None, } } diff --git a/codex-rs/core/src/rollout/policy.rs b/codex-rs/core/src/rollout/policy.rs index 9af54610e..92b607681 100644 --- a/codex-rs/core/src/rollout/policy.rs +++ b/codex-rs/core/src/rollout/policy.rs @@ -118,6 +118,7 @@ fn event_msg_persistence_mode(ev: &EventMsg) -> Option { | EventMsg::PatchApplyEnd(_) | EventMsg::McpToolCallEnd(_) | EventMsg::ViewImageToolCall(_) + | EventMsg::ImageGenerationEnd(_) | EventMsg::CollabAgentSpawnEnd(_) | EventMsg::CollabAgentInteractionEnd(_) | EventMsg::CollabWaitingEnd(_) @@ -172,6 +173,7 @@ fn event_msg_persistence_mode(ev: &EventMsg) -> Option { | EventMsg::CollabAgentInteractionBegin(_) | EventMsg::CollabWaitingBegin(_) | EventMsg::CollabCloseBegin(_) - | EventMsg::CollabResumeBegin(_) => None, + | EventMsg::CollabResumeBegin(_) + | EventMsg::ImageGenerationBegin(_) => None, } } diff --git a/codex-rs/core/src/stream_events_utils.rs b/codex-rs/core/src/stream_events_utils.rs index abb4aa86c..e44dacd42 100644 --- a/codex-rs/core/src/stream_events_utils.rs +++ b/codex-rs/core/src/stream_events_utils.rs @@ -159,8 +159,14 @@ pub(crate) async fn handle_output_item_done( Ok(None) => { if let Some(turn_item) = handle_non_tool_response_item(&item, plan_mode) { if previously_active_item.is_none() { + let mut started_item = turn_item.clone(); + if let TurnItem::ImageGeneration(item) = &mut started_item { + item.status = "in_progress".to_string(); + item.revised_prompt = None; + item.result.clear(); + } ctx.sess - .emit_turn_item_started(&ctx.turn_context, &turn_item) + .emit_turn_item_started(&ctx.turn_context, &started_item) .await; } @@ -243,7 +249,8 @@ pub(crate) fn handle_non_tool_response_item( match item { ResponseItem::Message { .. } | ResponseItem::Reasoning { .. } - | ResponseItem::WebSearchCall { .. } => { + | ResponseItem::WebSearchCall { .. } + | ResponseItem::ImageGenerationCall { .. } => { let mut turn_item = parse_turn_item(item)?; if let TurnItem::AgentMessage(agent_message) = &mut turn_item { let combined = agent_message diff --git a/codex-rs/core/tests/suite/items.rs b/codex-rs/core/tests/suite/items.rs index af1a83f53..56c1b374a 100644 --- a/codex-rs/core/tests/suite/items.rs +++ b/codex-rs/core/tests/suite/items.rs @@ -16,6 +16,7 @@ use codex_protocol::user_input::TextElement; use codex_protocol::user_input::UserInput; use core_test_support::responses::ev_assistant_message; use core_test_support::responses::ev_completed; +use core_test_support::responses::ev_image_generation_call; use core_test_support::responses::ev_message_item_added; use core_test_support::responses::ev_output_text_delta; use core_test_support::responses::ev_reasoning_item; @@ -262,6 +263,51 @@ async fn web_search_item_is_emitted() -> anyhow::Result<()> { Ok(()) } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn image_generation_call_event_is_emitted() -> anyhow::Result<()> { + skip_if_no_network!(Ok(())); + + let server = start_mock_server().await; + + let TestCodex { codex, .. } = test_codex().build(&server).await?; + + let first_response = sse(vec![ + ev_response_created("resp-1"), + ev_image_generation_call("ig_123", "completed", "A tiny blue square", "Zm9v"), + ev_completed("resp-1"), + ]); + mount_sse_once(&server, first_response).await; + + codex + .submit(Op::UserInput { + items: vec![UserInput::Text { + text: "generate a tiny blue square".into(), + text_elements: Vec::new(), + }], + final_output_json_schema: None, + }) + .await?; + + let begin = wait_for_event_match(&codex, |ev| match ev { + EventMsg::ImageGenerationBegin(event) => Some(event.clone()), + _ => None, + }) + .await; + let end = wait_for_event_match(&codex, |ev| match ev { + EventMsg::ImageGenerationEnd(event) => Some(event.clone()), + _ => None, + }) + .await; + + assert_eq!(begin.call_id, "ig_123"); + assert_eq!(end.call_id, "ig_123"); + assert_eq!(end.status, "completed"); + assert_eq!(end.revised_prompt, Some("A tiny blue square".to_string())); + assert_eq!(end.result, "Zm9v"); + + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn agent_message_content_delta_has_item_metadata() -> anyhow::Result<()> { skip_if_no_network!(Ok(())); diff --git a/codex-rs/exec/src/event_processor_with_human_output.rs b/codex-rs/exec/src/event_processor_with_human_output.rs index ff2699eed..584f3acc3 100644 --- a/codex-rs/exec/src/event_processor_with_human_output.rs +++ b/codex-rs/exec/src/event_processor_with_human_output.rs @@ -448,6 +448,22 @@ impl EventProcessor for EventProcessorWithHumanOutput { ts_msg!(self, "🌐 Searched: {detail}"); } } + EventMsg::ImageGenerationBegin(generated) => { + ts_msg!( + self, + "{} {}", + "image generation started".style(self.magenta), + generated.call_id + ); + } + EventMsg::ImageGenerationEnd(generated) => { + ts_msg!( + self, + "{} {}", + "generated image".style(self.magenta), + generated.call_id + ); + } EventMsg::PatchApplyBegin(PatchApplyBeginEvent { call_id, auto_approved, diff --git a/codex-rs/mcp-server/src/codex_tool_runner.rs b/codex-rs/mcp-server/src/codex_tool_runner.rs index 24896b042..d0cfaf909 100644 --- a/codex-rs/mcp-server/src/codex_tool_runner.rs +++ b/codex-rs/mcp-server/src/codex_tool_runner.rs @@ -352,6 +352,8 @@ async fn run_codex_tool_session_inner( | EventMsg::UserMessage(_) | EventMsg::ShutdownComplete | EventMsg::ViewImageToolCall(_) + | EventMsg::ImageGenerationBegin(_) + | EventMsg::ImageGenerationEnd(_) | EventMsg::RawResponseItem(_) | EventMsg::EnteredReviewMode(_) | EventMsg::ItemStarted(_) diff --git a/codex-rs/protocol/src/items.rs b/codex-rs/protocol/src/items.rs index 9284bbddf..9b904c900 100644 --- a/codex-rs/protocol/src/items.rs +++ b/codex-rs/protocol/src/items.rs @@ -5,6 +5,7 @@ use crate::protocol::AgentReasoningEvent; use crate::protocol::AgentReasoningRawContentEvent; use crate::protocol::ContextCompactedEvent; use crate::protocol::EventMsg; +use crate::protocol::ImageGenerationEndEvent; use crate::protocol::UserMessageEvent; use crate::protocol::WebSearchEndEvent; use crate::user_input::ByteRange; @@ -24,6 +25,7 @@ pub enum TurnItem { Plan(PlanItem), Reasoning(ReasoningItem), WebSearch(WebSearchItem), + ImageGeneration(ImageGenerationItem), ContextCompaction(ContextCompactionItem), } @@ -79,6 +81,16 @@ pub struct WebSearchItem { pub action: WebSearchAction, } +#[derive(Debug, Clone, Deserialize, Serialize, TS, JsonSchema, PartialEq)] +pub struct ImageGenerationItem { + pub id: String, + pub status: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional)] + pub revised_prompt: Option, + pub result: String, +} + #[derive(Debug, Clone, Deserialize, Serialize, TS, JsonSchema)] pub struct ContextCompactionItem { pub id: String, @@ -235,6 +247,17 @@ impl WebSearchItem { } } +impl ImageGenerationItem { + pub fn as_legacy_event(&self) -> EventMsg { + EventMsg::ImageGenerationEnd(ImageGenerationEndEvent { + call_id: self.id.clone(), + status: self.status.clone(), + revised_prompt: self.revised_prompt.clone(), + result: self.result.clone(), + }) + } +} + impl TurnItem { pub fn id(&self) -> String { match self { @@ -243,6 +266,7 @@ impl TurnItem { TurnItem::Plan(item) => item.id.clone(), TurnItem::Reasoning(item) => item.id.clone(), TurnItem::WebSearch(item) => item.id.clone(), + TurnItem::ImageGeneration(item) => item.id.clone(), TurnItem::ContextCompaction(item) => item.id.clone(), } } @@ -253,6 +277,7 @@ impl TurnItem { TurnItem::AgentMessage(item) => item.as_legacy_events(), TurnItem::Plan(_) => Vec::new(), TurnItem::WebSearch(item) => vec![item.as_legacy_event()], + TurnItem::ImageGeneration(item) => vec![item.as_legacy_event()], TurnItem::Reasoning(item) => item.as_legacy_events(show_raw_agent_reasoning), TurnItem::ContextCompaction(item) => vec![item.as_legacy_event()], } diff --git a/codex-rs/protocol/src/protocol.rs b/codex-rs/protocol/src/protocol.rs index 55f19255b..dbae3f3b5 100644 --- a/codex-rs/protocol/src/protocol.rs +++ b/codex-rs/protocol/src/protocol.rs @@ -1086,6 +1086,10 @@ pub enum EventMsg { WebSearchEnd(WebSearchEndEvent), + ImageGenerationBegin(ImageGenerationBeginEvent), + + ImageGenerationEnd(ImageGenerationEndEvent), + /// Notification that the server is about to execute a command. ExecCommandBegin(ExecCommandBeginEvent), @@ -1367,6 +1371,11 @@ impl HasLegacyEvent for ItemStartedEvent { TurnItem::WebSearch(item) => vec![EventMsg::WebSearchBegin(WebSearchBeginEvent { call_id: item.id.clone(), })], + TurnItem::ImageGeneration(item) => { + vec![EventMsg::ImageGenerationBegin(ImageGenerationBeginEvent { + call_id: item.id.clone(), + })] + } _ => Vec::new(), } } @@ -1876,6 +1885,21 @@ pub struct WebSearchEndEvent { pub action: WebSearchAction, } +#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, TS)] +pub struct ImageGenerationBeginEvent { + pub call_id: String, +} + +#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, TS)] +pub struct ImageGenerationEndEvent { + pub call_id: String, + pub status: String, + #[serde(skip_serializing_if = "Option::is_none")] + #[ts(optional)] + pub revised_prompt: Option, + pub result: String, +} + // Conversation kept for backward compatibility. /// Response payload for `Op::GetHistory` containing the current session's /// in-memory transcript. @@ -3115,6 +3139,7 @@ pub struct CollabResumeEndEvent { #[cfg(test)] mod tests { use super::*; + use crate::items::ImageGenerationItem; use crate::items::UserMessageItem; use crate::items::WebSearchItem; use anyhow::Result; @@ -3235,6 +3260,53 @@ mod tests { assert!(event.as_legacy_events(false).is_empty()); } + #[test] + fn item_started_event_from_image_generation_emits_begin_event() { + let event = ItemStartedEvent { + thread_id: ThreadId::new(), + turn_id: "turn-1".into(), + item: TurnItem::ImageGeneration(ImageGenerationItem { + id: "ig-1".into(), + status: "in_progress".into(), + revised_prompt: None, + result: String::new(), + }), + }; + + let legacy_events = event.as_legacy_events(false); + assert_eq!(legacy_events.len(), 1); + match &legacy_events[0] { + EventMsg::ImageGenerationBegin(event) => assert_eq!(event.call_id, "ig-1"), + _ => panic!("expected ImageGenerationBegin event"), + } + } + + #[test] + fn item_completed_event_from_image_generation_emits_end_event() { + let event = ItemCompletedEvent { + thread_id: ThreadId::new(), + turn_id: "turn-1".into(), + item: TurnItem::ImageGeneration(ImageGenerationItem { + id: "ig-1".into(), + status: "completed".into(), + revised_prompt: Some("A tiny blue square".into()), + result: "Zm9v".into(), + }), + }; + + let legacy_events = event.as_legacy_events(false); + assert_eq!(legacy_events.len(), 1); + match &legacy_events[0] { + EventMsg::ImageGenerationEnd(event) => { + assert_eq!(event.call_id, "ig-1"); + assert_eq!(event.status, "completed"); + assert_eq!(event.revised_prompt.as_deref(), Some("A tiny blue square")); + assert_eq!(event.result, "Zm9v"); + } + _ => panic!("expected ImageGenerationEnd event"), + } + } + #[test] fn rollback_failed_error_does_not_affect_turn_status() { let event = ErrorEvent { diff --git a/codex-rs/tui/src/chatwidget.rs b/codex-rs/tui/src/chatwidget.rs index 120ec3d57..317464c40 100644 --- a/codex-rs/tui/src/chatwidget.rs +++ b/codex-rs/tui/src/chatwidget.rs @@ -111,6 +111,8 @@ use codex_protocol::protocol::ExecCommandEndEvent; use codex_protocol::protocol::ExecCommandOutputDeltaEvent; use codex_protocol::protocol::ExecCommandSource; use codex_protocol::protocol::ExitedReviewModeEvent; +use codex_protocol::protocol::ImageGenerationBeginEvent; +use codex_protocol::protocol::ImageGenerationEndEvent; use codex_protocol::protocol::ListCustomPromptsResponseEvent; use codex_protocol::protocol::ListSkillsResponseEvent; use codex_protocol::protocol::McpListToolsResponseEvent; @@ -2282,6 +2284,20 @@ impl ChatWidget { self.request_redraw(); } + fn on_image_generation_begin(&mut self, _event: ImageGenerationBeginEvent) { + self.flush_answer_stream_with_separator(); + } + + fn on_image_generation_end(&mut self, event: ImageGenerationEndEvent) { + self.flush_answer_stream_with_separator(); + self.add_to_history(history_cell::new_image_generation_call( + event.call_id, + event.status, + event.revised_prompt, + )); + self.request_redraw(); + } + fn on_patch_apply_end(&mut self, event: codex_protocol::protocol::PatchApplyEndEvent) { let ev2 = event.clone(); self.defer_or_handle( @@ -4738,6 +4754,8 @@ impl ChatWidget { EventMsg::PatchApplyEnd(ev) => self.on_patch_apply_end(ev), EventMsg::ExecCommandEnd(ev) => self.on_exec_command_end(ev), EventMsg::ViewImageToolCall(ev) => self.on_view_image_tool_call(ev), + EventMsg::ImageGenerationBegin(ev) => self.on_image_generation_begin(ev), + EventMsg::ImageGenerationEnd(ev) => self.on_image_generation_end(ev), EventMsg::McpToolCallBegin(ev) => self.on_mcp_tool_call_begin(ev), EventMsg::McpToolCallEnd(ev) => self.on_mcp_tool_call_end(ev), EventMsg::WebSearchBegin(ev) => self.on_web_search_begin(ev), diff --git a/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__image_generation_call_history_snapshot.snap b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__image_generation_call_history_snapshot.snap new file mode 100644 index 000000000..9ed57a368 --- /dev/null +++ b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__image_generation_call_history_snapshot.snap @@ -0,0 +1,7 @@ +--- +source: tui/src/chatwidget/tests.rs +assertion_line: 5889 +expression: combined +--- +• Generated Image + └ completed: A tiny blue square diff --git a/codex-rs/tui/src/chatwidget/tests.rs b/codex-rs/tui/src/chatwidget/tests.rs index 6d74473e3..72fa61c13 100644 --- a/codex-rs/tui/src/chatwidget/tests.rs +++ b/codex-rs/tui/src/chatwidget/tests.rs @@ -72,6 +72,7 @@ use codex_protocol::protocol::ExecCommandStatus as CoreExecCommandStatus; use codex_protocol::protocol::ExecPolicyAmendment; use codex_protocol::protocol::ExitedReviewModeEvent; use codex_protocol::protocol::FileChange; +use codex_protocol::protocol::ImageGenerationEndEvent; use codex_protocol::protocol::ItemCompletedEvent; use codex_protocol::protocol::McpStartupCompleteEvent; use codex_protocol::protocol::McpStartupStatus; @@ -6041,6 +6042,26 @@ async fn view_image_tool_call_adds_history_cell() { assert_snapshot!("local_image_attachment_history_snapshot", combined); } +#[tokio::test] +async fn image_generation_call_adds_history_cell() { + let (mut chat, mut rx, _op_rx) = make_chatwidget_manual(None).await; + + chat.handle_codex_event(Event { + id: "sub-image-generation".into(), + msg: EventMsg::ImageGenerationEnd(ImageGenerationEndEvent { + call_id: "call-image-generation".into(), + status: "completed".into(), + revised_prompt: Some("A tiny blue square".into()), + result: "Zm9v".into(), + }), + }); + + let cells = drain_insert_history(&mut rx); + assert_eq!(cells.len(), 1, "expected a single history cell"); + let combined = lines_to_single_string(&cells[0]); + assert_snapshot!("image_generation_call_history_snapshot", combined); +} + // Snapshot test: interrupting a running exec finalizes the active cell with a red ✗ // marker (replacing the spinner) and flushes it into history. #[tokio::test] diff --git a/codex-rs/tui/src/history_cell.rs b/codex-rs/tui/src/history_cell.rs index 76e92ca0e..623680519 100644 --- a/codex-rs/tui/src/history_cell.rs +++ b/codex-rs/tui/src/history_cell.rs @@ -2196,6 +2196,21 @@ pub(crate) fn new_view_image_tool_call(path: PathBuf, cwd: &Path) -> PlainHistor PlainHistoryCell { lines } } +pub(crate) fn new_image_generation_call( + call_id: String, + status: String, + revised_prompt: Option, +) -> PlainHistoryCell { + let detail = revised_prompt.unwrap_or_else(|| call_id.clone()); + + let lines: Vec> = vec![ + vec!["• ".dim(), "Generated Image".bold()].into(), + vec![" └ ".dim(), format!("{status}: {detail}").dim()].into(), + ]; + + PlainHistoryCell { lines } +} + pub(crate) fn new_reasoning_summary_block(full_reasoning_buffer: String) -> Box { let full_reasoning_buffer = full_reasoning_buffer.trim(); if let Some(open) = full_reasoning_buffer.find("**") {