Preserve image detail in app-server inputs (#20693)

## Summary - Add optional image detail to user image inputs across core, app-server v2, thread history/event mapping, and the generated app-server schemas/types. - Preserve requested detail when serializing Responses image inputs: omitted detail stays on the existing `high` default, while explicit `original` keeps local images on the original-resolution path. - Support `high`/`original` consistently for tool image outputs, including MCP `codex/imageDetail`, code-mode image helpers, and `view_image`.
2026-07-01 00:31:56 +08:00 · 2026-05-15 15:04:04 -07:00
parent 249d50aafc
commit 8543e39885
81 changed files with 1302 additions and 156 deletions
@@ -1077,12 +1077,18 @@ impl ThreadHistoryBuilder {
            });
        }
        if let Some(images) = &payload.images {
-            for image in images {
-                content.push(UserInput::Image { url: image.clone() });
+            for (idx, image) in images.iter().enumerate() {
+                content.push(UserInput::Image {
+                    url: image.clone(),
+                    detail: payload.image_details.get(idx).copied().flatten(),
+                });
            }
        }
-        for path in &payload.local_images {
-            content.push(UserInput::LocalImage { path: path.clone() });
+        for (idx, path) in payload.local_images.iter().enumerate() {
+            content.push(UserInput::LocalImage {
+                path: path.clone(),
+                detail: payload.local_image_details.get(idx).copied().flatten(),
+            });
        }
        content
    }
@@ -1203,6 +1209,7 @@ mod tests {
    use codex_protocol::items::UserMessageItem as CoreUserMessageItem;
    use codex_protocol::items::build_hook_prompt_message;
    use codex_protocol::mcp::CallToolResult;
+    use codex_protocol::models::ImageDetail;
    use codex_protocol::models::MessagePhase as CoreMessagePhase;
    use codex_protocol::models::WebSearchAction as CoreWebSearchAction;
    use codex_protocol::parse_command::ParsedCommand;
@@ -1241,6 +1248,7 @@ mod tests {
                images: Some(vec!["https://example.com/one.png".into()]),
                text_elements: Vec::new(),
                local_images: Vec::new(),
+                ..Default::default()
            }),
            EventMsg::AgentMessage(AgentMessageEvent {
                message: "Hi there".into(),
@@ -1258,6 +1266,7 @@ mod tests {
                images: None,
                text_elements: Vec::new(),
                local_images: Vec::new(),
+                ..Default::default()
            }),
            EventMsg::AgentMessage(AgentMessageEvent {
                message: "Reply two".into(),
@@ -1288,6 +1297,7 @@ mod tests {
                    },
                    UserInput::Image {
                        url: "https://example.com/one.png".into(),
+                        detail: None,
                    }
                ],
            }
@@ -1335,6 +1345,45 @@ mod tests {
        );
    }

+    #[test]
+    fn rebuilds_user_message_image_details_from_legacy_events() {
+        let local_path = PathBuf::from("/tmp/local.png");
+        let events = vec![RolloutItem::EventMsg(EventMsg::UserMessage(
+            UserMessageEvent {
+                message: "inspect these".into(),
+                images: Some(vec!["https://example.com/image.png".into()]),
+                image_details: vec![Some(ImageDetail::Original)],
+                local_images: vec![local_path.clone()],
+                local_image_details: vec![Some(ImageDetail::Original)],
+                text_elements: Vec::new(),
+            },
+        ))];
+
+        let turns = build_turns_from_rollout_items(&events);
+
+        assert_eq!(turns.len(), 1);
+        assert_eq!(
+            turns[0].items[0],
+            ThreadItem::UserMessage {
+                id: "item-1".into(),
+                content: vec![
+                    UserInput::Text {
+                        text: "inspect these".into(),
+                        text_elements: Vec::new(),
+                    },
+                    UserInput::Image {
+                        url: "https://example.com/image.png".into(),
+                        detail: Some(ImageDetail::Original),
+                    },
+                    UserInput::LocalImage {
+                        path: local_path,
+                        detail: Some(ImageDetail::Original),
+                    },
+                ],
+            }
+        );
+    }
+
    #[test]
    fn ignores_non_plan_item_lifecycle_events() {
        let turn_id = "turn-1";
@@ -1351,6 +1400,7 @@ mod tests {
                images: None,
                text_elements: Vec::new(),
                local_images: Vec::new(),
+                ..Default::default()
            }),
            EventMsg::ItemStarted(ItemStartedEvent {
                thread_id,
@@ -1428,6 +1478,7 @@ mod tests {
                images: None,
                text_elements: Vec::new(),
                local_images: Vec::new(),
+                ..Default::default()
            })),
            RolloutItem::EventMsg(EventMsg::ImageGenerationEnd(ImageGenerationEndEvent {
                call_id: "ig_123".into(),
@@ -1485,6 +1536,7 @@ mod tests {
                images: None,
                text_elements: Vec::new(),
                local_images: Vec::new(),
+                ..Default::default()
            }),
            EventMsg::AgentReasoning(AgentReasoningEvent {
                text: "first summary".into(),
@@ -1537,6 +1589,7 @@ mod tests {
                images: None,
                text_elements: Vec::new(),
                local_images: Vec::new(),
+                ..Default::default()
            }),
            EventMsg::AgentMessage(AgentMessageEvent {
                message: "Working...".into(),
@@ -1554,6 +1607,7 @@ mod tests {
                images: None,
                text_elements: Vec::new(),
                local_images: Vec::new(),
+                ..Default::default()
            }),
            EventMsg::AgentMessage(AgentMessageEvent {
                message: "Second attempt complete.".into(),
@@ -1624,6 +1678,7 @@ mod tests {
                images: None,
                text_elements: Vec::new(),
                local_images: Vec::new(),
+                ..Default::default()
            }),
            EventMsg::AgentMessage(AgentMessageEvent {
                message: "A1".into(),
@@ -1635,6 +1690,7 @@ mod tests {
                images: None,
                text_elements: Vec::new(),
                local_images: Vec::new(),
+                ..Default::default()
            }),
            EventMsg::AgentMessage(AgentMessageEvent {
                message: "A2".into(),
@@ -1647,6 +1703,7 @@ mod tests {
                images: None,
                text_elements: Vec::new(),
                local_images: Vec::new(),
+                ..Default::default()
            }),
            EventMsg::AgentMessage(AgentMessageEvent {
                message: "A3".into(),
@@ -1712,6 +1769,7 @@ mod tests {
                images: None,
                text_elements: Vec::new(),
                local_images: Vec::new(),
+                ..Default::default()
            }),
            EventMsg::AgentMessage(AgentMessageEvent {
                message: "A1".into(),
@@ -1723,6 +1781,7 @@ mod tests {
                images: None,
                text_elements: Vec::new(),
                local_images: Vec::new(),
+                ..Default::default()
            }),
            EventMsg::AgentMessage(AgentMessageEvent {
                message: "A2".into(),
@@ -1754,12 +1813,14 @@ mod tests {
                images: None,
                text_elements: Vec::new(),
                local_images: Vec::new(),
+                ..Default::default()
            }),
            EventMsg::UserMessage(UserMessageEvent {
                message: "Steer".into(),
                images: None,
                text_elements: Vec::new(),
                local_images: Vec::new(),
+                ..Default::default()
            }),
            EventMsg::TurnComplete(TurnCompleteEvent {
                turn_id: "turn-a".into(),
@@ -1812,6 +1873,7 @@ mod tests {
                images: None,
                text_elements: Vec::new(),
                local_images: Vec::new(),
+                ..Default::default()
            }),
            EventMsg::WebSearchEnd(WebSearchEndEvent {
                call_id: "search-1".into(),
@@ -1984,6 +2046,7 @@ mod tests {
                images: None,
                text_elements: Vec::new(),
                local_images: Vec::new(),
+                ..Default::default()
            }),
            EventMsg::DynamicToolCallRequest(
                codex_protocol::dynamic_tools::DynamicToolCallRequest {
@@ -2049,6 +2112,7 @@ mod tests {
                images: None,
                text_elements: Vec::new(),
                local_images: Vec::new(),
+                ..Default::default()
            }),
            EventMsg::ExecCommandEnd(ExecCommandEndEvent {
                call_id: "exec-declined".into(),
@@ -2138,6 +2202,7 @@ mod tests {
                images: None,
                text_elements: Vec::new(),
                local_images: Vec::new(),
+                ..Default::default()
            }),
            EventMsg::GuardianAssessment(GuardianAssessmentEvent {
                id: "review-guardian-exec".into(),
@@ -2221,6 +2286,7 @@ mod tests {
                images: None,
                text_elements: Vec::new(),
                local_images: Vec::new(),
+                ..Default::default()
            }),
            EventMsg::GuardianAssessment(GuardianAssessmentEvent {
                id: "review-guardian-execve".into(),
@@ -2284,6 +2350,7 @@ mod tests {
                images: None,
                text_elements: Vec::new(),
                local_images: Vec::new(),
+                ..Default::default()
            }),
            EventMsg::TurnComplete(TurnCompleteEvent {
                turn_id: "turn-a".into(),
@@ -2303,6 +2370,7 @@ mod tests {
                images: None,
                text_elements: Vec::new(),
                local_images: Vec::new(),
+                ..Default::default()
            }),
            EventMsg::ExecCommandEnd(ExecCommandEndEvent {
                call_id: "exec-late".into(),
@@ -2376,6 +2444,7 @@ mod tests {
                images: None,
                text_elements: Vec::new(),
                local_images: Vec::new(),
+                ..Default::default()
            }),
            EventMsg::TurnComplete(TurnCompleteEvent {
                turn_id: "turn-a".into(),
@@ -2395,6 +2464,7 @@ mod tests {
                images: None,
                text_elements: Vec::new(),
                local_images: Vec::new(),
+                ..Default::default()
            }),
            EventMsg::ExecCommandEnd(ExecCommandEndEvent {
                call_id: "exec-unknown-turn".into(),
@@ -2463,6 +2533,7 @@ mod tests {
                images: None,
                text_elements: Vec::new(),
                local_images: Vec::new(),
+                ..Default::default()
            }),
            EventMsg::PatchApplyBegin(PatchApplyBeginEvent {
                call_id: "patch-call".into(),
@@ -2527,6 +2598,7 @@ mod tests {
                images: None,
                text_elements: Vec::new(),
                local_images: Vec::new(),
+                ..Default::default()
            }),
            EventMsg::ApplyPatchApprovalRequest(ApplyPatchApprovalRequestEvent {
                call_id: "patch-call".into(),
@@ -2591,6 +2663,7 @@ mod tests {
                images: None,
                text_elements: Vec::new(),
                local_images: Vec::new(),
+                ..Default::default()
            }),
            EventMsg::TurnComplete(TurnCompleteEvent {
                turn_id: "turn-a".into(),
@@ -2610,6 +2683,7 @@ mod tests {
                images: None,
                text_elements: Vec::new(),
                local_images: Vec::new(),
+                ..Default::default()
            }),
            EventMsg::TurnComplete(TurnCompleteEvent {
                turn_id: "turn-a".into(),
@@ -2657,6 +2731,7 @@ mod tests {
                images: None,
                text_elements: Vec::new(),
                local_images: Vec::new(),
+                ..Default::default()
            }),
            EventMsg::TurnComplete(TurnCompleteEvent {
                turn_id: "turn-a".into(),
@@ -2676,6 +2751,7 @@ mod tests {
                images: None,
                text_elements: Vec::new(),
                local_images: Vec::new(),
+                ..Default::default()
            }),
            EventMsg::TurnAborted(TurnAbortedEvent {
                turn_id: Some("turn-a".into()),
@@ -2748,6 +2824,7 @@ mod tests {
                images: None,
                text_elements: Vec::new(),
                local_images: Vec::new(),
+                ..Default::default()
            }),
            EventMsg::CollabResumeEnd(codex_protocol::protocol::CollabResumeEndEvent {
                call_id: "resume-1".into(),
@@ -2805,6 +2882,7 @@ mod tests {
                images: None,
                text_elements: Vec::new(),
                local_images: Vec::new(),
+                ..Default::default()
            }),
            EventMsg::CollabAgentSpawnEnd(codex_protocol::protocol::CollabAgentSpawnEndEvent {
                call_id: "spawn-1".into(),
@@ -2866,6 +2944,7 @@ mod tests {
                images: None,
                text_elements: Vec::new(),
                local_images: Vec::new(),
+                ..Default::default()
            }),
            EventMsg::CollabAgentInteractionBegin(
                codex_protocol::protocol::CollabAgentInteractionBeginEvent {
@@ -2929,6 +3008,7 @@ mod tests {
                images: None,
                text_elements: Vec::new(),
                local_images: Vec::new(),
+                ..Default::default()
            }),
            EventMsg::AgentMessage(AgentMessageEvent {
                message: "done".into(),
@@ -2965,6 +3045,7 @@ mod tests {
                images: None,
                text_elements: Vec::new(),
                local_images: Vec::new(),
+                ..Default::default()
            }),
            EventMsg::TurnComplete(TurnCompleteEvent {
                turn_id: "turn-a".into(),
@@ -3020,6 +3101,7 @@ mod tests {
                images: None,
                text_elements: Vec::new(),
                local_images: Vec::new(),
+                ..Default::default()
            }),
            EventMsg::Error(ErrorEvent {
                message: "stream failure".into(),
@@ -3077,6 +3159,7 @@ mod tests {
                images: None,
                text_elements: Vec::new(),
                local_images: Vec::new(),
+                ..Default::default()
            })),
            RolloutItem::ResponseItem(hook_prompt),
            RolloutItem::EventMsg(EventMsg::TurnComplete(TurnCompleteEvent {
@@ -15,6 +15,7 @@ use codex_protocol::memory_citation::MemoryCitation as CoreMemoryCitation;
 use codex_protocol::memory_citation::MemoryCitationEntry as CoreMemoryCitationEntry;
 use codex_protocol::models::AdditionalPermissionProfile as CoreAdditionalPermissionProfile;
 use codex_protocol::models::FileSystemPermissions as CoreFileSystemPermissions;
+use codex_protocol::models::ImageDetail;
 use codex_protocol::models::ManagedFileSystemPermissions as CoreManagedFileSystemPermissions;
 use codex_protocol::models::MessagePhase;
 use codex_protocol::models::NetworkPermissions as CoreNetworkPermissions;
@@ -2319,9 +2320,11 @@ fn core_turn_item_into_thread_item_converts_supported_variants() {
            },
            CoreUserInput::Image {
                image_url: "https://example.com/image.png".to_string(),
+                detail: Some(ImageDetail::Original),
            },
            CoreUserInput::LocalImage {
                path: PathBuf::from("local/image.png"),
+                detail: Some(ImageDetail::Original),
            },
            CoreUserInput::Skill {
                name: "skill-creator".to_string(),
@@ -2345,9 +2348,11 @@ fn core_turn_item_into_thread_item_converts_supported_variants() {
                },
                UserInput::Image {
                    url: "https://example.com/image.png".to_string(),
+                    detail: Some(ImageDetail::Original),
                },
                UserInput::LocalImage {
                    path: PathBuf::from("local/image.png"),
+                    detail: Some(ImageDetail::Original),
                },
                UserInput::Skill {
                    name: "skill-creator".to_string(),
@@ -2562,6 +2567,33 @@ fn core_turn_item_into_thread_item_converts_supported_variants() {
    );
 }

+#[test]
+fn user_input_into_core_preserves_image_detail() {
+    assert_eq!(
+        UserInput::Image {
+            url: "https://example.com/image.png".to_string(),
+            detail: Some(ImageDetail::Original),
+        }
+        .into_core(),
+        CoreUserInput::Image {
+            image_url: "https://example.com/image.png".to_string(),
+            detail: Some(ImageDetail::Original),
+        }
+    );
+
+    assert_eq!(
+        UserInput::LocalImage {
+            path: PathBuf::from("local/image.png"),
+            detail: Some(ImageDetail::Original),
+        }
+        .into_core(),
+        CoreUserInput::LocalImage {
+            path: PathBuf::from("local/image.png"),
+            detail: Some(ImageDetail::Original),
+        }
+    );
+}
+
 #[test]
 fn skills_list_params_serialization_uses_force_reload() {
    assert_eq!(
@@ -7,6 +7,7 @@ use codex_experimental_api_macros::ExperimentalApi;
 use codex_protocol::config_types::CollaborationMode;
 use codex_protocol::config_types::Personality;
 use codex_protocol::config_types::ReasoningSummary;
+use codex_protocol::models::ImageDetail;
 use codex_protocol::openai_models::ReasoningEffort;
 use codex_protocol::plan_tool::PlanItemArg as CorePlanItemArg;
 use codex_protocol::plan_tool::StepStatus as CorePlanStepStatus;
@@ -249,9 +250,15 @@ pub enum UserInput {
        text_elements: Vec<TextElement>,
    },
    Image {
+        #[serde(default)]
+        #[ts(optional)]
+        detail: Option<ImageDetail>,
        url: String,
    },
    LocalImage {
+        #[serde(default)]
+        #[ts(optional)]
+        detail: Option<ImageDetail>,
        path: PathBuf,
    },
    Skill {
@@ -274,8 +281,11 @@ impl UserInput {
                text,
                text_elements: text_elements.into_iter().map(Into::into).collect(),
            },
-            UserInput::Image { url } => CoreUserInput::Image { image_url: url },
-            UserInput::LocalImage { path } => CoreUserInput::LocalImage { path },
+            UserInput::Image { url, detail } => CoreUserInput::Image {
+                image_url: url,
+                detail,
+            },
+            UserInput::LocalImage { path, detail } => CoreUserInput::LocalImage { path, detail },
            UserInput::Skill { name, path } => CoreUserInput::Skill { name, path },
            UserInput::Mention { name, path } => CoreUserInput::Mention { name, path },
        }
@@ -292,8 +302,11 @@ impl From<CoreUserInput> for UserInput {
                text,
                text_elements: text_elements.into_iter().map(Into::into).collect(),
            },
-            CoreUserInput::Image { image_url } => UserInput::Image { url: image_url },
-            CoreUserInput::LocalImage { path } => UserInput::LocalImage { path },
+            CoreUserInput::Image { image_url, detail } => UserInput::Image {
+                url: image_url,
+                detail,
+            },
+            CoreUserInput::LocalImage { path, detail } => UserInput::LocalImage { path, detail },
            CoreUserInput::Skill { name, path } => UserInput::Skill { name, path },
            CoreUserInput::Mention { name, path } => UserInput::Mention { name, path },
            _ => unreachable!("unsupported user input variant"),