mirror of
https://github.com/pchuan98/codex.git
synced 2026-07-01 00:31:56 +08:00
Route standalone image generation through host finalization md (#25176)
## Why Standalone image-generation extensions emitted turn items through the low-level event path, bypassing host-owned finalization such as image persistence and contributor processing. At the same time, the generated-image save-path hint must remain visible to the model through the extension tool's `FunctionCallOutput`, rather than the legacy built-in developer-message path. ## What changed - Extended `ExtensionTurnItem` to support image-generation items while keeping the extension-facing emitter API limited to `emit_started` and `emit_completed`. - Routed extension completion through core `finalize_turn_item`, so standalone image-generation items receive host-owned processing and persisted `saved_path` values before publication. - Kept legacy built-in image generation on its existing developer-message hint path, while standalone image generation returns its deterministic saved-path hint in `FunctionCallOutput`. - Shared the image artifact path and output-hint formatting used by core and the image-generation extension. - Passed thread identity through extension tool calls so standalone image generation can construct the same intended artifact path as core. - Added an app-server integration test covering real standalone image generation, saved artifact publication, model-visible output hint wiring, and absence of the legacy developer-message hint. ## Validation - `just fmt` - `just test -p codex-image-generation-extension` - `just test -p codex-web-search-extension` - `just test -p codex-goal-extension` - `just test -p codex-memories-extension` - Targeted `codex-core` tests for image save history, extension completion finalization, and contributor execution - `just test -p codex-app-server standalone_image_generation_returns_saved_path_hint_to_model` - `just fix -p codex-core` - `just fix -p codex-image-generation-extension` - `just bazel-lock-update` - `just bazel-lock-check`
This commit is contained in:
@@ -13,7 +13,6 @@ pub use capabilities::ResponseItemInjector;
|
||||
pub use codex_tools::ConversationHistory;
|
||||
pub use codex_tools::ExtensionTurnItem;
|
||||
pub use codex_tools::FunctionCallError;
|
||||
pub use codex_tools::ImageGenerationCompletionFuture;
|
||||
pub use codex_tools::JsonToolOutput;
|
||||
pub use codex_tools::NoopTurnItemEmitter;
|
||||
pub use codex_tools::ResponsesApiTool;
|
||||
|
||||
@@ -26,7 +26,6 @@ use crate::IMAGE_GEN_NAMESPACE;
|
||||
use crate::IMAGEGEN_TOOL_NAME;
|
||||
|
||||
const RESULT: &str = "cG5n";
|
||||
const OUTPUT_HINT: &str = "Generated images are saved to /tmp as /tmp/call-1.png by default.";
|
||||
|
||||
#[test]
|
||||
fn uses_reserved_image_gen_namespace() {
|
||||
@@ -55,10 +54,9 @@ fn generate_uses_fixed_request_defaults() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn generated_output_returns_image_input_and_output_hint() {
|
||||
fn generated_output_returns_image_input() {
|
||||
let output = GeneratedImageOutput {
|
||||
result: RESULT.to_string(),
|
||||
output_hint: Some(OUTPUT_HINT.to_string()),
|
||||
};
|
||||
|
||||
let ResponseInputItem::FunctionCallOutput {
|
||||
@@ -73,15 +71,10 @@ fn generated_output_returns_image_input_and_output_hint() {
|
||||
};
|
||||
assert_eq!(
|
||||
content_items,
|
||||
vec![
|
||||
FunctionCallOutputContentItem::InputImage {
|
||||
image_url: format!("data:image/png;base64,{RESULT}"),
|
||||
detail: Some(DEFAULT_IMAGE_DETAIL),
|
||||
},
|
||||
FunctionCallOutputContentItem::InputText {
|
||||
text: OUTPUT_HINT.to_string(),
|
||||
},
|
||||
]
|
||||
vec![FunctionCallOutputContentItem::InputImage {
|
||||
image_url: format!("data:image/png;base64,{RESULT}"),
|
||||
detail: Some(DEFAULT_IMAGE_DETAIL),
|
||||
}]
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -3,6 +3,7 @@ use codex_api::ImageEditRequest;
|
||||
use codex_api::ImageGenerationRequest;
|
||||
use codex_api::ImageQuality;
|
||||
use codex_api::ImageUrl;
|
||||
use codex_extension_api::ExtensionTurnItem;
|
||||
use codex_extension_api::FunctionCallError;
|
||||
use codex_extension_api::ToolCall;
|
||||
use codex_extension_api::ToolExecutor;
|
||||
@@ -11,6 +12,7 @@ use codex_extension_api::ToolOutput;
|
||||
use codex_extension_api::ToolPayload;
|
||||
use codex_extension_api::ToolSpec;
|
||||
use codex_extension_api::parse_tool_input_schema;
|
||||
use codex_protocol::items::ImageGenerationItem;
|
||||
use codex_protocol::models::ContentItem;
|
||||
use codex_protocol::models::DEFAULT_IMAGE_DETAIL;
|
||||
use codex_protocol::models::FunctionCallOutputBody;
|
||||
@@ -84,6 +86,15 @@ impl ToolExecutor<ToolCall> for ImageGenerationTool {
|
||||
async fn handle(&self, call: ToolCall) -> Result<Box<dyn ToolOutput>, FunctionCallError> {
|
||||
let args = parse_args(&call)?;
|
||||
let request = request_for_action(&args, call.conversation_history.items())?;
|
||||
call.turn_item_emitter
|
||||
.emit_started(ExtensionTurnItem::ImageGeneration(ImageGenerationItem {
|
||||
id: call.call_id.clone(),
|
||||
status: "in_progress".to_string(),
|
||||
revised_prompt: None,
|
||||
result: String::new(),
|
||||
saved_path: None,
|
||||
}))
|
||||
.await;
|
||||
let response = match request {
|
||||
ImageRequest::Generate(request) => self.backend.generate(request).await,
|
||||
ImageRequest::Edit(request) => self.backend.edit(request).await,
|
||||
@@ -96,14 +107,16 @@ impl ToolExecutor<ToolCall> for ImageGenerationTool {
|
||||
"image generation returned no image data".to_string(),
|
||||
));
|
||||
};
|
||||
let output_hint = call
|
||||
.turn_item_emitter
|
||||
.image_generation_completed(call.call_id.clone(), args.prompt, result.clone())
|
||||
call.turn_item_emitter
|
||||
.emit_completed(ExtensionTurnItem::ImageGeneration(ImageGenerationItem {
|
||||
id: call.call_id.clone(),
|
||||
status: "completed".to_string(),
|
||||
revised_prompt: Some(args.prompt),
|
||||
result: result.clone(),
|
||||
saved_path: None,
|
||||
}))
|
||||
.await;
|
||||
Ok(Box::new(GeneratedImageOutput {
|
||||
result,
|
||||
output_hint,
|
||||
}))
|
||||
Ok(Box::new(GeneratedImageOutput { result }))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -284,7 +297,6 @@ fn imagegen_tool_spec() -> ToolSpec {
|
||||
|
||||
struct GeneratedImageOutput {
|
||||
result: String,
|
||||
output_hint: Option<String>,
|
||||
}
|
||||
|
||||
impl ToolOutput for GeneratedImageOutput {
|
||||
@@ -298,17 +310,12 @@ impl ToolOutput for GeneratedImageOutput {
|
||||
true
|
||||
}
|
||||
|
||||
/// Returns generated bytes and persisted-artifact context for model follow-up.
|
||||
/// Returns generated bytes for model follow-up.
|
||||
fn to_response_item(&self, call_id: &str, _payload: &ToolPayload) -> ResponseInputItem {
|
||||
let mut content = vec![FunctionCallOutputContentItem::InputImage {
|
||||
let content = vec![FunctionCallOutputContentItem::InputImage {
|
||||
image_url: format!("data:image/png;base64,{}", self.result),
|
||||
detail: Some(DEFAULT_IMAGE_DETAIL),
|
||||
}];
|
||||
if let Some(output_hint) = &self.output_hint {
|
||||
content.push(FunctionCallOutputContentItem::InputText {
|
||||
text: output_hint.clone(),
|
||||
});
|
||||
}
|
||||
ResponseInputItem::FunctionCallOutput {
|
||||
call_id: call_id.to_string(),
|
||||
output: FunctionCallOutputPayload {
|
||||
|
||||
Reference in New Issue
Block a user