mirror of
https://github.com/pchuan98/codex.git
synced 2026-07-01 00:31:56 +08:00
Add saved image path hint to standalone image generation (#25947)
## Why Standalone image generation returns image bytes to the model, but the model also needs the host artifact path to reference the generated file in follow-up work. ## What changed - Append the default saved-image path hint alongside the generated image tool output. - Reuse the existing core image-generation hint text. - Pass the thread ID and Codex home directory needed to compute the artifact path. - Add app-server and extension coverage for the model-visible hint. ## Validation - `just fmt` - `just bazel-lock-check` - `just test -p codex-app-server standalone_image_generation_returns_saved_path_hint_to_model`
This commit is contained in:
@@ -23,6 +23,7 @@ codex-model-provider = { workspace = true }
|
||||
codex-model-provider-info = { workspace = true }
|
||||
codex-protocol = { workspace = true }
|
||||
codex-tools = { workspace = true }
|
||||
codex-utils-absolute-path = { workspace = true }
|
||||
http = { workspace = true }
|
||||
schemars = { workspace = true }
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
|
||||
@@ -4,6 +4,7 @@ The `image_gen.imagegen` tool enables image generation from descriptions and edi
|
||||
- The user wants to modify an attached or previously generated image with specific changes, including adding or removing elements, altering colors, improving quality/resolution, or transforming the style (e.g., cartoon, oil painting).
|
||||
|
||||
Guidelines:
|
||||
- In code mode, pass the result to `generatedImage(result)`.
|
||||
- Set `action` to `generate` when the user asks for a brand new image.
|
||||
- Set `action` to `edit` when the user asks to modify an existing image from the conversation history.
|
||||
- Directly generate the image without reconfirmation or clarification.
|
||||
|
||||
@@ -13,6 +13,7 @@ use codex_features::Feature;
|
||||
use codex_login::AuthManager;
|
||||
use codex_model_provider::create_model_provider;
|
||||
use codex_model_provider_info::ModelProviderInfo;
|
||||
use codex_utils_absolute_path::AbsolutePathBuf;
|
||||
|
||||
use crate::backend::CodexImagesBackend;
|
||||
use crate::tool::ImageGenerationTool;
|
||||
@@ -26,6 +27,7 @@ struct ImageGenerationExtension {
|
||||
struct ImageGenerationExtensionConfig {
|
||||
enabled: bool,
|
||||
provider: ModelProviderInfo,
|
||||
codex_home: AbsolutePathBuf,
|
||||
}
|
||||
|
||||
impl From<&Config> for ImageGenerationExtensionConfig {
|
||||
@@ -35,6 +37,7 @@ impl From<&Config> for ImageGenerationExtensionConfig {
|
||||
enabled: config.features.enabled(Feature::ImageGenExt)
|
||||
&& config.model_provider.is_openai(),
|
||||
provider: config.model_provider.clone(),
|
||||
codex_home: config.codex_home.clone(),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -76,9 +79,14 @@ impl ToolContributor for ImageGenerationExtension {
|
||||
return Vec::new();
|
||||
}
|
||||
|
||||
vec![Arc::new(ImageGenerationTool::new(CodexImagesBackend::new(
|
||||
create_model_provider(config.provider.clone(), Some(self.auth_manager.clone())),
|
||||
)))]
|
||||
vec![Arc::new(ImageGenerationTool::new(
|
||||
CodexImagesBackend::new(create_model_provider(
|
||||
config.provider.clone(),
|
||||
Some(self.auth_manager.clone()),
|
||||
)),
|
||||
config.codex_home.clone(),
|
||||
thread_store.level_id().to_string(),
|
||||
))]
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -3,6 +3,7 @@ use codex_api::ImageEditRequest;
|
||||
use codex_api::ImageGenerationRequest;
|
||||
use codex_api::ImageQuality;
|
||||
use codex_api::ImageUrl;
|
||||
use codex_core::context::extension_image_generation_output_hint;
|
||||
use codex_extension_api::ToolOutput;
|
||||
use codex_extension_api::ToolPayload;
|
||||
use codex_extension_api::ToolSpec;
|
||||
@@ -54,9 +55,58 @@ fn generate_uses_fixed_request_defaults() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn generated_output_returns_image_input() {
|
||||
fn generated_output_returns_image_input_and_output_hint() {
|
||||
let output_hint =
|
||||
extension_image_generation_output_hint("/tmp", "/tmp/call-1.png").expect("hint should fit");
|
||||
let output = GeneratedImageOutput {
|
||||
result: RESULT.to_string(),
|
||||
output_hint: Some(output_hint.clone()),
|
||||
};
|
||||
|
||||
let ResponseInputItem::FunctionCallOutput {
|
||||
output: response_output,
|
||||
..
|
||||
} = output.to_response_item("call-1", &function_payload())
|
||||
else {
|
||||
panic!("imagegen should return function tool output");
|
||||
};
|
||||
let FunctionCallOutputBody::ContentItems(content_items) = response_output.body else {
|
||||
panic!("imagegen output should contain generated image bytes");
|
||||
};
|
||||
assert_eq!(
|
||||
content_items,
|
||||
vec![
|
||||
FunctionCallOutputContentItem::InputImage {
|
||||
image_url: format!("data:image/png;base64,{RESULT}"),
|
||||
detail: Some(DEFAULT_IMAGE_DETAIL),
|
||||
},
|
||||
FunctionCallOutputContentItem::InputText { text: output_hint },
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn generated_output_returns_generated_image_helper_input_in_code_mode() {
|
||||
let output = GeneratedImageOutput {
|
||||
result: RESULT.to_string(),
|
||||
output_hint: Some("generated image save hint".to_string()),
|
||||
};
|
||||
|
||||
assert_eq!(
|
||||
output.code_mode_result(&function_payload()),
|
||||
serde_json::json!({
|
||||
"image_url": format!("data:image/png;base64,{RESULT}"),
|
||||
"output_hint": "generated image save hint",
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn generated_output_omits_oversized_output_hint() {
|
||||
let long_path = "x".repeat(1024);
|
||||
let output = GeneratedImageOutput {
|
||||
result: RESULT.to_string(),
|
||||
output_hint: extension_image_generation_output_hint("/tmp", long_path),
|
||||
};
|
||||
|
||||
let ResponseInputItem::FunctionCallOutput {
|
||||
|
||||
@@ -3,6 +3,8 @@ use codex_api::ImageEditRequest;
|
||||
use codex_api::ImageGenerationRequest;
|
||||
use codex_api::ImageQuality;
|
||||
use codex_api::ImageUrl;
|
||||
use codex_core::context::extension_image_generation_output_hint;
|
||||
use codex_core::image_generation_artifact_path;
|
||||
use codex_extension_api::ExtensionTurnItem;
|
||||
use codex_extension_api::FunctionCallError;
|
||||
use codex_extension_api::ToolCall;
|
||||
@@ -25,6 +27,7 @@ use codex_tools::ResponsesApiNamespaceTool;
|
||||
use codex_tools::ResponsesApiTool;
|
||||
use codex_tools::ToolExposure;
|
||||
use codex_tools::default_namespace_description;
|
||||
use codex_utils_absolute_path::AbsolutePathBuf;
|
||||
use schemars::JsonSchema;
|
||||
use schemars::r#gen::SchemaSettings;
|
||||
use serde::Deserialize;
|
||||
@@ -42,12 +45,22 @@ const IMAGEGEN_DESCRIPTION: &str = include_str!("../imagegen_description.md");
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct ImageGenerationTool {
|
||||
backend: CodexImagesBackend,
|
||||
codex_home: AbsolutePathBuf,
|
||||
thread_id: String,
|
||||
}
|
||||
|
||||
impl ImageGenerationTool {
|
||||
/// Creates an image-generation tool backed by an image API executor.
|
||||
pub(crate) fn new(backend: CodexImagesBackend) -> Self {
|
||||
Self { backend }
|
||||
pub(crate) fn new(
|
||||
backend: CodexImagesBackend,
|
||||
codex_home: AbsolutePathBuf,
|
||||
thread_id: String,
|
||||
) -> Self {
|
||||
Self {
|
||||
backend,
|
||||
codex_home,
|
||||
thread_id,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -116,7 +129,17 @@ impl ToolExecutor<ToolCall> for ImageGenerationTool {
|
||||
saved_path: None,
|
||||
}))
|
||||
.await;
|
||||
Ok(Box::new(GeneratedImageOutput { result }))
|
||||
let output_path =
|
||||
image_generation_artifact_path(&self.codex_home, &self.thread_id, &call.call_id);
|
||||
let output_dir = output_path
|
||||
.parent()
|
||||
.unwrap_or_else(|| self.codex_home.clone());
|
||||
let output_hint =
|
||||
extension_image_generation_output_hint(output_dir.display(), output_path.display());
|
||||
Ok(Box::new(GeneratedImageOutput {
|
||||
result,
|
||||
output_hint,
|
||||
}))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -297,6 +320,7 @@ fn imagegen_tool_spec() -> ToolSpec {
|
||||
|
||||
struct GeneratedImageOutput {
|
||||
result: String,
|
||||
output_hint: Option<String>,
|
||||
}
|
||||
|
||||
impl ToolOutput for GeneratedImageOutput {
|
||||
@@ -310,12 +334,32 @@ impl ToolOutput for GeneratedImageOutput {
|
||||
true
|
||||
}
|
||||
|
||||
/// Returns generated bytes for model follow-up.
|
||||
/// Returns the object consumed by the code-mode `generatedImage()` helper.
|
||||
fn code_mode_result(&self, _payload: &ToolPayload) -> Value {
|
||||
let mut result = Map::from_iter([(
|
||||
"image_url".to_string(),
|
||||
Value::String(format!("data:image/png;base64,{}", self.result)),
|
||||
)]);
|
||||
if let Some(output_hint) = &self.output_hint {
|
||||
result.insert(
|
||||
"output_hint".to_string(),
|
||||
Value::String(output_hint.clone()),
|
||||
);
|
||||
}
|
||||
Value::Object(result)
|
||||
}
|
||||
|
||||
/// Returns generated bytes and persisted-artifact context for model follow-up.
|
||||
fn to_response_item(&self, call_id: &str, _payload: &ToolPayload) -> ResponseInputItem {
|
||||
let content = vec![FunctionCallOutputContentItem::InputImage {
|
||||
let mut content = vec![FunctionCallOutputContentItem::InputImage {
|
||||
image_url: format!("data:image/png;base64,{}", self.result),
|
||||
detail: Some(DEFAULT_IMAGE_DETAIL),
|
||||
}];
|
||||
if let Some(output_hint) = &self.output_hint {
|
||||
content.push(FunctionCallOutputContentItem::InputText {
|
||||
text: output_hint.clone(),
|
||||
});
|
||||
}
|
||||
ResponseInputItem::FunctionCallOutput {
|
||||
call_id: call_id.to_string(),
|
||||
output: FunctionCallOutputPayload {
|
||||
|
||||
Reference in New Issue
Block a user