Add saved image path hint to standalone image generation (#25947)

## Why

Standalone image generation returns image bytes to the model, but the
model also needs the host artifact path to reference the generated file
in follow-up work.

## What changed

- Append the default saved-image path hint alongside the generated image
tool output.
- Reuse the existing core image-generation hint text.
- Pass the thread ID and Codex home directory needed to compute the
artifact path.
- Add app-server and extension coverage for the model-visible hint.

## Validation

- `just fmt`
- `just bazel-lock-check`
- `just test -p codex-app-server
standalone_image_generation_returns_saved_path_hint_to_model`
This commit is contained in:
Won Park
2026-06-04 09:39:20 -07:00
committed by GitHub
parent 68db0bb5ec
commit 12e8764a9c
15 changed files with 249 additions and 18 deletions
+1
View File
@@ -23,6 +23,7 @@ codex-model-provider = { workspace = true }
codex-model-provider-info = { workspace = true }
codex-protocol = { workspace = true }
codex-tools = { workspace = true }
codex-utils-absolute-path = { workspace = true }
http = { workspace = true }
schemars = { workspace = true }
serde = { workspace = true, features = ["derive"] }
@@ -4,6 +4,7 @@ The `image_gen.imagegen` tool enables image generation from descriptions and edi
- The user wants to modify an attached or previously generated image with specific changes, including adding or removing elements, altering colors, improving quality/resolution, or transforming the style (e.g., cartoon, oil painting).
Guidelines:
- In code mode, pass the result to `generatedImage(result)`.
- Set `action` to `generate` when the user asks for a brand new image.
- Set `action` to `edit` when the user asks to modify an existing image from the conversation history.
- Directly generate the image without reconfirmation or clarification.
+11 -3
View File
@@ -13,6 +13,7 @@ use codex_features::Feature;
use codex_login::AuthManager;
use codex_model_provider::create_model_provider;
use codex_model_provider_info::ModelProviderInfo;
use codex_utils_absolute_path::AbsolutePathBuf;
use crate::backend::CodexImagesBackend;
use crate::tool::ImageGenerationTool;
@@ -26,6 +27,7 @@ struct ImageGenerationExtension {
struct ImageGenerationExtensionConfig {
enabled: bool,
provider: ModelProviderInfo,
codex_home: AbsolutePathBuf,
}
impl From<&Config> for ImageGenerationExtensionConfig {
@@ -35,6 +37,7 @@ impl From<&Config> for ImageGenerationExtensionConfig {
enabled: config.features.enabled(Feature::ImageGenExt)
&& config.model_provider.is_openai(),
provider: config.model_provider.clone(),
codex_home: config.codex_home.clone(),
}
}
}
@@ -76,9 +79,14 @@ impl ToolContributor for ImageGenerationExtension {
return Vec::new();
}
vec![Arc::new(ImageGenerationTool::new(CodexImagesBackend::new(
create_model_provider(config.provider.clone(), Some(self.auth_manager.clone())),
)))]
vec![Arc::new(ImageGenerationTool::new(
CodexImagesBackend::new(create_model_provider(
config.provider.clone(),
Some(self.auth_manager.clone()),
)),
config.codex_home.clone(),
thread_store.level_id().to_string(),
))]
}
}
+51 -1
View File
@@ -3,6 +3,7 @@ use codex_api::ImageEditRequest;
use codex_api::ImageGenerationRequest;
use codex_api::ImageQuality;
use codex_api::ImageUrl;
use codex_core::context::extension_image_generation_output_hint;
use codex_extension_api::ToolOutput;
use codex_extension_api::ToolPayload;
use codex_extension_api::ToolSpec;
@@ -54,9 +55,58 @@ fn generate_uses_fixed_request_defaults() {
}
#[test]
fn generated_output_returns_image_input() {
fn generated_output_returns_image_input_and_output_hint() {
let output_hint =
extension_image_generation_output_hint("/tmp", "/tmp/call-1.png").expect("hint should fit");
let output = GeneratedImageOutput {
result: RESULT.to_string(),
output_hint: Some(output_hint.clone()),
};
let ResponseInputItem::FunctionCallOutput {
output: response_output,
..
} = output.to_response_item("call-1", &function_payload())
else {
panic!("imagegen should return function tool output");
};
let FunctionCallOutputBody::ContentItems(content_items) = response_output.body else {
panic!("imagegen output should contain generated image bytes");
};
assert_eq!(
content_items,
vec![
FunctionCallOutputContentItem::InputImage {
image_url: format!("data:image/png;base64,{RESULT}"),
detail: Some(DEFAULT_IMAGE_DETAIL),
},
FunctionCallOutputContentItem::InputText { text: output_hint },
]
);
}
#[test]
fn generated_output_returns_generated_image_helper_input_in_code_mode() {
let output = GeneratedImageOutput {
result: RESULT.to_string(),
output_hint: Some("generated image save hint".to_string()),
};
assert_eq!(
output.code_mode_result(&function_payload()),
serde_json::json!({
"image_url": format!("data:image/png;base64,{RESULT}"),
"output_hint": "generated image save hint",
})
);
}
#[test]
fn generated_output_omits_oversized_output_hint() {
let long_path = "x".repeat(1024);
let output = GeneratedImageOutput {
result: RESULT.to_string(),
output_hint: extension_image_generation_output_hint("/tmp", long_path),
};
let ResponseInputItem::FunctionCallOutput {
+49 -5
View File
@@ -3,6 +3,8 @@ use codex_api::ImageEditRequest;
use codex_api::ImageGenerationRequest;
use codex_api::ImageQuality;
use codex_api::ImageUrl;
use codex_core::context::extension_image_generation_output_hint;
use codex_core::image_generation_artifact_path;
use codex_extension_api::ExtensionTurnItem;
use codex_extension_api::FunctionCallError;
use codex_extension_api::ToolCall;
@@ -25,6 +27,7 @@ use codex_tools::ResponsesApiNamespaceTool;
use codex_tools::ResponsesApiTool;
use codex_tools::ToolExposure;
use codex_tools::default_namespace_description;
use codex_utils_absolute_path::AbsolutePathBuf;
use schemars::JsonSchema;
use schemars::r#gen::SchemaSettings;
use serde::Deserialize;
@@ -42,12 +45,22 @@ const IMAGEGEN_DESCRIPTION: &str = include_str!("../imagegen_description.md");
#[derive(Clone)]
pub(crate) struct ImageGenerationTool {
backend: CodexImagesBackend,
codex_home: AbsolutePathBuf,
thread_id: String,
}
impl ImageGenerationTool {
/// Creates an image-generation tool backed by an image API executor.
pub(crate) fn new(backend: CodexImagesBackend) -> Self {
Self { backend }
pub(crate) fn new(
backend: CodexImagesBackend,
codex_home: AbsolutePathBuf,
thread_id: String,
) -> Self {
Self {
backend,
codex_home,
thread_id,
}
}
}
@@ -116,7 +129,17 @@ impl ToolExecutor<ToolCall> for ImageGenerationTool {
saved_path: None,
}))
.await;
Ok(Box::new(GeneratedImageOutput { result }))
let output_path =
image_generation_artifact_path(&self.codex_home, &self.thread_id, &call.call_id);
let output_dir = output_path
.parent()
.unwrap_or_else(|| self.codex_home.clone());
let output_hint =
extension_image_generation_output_hint(output_dir.display(), output_path.display());
Ok(Box::new(GeneratedImageOutput {
result,
output_hint,
}))
}
}
@@ -297,6 +320,7 @@ fn imagegen_tool_spec() -> ToolSpec {
struct GeneratedImageOutput {
result: String,
output_hint: Option<String>,
}
impl ToolOutput for GeneratedImageOutput {
@@ -310,12 +334,32 @@ impl ToolOutput for GeneratedImageOutput {
true
}
/// Returns generated bytes for model follow-up.
/// Returns the object consumed by the code-mode `generatedImage()` helper.
fn code_mode_result(&self, _payload: &ToolPayload) -> Value {
let mut result = Map::from_iter([(
"image_url".to_string(),
Value::String(format!("data:image/png;base64,{}", self.result)),
)]);
if let Some(output_hint) = &self.output_hint {
result.insert(
"output_hint".to_string(),
Value::String(output_hint.clone()),
);
}
Value::Object(result)
}
/// Returns generated bytes and persisted-artifact context for model follow-up.
fn to_response_item(&self, call_id: &str, _payload: &ToolPayload) -> ResponseInputItem {
let content = vec![FunctionCallOutputContentItem::InputImage {
let mut content = vec![FunctionCallOutputContentItem::InputImage {
image_url: format!("data:image/png;base64,{}", self.result),
detail: Some(DEFAULT_IMAGE_DETAIL),
}];
if let Some(output_hint) = &self.output_hint {
content.push(FunctionCallOutputContentItem::InputText {
text: output_hint.clone(),
});
}
ResponseInputItem::FunctionCallOutput {
call_id: call_id.to_string(),
output: FunctionCallOutputPayload {