Add saved image path hint to standalone image generation (#25947)

## Why

Standalone image generation returns image bytes to the model, but the
model also needs the host artifact path to reference the generated file
in follow-up work.

## What changed

- Append the default saved-image path hint alongside the generated image
tool output.
- Reuse the existing core image-generation hint text.
- Pass the thread ID and Codex home directory needed to compute the
artifact path.
- Add app-server and extension coverage for the model-visible hint.

## Validation

- `just fmt`
- `just bazel-lock-check`
- `just test -p codex-app-server
standalone_image_generation_returns_saved_path_hint_to_model`
This commit is contained in:
Won Park
2026-06-04 09:39:20 -07:00
committed by GitHub
Unverified
parent 68db0bb5ec
commit 12e8764a9c
15 changed files with 249 additions and 18 deletions
+1
View File
@@ -3052,6 +3052,7 @@ dependencies = [
"codex-model-provider-info",
"codex-protocol",
"codex-tools",
"codex-utils-absolute-path",
"http 1.4.0",
"pretty_assertions",
"schemars 0.8.22",
@@ -44,7 +44,7 @@ const DEFAULT_READ_TIMEOUT: Duration = Duration::from_secs(60);
const DEFAULT_READ_TIMEOUT: Duration = Duration::from_secs(10);
#[tokio::test]
async fn standalone_image_generation_persists_image_and_returns_it_to_model() -> Result<()> {
async fn standalone_image_generation_returns_saved_path_hint_to_model() -> Result<()> {
let call_id = "image-run-1";
let server = responses::start_mock_server().await;
mount_image_response(&server).await;
@@ -124,7 +124,13 @@ async fn standalone_image_generation_persists_image_and_returns_it_to_model() ->
"detail": "high",
})
);
assert_eq!(output["output"].as_array().map(Vec::len), Some(1));
let output_hint = output["output"][1]["text"]
.as_str()
.context("image output should include model-visible path hint")?;
assert!(
output_hint.contains(&saved_path.display().to_string()),
"output hint should identify the path core saved"
);
assert!(
!requests[1]
.message_input_texts("developer")
@@ -199,7 +205,7 @@ const result = await tools.image_gen__imagegen({
action: "generate",
prompt: "paint a blue whale",
});
image(result);
generatedImage(result);
"#,
),
responses::ev_completed("resp-1"),
@@ -246,7 +252,12 @@ image(result);
"detail": "high",
})
);
assert_eq!(output["output"].as_array().map(Vec::len), Some(2));
assert!(
output["output"][2]["text"]
.as_str()
.is_some_and(|text| text.contains("Generated images are saved"))
);
assert_eq!(output["output"].as_array().map(Vec::len), Some(3));
Ok(())
}
+1
View File
@@ -25,6 +25,7 @@ const EXEC_DESCRIPTION_TEMPLATE: &str = r#"Run JavaScript code to orchestrate/co
- `exit()`: Immediately ends the current script successfully (like an early return from the top level).
- `text(value: string | number | boolean | undefined | null)`: Appends a text item. Non-string values are stringified with `JSON.stringify(...)` when possible.
- `image(imageUrlOrItem: string | { image_url: string; detail?: "auto" | "low" | "high" | "original" | null } | ImageContent, detail?: "auto" | "low" | "high" | "original" | null)`: Appends an image item. `image_url` can be an HTTPS URL or a base64-encoded `data:` URL. To forward an MCP tool image, pass an individual `ImageContent` block from `result.content`, for example `image(result.content[0])`. MCP image blocks may request detail with `_meta: { "codex/imageDetail": "original" }`. When provided, the second `detail` argument overrides any detail embedded in the first argument.
- `generatedImage(result: { image_url: string; output_hint?: string })`: Appends an image-generation result and its optional output hint.
- `store(key: string, value: any)`: stores a serializable value under a string key for later `exec` calls in the same session.
- `load(key: string)`: returns the stored value for a string key, or `undefined` if it is missing.
- `notify(value: string | number | boolean | undefined | null)`: immediately injects an extra `custom_tool_call_output` for the current `exec` call. Values are stringified like `text(...)`.
@@ -129,6 +129,58 @@ pub(super) fn image_callback(
retval.set(v8::undefined(scope).into());
}
pub(super) fn generated_image_callback(
scope: &mut v8::PinScope<'_, '_>,
args: v8::FunctionCallbackArguments,
mut retval: v8::ReturnValue<v8::Value>,
) {
let value = if args.length() == 0 {
v8::undefined(scope).into()
} else {
args.get(0)
};
let output_hint = match generated_image_output_hint(scope, value) {
Ok(output_hint) => output_hint,
Err(error_text) => {
throw_type_error(scope, &error_text);
return;
}
};
let image_item = match normalize_output_image(scope, value, /*detail_override*/ None) {
Ok(image_item) => image_item,
Err(()) => return,
};
if let Some(state) = scope.get_slot::<RuntimeState>() {
let _ = state.event_tx.send(RuntimeEvent::ContentItem(image_item));
if let Some(text) = output_hint {
let _ = state.event_tx.send(RuntimeEvent::ContentItem(
FunctionCallOutputContentItem::InputText { text },
));
}
}
retval.set(v8::undefined(scope).into());
}
fn generated_image_output_hint(
scope: &mut v8::PinScope<'_, '_>,
value: v8::Local<'_, v8::Value>,
) -> Result<Option<String>, String> {
let object = v8::Local::<v8::Object>::try_from(value)
.map_err(|_| "generatedImage expects an image generation result object".to_string())?;
let key = v8::String::new(scope, "output_hint")
.ok_or_else(|| "failed to allocate generatedImage helper keys".to_string())?;
let output_hint = object
.get(scope, key.into())
.ok_or_else(|| "failed to read generatedImage output_hint".to_string())?;
if output_hint.is_undefined() {
return Ok(None);
}
if !output_hint.is_string() {
return Err("generatedImage output_hint must be a string when provided".to_string());
}
Ok(Some(output_hint.to_rust_string_lossy(scope)))
}
pub(super) fn store_callback(
scope: &mut v8::PinScope<'_, '_>,
args: v8::FunctionCallbackArguments,
@@ -1,6 +1,7 @@
use super::RuntimeState;
use super::callbacks::clear_timeout_callback;
use super::callbacks::exit_callback;
use super::callbacks::generated_image_callback;
use super::callbacks::image_callback;
use super::callbacks::load_callback;
use super::callbacks::notify_callback;
@@ -23,6 +24,7 @@ pub(super) fn install_globals(scope: &mut v8::PinScope<'_, '_>) -> Result<(), St
let set_timeout = helper_function(scope, "setTimeout", set_timeout_callback)?;
let text = helper_function(scope, "text", text_callback)?;
let image = helper_function(scope, "image", image_callback)?;
let generated_image = helper_function(scope, "generatedImage", generated_image_callback)?;
let store = helper_function(scope, "store", store_callback)?;
let load = helper_function(scope, "load", load_callback)?;
let notify = helper_function(scope, "notify", notify_callback)?;
@@ -35,6 +37,7 @@ pub(super) fn install_globals(scope: &mut v8::PinScope<'_, '_>) -> Result<(), St
set_global(scope, global, "setTimeout", set_timeout.into())?;
set_global(scope, global, "text", text.into())?;
set_global(scope, global, "image", image.into())?;
set_global(scope, global, "generatedImage", generated_image.into())?;
set_global(scope, global, "store", store.into())?;
set_global(scope, global, "load", load.into())?;
set_global(scope, global, "notify", notify.into())?;
+38
View File
@@ -1563,6 +1563,44 @@ image({
);
}
#[tokio::test]
async fn generated_image_helper_appends_image_and_output_hint() {
let service = CodeModeService::new();
let response = execute(
&service,
ExecuteRequest {
source: r#"
generatedImage({
image_url: "https://example.com/image.jpg",
output_hint: "generated image save hint",
});
"#
.to_string(),
yield_time_ms: None,
..execute_request("")
},
)
.await;
assert_eq!(
response,
RuntimeResponse::Result {
cell_id: cell_id("1"),
content_items: vec![
FunctionCallOutputContentItem::InputImage {
image_url: "https://example.com/image.jpg".to_string(),
detail: Some(crate::DEFAULT_IMAGE_DETAIL),
},
FunctionCallOutputContentItem::InputText {
text: "generated image save hint".to_string(),
},
],
error_text: None,
}
);
}
#[tokio::test]
async fn image_helper_second_arg_overrides_explicit_object_detail() {
let service = CodeModeService::new();
@@ -1,6 +1,27 @@
use super::ContextualUserFragment;
use std::fmt::Display;
/// Maximum size of the extension's model-facing generated-image path hint.
const MAX_IMAGE_GENERATION_OUTPUT_HINT_BYTES: usize = 1024;
/// Returns the extension's model-facing hint, or omits it if the path makes it too large.
pub fn extension_image_generation_output_hint(
image_output_dir: impl Display,
image_output_path: impl Display,
) -> Option<String> {
let hint = image_generation_hint(image_output_dir, image_output_path);
(hint.len() <= MAX_IMAGE_GENERATION_OUTPUT_HINT_BYTES).then_some(hint)
}
fn image_generation_hint(
image_output_dir: impl Display,
image_output_path: impl Display,
) -> String {
format!(
"Generated images are saved to {image_output_dir} as {image_output_path} by default.\nIf you need to use a generated image at another path, copy it and leave the original in place unless the user explicitly asks you to delete it."
)
}
#[derive(Debug, Clone, PartialEq)]
pub(crate) struct ImageGenerationInstructions {
image_output_dir: String,
@@ -30,9 +51,6 @@ impl ContextualUserFragment for ImageGenerationInstructions {
}
fn body(&self) -> String {
format!(
"Generated images are saved to {} as {} by default.\nIf you need to use a generated image at another path, copy it and leave the original in place unless the user explicitly asks you to delete it.",
self.image_output_dir, self.image_output_path
)
image_generation_hint(&self.image_output_dir, &self.image_output_path)
}
}
+1
View File
@@ -44,6 +44,7 @@ pub(crate) use environment_context::EnvironmentContext;
pub(crate) use guardian_followup_review_reminder::GuardianFollowupReviewReminder;
pub(crate) use hook_additional_context::HookAdditionalContext;
pub(crate) use image_generation_instructions::ImageGenerationInstructions;
pub use image_generation_instructions::extension_image_generation_output_hint;
pub use internal_model_context::InternalContextSource;
pub use internal_model_context::InternalModelContextFragment;
pub use internal_model_context::InvalidInternalContextSource;
+1
View File
@@ -99,6 +99,7 @@ pub(crate) use skills::manager;
pub(crate) use skills::maybe_emit_implicit_skill_invocation;
pub(crate) use skills::skills_load_input_from_config;
mod stream_events_utils;
pub use stream_events_utils::image_generation_artifact_path;
pub mod test_support;
mod unified_exec;
pub mod windows_sandbox;
+2 -1
View File
@@ -38,7 +38,8 @@ use tracing::warn;
const GENERATED_IMAGE_ARTIFACTS_DIR: &str = "generated_images";
pub(crate) fn image_generation_artifact_path(
/// Returns the host-owned default artifact path for a generated image.
pub fn image_generation_artifact_path(
codex_home: &AbsolutePathBuf,
session_id: &str,
call_id: &str,
+1
View File
@@ -23,6 +23,7 @@ codex-model-provider = { workspace = true }
codex-model-provider-info = { workspace = true }
codex-protocol = { workspace = true }
codex-tools = { workspace = true }
codex-utils-absolute-path = { workspace = true }
http = { workspace = true }
schemars = { workspace = true }
serde = { workspace = true, features = ["derive"] }
@@ -4,6 +4,7 @@ The `image_gen.imagegen` tool enables image generation from descriptions and edi
- The user wants to modify an attached or previously generated image with specific changes, including adding or removing elements, altering colors, improving quality/resolution, or transforming the style (e.g., cartoon, oil painting).
Guidelines:
- In code mode, pass the result to `generatedImage(result)`.
- Set `action` to `generate` when the user asks for a brand new image.
- Set `action` to `edit` when the user asks to modify an existing image from the conversation history.
- Directly generate the image without reconfirmation or clarification.
+11 -3
View File
@@ -13,6 +13,7 @@ use codex_features::Feature;
use codex_login::AuthManager;
use codex_model_provider::create_model_provider;
use codex_model_provider_info::ModelProviderInfo;
use codex_utils_absolute_path::AbsolutePathBuf;
use crate::backend::CodexImagesBackend;
use crate::tool::ImageGenerationTool;
@@ -26,6 +27,7 @@ struct ImageGenerationExtension {
struct ImageGenerationExtensionConfig {
enabled: bool,
provider: ModelProviderInfo,
codex_home: AbsolutePathBuf,
}
impl From<&Config> for ImageGenerationExtensionConfig {
@@ -35,6 +37,7 @@ impl From<&Config> for ImageGenerationExtensionConfig {
enabled: config.features.enabled(Feature::ImageGenExt)
&& config.model_provider.is_openai(),
provider: config.model_provider.clone(),
codex_home: config.codex_home.clone(),
}
}
}
@@ -76,9 +79,14 @@ impl ToolContributor for ImageGenerationExtension {
return Vec::new();
}
vec![Arc::new(ImageGenerationTool::new(CodexImagesBackend::new(
create_model_provider(config.provider.clone(), Some(self.auth_manager.clone())),
)))]
vec![Arc::new(ImageGenerationTool::new(
CodexImagesBackend::new(create_model_provider(
config.provider.clone(),
Some(self.auth_manager.clone()),
)),
config.codex_home.clone(),
thread_store.level_id().to_string(),
))]
}
}
+51 -1
View File
@@ -3,6 +3,7 @@ use codex_api::ImageEditRequest;
use codex_api::ImageGenerationRequest;
use codex_api::ImageQuality;
use codex_api::ImageUrl;
use codex_core::context::extension_image_generation_output_hint;
use codex_extension_api::ToolOutput;
use codex_extension_api::ToolPayload;
use codex_extension_api::ToolSpec;
@@ -54,9 +55,58 @@ fn generate_uses_fixed_request_defaults() {
}
#[test]
fn generated_output_returns_image_input() {
fn generated_output_returns_image_input_and_output_hint() {
let output_hint =
extension_image_generation_output_hint("/tmp", "/tmp/call-1.png").expect("hint should fit");
let output = GeneratedImageOutput {
result: RESULT.to_string(),
output_hint: Some(output_hint.clone()),
};
let ResponseInputItem::FunctionCallOutput {
output: response_output,
..
} = output.to_response_item("call-1", &function_payload())
else {
panic!("imagegen should return function tool output");
};
let FunctionCallOutputBody::ContentItems(content_items) = response_output.body else {
panic!("imagegen output should contain generated image bytes");
};
assert_eq!(
content_items,
vec![
FunctionCallOutputContentItem::InputImage {
image_url: format!("data:image/png;base64,{RESULT}"),
detail: Some(DEFAULT_IMAGE_DETAIL),
},
FunctionCallOutputContentItem::InputText { text: output_hint },
]
);
}
#[test]
fn generated_output_returns_generated_image_helper_input_in_code_mode() {
let output = GeneratedImageOutput {
result: RESULT.to_string(),
output_hint: Some("generated image save hint".to_string()),
};
assert_eq!(
output.code_mode_result(&function_payload()),
serde_json::json!({
"image_url": format!("data:image/png;base64,{RESULT}"),
"output_hint": "generated image save hint",
})
);
}
#[test]
fn generated_output_omits_oversized_output_hint() {
let long_path = "x".repeat(1024);
let output = GeneratedImageOutput {
result: RESULT.to_string(),
output_hint: extension_image_generation_output_hint("/tmp", long_path),
};
let ResponseInputItem::FunctionCallOutput {
+49 -5
View File
@@ -3,6 +3,8 @@ use codex_api::ImageEditRequest;
use codex_api::ImageGenerationRequest;
use codex_api::ImageQuality;
use codex_api::ImageUrl;
use codex_core::context::extension_image_generation_output_hint;
use codex_core::image_generation_artifact_path;
use codex_extension_api::ExtensionTurnItem;
use codex_extension_api::FunctionCallError;
use codex_extension_api::ToolCall;
@@ -25,6 +27,7 @@ use codex_tools::ResponsesApiNamespaceTool;
use codex_tools::ResponsesApiTool;
use codex_tools::ToolExposure;
use codex_tools::default_namespace_description;
use codex_utils_absolute_path::AbsolutePathBuf;
use schemars::JsonSchema;
use schemars::r#gen::SchemaSettings;
use serde::Deserialize;
@@ -42,12 +45,22 @@ const IMAGEGEN_DESCRIPTION: &str = include_str!("../imagegen_description.md");
#[derive(Clone)]
pub(crate) struct ImageGenerationTool {
backend: CodexImagesBackend,
codex_home: AbsolutePathBuf,
thread_id: String,
}
impl ImageGenerationTool {
/// Creates an image-generation tool backed by an image API executor.
pub(crate) fn new(backend: CodexImagesBackend) -> Self {
Self { backend }
pub(crate) fn new(
backend: CodexImagesBackend,
codex_home: AbsolutePathBuf,
thread_id: String,
) -> Self {
Self {
backend,
codex_home,
thread_id,
}
}
}
@@ -116,7 +129,17 @@ impl ToolExecutor<ToolCall> for ImageGenerationTool {
saved_path: None,
}))
.await;
Ok(Box::new(GeneratedImageOutput { result }))
let output_path =
image_generation_artifact_path(&self.codex_home, &self.thread_id, &call.call_id);
let output_dir = output_path
.parent()
.unwrap_or_else(|| self.codex_home.clone());
let output_hint =
extension_image_generation_output_hint(output_dir.display(), output_path.display());
Ok(Box::new(GeneratedImageOutput {
result,
output_hint,
}))
}
}
@@ -297,6 +320,7 @@ fn imagegen_tool_spec() -> ToolSpec {
struct GeneratedImageOutput {
result: String,
output_hint: Option<String>,
}
impl ToolOutput for GeneratedImageOutput {
@@ -310,12 +334,32 @@ impl ToolOutput for GeneratedImageOutput {
true
}
/// Returns generated bytes for model follow-up.
/// Returns the object consumed by the code-mode `generatedImage()` helper.
fn code_mode_result(&self, _payload: &ToolPayload) -> Value {
let mut result = Map::from_iter([(
"image_url".to_string(),
Value::String(format!("data:image/png;base64,{}", self.result)),
)]);
if let Some(output_hint) = &self.output_hint {
result.insert(
"output_hint".to_string(),
Value::String(output_hint.clone()),
);
}
Value::Object(result)
}
/// Returns generated bytes and persisted-artifact context for model follow-up.
fn to_response_item(&self, call_id: &str, _payload: &ToolPayload) -> ResponseInputItem {
let content = vec![FunctionCallOutputContentItem::InputImage {
let mut content = vec![FunctionCallOutputContentItem::InputImage {
image_url: format!("data:image/png;base64,{}", self.result),
detail: Some(DEFAULT_IMAGE_DETAIL),
}];
if let Some(output_hint) = &self.output_hint {
content.push(FunctionCallOutputContentItem::InputText {
text: output_hint.clone(),
});
}
ResponseInputItem::FunctionCallOutput {
call_id: call_id.to_string(),
output: FunctionCallOutputPayload {