mirror of
https://github.com/pchuan98/codex.git
synced 2026-07-01 00:31:56 +08:00
Add saved image path hint to standalone image generation (#25947)
## Why Standalone image generation returns image bytes to the model, but the model also needs the host artifact path to reference the generated file in follow-up work. ## What changed - Append the default saved-image path hint alongside the generated image tool output. - Reuse the existing core image-generation hint text. - Pass the thread ID and Codex home directory needed to compute the artifact path. - Add app-server and extension coverage for the model-visible hint. ## Validation - `just fmt` - `just bazel-lock-check` - `just test -p codex-app-server standalone_image_generation_returns_saved_path_hint_to_model`
This commit is contained in:
committed by
GitHub
Unverified
parent
68db0bb5ec
commit
12e8764a9c
@@ -25,6 +25,7 @@ const EXEC_DESCRIPTION_TEMPLATE: &str = r#"Run JavaScript code to orchestrate/co
|
||||
- `exit()`: Immediately ends the current script successfully (like an early return from the top level).
|
||||
- `text(value: string | number | boolean | undefined | null)`: Appends a text item. Non-string values are stringified with `JSON.stringify(...)` when possible.
|
||||
- `image(imageUrlOrItem: string | { image_url: string; detail?: "auto" | "low" | "high" | "original" | null } | ImageContent, detail?: "auto" | "low" | "high" | "original" | null)`: Appends an image item. `image_url` can be an HTTPS URL or a base64-encoded `data:` URL. To forward an MCP tool image, pass an individual `ImageContent` block from `result.content`, for example `image(result.content[0])`. MCP image blocks may request detail with `_meta: { "codex/imageDetail": "original" }`. When provided, the second `detail` argument overrides any detail embedded in the first argument.
|
||||
- `generatedImage(result: { image_url: string; output_hint?: string })`: Appends an image-generation result and its optional output hint.
|
||||
- `store(key: string, value: any)`: stores a serializable value under a string key for later `exec` calls in the same session.
|
||||
- `load(key: string)`: returns the stored value for a string key, or `undefined` if it is missing.
|
||||
- `notify(value: string | number | boolean | undefined | null)`: immediately injects an extra `custom_tool_call_output` for the current `exec` call. Values are stringified like `text(...)`.
|
||||
|
||||
@@ -129,6 +129,58 @@ pub(super) fn image_callback(
|
||||
retval.set(v8::undefined(scope).into());
|
||||
}
|
||||
|
||||
pub(super) fn generated_image_callback(
|
||||
scope: &mut v8::PinScope<'_, '_>,
|
||||
args: v8::FunctionCallbackArguments,
|
||||
mut retval: v8::ReturnValue<v8::Value>,
|
||||
) {
|
||||
let value = if args.length() == 0 {
|
||||
v8::undefined(scope).into()
|
||||
} else {
|
||||
args.get(0)
|
||||
};
|
||||
let output_hint = match generated_image_output_hint(scope, value) {
|
||||
Ok(output_hint) => output_hint,
|
||||
Err(error_text) => {
|
||||
throw_type_error(scope, &error_text);
|
||||
return;
|
||||
}
|
||||
};
|
||||
let image_item = match normalize_output_image(scope, value, /*detail_override*/ None) {
|
||||
Ok(image_item) => image_item,
|
||||
Err(()) => return,
|
||||
};
|
||||
if let Some(state) = scope.get_slot::<RuntimeState>() {
|
||||
let _ = state.event_tx.send(RuntimeEvent::ContentItem(image_item));
|
||||
if let Some(text) = output_hint {
|
||||
let _ = state.event_tx.send(RuntimeEvent::ContentItem(
|
||||
FunctionCallOutputContentItem::InputText { text },
|
||||
));
|
||||
}
|
||||
}
|
||||
retval.set(v8::undefined(scope).into());
|
||||
}
|
||||
|
||||
fn generated_image_output_hint(
|
||||
scope: &mut v8::PinScope<'_, '_>,
|
||||
value: v8::Local<'_, v8::Value>,
|
||||
) -> Result<Option<String>, String> {
|
||||
let object = v8::Local::<v8::Object>::try_from(value)
|
||||
.map_err(|_| "generatedImage expects an image generation result object".to_string())?;
|
||||
let key = v8::String::new(scope, "output_hint")
|
||||
.ok_or_else(|| "failed to allocate generatedImage helper keys".to_string())?;
|
||||
let output_hint = object
|
||||
.get(scope, key.into())
|
||||
.ok_or_else(|| "failed to read generatedImage output_hint".to_string())?;
|
||||
if output_hint.is_undefined() {
|
||||
return Ok(None);
|
||||
}
|
||||
if !output_hint.is_string() {
|
||||
return Err("generatedImage output_hint must be a string when provided".to_string());
|
||||
}
|
||||
Ok(Some(output_hint.to_rust_string_lossy(scope)))
|
||||
}
|
||||
|
||||
pub(super) fn store_callback(
|
||||
scope: &mut v8::PinScope<'_, '_>,
|
||||
args: v8::FunctionCallbackArguments,
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
use super::RuntimeState;
|
||||
use super::callbacks::clear_timeout_callback;
|
||||
use super::callbacks::exit_callback;
|
||||
use super::callbacks::generated_image_callback;
|
||||
use super::callbacks::image_callback;
|
||||
use super::callbacks::load_callback;
|
||||
use super::callbacks::notify_callback;
|
||||
@@ -23,6 +24,7 @@ pub(super) fn install_globals(scope: &mut v8::PinScope<'_, '_>) -> Result<(), St
|
||||
let set_timeout = helper_function(scope, "setTimeout", set_timeout_callback)?;
|
||||
let text = helper_function(scope, "text", text_callback)?;
|
||||
let image = helper_function(scope, "image", image_callback)?;
|
||||
let generated_image = helper_function(scope, "generatedImage", generated_image_callback)?;
|
||||
let store = helper_function(scope, "store", store_callback)?;
|
||||
let load = helper_function(scope, "load", load_callback)?;
|
||||
let notify = helper_function(scope, "notify", notify_callback)?;
|
||||
@@ -35,6 +37,7 @@ pub(super) fn install_globals(scope: &mut v8::PinScope<'_, '_>) -> Result<(), St
|
||||
set_global(scope, global, "setTimeout", set_timeout.into())?;
|
||||
set_global(scope, global, "text", text.into())?;
|
||||
set_global(scope, global, "image", image.into())?;
|
||||
set_global(scope, global, "generatedImage", generated_image.into())?;
|
||||
set_global(scope, global, "store", store.into())?;
|
||||
set_global(scope, global, "load", load.into())?;
|
||||
set_global(scope, global, "notify", notify.into())?;
|
||||
|
||||
@@ -1563,6 +1563,44 @@ image({
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn generated_image_helper_appends_image_and_output_hint() {
|
||||
let service = CodeModeService::new();
|
||||
|
||||
let response = execute(
|
||||
&service,
|
||||
ExecuteRequest {
|
||||
source: r#"
|
||||
generatedImage({
|
||||
image_url: "https://example.com/image.jpg",
|
||||
output_hint: "generated image save hint",
|
||||
});
|
||||
"#
|
||||
.to_string(),
|
||||
yield_time_ms: None,
|
||||
..execute_request("")
|
||||
},
|
||||
)
|
||||
.await;
|
||||
|
||||
assert_eq!(
|
||||
response,
|
||||
RuntimeResponse::Result {
|
||||
cell_id: cell_id("1"),
|
||||
content_items: vec![
|
||||
FunctionCallOutputContentItem::InputImage {
|
||||
image_url: "https://example.com/image.jpg".to_string(),
|
||||
detail: Some(crate::DEFAULT_IMAGE_DETAIL),
|
||||
},
|
||||
FunctionCallOutputContentItem::InputText {
|
||||
text: "generated image save hint".to_string(),
|
||||
},
|
||||
],
|
||||
error_text: None,
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn image_helper_second_arg_overrides_explicit_object_detail() {
|
||||
let service = CodeModeService::new();
|
||||
|
||||
Reference in New Issue
Block a user