mirror of
https://github.com/pchuan98/codex.git
synced 2026-07-01 00:31:56 +08:00
[code-mode] Reject remote image URLs from output helpers (#27732)
## Summary - reject HTTP(S) image URLs from the shared code-mode output-image normalization path - return a concise model-visible tool error so the model can recover on its next turn - apply the targeted rejection to both `image()` and `generatedImage()` - leave other non-empty image URL values to existing downstream handling The returned error is: > Tool call failed: remote image URLs are not supported in tool outputs. Pass a base64 data URI instead ## Why Responses Lite cannot lower a remote image URL emitted from a structured tool output. Rejecting HTTP(S) values in the Codex harness preserves the tool-call metadata and gives the model a recoverable next turn instead of invalidating the sample. ## Test coverage The regression is covered primarily by a `test_codex()` agent integration test that simulates the Responses API exchange and asserts the failed model-visible exec output. A supplemental runtime test covers both `http://` and `https://` inputs across both image output helpers. ## Test plan - `cd codex-rs && just test -p codex-code-mode` - `cd codex-rs && just test -p codex-code-mode-protocol` - `cd codex-rs && just test -p codex-core code_mode_image_helper_rejects_remote_url` - `cd codex-rs && just fmt` - `git diff --check origin/main...HEAD` Related context: https://github.com/openai/openai/pull/1022346
This commit is contained in:
committed by
GitHub
Unverified
parent
693082f3c4
commit
c09df9e353
@@ -5,6 +5,7 @@ use codex_code_mode_protocol::FunctionCallOutputContentItem;
|
||||
use codex_code_mode_protocol::ImageDetail;
|
||||
|
||||
const IMAGE_HELPER_EXPECTS_MESSAGE: &str = "image expects a non-empty image URL string, an object with image_url and optional detail, or a raw MCP image block";
|
||||
const REMOTE_IMAGE_URL_ERROR: &str = "Tool call failed: remote image URLs are not supported in tool outputs. Pass a base64 data URI instead";
|
||||
const CODEX_IMAGE_DETAIL_META_KEY: &str = "codex/imageDetail";
|
||||
|
||||
pub(super) fn serialize_output_text(
|
||||
@@ -59,11 +60,8 @@ pub(super) fn normalize_output_image(
|
||||
return Err(IMAGE_HELPER_EXPECTS_MESSAGE.to_string());
|
||||
}
|
||||
let lower = image_url.to_ascii_lowercase();
|
||||
if !(lower.starts_with("http://")
|
||||
|| lower.starts_with("https://")
|
||||
|| lower.starts_with("data:"))
|
||||
{
|
||||
return Err("image expects an http(s) or data URL".to_string());
|
||||
if lower.starts_with("http://") || lower.starts_with("https://") {
|
||||
return Err(REMOTE_IMAGE_URL_ERROR.to_string());
|
||||
}
|
||||
|
||||
let detail = detail_override.or(detail);
|
||||
|
||||
@@ -1405,7 +1405,7 @@ text(formatter.format(new Date("2025-01-02T03:04:05Z")));
|
||||
source: r#"
|
||||
const returnsUndefined = [
|
||||
text("first"),
|
||||
image("https://example.com/image.jpg"),
|
||||
image("data:image/png;base64,AAA"),
|
||||
notify("ping"),
|
||||
].map((value) => value === undefined);
|
||||
text(JSON.stringify(returnsUndefined));
|
||||
@@ -1426,7 +1426,7 @@ text(JSON.stringify(returnsUndefined));
|
||||
text: "first".to_string(),
|
||||
},
|
||||
FunctionCallOutputContentItem::InputImage {
|
||||
image_url: "https://example.com/image.jpg".to_string(),
|
||||
image_url: "data:image/png;base64,AAA".to_string(),
|
||||
detail: Some(crate::DEFAULT_IMAGE_DETAIL),
|
||||
},
|
||||
FunctionCallOutputContentItem::InputText {
|
||||
@@ -1482,7 +1482,7 @@ image({
|
||||
ExecuteRequest {
|
||||
source: r#"
|
||||
generatedImage({
|
||||
image_url: "https://example.com/image.jpg",
|
||||
image_url: "data:image/png;base64,AAA",
|
||||
output_hint: "generated image save hint",
|
||||
});
|
||||
"#
|
||||
@@ -1499,7 +1499,7 @@ generatedImage({
|
||||
cell_id: cell_id("1"),
|
||||
content_items: vec![
|
||||
FunctionCallOutputContentItem::InputImage {
|
||||
image_url: "https://example.com/image.jpg".to_string(),
|
||||
image_url: "data:image/png;base64,AAA".to_string(),
|
||||
detail: Some(crate::DEFAULT_IMAGE_DETAIL),
|
||||
},
|
||||
FunctionCallOutputContentItem::InputText {
|
||||
@@ -1521,7 +1521,7 @@ generatedImage({
|
||||
source: r#"
|
||||
image(
|
||||
{
|
||||
image_url: "https://example.com/image.jpg",
|
||||
image_url: "data:image/png;base64,AAA",
|
||||
detail: "high",
|
||||
},
|
||||
"original",
|
||||
@@ -1539,7 +1539,7 @@ image(
|
||||
RuntimeResponse::Result {
|
||||
cell_id: cell_id("1"),
|
||||
content_items: vec![FunctionCallOutputContentItem::InputImage {
|
||||
image_url: "https://example.com/image.jpg".to_string(),
|
||||
image_url: "data:image/png;base64,AAA".to_string(),
|
||||
detail: Some(crate::ImageDetail::Original),
|
||||
}],
|
||||
error_text: None,
|
||||
@@ -1594,7 +1594,7 @@ image(
|
||||
ExecuteRequest {
|
||||
source: r#"
|
||||
image({
|
||||
image_url: "https://example.com/image.jpg",
|
||||
image_url: "data:image/png;base64,AAA",
|
||||
detail: "low",
|
||||
});
|
||||
"#
|
||||
@@ -1610,7 +1610,7 @@ image({
|
||||
RuntimeResponse::Result {
|
||||
cell_id: cell_id("1"),
|
||||
content_items: vec![FunctionCallOutputContentItem::InputImage {
|
||||
image_url: "https://example.com/image.jpg".to_string(),
|
||||
image_url: "data:image/png;base64,AAA".to_string(),
|
||||
detail: Some(crate::ImageDetail::Low),
|
||||
}],
|
||||
error_text: None,
|
||||
@@ -1618,6 +1618,42 @@ image({
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn image_helpers_reject_remote_urls() {
|
||||
for image_url in [
|
||||
"http://example.com/image.jpg",
|
||||
"https://example.com/image.jpg",
|
||||
] {
|
||||
for source in [
|
||||
format!("image({image_url:?});"),
|
||||
format!("generatedImage({{ image_url: {image_url:?} }});"),
|
||||
] {
|
||||
let service = CodeModeService::new();
|
||||
|
||||
let response = execute(
|
||||
&service,
|
||||
ExecuteRequest {
|
||||
source,
|
||||
yield_time_ms: None,
|
||||
..execute_request("")
|
||||
},
|
||||
)
|
||||
.await;
|
||||
|
||||
assert_eq!(
|
||||
response,
|
||||
RuntimeResponse::Result {
|
||||
cell_id: cell_id("1"),
|
||||
content_items: Vec::new(),
|
||||
error_text: Some(
|
||||
"Tool call failed: remote image URLs are not supported in tool outputs. Pass a base64 data URI instead".to_string(),
|
||||
),
|
||||
}
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn image_helper_rejects_unsupported_detail() {
|
||||
let service = CodeModeService::new();
|
||||
@@ -1627,7 +1663,7 @@ image({
|
||||
ExecuteRequest {
|
||||
source: r#"
|
||||
image({
|
||||
image_url: "https://example.com/image.jpg",
|
||||
image_url: "data:image/png;base64,AAA",
|
||||
detail: "medium",
|
||||
});
|
||||
"#
|
||||
|
||||
Reference in New Issue
Block a user