mirror of
https://github.com/pchuan98/codex.git
synced 2026-07-01 00:31:56 +08:00
Owen Lin
040dafa32d
## Description This PR adds an optional `metadata` field to `ResponseItem` for Responses API calls. Only mechanical plumbing, no actual values populated and sent yet. Turns out just adding a new field to `ResponseItem` has quite a large blast radius already. This change is backwards compatible because `metadata` is optional and omitted when absent, so existing response items and rollout history without it still deserialize and requests that do not set it keep the same wire shape. For provider compatibility, we strip out `metadata` before non-OpenAI Responses requests so Azure and AWS Bedrock never see this field. My followup PR here will actually make use of it to start storing and passing along `turn_id`: https://github.com/openai/codex/pull/28360 ## What changed - Added `ResponseItemMetadata` with optional `turn_id`, plus optional `metadata` on Responses API item variants and inter-agent communication. - Preserved item metadata through response-item rewrites such as truncation, missing tool-output synthesis, compaction history rebuilding, visible-history conversion, rollout/resume, and generated app-server schemas/types. - Strip item metadata from non-OpenAI Responses requests while preserving it for OpenAI-shaped requests. - Updated the mechanical fixture/test construction churn required by the new optional field.
279 lines
9.6 KiB
Rust
279 lines
9.6 KiB
Rust
use anyhow::Context;
|
|
use codex_protocol::models::ContentItem;
|
|
use codex_protocol::models::DEFAULT_IMAGE_DETAIL;
|
|
use codex_protocol::models::PermissionProfile;
|
|
use codex_protocol::models::ResponseItem;
|
|
use codex_protocol::protocol::AskForApproval;
|
|
use codex_protocol::protocol::EventMsg;
|
|
use codex_protocol::protocol::Op;
|
|
use codex_protocol::protocol::RolloutItem;
|
|
use codex_protocol::protocol::RolloutLine;
|
|
use codex_protocol::user_input::UserInput;
|
|
use core_test_support::TempDirExt;
|
|
use core_test_support::responses;
|
|
use core_test_support::responses::ev_assistant_message;
|
|
use core_test_support::responses::ev_completed;
|
|
use core_test_support::responses::ev_response_created;
|
|
use core_test_support::responses::sse;
|
|
use core_test_support::responses::start_mock_server;
|
|
use core_test_support::skip_if_no_network;
|
|
use core_test_support::test_codex::TestCodex;
|
|
use core_test_support::test_codex::local_selections;
|
|
use core_test_support::test_codex::test_codex;
|
|
use core_test_support::test_codex::turn_permission_fields;
|
|
use core_test_support::wait_for_event;
|
|
use image::ImageBuffer;
|
|
use image::Rgba;
|
|
use pretty_assertions::assert_eq;
|
|
use std::path::Path;
|
|
use std::time::Duration;
|
|
|
|
fn find_user_message_with_image(text: &str) -> Option<ResponseItem> {
|
|
for line in text.lines() {
|
|
let trimmed = line.trim();
|
|
if trimmed.is_empty() {
|
|
continue;
|
|
}
|
|
let rollout: RolloutLine = match serde_json::from_str(trimmed) {
|
|
Ok(rollout) => rollout,
|
|
Err(_) => continue,
|
|
};
|
|
if let RolloutItem::ResponseItem(ResponseItem::Message { role, content, .. }) =
|
|
&rollout.item
|
|
&& role == "user"
|
|
&& content
|
|
.iter()
|
|
.any(|span| matches!(span, ContentItem::InputImage { .. }))
|
|
&& let RolloutItem::ResponseItem(item) = rollout.item.clone()
|
|
{
|
|
return Some(item);
|
|
}
|
|
}
|
|
None
|
|
}
|
|
|
|
fn extract_image_url(item: &ResponseItem) -> Option<String> {
|
|
match item {
|
|
ResponseItem::Message { content, .. } => content.iter().find_map(|span| match span {
|
|
ContentItem::InputImage { image_url, .. } => Some(image_url.clone()),
|
|
_ => None,
|
|
}),
|
|
_ => None,
|
|
}
|
|
}
|
|
|
|
async fn read_rollout_text(path: &Path) -> anyhow::Result<String> {
|
|
for _ in 0..50 {
|
|
if path.exists()
|
|
&& let Ok(text) = std::fs::read_to_string(path)
|
|
&& !text.trim().is_empty()
|
|
{
|
|
return Ok(text);
|
|
}
|
|
tokio::time::sleep(Duration::from_millis(20)).await;
|
|
}
|
|
std::fs::read_to_string(path)
|
|
.with_context(|| format!("read rollout file at {}", path.display()))
|
|
}
|
|
|
|
fn write_test_png(path: &Path, color: [u8; 4]) -> anyhow::Result<()> {
|
|
if let Some(parent) = path.parent() {
|
|
std::fs::create_dir_all(parent)?;
|
|
}
|
|
let image = ImageBuffer::from_pixel(2, 2, Rgba(color));
|
|
image.save(path)?;
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn copy_paste_local_image_persists_rollout_request_shape() -> anyhow::Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = start_mock_server().await;
|
|
|
|
let TestCodex {
|
|
codex,
|
|
cwd,
|
|
session_configured,
|
|
home: _home,
|
|
..
|
|
} = test_codex().build(&server).await?;
|
|
|
|
let rel_path = "images/paste.png";
|
|
let abs_path = cwd.path().join(rel_path);
|
|
write_test_png(&abs_path, [12, 34, 56, 255])?;
|
|
|
|
let response = sse(vec![
|
|
ev_response_created("resp-1"),
|
|
ev_assistant_message("msg-1", "done"),
|
|
ev_completed("resp-1"),
|
|
]);
|
|
responses::mount_sse_once(&server, response).await;
|
|
|
|
let session_model = session_configured.model.clone();
|
|
let (sandbox_policy, permission_profile) =
|
|
turn_permission_fields(PermissionProfile::Disabled, cwd.path());
|
|
|
|
codex
|
|
.submit(Op::UserInput {
|
|
items: vec![
|
|
UserInput::LocalImage {
|
|
path: abs_path.clone(),
|
|
detail: None,
|
|
},
|
|
UserInput::Text {
|
|
text: "pasted image".to_string(),
|
|
text_elements: Vec::new(),
|
|
},
|
|
],
|
|
final_output_json_schema: None,
|
|
responsesapi_client_metadata: None,
|
|
additional_context: Default::default(),
|
|
thread_settings: codex_protocol::protocol::ThreadSettingsOverrides {
|
|
environments: Some(local_selections(cwd.abs())),
|
|
approval_policy: Some(AskForApproval::Never),
|
|
sandbox_policy: Some(sandbox_policy),
|
|
permission_profile,
|
|
collaboration_mode: Some(codex_protocol::config_types::CollaborationMode {
|
|
mode: codex_protocol::config_types::ModeKind::Default,
|
|
settings: codex_protocol::config_types::Settings {
|
|
model: session_model,
|
|
reasoning_effort: None,
|
|
developer_instructions: None,
|
|
},
|
|
}),
|
|
..Default::default()
|
|
},
|
|
})
|
|
.await?;
|
|
|
|
wait_for_event(&codex, |event| matches!(event, EventMsg::TurnComplete(_))).await;
|
|
codex.submit(Op::Shutdown).await?;
|
|
wait_for_event(&codex, |event| matches!(event, EventMsg::ShutdownComplete)).await;
|
|
|
|
let rollout_path = codex.rollout_path().expect("rollout path");
|
|
let rollout_text = read_rollout_text(&rollout_path).await?;
|
|
let actual = find_user_message_with_image(&rollout_text)
|
|
.expect("expected user message with input image in rollout");
|
|
|
|
let image_url = extract_image_url(&actual).expect("expected image url in rollout");
|
|
let expected = ResponseItem::Message {
|
|
id: None,
|
|
role: "user".to_string(),
|
|
content: vec![
|
|
ContentItem::InputText {
|
|
text: codex_protocol::models::local_image_open_tag_text_with_path(
|
|
/*label_number*/ 1, &abs_path,
|
|
),
|
|
},
|
|
ContentItem::InputImage {
|
|
image_url,
|
|
detail: Some(DEFAULT_IMAGE_DETAIL),
|
|
},
|
|
ContentItem::InputText {
|
|
text: codex_protocol::models::image_close_tag_text(),
|
|
},
|
|
ContentItem::InputText {
|
|
text: "pasted image".to_string(),
|
|
},
|
|
],
|
|
phase: None,
|
|
metadata: None,
|
|
};
|
|
|
|
assert_eq!(actual, expected);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn drag_drop_image_persists_rollout_request_shape() -> anyhow::Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = start_mock_server().await;
|
|
|
|
let TestCodex {
|
|
codex,
|
|
cwd,
|
|
session_configured,
|
|
home: _home,
|
|
..
|
|
} = test_codex().build(&server).await?;
|
|
|
|
let image_url = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR4nGNgYAAAAAMAASsJTYQAAAAASUVORK5CYII=".to_string();
|
|
|
|
let response = sse(vec![
|
|
ev_response_created("resp-1"),
|
|
ev_assistant_message("msg-1", "done"),
|
|
ev_completed("resp-1"),
|
|
]);
|
|
responses::mount_sse_once(&server, response).await;
|
|
|
|
let session_model = session_configured.model.clone();
|
|
let (sandbox_policy, permission_profile) =
|
|
turn_permission_fields(PermissionProfile::Disabled, cwd.path());
|
|
|
|
codex
|
|
.submit(Op::UserInput {
|
|
items: vec![
|
|
UserInput::Image {
|
|
image_url: image_url.clone(),
|
|
detail: None,
|
|
},
|
|
UserInput::Text {
|
|
text: "dropped image".to_string(),
|
|
text_elements: Vec::new(),
|
|
},
|
|
],
|
|
final_output_json_schema: None,
|
|
responsesapi_client_metadata: None,
|
|
additional_context: Default::default(),
|
|
thread_settings: codex_protocol::protocol::ThreadSettingsOverrides {
|
|
environments: Some(local_selections(cwd.abs())),
|
|
approval_policy: Some(AskForApproval::Never),
|
|
sandbox_policy: Some(sandbox_policy),
|
|
permission_profile,
|
|
collaboration_mode: Some(codex_protocol::config_types::CollaborationMode {
|
|
mode: codex_protocol::config_types::ModeKind::Default,
|
|
settings: codex_protocol::config_types::Settings {
|
|
model: session_model,
|
|
reasoning_effort: None,
|
|
developer_instructions: None,
|
|
},
|
|
}),
|
|
..Default::default()
|
|
},
|
|
})
|
|
.await?;
|
|
|
|
wait_for_event(&codex, |event| matches!(event, EventMsg::TurnComplete(_))).await;
|
|
codex.submit(Op::Shutdown).await?;
|
|
wait_for_event(&codex, |event| matches!(event, EventMsg::ShutdownComplete)).await;
|
|
|
|
let rollout_path = codex.rollout_path().expect("rollout path");
|
|
let rollout_text = read_rollout_text(&rollout_path).await?;
|
|
let actual = find_user_message_with_image(&rollout_text)
|
|
.expect("expected user message with input image in rollout");
|
|
|
|
let image_url = extract_image_url(&actual).expect("expected image url in rollout");
|
|
let expected = ResponseItem::Message {
|
|
id: None,
|
|
role: "user".to_string(),
|
|
content: vec![
|
|
ContentItem::InputImage {
|
|
image_url,
|
|
detail: Some(DEFAULT_IMAGE_DETAIL),
|
|
},
|
|
ContentItem::InputText {
|
|
text: "dropped image".to_string(),
|
|
},
|
|
],
|
|
phase: None,
|
|
metadata: None,
|
|
};
|
|
|
|
assert_eq!(actual, expected);
|
|
|
|
Ok(())
|
|
}
|