Clarify realtime v2 context and handoff messages (#17896)

## Summary
- wrap realtime startup context in
`<startup_context>...</startup_context>` tags
- prefix V2 mirrored user text and relayed backend text with `[USER]` /
`[BACKEND]`
- remove the V2 progress suffix and replace the final V2 handoff output
with a short completion acknowledgement while preserving the existing V1
wrapper

## Testing
- cargo test -p codex-api
realtime_v2_session_update_includes_background_agent_tool_and_handoff_output_item
-- --exact
- cargo test -p codex-app-server webrtc_v2_background_agent_
- cargo test -p codex-app-server webrtc_v2_text_input_is_
- cargo test -p codex-core conversation_user_text_turn_is_
This commit is contained in:
bxie-openai
2026-04-15 16:26:20 -07:00
committed by GitHub
Unverified
parent 18d61f6923
commit c2bdb7812c
10 changed files with 133 additions and 27 deletions
@@ -75,6 +75,8 @@ const DEFAULT_TIMEOUT: Duration = Duration::from_secs(10);
const STARTUP_CONTEXT_HEADER: &str = "Startup context from Codex.";
const V2_STEERING_ACKNOWLEDGEMENT: &str =
"This was sent to steer the previous background agent task.";
const V2_HANDOFF_COMPLETE_ACKNOWLEDGEMENT: &str =
"Background agent finished. Use the preceding [BACKEND] messages as the result.";
#[derive(Debug, Clone, Copy)]
enum StartupContextConfig<'a> {
@@ -1359,7 +1361,8 @@ async fn webrtc_v2_forwards_audio_and_text_between_client_and_sideband() -> Resu
request["type"] == "conversation.item.create"
&& request["item"]["type"] == "message"
&& request["item"]["role"] == "user"
&& request["item"]["content"][0]["text"] == "hello"
&& request["item"]["content"][0]["type"] == "input_text"
&& request["item"]["content"][0]["text"] == "[USER] hello"
}),
"sideband requests should include user text item: {requests:?}"
);
@@ -1558,7 +1561,7 @@ async fn webrtc_v2_background_agent_tool_call_delegates_and_returns_function_out
assert_v2_progress_update(&progress, "delegated from v2");
let tool_output = harness.sideband_outbound_request(/*request_index*/ 2).await;
assert_v2_function_call_output(&tool_output, "call_v2", "delegated from v2");
assert_v2_function_call_output(&tool_output, "call_v2", V2_HANDOFF_COMPLETE_ACKNOWLEDGEMENT);
assert_eq!(
function_call_output_sideband_requests(&harness.realtime_server).len(),
1
@@ -1693,7 +1696,11 @@ async fn webrtc_v2_background_agent_progress_is_sent_before_function_output() ->
assert_v2_progress_update(&progress, "progress before final");
let tool_output = harness.sideband_outbound_request(/*request_index*/ 2).await;
assert_v2_function_call_output(&tool_output, "call_progress_order", "progress before final");
assert_v2_function_call_output(
&tool_output,
"call_progress_order",
V2_HANDOFF_COMPLETE_ACKNOWLEDGEMENT,
);
harness.shutdown().await;
Ok(())
@@ -1777,7 +1784,11 @@ async fn webrtc_v2_tool_call_delegated_turn_can_execute_shell_tool() -> Result<(
assert_v2_progress_update(&progress, "shell tool finished");
let tool_output = harness.sideband_outbound_request(/*request_index*/ 2).await;
assert_v2_function_call_output(&tool_output, "call_shell", "shell tool finished");
assert_v2_function_call_output(
&tool_output,
"call_shell",
V2_HANDOFF_COMPLETE_ACKNOWLEDGEMENT,
);
assert_eq!(
function_call_output_sideband_requests(&harness.realtime_server).len(),
1
@@ -1857,7 +1868,11 @@ async fn webrtc_v2_tool_call_does_not_block_sideband_audio() -> Result<()> {
assert_v2_progress_update(&progress, "late delegated result");
let tool_output = harness.sideband_outbound_request(/*request_index*/ 2).await;
assert_v2_function_call_output(&tool_output, "call_audio", "late delegated result");
assert_v2_function_call_output(
&tool_output,
"call_audio",
V2_HANDOFF_COMPLETE_ACKNOWLEDGEMENT,
);
harness.shutdown().await;
Ok(())
@@ -2090,7 +2105,7 @@ fn assert_v2_function_call_output(request: &Value, call_id: &str, expected_outpu
"item": {
"type": "function_call_output",
"call_id": call_id,
"output": format!("\"Agent Final Message\":\n\n{expected_output}"),
"output": expected_output,
}
})
);
@@ -2106,7 +2121,7 @@ fn assert_v2_progress_update(request: &Value, expected_text: &str) {
"role": "user",
"content": [{
"type": "input_text",
"text": format!("{expected_text}\n\nUpdate from background agent (task hasn't finished yet):")
"text": format!("[BACKEND] {expected_text}")
}]
}
})
@@ -2123,7 +2138,7 @@ fn assert_v2_user_text_item(request: &Value, expected_text: &str) {
"role": "user",
"content": [{
"type": "input_text",
"text": expected_text
"text": format!("[USER] {expected_text}")
}]
}
})
@@ -1670,7 +1670,7 @@ mod tests {
);
assert_eq!(
third_json["item"]["output"],
Value::String("\"Agent Final Message\":\n\ndelegated result".to_string())
Value::String("delegated result".to_string())
);
});
@@ -45,9 +45,11 @@ pub(super) fn conversation_handoff_append_message(
handoff_id: String,
output_text: String,
) -> RealtimeOutboundMessage {
let output_text = format!("{AGENT_FINAL_MESSAGE_PREFIX}{output_text}");
match event_parser {
RealtimeEventParser::V1 => v1_conversation_handoff_append_message(handoff_id, output_text),
RealtimeEventParser::V1 => v1_conversation_handoff_append_message(
handoff_id,
format!("{AGENT_FINAL_MESSAGE_PREFIX}{output_text}"),
),
RealtimeEventParser::RealtimeV2 => {
v2_conversation_handoff_append_message(handoff_id, output_text)
}
+7
View File
@@ -4944,6 +4944,8 @@ mod handlers {
use crate::config_loader::load_config_layers_state;
use crate::realtime_context::REALTIME_TURN_TOKEN_BUDGET;
use crate::realtime_context::truncate_realtime_text_to_token_budget;
use crate::realtime_conversation::REALTIME_USER_TEXT_PREFIX;
use crate::realtime_conversation::prefix_realtime_v2_text;
use codex_features::Feature;
use codex_utils_absolute_path::AbsolutePathBuf;
@@ -5164,6 +5166,11 @@ mod handlers {
if text.is_empty() {
return;
}
let text = if sess.conversation.is_running_v2().await {
prefix_realtime_v2_text(text, REALTIME_USER_TEXT_PREFIX)
} else {
text
};
let text = truncate_realtime_text_to_token_budget(&text, REALTIME_TURN_TOKEN_BUDGET);
if text.is_empty() {
return;
+24 -1
View File
@@ -25,6 +25,8 @@ use tracing::info;
use tracing::warn;
const STARTUP_CONTEXT_HEADER: &str = "Startup context from Codex.\nThis is background context about recent work and machine/workspace layout. It may be incomplete or stale. Use it to inform responses, and do not repeat it back unless relevant.";
const STARTUP_CONTEXT_OPEN_TAG: &str = "<startup_context>";
const STARTUP_CONTEXT_CLOSE_TAG: &str = "</startup_context>";
const CURRENT_THREAD_SECTION_TOKEN_BUDGET: usize = 1_200;
const RECENT_WORK_SECTION_TOKEN_BUDGET: usize = 2_200;
const WORKSPACE_SECTION_TOKEN_BUDGET: usize = 1_600;
@@ -106,7 +108,7 @@ pub(crate) async fn build_realtime_startup_context(
parts.push(section);
}
let context = truncate_text(&parts.join("\n\n"), TruncationPolicy::Tokens(budget_tokens));
let context = format_startup_context_blob(&parts.join("\n\n"), budget_tokens);
debug!(
approx_tokens = approx_token_count(&context),
bytes = context.len(),
@@ -443,6 +445,27 @@ fn format_section(title: &str, body: Option<String>, budget_tokens: usize) -> Op
))
}
fn format_startup_context_blob(body: &str, budget_tokens: usize) -> String {
let wrapper = format!("{STARTUP_CONTEXT_OPEN_TAG}\n\n{STARTUP_CONTEXT_CLOSE_TAG}");
let mut body_budget = budget_tokens.saturating_sub(approx_token_count(&wrapper));
loop {
let body = truncate_text(body, TruncationPolicy::Tokens(body_budget));
let wrapped = format!("{STARTUP_CONTEXT_OPEN_TAG}\n{body}\n{STARTUP_CONTEXT_CLOSE_TAG}");
let wrapped_tokens = approx_token_count(&wrapped);
if wrapped_tokens <= budget_tokens || body_budget == 0 {
return wrapped;
}
let excess_tokens = wrapped_tokens.saturating_sub(budget_tokens);
let next_budget = body_budget.saturating_sub(excess_tokens.max(1));
if next_budget == body_budget {
return wrapped;
}
body_budget = next_budget;
}
}
fn format_thread_group(
current_group: &Path,
group: &Path,
@@ -1,6 +1,7 @@
use super::build_current_thread_section;
use super::build_recent_work_section;
use super::build_workspace_section_with_user_root;
use super::format_startup_context_blob;
use chrono::TimeZone;
use chrono::Utc;
use codex_git_utils::GitSha;
@@ -170,6 +171,23 @@ fn current_thread_section_keeps_latest_turns_when_history_exceeds_budget() {
);
}
#[test]
fn startup_context_blob_is_wrapped_in_tags_and_fits_budget() {
let body = format!(
"Startup context from Codex.\n{}\n{}",
"recent work ".repeat(1_200),
"workspace tree ".repeat(800),
);
let wrapped = format_startup_context_blob(&body, /*budget_tokens*/ 200);
assert!(wrapped.starts_with("<startup_context>\n"));
assert!(wrapped.ends_with("\n</startup_context>"));
assert!(wrapped.contains("Startup context from Codex."));
assert!(wrapped.contains("tokens truncated"));
assert!(wrapped.len().div_ceil(4) <= 200);
}
#[test]
fn workspace_section_requires_meaningful_structure() {
let cwd = TempDir::new().expect("tempdir");
+43 -11
View File
@@ -65,8 +65,10 @@ const HANDOFF_OUT_QUEUE_CAPACITY: usize = 64;
const OUTPUT_EVENTS_QUEUE_CAPACITY: usize = 256;
const REALTIME_STARTUP_CONTEXT_TOKEN_BUDGET: usize = 5_000;
const DEFAULT_REALTIME_MODEL: &str = "gpt-realtime-1.5";
const REALTIME_V2_PROGRESS_UPDATE_SUFFIX: &str =
"\n\nUpdate from background agent (task hasn't finished yet):";
pub(crate) const REALTIME_USER_TEXT_PREFIX: &str = "[USER] ";
pub(crate) const REALTIME_BACKEND_TEXT_PREFIX: &str = "[BACKEND] ";
const REALTIME_V2_HANDOFF_COMPLETE_ACKNOWLEDGEMENT: &str =
"Background agent finished. Use the preceding [BACKEND] messages as the result.";
const REALTIME_V2_STEER_ACKNOWLEDGEMENT: &str =
"This was sent to steer the previous background agent task.";
const REALTIME_ACTIVE_RESPONSE_ERROR_PREFIX: &str =
@@ -209,6 +211,7 @@ impl RealtimeHandoffState {
struct ConversationState {
audio_tx: Sender<RealtimeAudioFrame>,
user_text_tx: Sender<String>,
session_kind: RealtimeSessionKind,
writer: RealtimeWebsocketWriter,
handoff: RealtimeHandoffState,
input_task: JoinHandle<()>,
@@ -245,6 +248,16 @@ impl RealtimeConversationManager {
.and_then(|state| state.realtime_active.load(Ordering::Relaxed).then_some(()))
}
pub(crate) async fn is_running_v2(&self) -> bool {
let state = self.state.lock().await;
matches!(
state.as_ref(),
Some(state)
if state.realtime_active.load(Ordering::Relaxed)
&& state.session_kind == RealtimeSessionKind::V2
)
}
async fn start(&self, start: RealtimeStart) -> CodexResult<RealtimeStartOutput> {
let previous_state = {
let mut guard = self.state.lock().await;
@@ -331,6 +344,7 @@ impl RealtimeConversationManager {
*guard = Some(ConversationState {
audio_tx,
user_text_tx,
session_kind,
writer,
handoff,
input_task: task,
@@ -406,15 +420,18 @@ impl RealtimeConversationManager {
pub(crate) async fn text_in(&self, text: String) -> CodexResult<()> {
let sender = {
let guard = self.state.lock().await;
guard.as_ref().map(|state| state.user_text_tx.clone())
guard
.as_ref()
.map(|state| (state.user_text_tx.clone(), state.session_kind))
};
let Some(sender) = sender else {
let Some((sender, session_kind)) = sender else {
return Err(CodexErr::InvalidRequest(
"conversation is not running".to_string(),
));
};
let text = prefix_realtime_text(text, REALTIME_USER_TEXT_PREFIX, session_kind);
sender
.send(text)
.await
@@ -437,6 +454,11 @@ impl RealtimeConversationManager {
return Ok(());
};
let output_text = prefix_realtime_text(
output_text,
REALTIME_BACKEND_TEXT_PREFIX,
handoff.session_kind,
);
*handoff.last_output_text.lock().await = Some(output_text.clone());
handoff
.output_tx
@@ -695,6 +717,17 @@ fn default_realtime_voice(version: RealtimeWsVersion) -> RealtimeVoice {
}
}
fn prefix_realtime_text(text: String, prefix: &str, session_kind: RealtimeSessionKind) -> String {
if session_kind != RealtimeSessionKind::V2 || text.is_empty() || text.starts_with(prefix) {
return text;
}
format!("{prefix}{text}")
}
pub(crate) fn prefix_realtime_v2_text(text: String, prefix: &str) -> String {
prefix_realtime_text(text, prefix, RealtimeSessionKind::V2)
}
fn validate_realtime_voice(version: RealtimeWsVersion, voice: RealtimeVoice) -> CodexResult<()> {
let voices = RealtimeVoicesList::builtin();
let allowed = match version {
@@ -1074,18 +1107,17 @@ async fn handle_handoff_output(
return Ok(());
}
}
writer
.send_conversation_item_create(format!(
"{output_text}{REALTIME_V2_PROGRESS_UPDATE_SUFFIX}"
))
.await
writer.send_conversation_item_create(output_text).await
}
HandoffOutput::FinalUpdate {
handoff_id,
output_text,
output_text: _,
} => {
if let Err(err) = writer
.send_conversation_handoff_append(handoff_id, output_text)
.send_conversation_handoff_append(
handoff_id,
REALTIME_V2_HANDOFF_COMPLETE_ACKNOWLEDGEMENT.to_string(),
)
.await
{
Err(err)
@@ -57,6 +57,8 @@ use wiremock::matchers::method;
use wiremock::matchers::path_regex;
const STARTUP_CONTEXT_HEADER: &str = "Startup context from Codex.";
const STARTUP_CONTEXT_OPEN_TAG: &str = "<startup_context>";
const STARTUP_CONTEXT_CLOSE_TAG: &str = "</startup_context>";
const REALTIME_BACKEND_PROMPT: &str = include_str!("../../templates/realtime/backend_prompt.md");
const USER_FIRST_NAME_PLACEHOLDER: &str = "{{ user_first_name }}";
const MEMORY_PROMPT_PHRASE: &str =
@@ -1530,6 +1532,8 @@ async fn conversation_start_injects_startup_context_from_thread_history() -> Res
let startup_context = websocket_request_instructions(&startup_context_request)
.expect("startup context request should contain instructions");
assert!(startup_context.contains(STARTUP_CONTEXT_OPEN_TAG));
assert!(startup_context.contains(STARTUP_CONTEXT_CLOSE_TAG));
assert!(startup_context.contains(STARTUP_CONTEXT_HEADER));
assert!(!startup_context.contains("## User"));
assert!(startup_context.contains("### "));
@@ -1747,6 +1751,8 @@ async fn conversation_startup_context_falls_back_to_workspace_map() -> Result<()
let startup_context = websocket_request_instructions(&startup_context_request)
.expect("startup context request should contain instructions");
assert!(startup_context.contains(STARTUP_CONTEXT_OPEN_TAG));
assert!(startup_context.contains(STARTUP_CONTEXT_CLOSE_TAG));
assert!(startup_context.contains(STARTUP_CONTEXT_HEADER));
assert!(startup_context.contains("## Machine / Workspace Map"));
assert!(startup_context.contains("notes.txt"));
@@ -1801,6 +1807,8 @@ async fn conversation_startup_context_is_truncated_and_sent_once_per_start() ->
.await;
let startup_context = websocket_request_instructions(&startup_context_request)
.expect("startup context request should contain instructions");
assert!(startup_context.contains(STARTUP_CONTEXT_OPEN_TAG));
assert!(startup_context.contains(STARTUP_CONTEXT_CLOSE_TAG));
assert!(startup_context.contains(STARTUP_CONTEXT_HEADER));
assert!(startup_context.len() <= 20_500);
@@ -1879,6 +1887,7 @@ async fn conversation_user_text_turn_is_sent_to_realtime_when_active() -> Result
assert_eq!(session_updated, "sess_user_text");
let user_text = "typed follow-up for realtime";
let prefixed_user_text = format!("[USER] {user_text}");
test.codex
.submit(Op::UserInput {
items: vec![UserInput::Text {
@@ -1898,7 +1907,7 @@ async fn conversation_user_text_turn_is_sent_to_realtime_when_active() -> Result
let realtime_text_request = wait_for_matching_websocket_request(
&realtime_server,
"normal user turn text mirrored to realtime",
|request| websocket_request_text(request).as_deref() == Some(user_text),
|request| websocket_request_text(request).as_deref() == Some(prefixed_user_text.as_str()),
)
.await;
let model_user_texts = response_mock.single_request().message_input_texts("user");
@@ -1907,7 +1916,7 @@ async fn conversation_user_text_turn_is_sent_to_realtime_when_active() -> Result
model_user_texts.iter().any(|text| text == user_text),
websocket_request_text(&realtime_text_request),
),
(true, Some(user_text.to_string())),
(true, Some(prefixed_user_text)),
);
let realtime_response_create = timeout(Duration::from_millis(200), async {
wait_for_matching_websocket_request(
@@ -9,4 +9,4 @@ content[0].type: input_text
model_has_full_user_text: true
realtime_text_equal_full_user_text: false
realtime_text_approx_tokens: 300
content[0].text: mirror-head alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alph…2417 tokens truncated…ega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega mirror-tail
content[0].text: [USER] mirror-head alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alp…2419 tokens truncated…ega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega mirror-tail
@@ -6,5 +6,5 @@ type: conversation.item.create
item.type: message
item.role: user
content[0].type: input_text
content[0].text: typed follow-up for realtime
content[0].text: [USER] typed follow-up for realtime
response.create: false