Add realtime speech append control (#27917)

## Why

Realtime voice harness tuning needs app-side control over what backend
Codex text is spoken. Backend orchestrator text is written for a reading
UI, so automatically speaking every preamble, progress update, or final
assistant message can make the realtime voice model too chatty.

For experimentation, clients need two simple controls: keep app/client
text-item injection on the existing item-create path, and add an
explicit speakable path that app code can call only when it wants
realtime to speak. Automatic Codex output also needs an opt-in way to
switch from the protocol's default speakable path to regular realtime
items, with a caller-provided prefix so prompt wording can be tuned
outside core.

The default remains unchanged: if a client omits the new start fields
and never calls `appendSpeech`, automatic backend output continues down
the existing speakable path for the selected realtime protocol.

## What Changed

- Adds experimental `thread/realtime/appendSpeech` for app-provided
speakable text.
- Keeps existing `thread/realtime/appendText` as the item-create API for
app-provided realtime text items.
- Adds `codexResponsesAsItems` / `codex_responses_as_items` on
`thread/realtime/start` to send automatic Codex responses with
`conversation.item.create` instead of the protocol's default speakable
output path.
- Adds `codexResponseItemPrefix` / `codex_response_item_prefix` so
clients can prepend experiment instructions to those automatic Codex
response items.
- Keeps literal `conversation.handoff.append` routing scoped to the v1
speakable path; v2 default speech uses its item/function-output plus
`response.create` behavior.
- Removes the earlier public silent-context API and hardcoded
silent-context prefix.
- Updates realtime tests to cover default automatic speakable behavior,
opt-in automatic item-create behavior, and explicit `appendSpeech`
behavior.

## Validation

- `cargo check -p codex-core -p codex-app-server -p codex-api`
- `just test -p codex-app-server realtime_conversation`
- `just test -p codex-core realtime_conversation` (50/51 passed in the
filtered parallel run; the lone failure passed when rerun in isolation)
- `just test -p codex-core
conversation_mirrors_assistant_message_text_to_realtime_handoff`
- `just test -p codex-api
e2e_connect_and_exchange_events_against_mock_ws_server`
- `just fix -p codex-core`
- `just fix -p codex-app-server`
- `cargo build -p codex-cli`
This commit is contained in:
guinness-oai
2026-06-15 16:15:58 -07:00
committed by GitHub
Unverified
parent 9728992fab
commit 1d8ff89aa3
15 changed files with 783 additions and 220 deletions
@@ -831,6 +831,12 @@ client_request_definitions! {
serialization: thread_id(params.thread_id),
response: v2::ThreadRealtimeAppendTextResponse,
},
#[experimental("thread/realtime/appendSpeech")]
ThreadRealtimeAppendSpeech => "thread/realtime/appendSpeech" {
params: v2::ThreadRealtimeAppendSpeechParams,
serialization: thread_id(params.thread_id),
response: v2::ThreadRealtimeAppendSpeechResponse,
},
#[experimental("thread/realtime/stop")]
ThreadRealtimeStop => "thread/realtime/stop" {
params: v2::ThreadRealtimeStopParams,
@@ -3032,6 +3038,8 @@ mod tests {
request_id: RequestId::Integer(9),
params: v2::ThreadRealtimeStartParams {
architecture: Some(RealtimeConversationArchitecture::Avas),
codex_responses_as_items: None,
codex_response_item_prefix: None,
thread_id: "thr_123".to_string(),
model: Some("realtime-treatment-model".to_string()),
output_modality: RealtimeOutputModality::Audio,
@@ -3049,6 +3057,8 @@ mod tests {
"params": {
"architecture": "avas",
"threadId": "thr_123",
"codexResponsesAsItems": null,
"codexResponseItemPrefix": null,
"model": "realtime-treatment-model",
"outputModality": "audio",
"prompt": "You are on a call",
@@ -3069,6 +3079,8 @@ mod tests {
request_id: RequestId::Integer(9),
params: v2::ThreadRealtimeStartParams {
architecture: None,
codex_responses_as_items: None,
codex_response_item_prefix: None,
thread_id: "thr_123".to_string(),
model: None,
output_modality: RealtimeOutputModality::Audio,
@@ -3086,6 +3098,8 @@ mod tests {
"params": {
"architecture": null,
"threadId": "thr_123",
"codexResponsesAsItems": null,
"codexResponseItemPrefix": null,
"model": null,
"outputModality": "audio",
"realtimeSessionId": null,
@@ -3101,6 +3115,8 @@ mod tests {
request_id: RequestId::Integer(9),
params: v2::ThreadRealtimeStartParams {
architecture: None,
codex_responses_as_items: None,
codex_response_item_prefix: None,
thread_id: "thr_123".to_string(),
model: None,
output_modality: RealtimeOutputModality::Audio,
@@ -3118,6 +3134,8 @@ mod tests {
"params": {
"architecture": null,
"threadId": "thr_123",
"codexResponsesAsItems": null,
"codexResponseItemPrefix": null,
"model": null,
"outputModality": "audio",
"prompt": null,
@@ -3166,6 +3184,29 @@ mod tests {
Ok(())
}
#[test]
fn serialize_thread_realtime_append_speech() -> Result<()> {
let request = ClientRequest::ThreadRealtimeAppendSpeech {
request_id: RequestId::Integer(10),
params: v2::ThreadRealtimeAppendSpeechParams {
thread_id: "thr_123".to_string(),
text: "Short voice update".to_string(),
},
};
assert_eq!(
json!({
"method": "thread/realtime/appendSpeech",
"id": 10,
"params": {
"threadId": "thr_123",
"text": "Short voice update"
}
}),
serde_json::to_value(&request)?,
);
Ok(())
}
#[test]
fn serialize_thread_status_changed_notification() -> Result<()> {
let notification =
@@ -3276,6 +3317,8 @@ mod tests {
request_id: RequestId::Integer(1),
params: v2::ThreadRealtimeStartParams {
architecture: None,
codex_responses_as_items: None,
codex_response_item_prefix: None,
thread_id: "thr_123".to_string(),
model: None,
output_modality: RealtimeOutputModality::Audio,