mirror of
https://github.com/pchuan98/codex.git
synced 2026-07-01 00:31:56 +08:00
Add realtime speech append control (#27917)
## Why Realtime voice harness tuning needs app-side control over what backend Codex text is spoken. Backend orchestrator text is written for a reading UI, so automatically speaking every preamble, progress update, or final assistant message can make the realtime voice model too chatty. For experimentation, clients need two simple controls: keep app/client text-item injection on the existing item-create path, and add an explicit speakable path that app code can call only when it wants realtime to speak. Automatic Codex output also needs an opt-in way to switch from the protocol's default speakable path to regular realtime items, with a caller-provided prefix so prompt wording can be tuned outside core. The default remains unchanged: if a client omits the new start fields and never calls `appendSpeech`, automatic backend output continues down the existing speakable path for the selected realtime protocol. ## What Changed - Adds experimental `thread/realtime/appendSpeech` for app-provided speakable text. - Keeps existing `thread/realtime/appendText` as the item-create API for app-provided realtime text items. - Adds `codexResponsesAsItems` / `codex_responses_as_items` on `thread/realtime/start` to send automatic Codex responses with `conversation.item.create` instead of the protocol's default speakable output path. - Adds `codexResponseItemPrefix` / `codex_response_item_prefix` so clients can prepend experiment instructions to those automatic Codex response items. - Keeps literal `conversation.handoff.append` routing scoped to the v1 speakable path; v2 default speech uses its item/function-output plus `response.create` behavior. - Removes the earlier public silent-context API and hardcoded silent-context prefix. - Updates realtime tests to cover default automatic speakable behavior, opt-in automatic item-create behavior, and explicit `appendSpeech` behavior. ## Validation - `cargo check -p codex-core -p codex-app-server -p codex-api` - `just test -p codex-app-server realtime_conversation` - `just test -p codex-core realtime_conversation` (50/51 passed in the filtered parallel run; the lone failure passed when rerun in isolation) - `just test -p codex-core conversation_mirrors_assistant_message_text_to_realtime_handoff` - `just test -p codex-api e2e_connect_and_exchange_events_against_mock_ws_server` - `just fix -p codex-core` - `just fix -p codex-app-server` - `cargo build -p codex-cli`
This commit is contained in:
committed by
GitHub
Unverified
parent
9728992fab
commit
1d8ff89aa3
@@ -831,6 +831,12 @@ client_request_definitions! {
|
||||
serialization: thread_id(params.thread_id),
|
||||
response: v2::ThreadRealtimeAppendTextResponse,
|
||||
},
|
||||
#[experimental("thread/realtime/appendSpeech")]
|
||||
ThreadRealtimeAppendSpeech => "thread/realtime/appendSpeech" {
|
||||
params: v2::ThreadRealtimeAppendSpeechParams,
|
||||
serialization: thread_id(params.thread_id),
|
||||
response: v2::ThreadRealtimeAppendSpeechResponse,
|
||||
},
|
||||
#[experimental("thread/realtime/stop")]
|
||||
ThreadRealtimeStop => "thread/realtime/stop" {
|
||||
params: v2::ThreadRealtimeStopParams,
|
||||
@@ -3032,6 +3038,8 @@ mod tests {
|
||||
request_id: RequestId::Integer(9),
|
||||
params: v2::ThreadRealtimeStartParams {
|
||||
architecture: Some(RealtimeConversationArchitecture::Avas),
|
||||
codex_responses_as_items: None,
|
||||
codex_response_item_prefix: None,
|
||||
thread_id: "thr_123".to_string(),
|
||||
model: Some("realtime-treatment-model".to_string()),
|
||||
output_modality: RealtimeOutputModality::Audio,
|
||||
@@ -3049,6 +3057,8 @@ mod tests {
|
||||
"params": {
|
||||
"architecture": "avas",
|
||||
"threadId": "thr_123",
|
||||
"codexResponsesAsItems": null,
|
||||
"codexResponseItemPrefix": null,
|
||||
"model": "realtime-treatment-model",
|
||||
"outputModality": "audio",
|
||||
"prompt": "You are on a call",
|
||||
@@ -3069,6 +3079,8 @@ mod tests {
|
||||
request_id: RequestId::Integer(9),
|
||||
params: v2::ThreadRealtimeStartParams {
|
||||
architecture: None,
|
||||
codex_responses_as_items: None,
|
||||
codex_response_item_prefix: None,
|
||||
thread_id: "thr_123".to_string(),
|
||||
model: None,
|
||||
output_modality: RealtimeOutputModality::Audio,
|
||||
@@ -3086,6 +3098,8 @@ mod tests {
|
||||
"params": {
|
||||
"architecture": null,
|
||||
"threadId": "thr_123",
|
||||
"codexResponsesAsItems": null,
|
||||
"codexResponseItemPrefix": null,
|
||||
"model": null,
|
||||
"outputModality": "audio",
|
||||
"realtimeSessionId": null,
|
||||
@@ -3101,6 +3115,8 @@ mod tests {
|
||||
request_id: RequestId::Integer(9),
|
||||
params: v2::ThreadRealtimeStartParams {
|
||||
architecture: None,
|
||||
codex_responses_as_items: None,
|
||||
codex_response_item_prefix: None,
|
||||
thread_id: "thr_123".to_string(),
|
||||
model: None,
|
||||
output_modality: RealtimeOutputModality::Audio,
|
||||
@@ -3118,6 +3134,8 @@ mod tests {
|
||||
"params": {
|
||||
"architecture": null,
|
||||
"threadId": "thr_123",
|
||||
"codexResponsesAsItems": null,
|
||||
"codexResponseItemPrefix": null,
|
||||
"model": null,
|
||||
"outputModality": "audio",
|
||||
"prompt": null,
|
||||
@@ -3166,6 +3184,29 @@ mod tests {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serialize_thread_realtime_append_speech() -> Result<()> {
|
||||
let request = ClientRequest::ThreadRealtimeAppendSpeech {
|
||||
request_id: RequestId::Integer(10),
|
||||
params: v2::ThreadRealtimeAppendSpeechParams {
|
||||
thread_id: "thr_123".to_string(),
|
||||
text: "Short voice update".to_string(),
|
||||
},
|
||||
};
|
||||
assert_eq!(
|
||||
json!({
|
||||
"method": "thread/realtime/appendSpeech",
|
||||
"id": 10,
|
||||
"params": {
|
||||
"threadId": "thr_123",
|
||||
"text": "Short voice update"
|
||||
}
|
||||
}),
|
||||
serde_json::to_value(&request)?,
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serialize_thread_status_changed_notification() -> Result<()> {
|
||||
let notification =
|
||||
@@ -3276,6 +3317,8 @@ mod tests {
|
||||
request_id: RequestId::Integer(1),
|
||||
params: v2::ThreadRealtimeStartParams {
|
||||
architecture: None,
|
||||
codex_responses_as_items: None,
|
||||
codex_response_item_prefix: None,
|
||||
thread_id: "thr_123".to_string(),
|
||||
model: None,
|
||||
output_modality: RealtimeOutputModality::Audio,
|
||||
|
||||
Reference in New Issue
Block a user