From e922f46a0f863a9630ddf7dc60dcbadb36a6c28a Mon Sep 17 00:00:00 2001 From: guinness-oai Date: Wed, 17 Jun 2026 20:57:13 -0700 Subject: [PATCH] [codex] Support assistant realtime append text (#28836) ## Why Frontend realtime voice continuity needs to replay a tiny previous-session overlap as actual conversation items, including assistant text. The app-server `thread/realtime/appendText` API already carries a role through to the Rust realtime websocket layer, but the shared role enum only accepted `user` and `developer`. ## What Changed - Added `assistant` to `ConversationTextRole` and regenerated the app-server schema/type fixtures. - Added `output_text` as a realtime conversation content type. - Updated realtime websocket item creation so assistant appendText emits `content: [{ type: "output_text", text }]`, while user and developer continue to emit `input_text`. - Updated app-server docs and tests to cover assistant appendText alongside the existing developer role behavior. ## Validation - `just write-app-server-schema` - `just fmt` (first sandboxed attempt failed because `uv` could not access `~/.cache/uv`; reran with filesystem access and passed) - `just test -p codex-api` passed: 126/126 - `just test -p codex-app-server-protocol` passed: 239/239, including generated JSON/TypeScript fixture checks - `just test -p codex-app-server` was started locally but stopped per request after unrelated local sandbox/Seatbelt failures (`sandbox-exec: sandbox_apply: Operation not permitted`) and one missing local `codex` binary failure; CI should be faster and more authoritative for the full suite. --- .../schema/json/ClientRequest.json | 3 +- .../codex_app_server_protocol.schemas.json | 3 +- .../codex_app_server_protocol.v2.schemas.json | 3 +- .../schema/typescript/ConversationTextRole.ts | 2 +- codex-rs/app-server/README.md | 2 +- .../tests/suite/v2/realtime_conversation.rs | 44 +++++++++++-- .../endpoint/realtime_websocket/methods.rs | 64 +++++++++++++++++-- .../endpoint/realtime_websocket/methods_v1.rs | 9 ++- .../endpoint/realtime_websocket/methods_v2.rs | 9 ++- .../endpoint/realtime_websocket/protocol.rs | 1 + codex-rs/protocol/src/protocol.rs | 1 + 11 files changed, 123 insertions(+), 18 deletions(-) diff --git a/codex-rs/app-server-protocol/schema/json/ClientRequest.json b/codex-rs/app-server-protocol/schema/json/ClientRequest.json index 749899498..5ebb861de 100644 --- a/codex-rs/app-server-protocol/schema/json/ClientRequest.json +++ b/codex-rs/app-server-protocol/schema/json/ClientRequest.json @@ -643,7 +643,8 @@ "ConversationTextRole": { "enum": [ "user", - "developer" + "developer", + "assistant" ], "type": "string" }, diff --git a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json index 5b0c59c42..9a13e8652 100644 --- a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json +++ b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json @@ -8707,7 +8707,8 @@ "ConversationTextRole": { "enum": [ "user", - "developer" + "developer", + "assistant" ], "type": "string" }, diff --git a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json index 45cfbd95c..0f6084a99 100644 --- a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json +++ b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json @@ -4992,7 +4992,8 @@ "ConversationTextRole": { "enum": [ "user", - "developer" + "developer", + "assistant" ], "type": "string" }, diff --git a/codex-rs/app-server-protocol/schema/typescript/ConversationTextRole.ts b/codex-rs/app-server-protocol/schema/typescript/ConversationTextRole.ts index 9cba89f83..a4d574b4d 100644 --- a/codex-rs/app-server-protocol/schema/typescript/ConversationTextRole.ts +++ b/codex-rs/app-server-protocol/schema/typescript/ConversationTextRole.ts @@ -2,4 +2,4 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. -export type ConversationTextRole = "user" | "developer"; +export type ConversationTextRole = "user" | "developer" | "assistant"; diff --git a/codex-rs/app-server/README.md b/codex-rs/app-server/README.md index 7dde50d19..a6c5ddab7 100644 --- a/codex-rs/app-server/README.md +++ b/codex-rs/app-server/README.md @@ -167,7 +167,7 @@ Example with notification opt-out: - `turn/interrupt` — request cancellation of an in-flight turn by `(thread_id, turn_id)`; success is an empty `{}` response and the turn finishes with `status: "interrupted"`. - `thread/realtime/start` — start a thread-scoped realtime session (experimental); pass `outputModality: "text"` or `outputModality: "audio"` to choose model output, optionally pass `model` and `version` to override configured realtime selection for this session only, and pass `includeStartupContext: false` to omit Codex's generated startup context. By default, automatic Codex text follows the protocol's speakable output path. Pass `clientManagedHandoffs: true` to disable automatic Codex response delivery so only the client's explicit append calls produce handoffs. Pass `codexResponsesAsItems: true` to send automatic Codex responses as realtime conversation items instead, and optionally pass `codexResponseItemPrefix` to prepend experiment instructions to those items. For V1 sessions, pass `codexResponseHandoffPrefix` while item mode is disabled to route automatic Codex commentary through `conversation.handoff.append` with that prefix; final answers remain unprefixed. Returns `{}` and streams `thread/realtime/*` notifications. Omit `transport` for the websocket transport, or pass `{ "type": "webrtc", "sdp": "..." }` to create a WebRTC session from a browser-generated SDP offer; the remote answer SDP is emitted as `thread/realtime/sdp`. - `thread/realtime/appendAudio` — append an input audio chunk to the active realtime session (experimental); returns `{}`. -- `thread/realtime/appendText` — append text input to the active realtime session with a required `role` of `user` or `developer` (experimental); returns `{}`. Older clients that omit `role` default to `user`. +- `thread/realtime/appendText` — append text input to the active realtime session with a required `role` of `user`, `developer`, or `assistant` (experimental); returns `{}`. Older clients that omit `role` default to `user`. - `thread/realtime/appendSpeech` — append text that the realtime model should speak to the user (experimental); returns `{}`. - `thread/realtime/stop` — stop the active realtime session for the thread (experimental); returns `{}`. - `review/start` — kick off Codex’s automated reviewer for a thread; responds like `turn/start` and emits `item/started`/`item/completed` notifications with `enteredReviewMode` and `exitedReviewMode` items, plus a final assistant `agentMessage` containing the review. diff --git a/codex-rs/app-server/tests/suite/v2/realtime_conversation.rs b/codex-rs/app-server/tests/suite/v2/realtime_conversation.rs index 274ee9b90..c09278bb8 100644 --- a/codex-rs/app-server/tests/suite/v2/realtime_conversation.rs +++ b/codex-rs/app-server/tests/suite/v2/realtime_conversation.rs @@ -709,6 +709,20 @@ async fn realtime_conversation_streams_v2_notifications() -> Result<()> { .await??; let _: ThreadRealtimeAppendTextResponse = to_response(text_append_response)?; + let assistant_append_request_id = mcp + .send_thread_realtime_append_text_request(ThreadRealtimeAppendTextParams { + thread_id: started.thread_id.clone(), + text: "welcome back".to_string(), + role: ConversationTextRole::Assistant, + }) + .await?; + let assistant_append_response: JSONRPCResponse = timeout( + DEFAULT_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(assistant_append_request_id)), + ) + .await??; + let _: ThreadRealtimeAppendTextResponse = to_response(assistant_append_response)?; + let output_audio = read_notification::( &mut mcp, "thread/realtime/outputAudio/delta", @@ -790,7 +804,7 @@ async fn realtime_conversation_streams_v2_notifications() -> Result<()> { let connections = realtime_server.connections(); assert_eq!(connections.len(), 1); let connection = &connections[0]; - assert_eq!(connection.len(), 3); + assert_eq!(connection.len(), 4); assert_eq!( connection[0].body_json()["type"].as_str(), Some("session.update") @@ -799,13 +813,14 @@ async fn realtime_conversation_streams_v2_notifications() -> Result<()> { connection[0].body_json()["session"]["instructions"].as_str(), Some(startup_context_instructions.as_str()), ); - let text_request = connection + let text_requests = connection .iter() .map(WebSocketRequest::body_json) - .find(|request| request["type"] == "conversation.item.create") - .context("expected conversation item request")?; + .filter(|request| request["type"] == "conversation.item.create") + .collect::>(); + assert_eq!(text_requests.len(), 2); assert_eq!( - text_request, + text_requests[0], json!({ "type": "conversation.item.create", "item": { @@ -818,6 +833,20 @@ async fn realtime_conversation_streams_v2_notifications() -> Result<()> { }, }) ); + assert_eq!( + text_requests[1], + json!({ + "type": "conversation.item.create", + "item": { + "type": "message", + "role": "assistant", + "content": [{ + "type": "output_text", + "text": "welcome back", + }], + }, + }) + ); let mut request_types = [ connection[1].body_json()["type"] .as_str() @@ -827,11 +856,16 @@ async fn realtime_conversation_streams_v2_notifications() -> Result<()> { .as_str() .context("expected websocket request type")? .to_string(), + connection[3].body_json()["type"] + .as_str() + .context("expected websocket request type")? + .to_string(), ]; request_types.sort(); assert_eq!( request_types, [ + "conversation.item.create".to_string(), "conversation.item.create".to_string(), "input_audio_buffer.append".to_string(), ] diff --git a/codex-rs/codex-api/src/endpoint/realtime_websocket/methods.rs b/codex-rs/codex-api/src/endpoint/realtime_websocket/methods.rs index 0aa2feb04..499635579 100644 --- a/codex-rs/codex-api/src/endpoint/realtime_websocket/methods.rs +++ b/codex-rs/codex-api/src/endpoint/realtime_websocket/methods.rs @@ -1637,10 +1637,29 @@ mod tests { .into_text() .expect("text"); let fourth_json: Value = serde_json::from_str(&fourth).expect("json"); - assert_eq!(fourth_json["type"], "conversation.handoff.append"); - assert_eq!(fourth_json["handoff_id"], "handoff_1"); + assert_eq!(fourth_json["type"], "conversation.item.create"); + assert_eq!(fourth_json["item"]["role"], "assistant"); assert_eq!( - fourth_json["output_text"], + fourth_json["item"]["content"][0]["type"], + Value::String("output_text".to_string()) + ); + assert_eq!( + fourth_json["item"]["content"][0]["text"], + Value::String("assistant context".to_string()) + ); + + let fifth = ws + .next() + .await + .expect("fifth msg") + .expect("fifth msg ok") + .into_text() + .expect("text"); + let fifth_json: Value = serde_json::from_str(&fifth).expect("json"); + assert_eq!(fifth_json["type"], "conversation.handoff.append"); + assert_eq!(fifth_json["handoff_id"], "handoff_1"); + assert_eq!( + fifth_json["output_text"], "\"Agent Final Message\":\n\nhello from background agent" ); @@ -1766,6 +1785,13 @@ mod tests { ) .await .expect("send item"); + connection + .send_conversation_item_create( + "assistant context".to_string(), + ConversationTextRole::Assistant, + ) + .await + .expect("send assistant item"); connection .send_conversation_function_call_output( "handoff_1".to_string(), @@ -1988,16 +2014,35 @@ mod tests { .expect("text"); let third_json: Value = serde_json::from_str(&third).expect("json"); assert_eq!(third_json["type"], "conversation.item.create"); + assert_eq!(third_json["item"]["role"], "assistant"); assert_eq!( - third_json["item"]["type"], + third_json["item"]["content"][0]["type"], + Value::String("output_text".to_string()) + ); + assert_eq!( + third_json["item"]["content"][0]["text"], + Value::String("assistant context".to_string()) + ); + + let fourth = ws + .next() + .await + .expect("fourth msg") + .expect("fourth msg ok") + .into_text() + .expect("text"); + let fourth_json: Value = serde_json::from_str(&fourth).expect("json"); + assert_eq!(fourth_json["type"], "conversation.item.create"); + assert_eq!( + fourth_json["item"]["type"], Value::String("function_call_output".to_string()) ); assert_eq!( - third_json["item"]["call_id"], + fourth_json["item"]["call_id"], Value::String("call_1".to_string()) ); assert_eq!( - third_json["item"]["output"], + fourth_json["item"]["output"], Value::String("delegated result".to_string()) ); }); @@ -2054,6 +2099,13 @@ mod tests { ) .await .expect("send text item"); + connection + .send_conversation_item_create( + "assistant context".to_string(), + ConversationTextRole::Assistant, + ) + .await + .expect("send assistant item"); connection .send_conversation_function_call_output( "call_1".to_string(), diff --git a/codex-rs/codex-api/src/endpoint/realtime_websocket/methods_v1.rs b/codex-rs/codex-api/src/endpoint/realtime_websocket/methods_v1.rs index a7f73d82f..aa063d07c 100644 --- a/codex-rs/codex-api/src/endpoint/realtime_websocket/methods_v1.rs +++ b/codex-rs/codex-api/src/endpoint/realtime_websocket/methods_v1.rs @@ -19,12 +19,19 @@ pub(super) fn conversation_item_create_message( text: String, role: ConversationTextRole, ) -> RealtimeOutboundMessage { + let content_type = match role { + ConversationTextRole::Assistant => ConversationContentType::OutputText, + ConversationTextRole::User | ConversationTextRole::Developer => { + ConversationContentType::InputText + } + }; + RealtimeOutboundMessage::ConversationItemCreate { item: ConversationItemPayload::Message(ConversationMessageItem { r#type: ConversationItemType::Message, role, content: vec![ConversationItemContent { - r#type: ConversationContentType::InputText, + r#type: content_type, text, }], }), diff --git a/codex-rs/codex-api/src/endpoint/realtime_websocket/methods_v2.rs b/codex-rs/codex-api/src/endpoint/realtime_websocket/methods_v2.rs index 702b4be86..ee5d5031a 100644 --- a/codex-rs/codex-api/src/endpoint/realtime_websocket/methods_v2.rs +++ b/codex-rs/codex-api/src/endpoint/realtime_websocket/methods_v2.rs @@ -40,12 +40,19 @@ pub(super) fn conversation_item_create_message( text: String, role: ConversationTextRole, ) -> RealtimeOutboundMessage { + let content_type = match role { + ConversationTextRole::Assistant => ConversationContentType::OutputText, + ConversationTextRole::User | ConversationTextRole::Developer => { + ConversationContentType::InputText + } + }; + RealtimeOutboundMessage::ConversationItemCreate { item: ConversationItemPayload::Message(ConversationMessageItem { r#type: ConversationItemType::Message, role, content: vec![ConversationItemContent { - r#type: ConversationContentType::InputText, + r#type: content_type, text, }], }), diff --git a/codex-rs/codex-api/src/endpoint/realtime_websocket/protocol.rs b/codex-rs/codex-api/src/endpoint/realtime_websocket/protocol.rs index 8f61fc1d1..48f89b0d3 100644 --- a/codex-rs/codex-api/src/endpoint/realtime_websocket/protocol.rs +++ b/codex-rs/codex-api/src/endpoint/realtime_websocket/protocol.rs @@ -195,6 +195,7 @@ pub(super) struct ConversationItemContent { #[serde(rename_all = "snake_case")] pub(super) enum ConversationContentType { InputText, + OutputText, } #[derive(Debug, Clone, Serialize)] diff --git a/codex-rs/protocol/src/protocol.rs b/codex-rs/protocol/src/protocol.rs index f87d71b21..e77590c5e 100644 --- a/codex-rs/protocol/src/protocol.rs +++ b/codex-rs/protocol/src/protocol.rs @@ -417,6 +417,7 @@ pub enum ConversationTextRole { #[default] User, Developer, + Assistant, } #[derive(Debug, Clone, PartialEq)]