Add realtime output modality and transcript events (#17701)

- Add outputModality to thread/realtime/start and wire text/audio output
selection through app-server, core, API, and TUI.\n- Rename the realtime
transcript delta notification and add a separate transcript done
notification that forwards final text from item done without correlating
it with deltas.
This commit is contained in:
Ahmed Ibrahim
2026-04-14 00:13:13 -07:00
committed by GitHub
Unverified
parent a6b03a22cc
commit 2f6fc7c137
38 changed files with 711 additions and 77 deletions
+20 -2
View File
@@ -42,6 +42,7 @@ use codex_protocol::protocol::RealtimeConversationRealtimeEvent;
use codex_protocol::protocol::RealtimeConversationSdpEvent;
use codex_protocol::protocol::RealtimeConversationStartedEvent;
use codex_protocol::protocol::RealtimeHandoffRequested;
use codex_protocol::protocol::RealtimeOutputModality;
use codex_protocol::protocol::RealtimeVoice;
use codex_protocol::protocol::RealtimeVoicesList;
use http::HeaderMap;
@@ -593,8 +594,14 @@ async fn prepare_realtime_start(
api_provider.base_url = realtime_ws_base_url.clone();
}
let version = config.realtime.version;
let session_config =
build_realtime_session_config(sess, params.prompt, params.session_id, params.voice).await?;
let session_config = build_realtime_session_config(
sess,
params.prompt,
params.session_id,
params.output_modality,
params.voice,
)
.await?;
let requested_session_id = session_config.session_id.clone();
let extra_headers = match transport {
ConversationStartTransport::Websocket => {
@@ -622,6 +629,7 @@ pub(crate) async fn build_realtime_session_config(
sess: &Arc<Session>,
prompt: Option<Option<String>>,
session_id: Option<String>,
output_modality: RealtimeOutputModality,
voice: Option<RealtimeVoice>,
) -> CodexResult<RealtimeSessionConfig> {
let config = sess.get_config().await;
@@ -653,6 +661,13 @@ pub(crate) async fn build_realtime_session_config(
RealtimeWsVersion::V1 => RealtimeEventParser::V1,
RealtimeWsVersion::V2 => RealtimeEventParser::RealtimeV2,
};
if config.realtime.version == RealtimeWsVersion::V1
&& matches!(output_modality, RealtimeOutputModality::Text)
{
return Err(CodexErr::InvalidRequest(
"text realtime output modality requires realtime v2".to_string(),
));
}
let session_mode = match config.realtime.session_type {
RealtimeWsMode::Conversational => RealtimeSessionMode::Conversational,
RealtimeWsMode::Transcription => RealtimeSessionMode::Transcription,
@@ -667,6 +682,7 @@ pub(crate) async fn build_realtime_session_config(
session_id: Some(session_id.unwrap_or_else(|| sess.conversation_id.to_string())),
event_parser,
session_mode,
output_modality,
voice,
})
}
@@ -1216,7 +1232,9 @@ async fn handle_realtime_server_event(
RealtimeEvent::Error(_) => true,
RealtimeEvent::SessionUpdated { .. }
| RealtimeEvent::InputTranscriptDelta(_)
| RealtimeEvent::InputTranscriptDone(_)
| RealtimeEvent::OutputTranscriptDelta(_)
| RealtimeEvent::OutputTranscriptDone(_)
| RealtimeEvent::ConversationItemAdded(_)
| RealtimeEvent::ConversationItemDone { .. } => false,
};