Add realtime output modality and transcript events (#17701)

- Add outputModality to thread/realtime/start and wire text/audio output
selection through app-server, core, API, and TUI.\n- Rename the realtime
transcript delta notification and add a separate transcript done
notification that forwards final text from item done without correlating
it with deltas.
This commit is contained in:
Ahmed Ibrahim
2026-04-14 00:13:13 -07:00
committed by GitHub
parent a6b03a22cc
commit 2f6fc7c137
38 changed files with 711 additions and 77 deletions
+4 -1
View File
@@ -385,7 +385,10 @@ fn server_notification_thread_target(
ServerNotification::ThreadRealtimeItemAdded(notification) => {
Some(notification.thread_id.as_str())
}
ServerNotification::ThreadRealtimeTranscriptUpdated(notification) => {
ServerNotification::ThreadRealtimeTranscriptDelta(notification) => {
Some(notification.thread_id.as_str())
}
ServerNotification::ThreadRealtimeTranscriptDone(notification) => {
Some(notification.thread_id.as_str())
}
ServerNotification::ThreadRealtimeOutputAudioDelta(notification) => {
+1
View File
@@ -657,6 +657,7 @@ impl AppServerSession {
request_id,
params: ThreadRealtimeStartParams {
thread_id: thread_id.to_string(),
output_modality: params.output_modality,
prompt: params.prompt,
session_id: params.session_id,
voice: params.voice,
+2 -1
View File
@@ -6272,7 +6272,8 @@ impl ChatWidget {
| ServerNotification::FsChanged(_)
| ServerNotification::FuzzyFileSearchSessionUpdated(_)
| ServerNotification::FuzzyFileSearchSessionCompleted(_)
| ServerNotification::ThreadRealtimeTranscriptUpdated(_)
| ServerNotification::ThreadRealtimeTranscriptDelta(_)
| ServerNotification::ThreadRealtimeTranscriptDone(_)
| ServerNotification::WindowsWorldWritableWarning(_)
| ServerNotification::WindowsSandboxSetupCompleted(_)
| ServerNotification::AccountLoginCompleted(_) => {}
+4
View File
@@ -7,6 +7,7 @@ use codex_protocol::protocol::RealtimeConversationClosedEvent;
use codex_protocol::protocol::RealtimeConversationRealtimeEvent;
use codex_protocol::protocol::RealtimeConversationStartedEvent;
use codex_protocol::protocol::RealtimeEvent;
use codex_protocol::protocol::RealtimeOutputModality;
use codex_realtime_webrtc::RealtimeWebrtcEvent;
use codex_realtime_webrtc::RealtimeWebrtcSession;
use codex_realtime_webrtc::RealtimeWebrtcSessionHandle;
@@ -236,6 +237,7 @@ impl ChatWidget {
) {
self.submit_op(AppCommand::realtime_conversation_start(
ConversationStartParams {
output_modality: RealtimeOutputModality::Audio,
prompt: None,
session_id: None,
transport,
@@ -327,7 +329,9 @@ impl ChatWidget {
}
RealtimeEvent::InputAudioSpeechStarted(_) => self.interrupt_realtime_audio_playback(),
RealtimeEvent::InputTranscriptDelta(_) => {}
RealtimeEvent::InputTranscriptDone(_) => {}
RealtimeEvent::OutputTranscriptDelta(_) => {}
RealtimeEvent::OutputTranscriptDone(_) => {}
RealtimeEvent::AudioOut(frame) => self.enqueue_realtime_audio_out(&frame),
RealtimeEvent::ResponseCreated(_) => {}
RealtimeEvent::ResponseCancelled(_) => self.interrupt_realtime_audio_playback(),