use codex_protocol::protocol::ConversationTextRole; use codex_protocol::protocol::RealtimeAudioFrame as CoreRealtimeAudioFrame; use codex_protocol::protocol::RealtimeConversationArchitecture; use codex_protocol::protocol::RealtimeConversationVersion; use codex_protocol::protocol::RealtimeOutputModality; use codex_protocol::protocol::RealtimeVoice; use codex_protocol::protocol::RealtimeVoicesList; use schemars::JsonSchema; use serde::Deserialize; use serde::Serialize; use serde_json::Value as JsonValue; use ts_rs::TS; /// EXPERIMENTAL - thread realtime audio chunk. #[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq, JsonSchema, TS)] #[serde(rename_all = "camelCase")] #[ts(export_to = "v2/")] pub struct ThreadRealtimeAudioChunk { pub data: String, pub sample_rate: u32, pub num_channels: u16, pub samples_per_channel: Option, pub item_id: Option, } impl From for ThreadRealtimeAudioChunk { fn from(value: CoreRealtimeAudioFrame) -> Self { let CoreRealtimeAudioFrame { data, sample_rate, num_channels, samples_per_channel, item_id, } = value; Self { data, sample_rate, num_channels, samples_per_channel, item_id, } } } impl From for CoreRealtimeAudioFrame { fn from(value: ThreadRealtimeAudioChunk) -> Self { let ThreadRealtimeAudioChunk { data, sample_rate, num_channels, samples_per_channel, item_id, } = value; Self { data, sample_rate, num_channels, samples_per_channel, item_id, } } } /// EXPERIMENTAL - start a thread-scoped realtime session. #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] #[serde(rename_all = "camelCase")] #[ts(export_to = "v2/")] pub struct ThreadRealtimeStartParams { pub thread_id: String, /// Overrides the configured realtime architecture for this session only. #[ts(optional = nullable)] pub architecture: Option, /// Sends automatic Codex responses as realtime conversation items instead of handoff appends. #[ts(optional = nullable)] pub codex_responses_as_items: Option, /// Optional prefix added to automatic Codex response items when `codexResponsesAsItems` is true. #[ts(optional = nullable)] pub codex_response_item_prefix: Option, /// Overrides the configured realtime model for this session only. #[ts(optional = nullable)] pub model: Option, /// Selects text or audio output for the realtime session. Transport and voice stay /// independent so clients can choose how they connect separately from what the model emits. pub output_modality: RealtimeOutputModality, /// Set to false to start without Codex's startup context. Omitted or null includes it. #[ts(optional = nullable)] pub include_startup_context: Option, #[serde( default, deserialize_with = "crate::protocol::serde_helpers::deserialize_double_option", serialize_with = "crate::protocol::serde_helpers::serialize_double_option", skip_serializing_if = "Option::is_none" )] #[ts(optional = nullable)] pub prompt: Option>, #[ts(optional = nullable)] pub realtime_session_id: Option, #[ts(optional = nullable)] pub transport: Option, /// Overrides the configured realtime protocol version for this session only. #[ts(optional = nullable)] pub version: Option, #[ts(optional = nullable)] pub voice: Option, } /// EXPERIMENTAL - transport used by thread realtime. #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] #[serde(tag = "type", rename_all = "camelCase")] #[ts(export_to = "v2/", tag = "type")] pub enum ThreadRealtimeStartTransport { Websocket, Webrtc { /// SDP offer generated by a WebRTC RTCPeerConnection after configuring audio and the /// realtime events data channel. sdp: String, }, } /// EXPERIMENTAL - response for starting thread realtime. #[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq, JsonSchema, TS)] #[serde(rename_all = "camelCase")] #[ts(export_to = "v2/")] pub struct ThreadRealtimeStartResponse {} /// EXPERIMENTAL - append audio input to thread realtime. #[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq, JsonSchema, TS)] #[serde(rename_all = "camelCase")] #[ts(export_to = "v2/")] pub struct ThreadRealtimeAppendAudioParams { pub thread_id: String, pub audio: ThreadRealtimeAudioChunk, } /// EXPERIMENTAL - response for appending realtime audio input. #[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq, JsonSchema, TS)] #[serde(rename_all = "camelCase")] #[ts(export_to = "v2/")] pub struct ThreadRealtimeAppendAudioResponse {} /// EXPERIMENTAL - append text input to thread realtime. #[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq, JsonSchema, TS)] #[serde(rename_all = "camelCase")] #[ts(export_to = "v2/")] pub struct ThreadRealtimeAppendTextParams { pub thread_id: String, pub text: String, #[serde(default)] pub role: ConversationTextRole, } /// EXPERIMENTAL - response for appending realtime text input. #[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq, JsonSchema, TS)] #[serde(rename_all = "camelCase")] #[ts(export_to = "v2/")] pub struct ThreadRealtimeAppendTextResponse {} /// EXPERIMENTAL - append speakable text to thread realtime. #[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq, JsonSchema, TS)] #[serde(rename_all = "camelCase")] #[ts(export_to = "v2/")] pub struct ThreadRealtimeAppendSpeechParams { pub thread_id: String, pub text: String, } /// EXPERIMENTAL - response for appending realtime speech. #[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq, JsonSchema, TS)] #[serde(rename_all = "camelCase")] #[ts(export_to = "v2/")] pub struct ThreadRealtimeAppendSpeechResponse {} /// EXPERIMENTAL - stop thread realtime. #[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq, JsonSchema, TS)] #[serde(rename_all = "camelCase")] #[ts(export_to = "v2/")] pub struct ThreadRealtimeStopParams { pub thread_id: String, } /// EXPERIMENTAL - response for stopping thread realtime. #[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq, JsonSchema, TS)] #[serde(rename_all = "camelCase")] #[ts(export_to = "v2/")] pub struct ThreadRealtimeStopResponse {} /// EXPERIMENTAL - list voices supported by thread realtime. #[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq, JsonSchema, TS)] #[serde(rename_all = "camelCase")] #[ts(export_to = "v2/")] pub struct ThreadRealtimeListVoicesParams {} /// EXPERIMENTAL - response for listing supported realtime voices. #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, JsonSchema, TS)] #[serde(rename_all = "camelCase")] #[ts(export_to = "v2/")] pub struct ThreadRealtimeListVoicesResponse { pub voices: RealtimeVoicesList, } /// EXPERIMENTAL - emitted when thread realtime startup is accepted. #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] #[serde(rename_all = "camelCase")] #[ts(export_to = "v2/")] pub struct ThreadRealtimeStartedNotification { pub thread_id: String, pub realtime_session_id: Option, pub version: RealtimeConversationVersion, } /// EXPERIMENTAL - raw non-audio thread realtime item emitted by the backend. #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] #[serde(rename_all = "camelCase")] #[ts(export_to = "v2/")] pub struct ThreadRealtimeItemAddedNotification { pub thread_id: String, pub item: JsonValue, } /// EXPERIMENTAL - flat transcript delta emitted whenever realtime /// transcript text changes. #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, JsonSchema, TS)] #[serde(rename_all = "camelCase")] #[ts(export_to = "v2/")] pub struct ThreadRealtimeTranscriptDeltaNotification { pub thread_id: String, pub role: String, /// Live transcript delta from the realtime event. pub delta: String, } /// EXPERIMENTAL - final transcript text emitted when realtime completes /// a transcript part. #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, JsonSchema, TS)] #[serde(rename_all = "camelCase")] #[ts(export_to = "v2/")] pub struct ThreadRealtimeTranscriptDoneNotification { pub thread_id: String, pub role: String, /// Final complete text for the transcript part. pub text: String, } /// EXPERIMENTAL - streamed output audio emitted by thread realtime. #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] #[serde(rename_all = "camelCase")] #[ts(export_to = "v2/")] pub struct ThreadRealtimeOutputAudioDeltaNotification { pub thread_id: String, pub audio: ThreadRealtimeAudioChunk, } /// EXPERIMENTAL - emitted with the remote SDP for a WebRTC realtime session. #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] #[serde(rename_all = "camelCase")] #[ts(export_to = "v2/")] pub struct ThreadRealtimeSdpNotification { pub thread_id: String, pub sdp: String, } /// EXPERIMENTAL - emitted when thread realtime encounters an error. #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] #[serde(rename_all = "camelCase")] #[ts(export_to = "v2/")] pub struct ThreadRealtimeErrorNotification { pub thread_id: String, pub message: String, } /// EXPERIMENTAL - emitted when thread realtime transport closes. #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] #[serde(rename_all = "camelCase")] #[ts(export_to = "v2/")] pub struct ThreadRealtimeClosedNotification { pub thread_id: String, pub reason: Option, }