diff --git a/codex-rs/app-server-protocol/schema/json/ClientRequest.json b/codex-rs/app-server-protocol/schema/json/ClientRequest.json index 7247794ea..1c7d3d0df 100644 --- a/codex-rs/app-server-protocol/schema/json/ClientRequest.json +++ b/codex-rs/app-server-protocol/schema/json/ClientRequest.json @@ -1993,6 +1993,13 @@ ], "type": "object" }, + "RealtimeConversationArchitecture": { + "enum": [ + "realtimeapi", + "avas" + ], + "type": "string" + }, "RealtimeConversationVersion": { "enum": [ "v1", diff --git a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json index e40c81f5d..8dd77d38d 100644 --- a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json +++ b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json @@ -13799,6 +13799,13 @@ "title": "RawResponseItemCompletedNotification", "type": "object" }, + "RealtimeConversationArchitecture": { + "enum": [ + "realtimeapi", + "avas" + ], + "type": "string" + }, "RealtimeConversationVersion": { "enum": [ "v1", diff --git a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json index 3a36c1022..de2d8ad74 100644 --- a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json +++ b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json @@ -10272,6 +10272,13 @@ "title": "RawResponseItemCompletedNotification", "type": "object" }, + "RealtimeConversationArchitecture": { + "enum": [ + "realtimeapi", + "avas" + ], + "type": "string" + }, "RealtimeConversationVersion": { "enum": [ "v1", diff --git a/codex-rs/app-server-protocol/schema/typescript/RealtimeConversationArchitecture.ts b/codex-rs/app-server-protocol/schema/typescript/RealtimeConversationArchitecture.ts new file mode 100644 index 000000000..4467e4a0f --- /dev/null +++ b/codex-rs/app-server-protocol/schema/typescript/RealtimeConversationArchitecture.ts @@ -0,0 +1,5 @@ +// GENERATED CODE! DO NOT MODIFY BY HAND! + +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +export type RealtimeConversationArchitecture = "realtimeapi" | "avas"; diff --git a/codex-rs/app-server-protocol/schema/typescript/index.ts b/codex-rs/app-server-protocol/schema/typescript/index.ts index 149b3aec0..7aa64ab00 100644 --- a/codex-rs/app-server-protocol/schema/typescript/index.ts +++ b/codex-rs/app-server-protocol/schema/typescript/index.ts @@ -51,6 +51,7 @@ export type { NetworkPolicyRuleAction } from "./NetworkPolicyRuleAction"; export type { ParsedCommand } from "./ParsedCommand"; export type { Personality } from "./Personality"; export type { PlanType } from "./PlanType"; +export type { RealtimeConversationArchitecture } from "./RealtimeConversationArchitecture"; export type { RealtimeConversationVersion } from "./RealtimeConversationVersion"; export type { RealtimeOutputModality } from "./RealtimeOutputModality"; export type { RealtimeVoice } from "./RealtimeVoice"; diff --git a/codex-rs/app-server-protocol/src/protocol/common.rs b/codex-rs/app-server-protocol/src/protocol/common.rs index 0c2c88c74..b036532dd 100644 --- a/codex-rs/app-server-protocol/src/protocol/common.rs +++ b/codex-rs/app-server-protocol/src/protocol/common.rs @@ -1647,6 +1647,7 @@ mod tests { use codex_protocol::account::PlanType; use codex_protocol::models::BUILT_IN_PERMISSION_PROFILE_READ_ONLY; use codex_protocol::parse_command::ParsedCommand; + use codex_protocol::protocol::RealtimeConversationArchitecture; use codex_protocol::protocol::RealtimeConversationVersion; use codex_protocol::protocol::RealtimeOutputModality; use codex_protocol::protocol::RealtimeVoice; @@ -3012,6 +3013,7 @@ mod tests { let request = ClientRequest::ThreadRealtimeStart { request_id: RequestId::Integer(9), params: v2::ThreadRealtimeStartParams { + architecture: Some(RealtimeConversationArchitecture::Avas), thread_id: "thr_123".to_string(), model: Some("realtime-treatment-model".to_string()), output_modality: RealtimeOutputModality::Audio, @@ -3027,6 +3029,7 @@ mod tests { "method": "thread/realtime/start", "id": 9, "params": { + "architecture": "avas", "threadId": "thr_123", "model": "realtime-treatment-model", "outputModality": "audio", @@ -3047,6 +3050,7 @@ mod tests { let default_prompt_request = ClientRequest::ThreadRealtimeStart { request_id: RequestId::Integer(9), params: v2::ThreadRealtimeStartParams { + architecture: None, thread_id: "thr_123".to_string(), model: None, output_modality: RealtimeOutputModality::Audio, @@ -3062,6 +3066,7 @@ mod tests { "method": "thread/realtime/start", "id": 9, "params": { + "architecture": null, "threadId": "thr_123", "model": null, "outputModality": "audio", @@ -3077,6 +3082,7 @@ mod tests { let null_prompt_request = ClientRequest::ThreadRealtimeStart { request_id: RequestId::Integer(9), params: v2::ThreadRealtimeStartParams { + architecture: None, thread_id: "thr_123".to_string(), model: None, output_modality: RealtimeOutputModality::Audio, @@ -3092,6 +3098,7 @@ mod tests { "method": "thread/realtime/start", "id": 9, "params": { + "architecture": null, "threadId": "thr_123", "model": null, "outputModality": "audio", @@ -3250,6 +3257,7 @@ mod tests { let request = ClientRequest::ThreadRealtimeStart { request_id: RequestId::Integer(1), params: v2::ThreadRealtimeStartParams { + architecture: None, thread_id: "thr_123".to_string(), model: None, output_modality: RealtimeOutputModality::Audio, diff --git a/codex-rs/app-server-protocol/src/protocol/v2/realtime.rs b/codex-rs/app-server-protocol/src/protocol/v2/realtime.rs index a86cd8067..faeef68fc 100644 --- a/codex-rs/app-server-protocol/src/protocol/v2/realtime.rs +++ b/codex-rs/app-server-protocol/src/protocol/v2/realtime.rs @@ -1,4 +1,5 @@ use codex_protocol::protocol::RealtimeAudioFrame as CoreRealtimeAudioFrame; +use codex_protocol::protocol::RealtimeConversationArchitecture; use codex_protocol::protocol::RealtimeConversationVersion; use codex_protocol::protocol::RealtimeOutputModality; use codex_protocol::protocol::RealtimeVoice; @@ -65,6 +66,9 @@ impl From for CoreRealtimeAudioFrame { #[ts(export_to = "v2/")] pub struct ThreadRealtimeStartParams { pub thread_id: String, + /// Overrides the configured realtime architecture for this session only. + #[ts(optional = nullable)] + pub architecture: Option, /// Overrides the configured realtime model for this session only. #[ts(optional = nullable)] pub model: Option, diff --git a/codex-rs/app-server/src/request_processors/turn_processor.rs b/codex-rs/app-server/src/request_processors/turn_processor.rs index fe5d3278e..f25c589e5 100644 --- a/codex-rs/app-server/src/request_processors/turn_processor.rs +++ b/codex-rs/app-server/src/request_processors/turn_processor.rs @@ -934,6 +934,7 @@ impl TurnRequestProcessor { request_id, thread.as_ref(), Op::RealtimeConversationStart(ConversationStartParams { + architecture: params.architecture, model: params.model, output_modality: params.output_modality, prompt: params.prompt, diff --git a/codex-rs/app-server/tests/suite/v2/experimental_api.rs b/codex-rs/app-server/tests/suite/v2/experimental_api.rs index 03c9d455f..9e45cf256 100644 --- a/codex-rs/app-server/tests/suite/v2/experimental_api.rs +++ b/codex-rs/app-server/tests/suite/v2/experimental_api.rs @@ -79,6 +79,7 @@ async fn realtime_conversation_start_requires_experimental_api_capability() -> R let request_id = mcp .send_thread_realtime_start_request(ThreadRealtimeStartParams { + architecture: None, thread_id: "thr_123".to_string(), model: None, output_modality: RealtimeOutputModality::Audio, @@ -187,6 +188,7 @@ async fn realtime_webrtc_start_requires_experimental_api_capability() -> Result< let request_id = mcp .send_thread_realtime_start_request(ThreadRealtimeStartParams { + architecture: None, thread_id: "thr_123".to_string(), model: None, output_modality: RealtimeOutputModality::Audio, diff --git a/codex-rs/app-server/tests/suite/v2/realtime_conversation.rs b/codex-rs/app-server/tests/suite/v2/realtime_conversation.rs index ddc0eae40..50ed876ec 100644 --- a/codex-rs/app-server/tests/suite/v2/realtime_conversation.rs +++ b/codex-rs/app-server/tests/suite/v2/realtime_conversation.rs @@ -313,6 +313,7 @@ impl RealtimeE2eHarness { let start_request_id = self .mcp .send_thread_realtime_start_request(ThreadRealtimeStartParams { + architecture: None, thread_id: self.thread_id.clone(), model: None, output_modality: RealtimeOutputModality::Audio, @@ -560,6 +561,7 @@ async fn realtime_conversation_streams_v2_notifications() -> Result<()> { let start_request_id = mcp .send_thread_realtime_start_request(ThreadRealtimeStartParams { + architecture: None, thread_id: thread_start.thread.id.clone(), model: Some("realtime-treatment-model".to_string()), output_modality: RealtimeOutputModality::Audio, @@ -815,6 +817,7 @@ async fn realtime_text_output_modality_requests_text_output_and_final_transcript let start_request_id = mcp .send_thread_realtime_start_request(ThreadRealtimeStartParams { + architecture: None, thread_id: thread_start.thread.id.clone(), model: None, output_modality: RealtimeOutputModality::Text, @@ -991,6 +994,7 @@ async fn realtime_conversation_stop_emits_closed_notification() -> Result<()> { let start_request_id = mcp .send_thread_realtime_start_request(ThreadRealtimeStartParams { + architecture: None, thread_id: thread_start.thread.id.clone(), model: None, output_modality: RealtimeOutputModality::Audio, @@ -1090,6 +1094,7 @@ async fn realtime_webrtc_start_emits_sdp_notification() -> Result<()> { let thread_id = thread_start.thread.id; let start_request_id = mcp .send_thread_realtime_start_request(ThreadRealtimeStartParams { + architecture: None, thread_id: thread_id.clone(), model: None, output_modality: RealtimeOutputModality::Audio, @@ -2137,6 +2142,7 @@ async fn realtime_webrtc_start_surfaces_backend_error() -> Result<()> { let start_request_id = mcp .send_thread_realtime_start_request(ThreadRealtimeStartParams { + architecture: None, thread_id: thread_start.thread.id, model: None, output_modality: RealtimeOutputModality::Audio, @@ -2198,6 +2204,7 @@ async fn realtime_conversation_requires_feature_flag() -> Result<()> { let start_request_id = mcp .send_thread_realtime_start_request(ThreadRealtimeStartParams { + architecture: None, thread_id: thread_start.thread.id.clone(), model: None, output_modality: RealtimeOutputModality::Audio, diff --git a/codex-rs/codex-api/src/endpoint/realtime_call.rs b/codex-rs/codex-api/src/endpoint/realtime_call.rs index 227b85c70..e2ade0266 100644 --- a/codex-rs/codex-api/src/endpoint/realtime_call.rs +++ b/codex-rs/codex-api/src/endpoint/realtime_call.rs @@ -6,8 +6,10 @@ use crate::error::ApiError; use crate::provider::Provider; use bytes::Bytes; use codex_client::HttpTransport; +use codex_client::Request; use codex_client::RequestBody; use codex_client::RequestTelemetry; +use codex_protocol::protocol::RealtimeConversationArchitecture; use http::HeaderMap; use http::HeaderValue; use http::Method; @@ -118,6 +120,22 @@ impl RealtimeCallClient { sdp: String, session_config: RealtimeSessionConfig, extra_headers: HeaderMap, + ) -> Result { + self.create_with_session_architecture_and_headers( + sdp, + session_config, + RealtimeConversationArchitecture::RealtimeApi, + extra_headers, + ) + .await + } + + pub async fn create_with_session_architecture_and_headers( + &self, + sdp: String, + session_config: RealtimeSessionConfig, + architecture: RealtimeConversationArchitecture, + extra_headers: HeaderMap, ) -> Result { trace!(target: "codex_api::realtime_websocket::wire", "realtime call request SDP: {sdp}"); // WebRTC can begin inference as soon as the peer connection comes up, so the initial @@ -136,7 +154,13 @@ impl RealtimeCallClient { .map_err(|err| ApiError::Stream(format!("failed to encode realtime call: {err}")))?; let resp = self .session - .execute(Method::POST, Self::path(), extra_headers, Some(body)) + .execute_with( + Method::POST, + Self::path(), + extra_headers, + Some(body), + |req| configure_realtime_call_request(req, architecture), + ) .await?; let sdp = decode_sdp_response(resp.body.as_ref())?; let call_id = decode_call_id_from_location(&resp.headers)?; @@ -167,6 +191,7 @@ impl RealtimeCallClient { extra_headers, /*body*/ None, |req| { + configure_realtime_call_request(req, architecture); req.headers.insert( CONTENT_TYPE, HeaderValue::from_static(MULTIPART_CONTENT_TYPE), @@ -183,6 +208,30 @@ impl RealtimeCallClient { } } +fn configure_realtime_call_request( + request: &mut Request, + architecture: RealtimeConversationArchitecture, +) { + match architecture { + RealtimeConversationArchitecture::RealtimeApi => {} + RealtimeConversationArchitecture::Avas => { + append_query_pair(&mut request.url, "intent", "quicksilver"); + append_query_pair(&mut request.url, "architecture", "avas"); + } + } +} + +fn append_query_pair(url: &mut String, key: &str, value: &str) { + if url.contains('?') { + url.push('&'); + } else { + url.push('?'); + } + url.push_str(key); + url.push('='); + url.push_str(value); +} + fn realtime_session_json(session_config: RealtimeSessionConfig) -> Result { session_update_session_json(session_config) .map_err(|err| ApiError::Stream(format!("failed to encode realtime call session: {err}"))) @@ -209,7 +258,7 @@ fn decode_call_id_from_location(headers: &HeaderMap) -> Result .next() .unwrap_or(location) .rsplit('/') - .find(|segment| segment.starts_with("rtc_") && segment.len() > "rtc_".len()) + .find(|segment| is_realtime_call_id_segment(segment)) .map(str::to_string) .ok_or_else(|| { ApiError::Stream(format!( @@ -218,6 +267,21 @@ fn decode_call_id_from_location(headers: &HeaderMap) -> Result }) } +fn is_realtime_call_id_segment(segment: &str) -> bool { + if segment.starts_with("rtc_") && segment.len() > "rtc_".len() { + return true; + } + + if segment.len() != 36 { + return false; + } + + segment.char_indices().all(|(index, ch)| match index { + 8 | 13 | 18 | 23 => ch == '-', + _ => ch.is_ascii_hexdigit(), + }) +} + #[cfg(test)] mod tests { use super::*; @@ -459,6 +523,41 @@ mod tests { ); } + #[tokio::test] + async fn sends_avas_session_call_query_params() { + let transport = CapturingTransport::new(); + let client = RealtimeCallClient::new( + transport.clone(), + provider("https://api.openai.com/v1"), + Arc::new(DummyAuth), + ); + + let response = client + .create_with_session_architecture_and_headers( + "v=offer\r\n".to_string(), + realtime_session_config("sess-api"), + RealtimeConversationArchitecture::Avas, + HeaderMap::new(), + ) + .await + .expect("request should succeed"); + + assert_eq!( + response, + RealtimeCallResponse { + sdp: "v=0\r\n".to_string(), + call_id: "rtc_test".to_string(), + } + ); + + let request = transport.last_request.lock().unwrap().clone().unwrap(); + assert_eq!(request.method, Method::POST); + assert_eq!( + request.url, + "https://api.openai.com/v1/realtime/calls?intent=quicksilver&architecture=avas" + ); + } + #[tokio::test] async fn sends_backend_session_call_as_json_body() { let transport = CapturingTransport::new(); @@ -541,4 +640,17 @@ mod tests { "stream error: realtime call Location does not contain a call id: /v1/realtime/calls" ); } + + #[test] + fn accepts_uuid_call_id_from_location() { + let mut headers = HeaderMap::new(); + headers.insert( + LOCATION, + HeaderValue::from_static("/v1/realtime/calls/019eb97d-8e9a-7ff3-94b0-ea019babd5d7"), + ); + + let call_id = decode_call_id_from_location(&headers).expect("UUID call id should parse"); + + assert_eq!(call_id, "019eb97d-8e9a-7ff3-94b0-ea019babd5d7"); + } } diff --git a/codex-rs/config/src/config_toml.rs b/codex-rs/config/src/config_toml.rs index 7c060cadb..7270c3f32 100644 --- a/codex-rs/config/src/config_toml.rs +++ b/codex-rs/config/src/config_toml.rs @@ -372,6 +372,10 @@ pub struct ConfigToml { /// `/v1/realtime` /// connection) without changing normal provider HTTP requests. pub experimental_realtime_ws_base_url: Option, + /// Experimental / do not use. Overrides only the WebRTC realtime call + /// creation base URL. This is separate from `experimental_realtime_ws_base_url` + /// because WebRTC call creation is HTTP, while sideband control is websocket. + pub experimental_realtime_webrtc_call_base_url: Option, /// Experimental / do not use. Selects the realtime websocket model/snapshot /// used for the `Op::RealtimeConversation` connection. pub experimental_realtime_ws_model: Option, @@ -583,12 +587,14 @@ pub enum RealtimeTransport { Websocket, } +pub use codex_protocol::protocol::RealtimeConversationArchitecture as RealtimeArchitecture; pub use codex_protocol::protocol::RealtimeConversationVersion as RealtimeWsVersion; pub use codex_protocol::protocol::RealtimeVoice; #[derive(Serialize, Deserialize, Debug, Clone, Default, PartialEq, Eq, JsonSchema)] #[schemars(deny_unknown_fields)] pub struct RealtimeConfig { + pub architecture: RealtimeArchitecture, pub version: RealtimeWsVersion, #[serde(rename = "type")] pub session_type: RealtimeWsMode, @@ -599,6 +605,7 @@ pub struct RealtimeConfig { #[derive(Serialize, Deserialize, Debug, Clone, Default, PartialEq, Eq, JsonSchema)] #[schemars(deny_unknown_fields)] pub struct RealtimeToml { + pub architecture: Option, pub version: Option, #[serde(rename = "type")] pub session_type: Option, diff --git a/codex-rs/config/src/loader/mod.rs b/codex-rs/config/src/loader/mod.rs index d47ad50b2..d154f87e1 100644 --- a/codex-rs/config/src/loader/mod.rs +++ b/codex-rs/config/src/loader/mod.rs @@ -68,6 +68,7 @@ const PROJECT_LOCAL_CONFIG_DENYLIST: &[&str] = &[ "notify", "profile", "profiles", + "experimental_realtime_webrtc_call_base_url", "experimental_realtime_ws_base_url", "otel", ]; diff --git a/codex-rs/core/config.schema.json b/codex-rs/core/config.schema.json index 78d667a63..068f482c7 100644 --- a/codex-rs/core/config.schema.json +++ b/codex-rs/core/config.schema.json @@ -2450,6 +2450,13 @@ }, "type": "object" }, + "RealtimeConversationArchitecture": { + "enum": [ + "realtimeapi", + "avas" + ], + "type": "string" + }, "RealtimeConversationVersion": { "enum": [ "v1", @@ -2460,6 +2467,9 @@ "RealtimeToml": { "additionalProperties": false, "properties": { + "architecture": { + "$ref": "#/definitions/RealtimeConversationArchitecture" + }, "transport": { "$ref": "#/definitions/RealtimeTransport" }, @@ -4496,6 +4506,10 @@ "description": "Experimental / do not use. Replaces the built-in realtime start instructions inserted into developer messages when realtime becomes active.", "type": "string" }, + "experimental_realtime_webrtc_call_base_url": { + "description": "Experimental / do not use. Overrides only the WebRTC realtime call creation base URL. This is separate from `experimental_realtime_ws_base_url` because WebRTC call creation is HTTP, while sideband control is websocket.", + "type": "string" + }, "experimental_realtime_ws_backend_prompt": { "description": "Experimental / do not use. Overrides only the realtime conversation websocket transport instructions (the `Op::RealtimeConversation` `/ws` session.update instructions) without changing normal prompts.", "type": "string" @@ -5254,4 +5268,4 @@ }, "title": "ConfigToml", "type": "object" -} \ No newline at end of file +} diff --git a/codex-rs/core/src/client.rs b/codex-rs/core/src/client.rs index 75ed2039a..6c4058bb0 100644 --- a/codex-rs/core/src/client.rs +++ b/codex-rs/core/src/client.rs @@ -77,6 +77,7 @@ use codex_protocol::models::ResponseItem; use codex_protocol::openai_models::ModelInfo; use codex_protocol::openai_models::ReasoningEffort as ReasoningEffortConfig; use codex_protocol::protocol::InternalSessionSource; +use codex_protocol::protocol::RealtimeConversationArchitecture; use codex_protocol::protocol::SessionSource; use codex_protocol::protocol::W3cTraceContext; use codex_rollout_trace::CompactionTraceContext; @@ -522,7 +523,9 @@ impl ModelClient { &self, sdp: String, session_config: ApiRealtimeSessionConfig, + architecture: RealtimeConversationArchitecture, mut extra_headers: ApiHeaderMap, + api_provider_override: Option, ) -> Result { // Create the media call over HTTP first, then retain matching auth so realtime can attach // the server-side control WebSocket to the call id from that HTTP response. @@ -535,11 +538,16 @@ impl ModelClient { client_setup.api_auth.as_ref(), )); let transport = ReqwestTransport::new(build_reqwest_client()); - let response = - ApiRealtimeCallClient::new(transport, client_setup.api_provider, client_setup.api_auth) - .create_with_session_and_headers(sdp, session_config, extra_headers) - .await - .map_err(map_api_error)?; + let api_provider = api_provider_override.unwrap_or(client_setup.api_provider); + let response = ApiRealtimeCallClient::new(transport, api_provider, client_setup.api_auth) + .create_with_session_architecture_and_headers( + sdp, + session_config, + architecture, + extra_headers, + ) + .await + .map_err(map_api_error)?; Ok(RealtimeWebrtcCallStart { sdp: response.sdp, call_id: response.call_id, diff --git a/codex-rs/core/src/config/config_tests.rs b/codex-rs/core/src/config/config_tests.rs index 9072cff3c..f422591d8 100644 --- a/codex-rs/core/src/config/config_tests.rs +++ b/codex-rs/core/src/config/config_tests.rs @@ -12,6 +12,7 @@ use codex_config::config_toml::AutoReviewToml; use codex_config::config_toml::ConfigToml; use codex_config::config_toml::ExperimentalRequestUserInput; use codex_config::config_toml::ProjectConfig; +use codex_config::config_toml::RealtimeArchitecture; use codex_config::config_toml::RealtimeConfig; use codex_config::config_toml::RealtimeToml; use codex_config::config_toml::RealtimeTransport; @@ -10488,8 +10489,8 @@ experimental_thread_config_endpoint = "http://127.0.0.1:8061" #[tokio::test] async fn experimental_realtime_ws_base_url_loads_from_config_toml() -> std::io::Result<()> { let cfg: ConfigToml = toml::from_str( - r#" -experimental_realtime_ws_base_url = "http://127.0.0.1:8011" + r#"experimental_realtime_ws_base_url = "http://127.0.0.1:8011" +experimental_realtime_webrtc_call_base_url = "http://127.0.0.1:8082/v1" "#, ) .expect("TOML deserialization should succeed"); @@ -10498,7 +10499,10 @@ experimental_realtime_ws_base_url = "http://127.0.0.1:8011" cfg.experimental_realtime_ws_base_url.as_deref(), Some("http://127.0.0.1:8011") ); - + assert_eq!( + cfg.experimental_realtime_webrtc_call_base_url.as_deref(), + Some("http://127.0.0.1:8082/v1") + ); let codex_home = TempDir::new()?; let config = Config::load_from_base_config_with_overrides( cfg, @@ -10511,6 +10515,10 @@ experimental_realtime_ws_base_url = "http://127.0.0.1:8011" config.experimental_realtime_ws_base_url.as_deref(), Some("http://127.0.0.1:8011") ); + assert_eq!( + config.experimental_realtime_webrtc_call_base_url.as_deref(), + Some("http://127.0.0.1:8082/v1") + ); Ok(()) } @@ -10634,6 +10642,7 @@ async fn realtime_loads_from_config_toml() -> std::io::Result<()> { let cfg: ConfigToml = toml::from_str( r#" [realtime] +architecture = "avas" version = "v2" type = "transcription" transport = "webrtc" @@ -10645,6 +10654,7 @@ voice = "cedar" assert_eq!( cfg.realtime, Some(RealtimeToml { + architecture: Some(RealtimeArchitecture::Avas), version: Some(RealtimeWsVersion::V2), session_type: Some(RealtimeWsMode::Transcription), transport: Some(RealtimeTransport::WebRtc), @@ -10663,6 +10673,7 @@ voice = "cedar" assert_eq!( config.realtime, RealtimeConfig { + architecture: RealtimeArchitecture::Avas, version: RealtimeWsVersion::V2, session_type: RealtimeWsMode::Transcription, transport: RealtimeTransport::WebRtc, diff --git a/codex-rs/core/src/config/mod.rs b/codex-rs/core/src/config/mod.rs index 1c3942853..cdb3aca04 100644 --- a/codex-rs/core/src/config/mod.rs +++ b/codex-rs/core/src/config/mod.rs @@ -945,6 +945,9 @@ pub struct Config { /// `/v1/realtime` /// connection) without changing normal provider HTTP requests. pub experimental_realtime_ws_base_url: Option, + /// Experimental / do not use. Overrides only the WebRTC realtime call + /// creation base URL. + pub experimental_realtime_webrtc_call_base_url: Option, /// Experimental / do not use. Selects the realtime websocket model/snapshot /// used for the `Op::RealtimeConversation` connection. pub experimental_realtime_ws_model: Option, @@ -3549,12 +3552,15 @@ impl Config { speaker: audio.speaker, }), experimental_realtime_ws_base_url: cfg.experimental_realtime_ws_base_url, + experimental_realtime_webrtc_call_base_url: cfg + .experimental_realtime_webrtc_call_base_url, experimental_realtime_ws_model: cfg.experimental_realtime_ws_model, realtime: cfg .realtime .map_or_else(RealtimeConfig::default, |realtime| { let defaults = RealtimeConfig::default(); RealtimeConfig { + architecture: realtime.architecture.unwrap_or(defaults.architecture), version: realtime.version.unwrap_or(defaults.version), session_type: realtime.session_type.unwrap_or(defaults.session_type), transport: realtime.transport.unwrap_or(defaults.transport), diff --git a/codex-rs/core/src/realtime_conversation.rs b/codex-rs/core/src/realtime_conversation.rs index 33568f6f3..c14bf0af5 100644 --- a/codex-rs/core/src/realtime_conversation.rs +++ b/codex-rs/core/src/realtime_conversation.rs @@ -38,6 +38,7 @@ use codex_protocol::protocol::ConversationTextParams; use codex_protocol::protocol::ErrorEvent; use codex_protocol::protocol::Event; use codex_protocol::protocol::EventMsg; +use codex_protocol::protocol::RealtimeConversationArchitecture; use codex_protocol::protocol::RealtimeConversationClosedEvent; use codex_protocol::protocol::RealtimeConversationRealtimeEvent; use codex_protocol::protocol::RealtimeConversationSdpEvent; @@ -232,7 +233,9 @@ struct ConversationState { struct RealtimeStart { api_provider: ApiProvider, + architecture: RealtimeConversationArchitecture, extra_headers: Option, + realtime_call_api_provider: Option, session_config: RealtimeSessionConfig, model_client: ModelClient, sdp: Option, @@ -284,7 +287,9 @@ impl RealtimeConversationManager { async fn start_inner(&self, start: RealtimeStart) -> CodexResult { let RealtimeStart { api_provider, + architecture, extra_headers, + realtime_call_api_provider, session_config, model_client, sdp, @@ -318,7 +323,9 @@ impl RealtimeConversationManager { .create_realtime_call_with_headers( sdp, session_config.clone(), + architecture, extra_headers.unwrap_or_default(), + realtime_call_api_provider, ) .await?; let task = spawn_webrtc_sideband_input_task(RealtimeWebrtcSidebandInputTask { @@ -613,7 +620,9 @@ pub(crate) async fn handle_start( struct PreparedRealtimeConversationStart { api_provider: ApiProvider, + architecture: RealtimeConversationArchitecture, extra_headers: Option, + realtime_call_api_provider: Option, requested_realtime_session_id: Option, version: RealtimeWsVersion, session_config: RealtimeSessionConfig, @@ -639,7 +648,23 @@ async fn prepare_realtime_start( if let Some(realtime_ws_base_url) = &config.experimental_realtime_ws_base_url { api_provider.base_url = realtime_ws_base_url.clone(); } + let realtime_call_api_provider = + if let Some(realtime_call_base_url) = &config.experimental_realtime_webrtc_call_base_url { + let mut api_provider = provider.to_api_provider(Some(AuthMode::ApiKey))?; + api_provider.base_url = realtime_call_base_url.clone(); + Some(api_provider) + } else { + None + }; let version = params.version.unwrap_or(config.realtime.version); + // TODO(pbakkum): Remove the realtimeapi/AVAS branch once WebRTC realtime sessions always use AVAS. + let architecture = params.architecture.unwrap_or(config.realtime.architecture); + validate_realtime_architecture( + architecture, + version, + &transport, + config.realtime.session_type, + )?; let session_config = build_realtime_session_config( sess, params.model, @@ -670,7 +695,9 @@ async fn prepare_realtime_start( }; Ok(PreparedRealtimeConversationStart { api_provider, + architecture, extra_headers, + realtime_call_api_provider, requested_realtime_session_id, version, session_config, @@ -678,6 +705,33 @@ async fn prepare_realtime_start( }) } +fn validate_realtime_architecture( + architecture: RealtimeConversationArchitecture, + version: RealtimeWsVersion, + transport: &ConversationStartTransport, + session_type: RealtimeWsMode, +) -> CodexResult<()> { + if architecture != RealtimeConversationArchitecture::Avas { + return Ok(()); + } + if version != RealtimeWsVersion::V1 { + return Err(CodexErr::InvalidRequest( + "AVAS realtime architecture requires realtime v1".to_string(), + )); + } + if !matches!(transport, ConversationStartTransport::Webrtc { .. }) { + return Err(CodexErr::InvalidRequest( + "AVAS realtime architecture requires WebRTC transport".to_string(), + )); + } + if session_type != RealtimeWsMode::Conversational { + return Err(CodexErr::InvalidRequest( + "AVAS realtime architecture requires conversational realtime".to_string(), + )); + } + Ok(()) +} + pub(crate) async fn build_realtime_session_config( sess: &Arc, model: Option, @@ -786,7 +840,9 @@ async fn handle_start_inner( ) -> CodexResult<()> { let PreparedRealtimeConversationStart { api_provider, + architecture, extra_headers, + realtime_call_api_provider, requested_realtime_session_id, version, session_config, @@ -799,7 +855,9 @@ async fn handle_start_inner( }; let start = RealtimeStart { api_provider, + architecture, extra_headers, + realtime_call_api_provider, session_config, model_client: sess.services.model_client.clone(), sdp, diff --git a/codex-rs/core/tests/suite/compact_remote.rs b/codex-rs/core/tests/suite/compact_remote.rs index a63397c55..3345cd72f 100644 --- a/codex-rs/core/tests/suite/compact_remote.rs +++ b/codex-rs/core/tests/suite/compact_remote.rs @@ -200,6 +200,7 @@ async fn start_remote_realtime_server() -> responses::WebSocketTestServer { async fn start_realtime_conversation(codex: &codex_core::CodexThread) -> Result<()> { codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + architecture: None, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), diff --git a/codex-rs/core/tests/suite/realtime_conversation.rs b/codex-rs/core/tests/suite/realtime_conversation.rs index 25f0516b0..387096944 100644 --- a/codex-rs/core/tests/suite/realtime_conversation.rs +++ b/codex-rs/core/tests/suite/realtime_conversation.rs @@ -18,6 +18,7 @@ use codex_protocol::protocol::EventMsg; use codex_protocol::protocol::InitialHistory; use codex_protocol::protocol::Op; use codex_protocol::protocol::RealtimeAudioFrame; +use codex_protocol::protocol::RealtimeConversationArchitecture; use codex_protocol::protocol::RealtimeConversationRealtimeEvent; use codex_protocol::protocol::RealtimeConversationVersion; use codex_protocol::protocol::RealtimeEvent; @@ -282,6 +283,7 @@ async fn conversation_start_audio_text_close_round_trip() -> Result<()> { test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + architecture: None, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -422,6 +424,7 @@ async fn conversation_start_defaults_to_v2_and_gpt_realtime_1_5() -> Result<()> test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + architecture: None, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -512,6 +515,7 @@ async fn conversation_webrtc_start_posts_generated_session() -> Result<()> { test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + architecture: None, model: Some("session-override-model".to_string()), output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -650,6 +654,200 @@ async fn conversation_webrtc_start_posts_generated_session() -> Result<()> { Ok(()) } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn conversation_webrtc_start_uses_avas_architecture_query() -> Result<()> { + skip_if_no_network!(Ok(())); + + let server = start_mock_server().await; + let capture = RealtimeCallRequestCapture::new(); + Mock::given(method("POST")) + .and(path_regex(".*/realtime/calls$")) + .and(capture.clone()) + .respond_with( + ResponseTemplate::new(200) + .insert_header("Location", "/v1/realtime/calls/calls/rtc_avas_test") + .set_body_string("v=answer\r\n"), + ) + .mount(&server) + .await; + let realtime_server = start_websocket_server_with_headers(vec![WebSocketConnectionConfig { + requests: vec![ + vec![json!({ + "type": "session.updated", + "session": { "id": "sess_webrtc", "instructions": "backend prompt" } + })], + vec![], + ], + response_headers: Vec::new(), + accept_delay: None, + close_after_requests: false, + }]) + .await; + + let realtime_ws_base_url = realtime_server.uri().to_string(); + let mut builder = test_codex().with_config(move |config| { + config.experimental_realtime_ws_backend_prompt = Some("backend prompt".to_string()); + config.experimental_realtime_ws_base_url = Some(realtime_ws_base_url); + config.realtime.version = RealtimeWsVersion::V1; + }); + let test = builder.build(&server).await?; + + test.codex + .submit(Op::RealtimeConversationStart(ConversationStartParams { + architecture: Some(RealtimeConversationArchitecture::Avas), + model: None, + output_modality: RealtimeOutputModality::Audio, + prompt: Some(Some("backend prompt".to_string())), + realtime_session_id: None, + transport: Some(ConversationStartTransport::Webrtc { + sdp: "v=offer\r\n".to_string(), + }), + version: None, + voice: None, + })) + .await?; + + let created = wait_for_event_match(&test.codex, |msg| match msg { + EventMsg::RealtimeConversationSdp(created) => Some(Ok(created.clone())), + EventMsg::Error(err) => Some(Err(err.clone())), + _ => None, + }) + .await + .unwrap_or_else(|err: ErrorEvent| panic!("conversation call create failed: {err:?}")); + assert_eq!(created.sdp, "v=answer\r\n"); + + let request = capture.single_request(); + assert_eq!(request.url.path(), "/v1/realtime/calls"); + assert_eq!( + request.url.query(), + Some("intent=quicksilver&architecture=avas") + ); + + let session_updated = wait_for_event_match(&test.codex, |msg| match msg { + EventMsg::RealtimeConversationRealtime(RealtimeConversationRealtimeEvent { + payload: + RealtimeEvent::SessionUpdated { + realtime_session_id, + .. + }, + }) => Some(realtime_session_id.clone()), + _ => None, + }) + .await; + assert_eq!(session_updated, "sess_webrtc"); + let handshake = realtime_server.single_handshake(); + assert_eq!( + handshake.uri(), + "/v1/realtime?intent=quicksilver&call_id=rtc_avas_test" + ); + assert_eq!( + handshake.header("authorization").as_deref(), + Some("Bearer dummy") + ); + + test.codex.submit(Op::RealtimeConversationClose).await?; + realtime_server.shutdown().await; + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn conversation_webrtc_start_uses_configured_call_base_url_for_avas() -> Result<()> { + skip_if_no_network!(Ok(())); + + let server = start_mock_server().await; + let capture = RealtimeCallRequestCapture::new(); + Mock::given(method("POST")) + .and(path_regex(".*/realtime/calls$")) + .and(capture.clone()) + .respond_with( + ResponseTemplate::new(200) + .insert_header("Location", "/v1/realtime/rtc_local_avas_test") + .set_body_string("v=answer\r\n"), + ) + .mount(&server) + .await; + let realtime_server = start_websocket_server_with_headers(vec![WebSocketConnectionConfig { + requests: vec![ + vec![json!({ + "type": "session.updated", + "session": { "id": "sess_webrtc", "instructions": "backend prompt" } + })], + vec![], + ], + response_headers: Vec::new(), + accept_delay: None, + close_after_requests: false, + }]) + .await; + + let realtime_ws_base_url = realtime_server.uri().to_string(); + let realtime_call_base_url = format!("{}/v1", server.uri()); + let mut builder = test_codex().with_config(move |config| { + config.experimental_realtime_ws_backend_prompt = Some("backend prompt".to_string()); + config.experimental_realtime_ws_base_url = Some(realtime_ws_base_url); + config.experimental_realtime_webrtc_call_base_url = Some(realtime_call_base_url); + config.realtime.version = RealtimeWsVersion::V1; + }); + let test = builder.build(&server).await?; + + test.codex + .submit(Op::RealtimeConversationStart(ConversationStartParams { + architecture: Some(RealtimeConversationArchitecture::Avas), + model: None, + output_modality: RealtimeOutputModality::Audio, + prompt: Some(Some("backend prompt".to_string())), + realtime_session_id: None, + transport: Some(ConversationStartTransport::Webrtc { + sdp: "v=offer\r\n".to_string(), + }), + version: None, + voice: None, + })) + .await?; + + let created = wait_for_event_match(&test.codex, |msg| match msg { + EventMsg::RealtimeConversationSdp(created) => Some(Ok(created.clone())), + EventMsg::Error(err) => Some(Err(err.clone())), + _ => None, + }) + .await + .unwrap_or_else(|err: ErrorEvent| panic!("conversation call create failed: {err:?}")); + assert_eq!(created.sdp, "v=answer\r\n"); + + let request = capture.single_request(); + assert_eq!(request.url.path(), "/v1/realtime/calls"); + assert_eq!( + request.url.query(), + Some("intent=quicksilver&architecture=avas") + ); + + let session_updated = wait_for_event_match(&test.codex, |msg| match msg { + EventMsg::RealtimeConversationRealtime(RealtimeConversationRealtimeEvent { + payload: + RealtimeEvent::SessionUpdated { + realtime_session_id, + .. + }, + }) => Some(realtime_session_id.clone()), + _ => None, + }) + .await; + assert_eq!(session_updated, "sess_webrtc"); + let handshake = realtime_server.single_handshake(); + assert_eq!( + handshake.uri(), + "/v1/realtime?intent=quicksilver&call_id=rtc_local_avas_test" + ); + assert_eq!( + handshake.header("authorization").as_deref(), + Some("Bearer dummy") + ); + + test.codex.submit(Op::RealtimeConversationClose).await?; + realtime_server.shutdown().await; + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn conversation_webrtc_close_while_sideband_connecting_drops_pending_join() -> Result<()> { skip_if_no_network!(Ok(())); @@ -684,6 +882,7 @@ async fn conversation_webrtc_close_while_sideband_connecting_drops_pending_join( test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + architecture: None, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -770,6 +969,7 @@ async fn conversation_webrtc_sideband_connect_failure_closes_with_error() -> Res test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + architecture: None, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -857,6 +1057,7 @@ async fn conversation_start_uses_openai_env_key_fallback_with_chatgpt_auth() -> test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + architecture: None, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -925,6 +1126,7 @@ async fn conversation_transport_close_emits_closed_event() -> Result<()> { test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + architecture: None, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -1017,6 +1219,7 @@ async fn conversation_start_preflight_failure_emits_realtime_error_only() -> Res test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + architecture: None, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -1063,6 +1266,7 @@ async fn conversation_start_connect_failure_emits_realtime_error_only() -> Resul test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + architecture: None, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -1156,6 +1360,7 @@ async fn conversation_second_start_replaces_runtime() -> Result<()> { test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + architecture: None, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("old".to_string())), @@ -1181,6 +1386,7 @@ async fn conversation_second_start_replaces_runtime() -> Result<()> { test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + architecture: None, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("new".to_string())), @@ -1277,6 +1483,7 @@ async fn conversation_uses_experimental_realtime_ws_base_url_override() -> Resul test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + architecture: None, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -1341,6 +1548,7 @@ async fn conversation_uses_default_realtime_backend_prompt() -> Result<()> { test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + architecture: None, model: None, output_modality: RealtimeOutputModality::Audio, prompt: None, @@ -1413,6 +1621,7 @@ async fn conversation_uses_empty_instructions_for_null_or_empty_prompt() -> Resu ] { test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + architecture: None, model: None, output_modality: RealtimeOutputModality::Audio, prompt, @@ -1478,6 +1687,7 @@ async fn conversation_uses_explicit_start_voice() -> Result<()> { test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + architecture: None, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -1535,6 +1745,7 @@ async fn conversation_uses_configured_realtime_voice() -> Result<()> { test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + architecture: None, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -1580,6 +1791,7 @@ async fn conversation_rejects_voice_for_wrong_realtime_version() -> Result<()> { test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + architecture: None, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -1626,6 +1838,7 @@ async fn conversation_uses_experimental_realtime_ws_backend_prompt_override() -> test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + architecture: None, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("prompt from op".to_string())), @@ -1698,6 +1911,7 @@ async fn conversation_uses_experimental_realtime_ws_startup_context_override() - test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + architecture: None, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("prompt from op".to_string())), @@ -1764,6 +1978,7 @@ async fn conversation_disables_realtime_startup_context_with_empty_override() -> test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + architecture: None, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("prompt from op".to_string())), @@ -1823,6 +2038,7 @@ async fn conversation_start_injects_startup_context_from_thread_history() -> Res test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + architecture: None, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -1934,6 +2150,7 @@ async fn conversation_startup_context_current_thread_selects_many_turns_by_budge codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + architecture: None, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -2041,6 +2258,7 @@ async fn conversation_startup_context_falls_back_to_workspace_map() -> Result<() test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + architecture: None, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -2100,6 +2318,7 @@ async fn conversation_startup_context_is_truncated_and_sent_once_per_start() -> test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + architecture: None, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -2179,6 +2398,7 @@ async fn conversation_user_text_turn_is_not_sent_to_realtime() -> Result<()> { test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + architecture: None, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -2275,6 +2495,7 @@ async fn realtime_v2_noop_tool_call_returns_empty_function_output_without_respon test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + architecture: None, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -2373,6 +2594,7 @@ async fn conversation_mirrors_assistant_message_text_to_realtime_handoff() -> Re test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + architecture: None, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -2509,6 +2731,7 @@ async fn conversation_handoff_persists_across_item_done_until_turn_complete() -> test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + architecture: None, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -2660,6 +2883,7 @@ async fn inbound_handoff_request_starts_turn() -> Result<()> { test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + architecture: None, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -2761,6 +2985,7 @@ async fn inbound_handoff_request_uses_active_transcript() -> Result<()> { test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + architecture: None, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -2863,6 +3088,7 @@ async fn inbound_handoff_request_sends_transcript_delta_after_each_handoff() -> test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + architecture: None, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -2963,6 +3189,7 @@ async fn inbound_conversation_item_does_not_start_turn_and_still_forwards_audio( test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + architecture: None, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -3085,6 +3312,7 @@ async fn delegated_turn_user_role_echo_does_not_redelegate_and_still_forwards_au test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + architecture: None, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -3237,6 +3465,7 @@ async fn inbound_handoff_request_does_not_block_realtime_event_forwarding() -> R test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + architecture: None, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -3373,6 +3602,7 @@ async fn inbound_handoff_request_steers_active_turn() -> Result<()> { test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + architecture: None, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -3525,6 +3755,7 @@ async fn inbound_handoff_request_starts_turn_and_does_not_block_realtime_audio() test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + architecture: None, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), diff --git a/codex-rs/protocol/src/protocol.rs b/codex-rs/protocol/src/protocol.rs index 845cb730d..07ebd3026 100644 --- a/codex-rs/protocol/src/protocol.rs +++ b/codex-rs/protocol/src/protocol.rs @@ -185,6 +185,8 @@ pub struct McpServerRefreshConfig { #[derive(Debug, Clone, PartialEq)] pub struct ConversationStartParams { + /// Overrides the configured realtime architecture for this session only. + pub architecture: Option, /// Overrides the configured realtime model for this session only. pub model: Option, /// Selects whether the realtime session should produce text or audio output. @@ -1495,6 +1497,15 @@ pub enum RealtimeConversationVersion { V2, } +#[derive(Debug, Clone, Copy, Default, Deserialize, Serialize, PartialEq, Eq, JsonSchema, TS)] +#[serde(rename_all = "snake_case")] +pub enum RealtimeConversationArchitecture { + #[default] + #[serde(rename = "realtimeapi")] + RealtimeApi, + Avas, +} + #[derive(Debug, Clone, Deserialize, Serialize, PartialEq, JsonSchema, TS)] pub struct RealtimeConversationStartedEvent { pub realtime_session_id: Option, diff --git a/codex-rs/thread-manager-sample/src/main.rs b/codex-rs/thread-manager-sample/src/main.rs index 5c8aae8b3..608c9a90a 100644 --- a/codex-rs/thread-manager-sample/src/main.rs +++ b/codex-rs/thread-manager-sample/src/main.rs @@ -258,6 +258,7 @@ fn new_config(model: Option, arg0_paths: Arg0DispatchPaths) -> anyhow::R apps_mcp_product_sku: None, realtime_audio: RealtimeAudioConfig::default(), experimental_realtime_ws_base_url: None, + experimental_realtime_webrtc_call_base_url: None, experimental_realtime_ws_model: None, realtime: RealtimeConfig::default(), experimental_realtime_ws_backend_prompt: None,