agent/codex
1
0
Fork 0
mirror of https://github.com/pchuan98/codex.git synced 2026-07-01 00:31:56 +08:00
Code Issues Packages Projects Releases Wiki Activity

realtime: add AVAS architecture override (#27720)

Browse Source 
## Summary

Adds a `RealtimeConversationArchitecture` option for realtime
conversation startup, with `realtimeapi` as the default and `avas` as an
opt-in architecture.

The AVAS path is limited to realtime v1 conversational WebRTC starts,
and WebRTC call creation appends `intent=quicksilver&architecture=avas`
to `/v1/realtime/calls`. The existing sideband websocket still joins by
`call_id`.

This also exposes the per-session architecture override through
app-server v2 `thread/realtime/start` params and updates the config
schema for `[realtime].architecture`.

## Validation

- `just fmt`
- `just write-config-schema`
- `just test -p codex-api sends_avas_session_call_query_params`
- `just test -p codex-core -E
'test(~conversation_webrtc_start_uses_avas_architecture_query)'`
- `just test -p codex-core -E 'test(realtime_loads_from_config_toml)'`
- `just test -p codex-app-server-protocol -E
'test(~serialize_thread_realtime_start) |
test(generated_ts_optional_nullable_fields_only_in_params)'`
- `just test -p codex-app-server -E
'test(realtime_webrtc_start_emits_sdp_notification)'`
This commit is contained in:
Peter Bakkum
2026-06-12 11:11:13 -07:00
committed by GitHub
Unverified
parent 94427aaf46
commit 6652e82dd0
22 changed files with 521 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
codex-rs/app-server-protocol/schema/json/ClientRequest.json
Generated
+7
View File
@@ -1993,6 +1993,13 @@
],
"type": "object"
},
"RealtimeConversationArchitecture": {
"enum": [
"realtimeapi",
"avas"
],
"type": "string"
},
"RealtimeConversationVersion": {
"enum": [
"v1",
codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json
Generated
+7
View File
@@ -13799,6 +13799,13 @@
"title": "RawResponseItemCompletedNotification",
"type": "object"
},
"RealtimeConversationArchitecture": {
"enum": [
"realtimeapi",
"avas"
],
"type": "string"
},
"RealtimeConversationVersion": {
"enum": [
"v1",
codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json
Generated
+7
View File
@@ -10272,6 +10272,13 @@
"title": "RawResponseItemCompletedNotification",
"type": "object"
},
"RealtimeConversationArchitecture": {
"enum": [
"realtimeapi",
"avas"
],
"type": "string"
},
"RealtimeConversationVersion": {
"enum": [
"v1",
codex-rs/app-server-protocol/schema/typescript/RealtimeConversationArchitecture.ts
Generated
+5
View File
@@ -0,0 +1,5 @@
// GENERATED CODE! DO NOT MODIFY BY HAND!
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
export type RealtimeConversationArchitecture = "realtimeapi" | "avas";
codex-rs/app-server-protocol/schema/typescript/index.ts
Generated
+1
View File
@@ -51,6 +51,7 @@ export type { NetworkPolicyRuleAction } from "./NetworkPolicyRuleAction";
export type { ParsedCommand } from "./ParsedCommand";
export type { Personality } from "./Personality";
export type { PlanType } from "./PlanType";
export type { RealtimeConversationArchitecture } from "./RealtimeConversationArchitecture";
export type { RealtimeConversationVersion } from "./RealtimeConversationVersion";
export type { RealtimeOutputModality } from "./RealtimeOutputModality";
export type { RealtimeVoice } from "./RealtimeVoice";
codex-rs/app-server-protocol/src/protocol/common.rs
+8
View File
@@ -1647,6 +1647,7 @@ mod tests {
use codex_protocol::account::PlanType;
use codex_protocol::models::BUILT_IN_PERMISSION_PROFILE_READ_ONLY;
use codex_protocol::parse_command::ParsedCommand;
use codex_protocol::protocol::RealtimeConversationArchitecture;
use codex_protocol::protocol::RealtimeConversationVersion;
use codex_protocol::protocol::RealtimeOutputModality;
use codex_protocol::protocol::RealtimeVoice;
@@ -3012,6 +3013,7 @@ mod tests {
let request = ClientRequest::ThreadRealtimeStart {
request_id: RequestId::Integer(9),
params: v2::ThreadRealtimeStartParams {
architecture: Some(RealtimeConversationArchitecture::Avas),
thread_id: "thr_123".to_string(),
model: Some("realtime-treatment-model".to_string()),
output_modality: RealtimeOutputModality::Audio,
@@ -3027,6 +3029,7 @@ mod tests {
"method": "thread/realtime/start",
"id": 9,
"params": {
"architecture": "avas",
"threadId": "thr_123",
"model": "realtime-treatment-model",
"outputModality": "audio",
@@ -3047,6 +3050,7 @@ mod tests {
let default_prompt_request = ClientRequest::ThreadRealtimeStart {
request_id: RequestId::Integer(9),
params: v2::ThreadRealtimeStartParams {
architecture: None,
thread_id: "thr_123".to_string(),
model: None,
output_modality: RealtimeOutputModality::Audio,
@@ -3062,6 +3066,7 @@ mod tests {
"method": "thread/realtime/start",
"id": 9,
"params": {
"architecture": null,
"threadId": "thr_123",
"model": null,
"outputModality": "audio",
@@ -3077,6 +3082,7 @@ mod tests {
let null_prompt_request = ClientRequest::ThreadRealtimeStart {
request_id: RequestId::Integer(9),
params: v2::ThreadRealtimeStartParams {
architecture: None,
thread_id: "thr_123".to_string(),
model: None,
output_modality: RealtimeOutputModality::Audio,
@@ -3092,6 +3098,7 @@ mod tests {
"method": "thread/realtime/start",
"id": 9,
"params": {
"architecture": null,
"threadId": "thr_123",
"model": null,
"outputModality": "audio",
@@ -3250,6 +3257,7 @@ mod tests {
let request = ClientRequest::ThreadRealtimeStart {
request_id: RequestId::Integer(1),
params: v2::ThreadRealtimeStartParams {
architecture: None,
thread_id: "thr_123".to_string(),
model: None,
output_modality: RealtimeOutputModality::Audio,
codex-rs/app-server-protocol/src/protocol/v2/realtime.rs
+4
View File
@@ -1,4 +1,5 @@
use codex_protocol::protocol::RealtimeAudioFrame as CoreRealtimeAudioFrame;
use codex_protocol::protocol::RealtimeConversationArchitecture;
use codex_protocol::protocol::RealtimeConversationVersion;
use codex_protocol::protocol::RealtimeOutputModality;
use codex_protocol::protocol::RealtimeVoice;
@@ -65,6 +66,9 @@ impl From<ThreadRealtimeAudioChunk> for CoreRealtimeAudioFrame {
#[ts(export_to = "v2/")]
pub struct ThreadRealtimeStartParams {
pub thread_id: String,
/// Overrides the configured realtime architecture for this session only.
#[ts(optional = nullable)]
pub architecture: Option<RealtimeConversationArchitecture>,
/// Overrides the configured realtime model for this session only.
#[ts(optional = nullable)]
pub model: Option<String>,
codex-rs/app-server/src/request_processors/turn_processor.rs
+1
View File
@@ -934,6 +934,7 @@ impl TurnRequestProcessor {
request_id,
thread.as_ref(),
Op::RealtimeConversationStart(ConversationStartParams {
architecture: params.architecture,
model: params.model,
output_modality: params.output_modality,
prompt: params.prompt,
codex-rs/app-server/tests/suite/v2/experimental_api.rs
+2
View File
@@ -79,6 +79,7 @@ async fn realtime_conversation_start_requires_experimental_api_capability() -> R
let request_id = mcp
.send_thread_realtime_start_request(ThreadRealtimeStartParams {
architecture: None,
thread_id: "thr_123".to_string(),
model: None,
output_modality: RealtimeOutputModality::Audio,
@@ -187,6 +188,7 @@ async fn realtime_webrtc_start_requires_experimental_api_capability() -> Result<
let request_id = mcp
.send_thread_realtime_start_request(ThreadRealtimeStartParams {
architecture: None,
thread_id: "thr_123".to_string(),
model: None,
output_modality: RealtimeOutputModality::Audio,
codex-rs/app-server/tests/suite/v2/realtime_conversation.rs
+7
View File
@@ -313,6 +313,7 @@ impl RealtimeE2eHarness {
let start_request_id = self
.mcp
.send_thread_realtime_start_request(ThreadRealtimeStartParams {
architecture: None,
thread_id: self.thread_id.clone(),
model: None,
output_modality: RealtimeOutputModality::Audio,
@@ -560,6 +561,7 @@ async fn realtime_conversation_streams_v2_notifications() -> Result<()> {
let start_request_id = mcp
.send_thread_realtime_start_request(ThreadRealtimeStartParams {
architecture: None,
thread_id: thread_start.thread.id.clone(),
model: Some("realtime-treatment-model".to_string()),
output_modality: RealtimeOutputModality::Audio,
@@ -815,6 +817,7 @@ async fn realtime_text_output_modality_requests_text_output_and_final_transcript
let start_request_id = mcp
.send_thread_realtime_start_request(ThreadRealtimeStartParams {
architecture: None,
thread_id: thread_start.thread.id.clone(),
model: None,
output_modality: RealtimeOutputModality::Text,
@@ -991,6 +994,7 @@ async fn realtime_conversation_stop_emits_closed_notification() -> Result<()> {
let start_request_id = mcp
.send_thread_realtime_start_request(ThreadRealtimeStartParams {
architecture: None,
thread_id: thread_start.thread.id.clone(),
model: None,
output_modality: RealtimeOutputModality::Audio,
@@ -1090,6 +1094,7 @@ async fn realtime_webrtc_start_emits_sdp_notification() -> Result<()> {
let thread_id = thread_start.thread.id;
let start_request_id = mcp
.send_thread_realtime_start_request(ThreadRealtimeStartParams {
architecture: None,
thread_id: thread_id.clone(),
model: None,
output_modality: RealtimeOutputModality::Audio,
@@ -2137,6 +2142,7 @@ async fn realtime_webrtc_start_surfaces_backend_error() -> Result<()> {
let start_request_id = mcp
.send_thread_realtime_start_request(ThreadRealtimeStartParams {
architecture: None,
thread_id: thread_start.thread.id,
model: None,
output_modality: RealtimeOutputModality::Audio,
@@ -2198,6 +2204,7 @@ async fn realtime_conversation_requires_feature_flag() -> Result<()> {
let start_request_id = mcp
.send_thread_realtime_start_request(ThreadRealtimeStartParams {
architecture: None,
thread_id: thread_start.thread.id.clone(),
model: None,
output_modality: RealtimeOutputModality::Audio,
codex-rs/codex-api/src/endpoint/realtime_call.rs
+114 -2
View File
@@ -6,8 +6,10 @@ use crate::error::ApiError;
use crate::provider::Provider;
use bytes::Bytes;
use codex_client::HttpTransport;
use codex_client::Request;
use codex_client::RequestBody;
use codex_client::RequestTelemetry;
use codex_protocol::protocol::RealtimeConversationArchitecture;
use http::HeaderMap;
use http::HeaderValue;
use http::Method;
@@ -118,6 +120,22 @@ impl<T: HttpTransport> RealtimeCallClient<T> {
sdp: String,
session_config: RealtimeSessionConfig,
extra_headers: HeaderMap,
) -> Result<RealtimeCallResponse, ApiError> {
self.create_with_session_architecture_and_headers(
sdp,
session_config,
RealtimeConversationArchitecture::RealtimeApi,
extra_headers,
)
.await
}
pub async fn create_with_session_architecture_and_headers(
&self,
sdp: String,
session_config: RealtimeSessionConfig,
architecture: RealtimeConversationArchitecture,
extra_headers: HeaderMap,
) -> Result<RealtimeCallResponse, ApiError> {
trace!(target: "codex_api::realtime_websocket::wire", "realtime call request SDP: {sdp}");
// WebRTC can begin inference as soon as the peer connection comes up, so the initial
@@ -136,7 +154,13 @@ impl<T: HttpTransport> RealtimeCallClient<T> {
.map_err(|err| ApiError::Stream(format!("failed to encode realtime call: {err}")))?;
let resp = self
.session
.execute(Method::POST, Self::path(), extra_headers, Some(body))
.execute_with(
Method::POST,
Self::path(),
extra_headers,
Some(body),
|req| configure_realtime_call_request(req, architecture),
)
.await?;
let sdp = decode_sdp_response(resp.body.as_ref())?;
let call_id = decode_call_id_from_location(&resp.headers)?;
@@ -167,6 +191,7 @@ impl<T: HttpTransport> RealtimeCallClient<T> {
extra_headers,
/*body*/ None,
|req| {
configure_realtime_call_request(req, architecture);
req.headers.insert(
CONTENT_TYPE,
HeaderValue::from_static(MULTIPART_CONTENT_TYPE),
@@ -183,6 +208,30 @@ impl<T: HttpTransport> RealtimeCallClient<T> {
}
}
fn configure_realtime_call_request(
request: &mut Request,
architecture: RealtimeConversationArchitecture,
) {
match architecture {
RealtimeConversationArchitecture::RealtimeApi => {}
RealtimeConversationArchitecture::Avas => {
append_query_pair(&mut request.url, "intent", "quicksilver");
append_query_pair(&mut request.url, "architecture", "avas");
}
}
}
fn append_query_pair(url: &mut String, key: &str, value: &str) {
if url.contains('?') {
url.push('&');
} else {
url.push('?');
}
url.push_str(key);
url.push('=');
url.push_str(value);
}
fn realtime_session_json(session_config: RealtimeSessionConfig) -> Result<Value, ApiError> {
session_update_session_json(session_config)
.map_err(|err| ApiError::Stream(format!("failed to encode realtime call session: {err}")))
@@ -209,7 +258,7 @@ fn decode_call_id_from_location(headers: &HeaderMap) -> Result<String, ApiError>
.next()
.unwrap_or(location)
.rsplit('/')
.find(|segment| segment.starts_with("rtc_") && segment.len() > "rtc_".len())
.find(|segment| is_realtime_call_id_segment(segment))
.map(str::to_string)
.ok_or_else(|| {
ApiError::Stream(format!(
@@ -218,6 +267,21 @@ fn decode_call_id_from_location(headers: &HeaderMap) -> Result<String, ApiError>
})
}
fn is_realtime_call_id_segment(segment: &str) -> bool {
if segment.starts_with("rtc_") && segment.len() > "rtc_".len() {
return true;
}
if segment.len() != 36 {
return false;
}
segment.char_indices().all(|(index, ch)| match index {
8 | 13 | 18 | 23 => ch == '-',
_ => ch.is_ascii_hexdigit(),
})
}
#[cfg(test)]
mod tests {
use super::*;
@@ -459,6 +523,41 @@ mod tests {
);
}
#[tokio::test]
async fn sends_avas_session_call_query_params() {
let transport = CapturingTransport::new();
let client = RealtimeCallClient::new(
transport.clone(),
provider("https://api.openai.com/v1"),
Arc::new(DummyAuth),
);
let response = client
.create_with_session_architecture_and_headers(
"v=offer\r\n".to_string(),
realtime_session_config("sess-api"),
RealtimeConversationArchitecture::Avas,
HeaderMap::new(),
)
.await
.expect("request should succeed");
assert_eq!(
response,
RealtimeCallResponse {
sdp: "v=0\r\n".to_string(),
call_id: "rtc_test".to_string(),
}
);
let request = transport.last_request.lock().unwrap().clone().unwrap();
assert_eq!(request.method, Method::POST);
assert_eq!(
request.url,
"https://api.openai.com/v1/realtime/calls?intent=quicksilver&architecture=avas"
);
}
#[tokio::test]
async fn sends_backend_session_call_as_json_body() {
let transport = CapturingTransport::new();
@@ -541,4 +640,17 @@ mod tests {
"stream error: realtime call Location does not contain a call id: /v1/realtime/calls"
);
}
#[test]
fn accepts_uuid_call_id_from_location() {
let mut headers = HeaderMap::new();
headers.insert(
LOCATION,
HeaderValue::from_static("/v1/realtime/calls/019eb97d-8e9a-7ff3-94b0-ea019babd5d7"),
);
let call_id = decode_call_id_from_location(&headers).expect("UUID call id should parse");
assert_eq!(call_id, "019eb97d-8e9a-7ff3-94b0-ea019babd5d7");
}
}
codex-rs/config/src/config_toml.rs
+7
View File
@@ -372,6 +372,10 @@ pub struct ConfigToml {
/// `/v1/realtime`
/// connection) without changing normal provider HTTP requests.
pub experimental_realtime_ws_base_url: Option<String>,
/// Experimental / do not use. Overrides only the WebRTC realtime call
/// creation base URL. This is separate from `experimental_realtime_ws_base_url`
/// because WebRTC call creation is HTTP, while sideband control is websocket.
pub experimental_realtime_webrtc_call_base_url: Option<String>,
/// Experimental / do not use. Selects the realtime websocket model/snapshot
/// used for the `Op::RealtimeConversation` connection.
pub experimental_realtime_ws_model: Option<String>,
@@ -583,12 +587,14 @@ pub enum RealtimeTransport {
Websocket,
}
pub use codex_protocol::protocol::RealtimeConversationArchitecture as RealtimeArchitecture;
pub use codex_protocol::protocol::RealtimeConversationVersion as RealtimeWsVersion;
pub use codex_protocol::protocol::RealtimeVoice;
#[derive(Serialize, Deserialize, Debug, Clone, Default, PartialEq, Eq, JsonSchema)]
#[schemars(deny_unknown_fields)]
pub struct RealtimeConfig {
pub architecture: RealtimeArchitecture,
pub version: RealtimeWsVersion,
#[serde(rename = "type")]
pub session_type: RealtimeWsMode,
@@ -599,6 +605,7 @@ pub struct RealtimeConfig {
#[derive(Serialize, Deserialize, Debug, Clone, Default, PartialEq, Eq, JsonSchema)]
#[schemars(deny_unknown_fields)]
pub struct RealtimeToml {
pub architecture: Option<RealtimeArchitecture>,
pub version: Option<RealtimeWsVersion>,
#[serde(rename = "type")]
pub session_type: Option<RealtimeWsMode>,
codex-rs/config/src/loader/mod.rs
+1
View File
@@ -68,6 +68,7 @@ const PROJECT_LOCAL_CONFIG_DENYLIST: &[&str] = &[
"notify",
"profile",
"profiles",
"experimental_realtime_webrtc_call_base_url",
"experimental_realtime_ws_base_url",
"otel",
];
codex-rs/core/config.schema.json
+15 -1
View File
@@ -2450,6 +2450,13 @@
},
"type": "object"
},
"RealtimeConversationArchitecture": {
"enum": [
"realtimeapi",
"avas"
],
"type": "string"
},
"RealtimeConversationVersion": {
"enum": [
"v1",
@@ -2460,6 +2467,9 @@
"RealtimeToml": {
"additionalProperties": false,
"properties": {
"architecture": {
"$ref": "#/definitions/RealtimeConversationArchitecture"
},
"transport": {
"$ref": "#/definitions/RealtimeTransport"
},
@@ -4496,6 +4506,10 @@
"description": "Experimental / do not use. Replaces the built-in realtime start instructions inserted into developer messages when realtime becomes active.",
"type": "string"
},
"experimental_realtime_webrtc_call_base_url": {
"description": "Experimental / do not use. Overrides only the WebRTC realtime call creation base URL. This is separate from `experimental_realtime_ws_base_url` because WebRTC call creation is HTTP, while sideband control is websocket.",
"type": "string"
},
"experimental_realtime_ws_backend_prompt": {
"description": "Experimental / do not use. Overrides only the realtime conversation websocket transport instructions (the `Op::RealtimeConversation` `/ws` session.update instructions) without changing normal prompts.",
"type": "string"
@@ -5254,4 +5268,4 @@
},
"title": "ConfigToml",
"type": "object"
}
}
codex-rs/core/src/client.rs
+13 -5
View File
@@ -77,6 +77,7 @@ use codex_protocol::models::ResponseItem;
use codex_protocol::openai_models::ModelInfo;
use codex_protocol::openai_models::ReasoningEffort as ReasoningEffortConfig;
use codex_protocol::protocol::InternalSessionSource;
use codex_protocol::protocol::RealtimeConversationArchitecture;
use codex_protocol::protocol::SessionSource;
use codex_protocol::protocol::W3cTraceContext;
use codex_rollout_trace::CompactionTraceContext;
@@ -522,7 +523,9 @@ impl ModelClient {
&self,
sdp: String,
session_config: ApiRealtimeSessionConfig,
architecture: RealtimeConversationArchitecture,
mut extra_headers: ApiHeaderMap,
api_provider_override: Option<ApiProvider>,
) -> Result<RealtimeWebrtcCallStart> {
// Create the media call over HTTP first, then retain matching auth so realtime can attach
// the server-side control WebSocket to the call id from that HTTP response.
@@ -535,11 +538,16 @@ impl ModelClient {
client_setup.api_auth.as_ref(),
));
let transport = ReqwestTransport::new(build_reqwest_client());
let response =
ApiRealtimeCallClient::new(transport, client_setup.api_provider, client_setup.api_auth)
.create_with_session_and_headers(sdp, session_config, extra_headers)
.await
.map_err(map_api_error)?;
let api_provider = api_provider_override.unwrap_or(client_setup.api_provider);
let response = ApiRealtimeCallClient::new(transport, api_provider, client_setup.api_auth)
.create_with_session_architecture_and_headers(
sdp,
session_config,
architecture,
extra_headers,
)
.await
.map_err(map_api_error)?;
Ok(RealtimeWebrtcCallStart {
sdp: response.sdp,
call_id: response.call_id,
codex-rs/core/src/config/config_tests.rs
+14 -3
View File
@@ -12,6 +12,7 @@ use codex_config::config_toml::AutoReviewToml;
use codex_config::config_toml::ConfigToml;
use codex_config::config_toml::ExperimentalRequestUserInput;
use codex_config::config_toml::ProjectConfig;
use codex_config::config_toml::RealtimeArchitecture;
use codex_config::config_toml::RealtimeConfig;
use codex_config::config_toml::RealtimeToml;
use codex_config::config_toml::RealtimeTransport;
@@ -10488,8 +10489,8 @@ experimental_thread_config_endpoint = "http://127.0.0.1:8061"
#[tokio::test]
async fn experimental_realtime_ws_base_url_loads_from_config_toml() -> std::io::Result<()> {
let cfg: ConfigToml = toml::from_str(
r#"
experimental_realtime_ws_base_url = "http://127.0.0.1:8011"
r#"experimental_realtime_ws_base_url = "http://127.0.0.1:8011"
experimental_realtime_webrtc_call_base_url = "http://127.0.0.1:8082/v1"
"#,
)
.expect("TOML deserialization should succeed");
@@ -10498,7 +10499,10 @@ experimental_realtime_ws_base_url = "http://127.0.0.1:8011"
cfg.experimental_realtime_ws_base_url.as_deref(),
Some("http://127.0.0.1:8011")
);
assert_eq!(
cfg.experimental_realtime_webrtc_call_base_url.as_deref(),
Some("http://127.0.0.1:8082/v1")
);
let codex_home = TempDir::new()?;
let config = Config::load_from_base_config_with_overrides(
cfg,
@@ -10511,6 +10515,10 @@ experimental_realtime_ws_base_url = "http://127.0.0.1:8011"
config.experimental_realtime_ws_base_url.as_deref(),
Some("http://127.0.0.1:8011")
);
assert_eq!(
config.experimental_realtime_webrtc_call_base_url.as_deref(),
Some("http://127.0.0.1:8082/v1")
);
Ok(())
}
@@ -10634,6 +10642,7 @@ async fn realtime_loads_from_config_toml() -> std::io::Result<()> {
let cfg: ConfigToml = toml::from_str(
r#"
[realtime]
architecture = "avas"
version = "v2"
type = "transcription"
transport = "webrtc"
@@ -10645,6 +10654,7 @@ voice = "cedar"
assert_eq!(
cfg.realtime,
Some(RealtimeToml {
architecture: Some(RealtimeArchitecture::Avas),
version: Some(RealtimeWsVersion::V2),
session_type: Some(RealtimeWsMode::Transcription),
transport: Some(RealtimeTransport::WebRtc),
@@ -10663,6 +10673,7 @@ voice = "cedar"
assert_eq!(
config.realtime,
RealtimeConfig {
architecture: RealtimeArchitecture::Avas,
version: RealtimeWsVersion::V2,
session_type: RealtimeWsMode::Transcription,
transport: RealtimeTransport::WebRtc,
codex-rs/core/src/config/mod.rs
+6
View File
@@ -945,6 +945,9 @@ pub struct Config {
/// `/v1/realtime`
/// connection) without changing normal provider HTTP requests.
pub experimental_realtime_ws_base_url: Option<String>,
/// Experimental / do not use. Overrides only the WebRTC realtime call
/// creation base URL.
pub experimental_realtime_webrtc_call_base_url: Option<String>,
/// Experimental / do not use. Selects the realtime websocket model/snapshot
/// used for the `Op::RealtimeConversation` connection.
pub experimental_realtime_ws_model: Option<String>,
@@ -3549,12 +3552,15 @@ impl Config {
speaker: audio.speaker,
}),
experimental_realtime_ws_base_url: cfg.experimental_realtime_ws_base_url,
experimental_realtime_webrtc_call_base_url: cfg
.experimental_realtime_webrtc_call_base_url,
experimental_realtime_ws_model: cfg.experimental_realtime_ws_model,
realtime: cfg
.realtime
.map_or_else(RealtimeConfig::default, |realtime| {
let defaults = RealtimeConfig::default();
RealtimeConfig {
architecture: realtime.architecture.unwrap_or(defaults.architecture),
version: realtime.version.unwrap_or(defaults.version),
session_type: realtime.session_type.unwrap_or(defaults.session_type),
transport: realtime.transport.unwrap_or(defaults.transport),
codex-rs/core/src/realtime_conversation.rs
+58
View File
@@ -38,6 +38,7 @@ use codex_protocol::protocol::ConversationTextParams;
use codex_protocol::protocol::ErrorEvent;
use codex_protocol::protocol::Event;
use codex_protocol::protocol::EventMsg;
use codex_protocol::protocol::RealtimeConversationArchitecture;
use codex_protocol::protocol::RealtimeConversationClosedEvent;
use codex_protocol::protocol::RealtimeConversationRealtimeEvent;
use codex_protocol::protocol::RealtimeConversationSdpEvent;
@@ -232,7 +233,9 @@ struct ConversationState {
struct RealtimeStart {
api_provider: ApiProvider,
architecture: RealtimeConversationArchitecture,
extra_headers: Option<HeaderMap>,
realtime_call_api_provider: Option<ApiProvider>,
session_config: RealtimeSessionConfig,
model_client: ModelClient,
sdp: Option<String>,
@@ -284,7 +287,9 @@ impl RealtimeConversationManager {
async fn start_inner(&self, start: RealtimeStart) -> CodexResult<RealtimeStartOutput> {
let RealtimeStart {
api_provider,
architecture,
extra_headers,
realtime_call_api_provider,
session_config,
model_client,
sdp,
@@ -318,7 +323,9 @@ impl RealtimeConversationManager {
.create_realtime_call_with_headers(
sdp,
session_config.clone(),
architecture,
extra_headers.unwrap_or_default(),
realtime_call_api_provider,
)
.await?;
let task = spawn_webrtc_sideband_input_task(RealtimeWebrtcSidebandInputTask {
@@ -613,7 +620,9 @@ pub(crate) async fn handle_start(
struct PreparedRealtimeConversationStart {
api_provider: ApiProvider,
architecture: RealtimeConversationArchitecture,
extra_headers: Option<HeaderMap>,
realtime_call_api_provider: Option<ApiProvider>,
requested_realtime_session_id: Option<String>,
version: RealtimeWsVersion,
session_config: RealtimeSessionConfig,
@@ -639,7 +648,23 @@ async fn prepare_realtime_start(
if let Some(realtime_ws_base_url) = &config.experimental_realtime_ws_base_url {
api_provider.base_url = realtime_ws_base_url.clone();
}
let realtime_call_api_provider =
if let Some(realtime_call_base_url) = &config.experimental_realtime_webrtc_call_base_url {
let mut api_provider = provider.to_api_provider(Some(AuthMode::ApiKey))?;
api_provider.base_url = realtime_call_base_url.clone();
Some(api_provider)
} else {
None
};
let version = params.version.unwrap_or(config.realtime.version);
// TODO(pbakkum): Remove the realtimeapi/AVAS branch once WebRTC realtime sessions always use AVAS.
let architecture = params.architecture.unwrap_or(config.realtime.architecture);
validate_realtime_architecture(
architecture,
version,
&transport,
config.realtime.session_type,
)?;
let session_config = build_realtime_session_config(
sess,
params.model,
@@ -670,7 +695,9 @@ async fn prepare_realtime_start(
};
Ok(PreparedRealtimeConversationStart {
api_provider,
architecture,
extra_headers,
realtime_call_api_provider,
requested_realtime_session_id,
version,
session_config,
@@ -678,6 +705,33 @@ async fn prepare_realtime_start(
})
}
fn validate_realtime_architecture(
architecture: RealtimeConversationArchitecture,
version: RealtimeWsVersion,
transport: &ConversationStartTransport,
session_type: RealtimeWsMode,
) -> CodexResult<()> {
if architecture != RealtimeConversationArchitecture::Avas {
return Ok(());
}
if version != RealtimeWsVersion::V1 {
return Err(CodexErr::InvalidRequest(
"AVAS realtime architecture requires realtime v1".to_string(),
));
}
if !matches!(transport, ConversationStartTransport::Webrtc { .. }) {
return Err(CodexErr::InvalidRequest(
"AVAS realtime architecture requires WebRTC transport".to_string(),
));
}
if session_type != RealtimeWsMode::Conversational {
return Err(CodexErr::InvalidRequest(
"AVAS realtime architecture requires conversational realtime".to_string(),
));
}
Ok(())
}
pub(crate) async fn build_realtime_session_config(
sess: &Arc<Session>,
model: Option<String>,
@@ -786,7 +840,9 @@ async fn handle_start_inner(
) -> CodexResult<()> {
let PreparedRealtimeConversationStart {
api_provider,
architecture,
extra_headers,
realtime_call_api_provider,
requested_realtime_session_id,
version,
session_config,
@@ -799,7 +855,9 @@ async fn handle_start_inner(
};
let start = RealtimeStart {
api_provider,
architecture,
extra_headers,
realtime_call_api_provider,
session_config,
model_client: sess.services.model_client.clone(),
sdp,
codex-rs/core/tests/suite/compact_remote.rs
+1
View File
@@ -200,6 +200,7 @@ async fn start_remote_realtime_server() -> responses::WebSocketTestServer {
async fn start_realtime_conversation(codex: &codex_core::CodexThread) -> Result<()> {
codex
.submit(Op::RealtimeConversationStart(ConversationStartParams {
architecture: None,
model: None,
output_modality: RealtimeOutputModality::Audio,
prompt: Some(Some("backend prompt".to_string())),
codex-rs/core/tests/suite/realtime_conversation.rs
+231
View File
@@ -18,6 +18,7 @@ use codex_protocol::protocol::EventMsg;
use codex_protocol::protocol::InitialHistory;
use codex_protocol::protocol::Op;
use codex_protocol::protocol::RealtimeAudioFrame;
use codex_protocol::protocol::RealtimeConversationArchitecture;
use codex_protocol::protocol::RealtimeConversationRealtimeEvent;
use codex_protocol::protocol::RealtimeConversationVersion;
use codex_protocol::protocol::RealtimeEvent;
@@ -282,6 +283,7 @@ async fn conversation_start_audio_text_close_round_trip() -> Result<()> {
test.codex
.submit(Op::RealtimeConversationStart(ConversationStartParams {
architecture: None,
model: None,
output_modality: RealtimeOutputModality::Audio,
prompt: Some(Some("backend prompt".to_string())),
@@ -422,6 +424,7 @@ async fn conversation_start_defaults_to_v2_and_gpt_realtime_1_5() -> Result<()>
test.codex
.submit(Op::RealtimeConversationStart(ConversationStartParams {
architecture: None,
model: None,
output_modality: RealtimeOutputModality::Audio,
prompt: Some(Some("backend prompt".to_string())),
@@ -512,6 +515,7 @@ async fn conversation_webrtc_start_posts_generated_session() -> Result<()> {
test.codex
.submit(Op::RealtimeConversationStart(ConversationStartParams {
architecture: None,
model: Some("session-override-model".to_string()),
output_modality: RealtimeOutputModality::Audio,
prompt: Some(Some("backend prompt".to_string())),
@@ -650,6 +654,200 @@ async fn conversation_webrtc_start_posts_generated_session() -> Result<()> {
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn conversation_webrtc_start_uses_avas_architecture_query() -> Result<()> {
skip_if_no_network!(Ok(()));
let server = start_mock_server().await;
let capture = RealtimeCallRequestCapture::new();
Mock::given(method("POST"))
.and(path_regex(".*/realtime/calls$"))
.and(capture.clone())
.respond_with(
ResponseTemplate::new(200)
.insert_header("Location", "/v1/realtime/calls/calls/rtc_avas_test")
.set_body_string("v=answer\r\n"),
)
.mount(&server)
.await;
let realtime_server = start_websocket_server_with_headers(vec![WebSocketConnectionConfig {
requests: vec![
vec![json!({
"type": "session.updated",
"session": { "id": "sess_webrtc", "instructions": "backend prompt" }
})],
vec![],
],
response_headers: Vec::new(),
accept_delay: None,
close_after_requests: false,
}])
.await;
let realtime_ws_base_url = realtime_server.uri().to_string();
let mut builder = test_codex().with_config(move |config| {
config.experimental_realtime_ws_backend_prompt = Some("backend prompt".to_string());
config.experimental_realtime_ws_base_url = Some(realtime_ws_base_url);
config.realtime.version = RealtimeWsVersion::V1;
});
let test = builder.build(&server).await?;
test.codex
.submit(Op::RealtimeConversationStart(ConversationStartParams {
architecture: Some(RealtimeConversationArchitecture::Avas),
model: None,
output_modality: RealtimeOutputModality::Audio,
prompt: Some(Some("backend prompt".to_string())),
realtime_session_id: None,
transport: Some(ConversationStartTransport::Webrtc {
sdp: "v=offer\r\n".to_string(),
}),
version: None,
voice: None,
}))
.await?;
let created = wait_for_event_match(&test.codex, |msg| match msg {
EventMsg::RealtimeConversationSdp(created) => Some(Ok(created.clone())),
EventMsg::Error(err) => Some(Err(err.clone())),
_ => None,
})
.await
.unwrap_or_else(|err: ErrorEvent| panic!("conversation call create failed: {err:?}"));
assert_eq!(created.sdp, "v=answer\r\n");
let request = capture.single_request();
assert_eq!(request.url.path(), "/v1/realtime/calls");
assert_eq!(
request.url.query(),
Some("intent=quicksilver&architecture=avas")
);
let session_updated = wait_for_event_match(&test.codex, |msg| match msg {
EventMsg::RealtimeConversationRealtime(RealtimeConversationRealtimeEvent {
payload:
RealtimeEvent::SessionUpdated {
realtime_session_id,
..
},
}) => Some(realtime_session_id.clone()),
_ => None,
})
.await;
assert_eq!(session_updated, "sess_webrtc");
let handshake = realtime_server.single_handshake();
assert_eq!(
handshake.uri(),
"/v1/realtime?intent=quicksilver&call_id=rtc_avas_test"
);
assert_eq!(
handshake.header("authorization").as_deref(),
Some("Bearer dummy")
);
test.codex.submit(Op::RealtimeConversationClose).await?;
realtime_server.shutdown().await;
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn conversation_webrtc_start_uses_configured_call_base_url_for_avas() -> Result<()> {
skip_if_no_network!(Ok(()));
let server = start_mock_server().await;
let capture = RealtimeCallRequestCapture::new();
Mock::given(method("POST"))
.and(path_regex(".*/realtime/calls$"))
.and(capture.clone())
.respond_with(
ResponseTemplate::new(200)
.insert_header("Location", "/v1/realtime/rtc_local_avas_test")
.set_body_string("v=answer\r\n"),
)
.mount(&server)
.await;
let realtime_server = start_websocket_server_with_headers(vec![WebSocketConnectionConfig {
requests: vec![
vec![json!({
"type": "session.updated",
"session": { "id": "sess_webrtc", "instructions": "backend prompt" }
})],
vec![],
],
response_headers: Vec::new(),
accept_delay: None,
close_after_requests: false,
}])
.await;
let realtime_ws_base_url = realtime_server.uri().to_string();
let realtime_call_base_url = format!("{}/v1", server.uri());
let mut builder = test_codex().with_config(move |config| {
config.experimental_realtime_ws_backend_prompt = Some("backend prompt".to_string());
config.experimental_realtime_ws_base_url = Some(realtime_ws_base_url);
config.experimental_realtime_webrtc_call_base_url = Some(realtime_call_base_url);
config.realtime.version = RealtimeWsVersion::V1;
});
let test = builder.build(&server).await?;
test.codex
.submit(Op::RealtimeConversationStart(ConversationStartParams {
architecture: Some(RealtimeConversationArchitecture::Avas),
model: None,
output_modality: RealtimeOutputModality::Audio,
prompt: Some(Some("backend prompt".to_string())),
realtime_session_id: None,
transport: Some(ConversationStartTransport::Webrtc {
sdp: "v=offer\r\n".to_string(),
}),
version: None,
voice: None,
}))
.await?;
let created = wait_for_event_match(&test.codex, |msg| match msg {
EventMsg::RealtimeConversationSdp(created) => Some(Ok(created.clone())),
EventMsg::Error(err) => Some(Err(err.clone())),
_ => None,
})
.await
.unwrap_or_else(|err: ErrorEvent| panic!("conversation call create failed: {err:?}"));
assert_eq!(created.sdp, "v=answer\r\n");
let request = capture.single_request();
assert_eq!(request.url.path(), "/v1/realtime/calls");
assert_eq!(
request.url.query(),
Some("intent=quicksilver&architecture=avas")
);
let session_updated = wait_for_event_match(&test.codex, |msg| match msg {
EventMsg::RealtimeConversationRealtime(RealtimeConversationRealtimeEvent {
payload:
RealtimeEvent::SessionUpdated {
realtime_session_id,
..
},
}) => Some(realtime_session_id.clone()),
_ => None,
})
.await;
assert_eq!(session_updated, "sess_webrtc");
let handshake = realtime_server.single_handshake();
assert_eq!(
handshake.uri(),
"/v1/realtime?intent=quicksilver&call_id=rtc_local_avas_test"
);
assert_eq!(
handshake.header("authorization").as_deref(),
Some("Bearer dummy")
);
test.codex.submit(Op::RealtimeConversationClose).await?;
realtime_server.shutdown().await;
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn conversation_webrtc_close_while_sideband_connecting_drops_pending_join() -> Result<()> {
skip_if_no_network!(Ok(()));
@@ -684,6 +882,7 @@ async fn conversation_webrtc_close_while_sideband_connecting_drops_pending_join(
test.codex
.submit(Op::RealtimeConversationStart(ConversationStartParams {
architecture: None,
model: None,
output_modality: RealtimeOutputModality::Audio,
prompt: Some(Some("backend prompt".to_string())),
@@ -770,6 +969,7 @@ async fn conversation_webrtc_sideband_connect_failure_closes_with_error() -> Res
test.codex
.submit(Op::RealtimeConversationStart(ConversationStartParams {
architecture: None,
model: None,
output_modality: RealtimeOutputModality::Audio,
prompt: Some(Some("backend prompt".to_string())),
@@ -857,6 +1057,7 @@ async fn conversation_start_uses_openai_env_key_fallback_with_chatgpt_auth() ->
test.codex
.submit(Op::RealtimeConversationStart(ConversationStartParams {
architecture: None,
model: None,
output_modality: RealtimeOutputModality::Audio,
prompt: Some(Some("backend prompt".to_string())),
@@ -925,6 +1126,7 @@ async fn conversation_transport_close_emits_closed_event() -> Result<()> {
test.codex
.submit(Op::RealtimeConversationStart(ConversationStartParams {
architecture: None,
model: None,
output_modality: RealtimeOutputModality::Audio,
prompt: Some(Some("backend prompt".to_string())),
@@ -1017,6 +1219,7 @@ async fn conversation_start_preflight_failure_emits_realtime_error_only() -> Res
test.codex
.submit(Op::RealtimeConversationStart(ConversationStartParams {
architecture: None,
model: None,
output_modality: RealtimeOutputModality::Audio,
prompt: Some(Some("backend prompt".to_string())),
@@ -1063,6 +1266,7 @@ async fn conversation_start_connect_failure_emits_realtime_error_only() -> Resul
test.codex
.submit(Op::RealtimeConversationStart(ConversationStartParams {
architecture: None,
model: None,
output_modality: RealtimeOutputModality::Audio,
prompt: Some(Some("backend prompt".to_string())),
@@ -1156,6 +1360,7 @@ async fn conversation_second_start_replaces_runtime() -> Result<()> {
test.codex
.submit(Op::RealtimeConversationStart(ConversationStartParams {
architecture: None,
model: None,
output_modality: RealtimeOutputModality::Audio,
prompt: Some(Some("old".to_string())),
@@ -1181,6 +1386,7 @@ async fn conversation_second_start_replaces_runtime() -> Result<()> {
test.codex
.submit(Op::RealtimeConversationStart(ConversationStartParams {
architecture: None,
model: None,
output_modality: RealtimeOutputModality::Audio,
prompt: Some(Some("new".to_string())),
@@ -1277,6 +1483,7 @@ async fn conversation_uses_experimental_realtime_ws_base_url_override() -> Resul
test.codex
.submit(Op::RealtimeConversationStart(ConversationStartParams {
architecture: None,
model: None,
output_modality: RealtimeOutputModality::Audio,
prompt: Some(Some("backend prompt".to_string())),
@@ -1341,6 +1548,7 @@ async fn conversation_uses_default_realtime_backend_prompt() -> Result<()> {
test.codex
.submit(Op::RealtimeConversationStart(ConversationStartParams {
architecture: None,
model: None,
output_modality: RealtimeOutputModality::Audio,
prompt: None,
@@ -1413,6 +1621,7 @@ async fn conversation_uses_empty_instructions_for_null_or_empty_prompt() -> Resu
] {
test.codex
.submit(Op::RealtimeConversationStart(ConversationStartParams {
architecture: None,
model: None,
output_modality: RealtimeOutputModality::Audio,
prompt,
@@ -1478,6 +1687,7 @@ async fn conversation_uses_explicit_start_voice() -> Result<()> {
test.codex
.submit(Op::RealtimeConversationStart(ConversationStartParams {
architecture: None,
model: None,
output_modality: RealtimeOutputModality::Audio,
prompt: Some(Some("backend prompt".to_string())),
@@ -1535,6 +1745,7 @@ async fn conversation_uses_configured_realtime_voice() -> Result<()> {
test.codex
.submit(Op::RealtimeConversationStart(ConversationStartParams {
architecture: None,
model: None,
output_modality: RealtimeOutputModality::Audio,
prompt: Some(Some("backend prompt".to_string())),
@@ -1580,6 +1791,7 @@ async fn conversation_rejects_voice_for_wrong_realtime_version() -> Result<()> {
test.codex
.submit(Op::RealtimeConversationStart(ConversationStartParams {
architecture: None,
model: None,
output_modality: RealtimeOutputModality::Audio,
prompt: Some(Some("backend prompt".to_string())),
@@ -1626,6 +1838,7 @@ async fn conversation_uses_experimental_realtime_ws_backend_prompt_override() ->
test.codex
.submit(Op::RealtimeConversationStart(ConversationStartParams {
architecture: None,
model: None,
output_modality: RealtimeOutputModality::Audio,
prompt: Some(Some("prompt from op".to_string())),
@@ -1698,6 +1911,7 @@ async fn conversation_uses_experimental_realtime_ws_startup_context_override() -
test.codex
.submit(Op::RealtimeConversationStart(ConversationStartParams {
architecture: None,
model: None,
output_modality: RealtimeOutputModality::Audio,
prompt: Some(Some("prompt from op".to_string())),
@@ -1764,6 +1978,7 @@ async fn conversation_disables_realtime_startup_context_with_empty_override() ->
test.codex
.submit(Op::RealtimeConversationStart(ConversationStartParams {
architecture: None,
model: None,
output_modality: RealtimeOutputModality::Audio,
prompt: Some(Some("prompt from op".to_string())),
@@ -1823,6 +2038,7 @@ async fn conversation_start_injects_startup_context_from_thread_history() -> Res
test.codex
.submit(Op::RealtimeConversationStart(ConversationStartParams {
architecture: None,
model: None,
output_modality: RealtimeOutputModality::Audio,
prompt: Some(Some("backend prompt".to_string())),
@@ -1934,6 +2150,7 @@ async fn conversation_startup_context_current_thread_selects_many_turns_by_budge
codex
.submit(Op::RealtimeConversationStart(ConversationStartParams {
architecture: None,
model: None,
output_modality: RealtimeOutputModality::Audio,
prompt: Some(Some("backend prompt".to_string())),
@@ -2041,6 +2258,7 @@ async fn conversation_startup_context_falls_back_to_workspace_map() -> Result<()
test.codex
.submit(Op::RealtimeConversationStart(ConversationStartParams {
architecture: None,
model: None,
output_modality: RealtimeOutputModality::Audio,
prompt: Some(Some("backend prompt".to_string())),
@@ -2100,6 +2318,7 @@ async fn conversation_startup_context_is_truncated_and_sent_once_per_start() ->
test.codex
.submit(Op::RealtimeConversationStart(ConversationStartParams {
architecture: None,
model: None,
output_modality: RealtimeOutputModality::Audio,
prompt: Some(Some("backend prompt".to_string())),
@@ -2179,6 +2398,7 @@ async fn conversation_user_text_turn_is_not_sent_to_realtime() -> Result<()> {
test.codex
.submit(Op::RealtimeConversationStart(ConversationStartParams {
architecture: None,
model: None,
output_modality: RealtimeOutputModality::Audio,
prompt: Some(Some("backend prompt".to_string())),
@@ -2275,6 +2495,7 @@ async fn realtime_v2_noop_tool_call_returns_empty_function_output_without_respon
test.codex
.submit(Op::RealtimeConversationStart(ConversationStartParams {
architecture: None,
model: None,
output_modality: RealtimeOutputModality::Audio,
prompt: Some(Some("backend prompt".to_string())),
@@ -2373,6 +2594,7 @@ async fn conversation_mirrors_assistant_message_text_to_realtime_handoff() -> Re
test.codex
.submit(Op::RealtimeConversationStart(ConversationStartParams {
architecture: None,
model: None,
output_modality: RealtimeOutputModality::Audio,
prompt: Some(Some("backend prompt".to_string())),
@@ -2509,6 +2731,7 @@ async fn conversation_handoff_persists_across_item_done_until_turn_complete() ->
test.codex
.submit(Op::RealtimeConversationStart(ConversationStartParams {
architecture: None,
model: None,
output_modality: RealtimeOutputModality::Audio,
prompt: Some(Some("backend prompt".to_string())),
@@ -2660,6 +2883,7 @@ async fn inbound_handoff_request_starts_turn() -> Result<()> {
test.codex
.submit(Op::RealtimeConversationStart(ConversationStartParams {
architecture: None,
model: None,
output_modality: RealtimeOutputModality::Audio,
prompt: Some(Some("backend prompt".to_string())),
@@ -2761,6 +2985,7 @@ async fn inbound_handoff_request_uses_active_transcript() -> Result<()> {
test.codex
.submit(Op::RealtimeConversationStart(ConversationStartParams {
architecture: None,
model: None,
output_modality: RealtimeOutputModality::Audio,
prompt: Some(Some("backend prompt".to_string())),
@@ -2863,6 +3088,7 @@ async fn inbound_handoff_request_sends_transcript_delta_after_each_handoff() ->
test.codex
.submit(Op::RealtimeConversationStart(ConversationStartParams {
architecture: None,
model: None,
output_modality: RealtimeOutputModality::Audio,
prompt: Some(Some("backend prompt".to_string())),
@@ -2963,6 +3189,7 @@ async fn inbound_conversation_item_does_not_start_turn_and_still_forwards_audio(
test.codex
.submit(Op::RealtimeConversationStart(ConversationStartParams {
architecture: None,
model: None,
output_modality: RealtimeOutputModality::Audio,
prompt: Some(Some("backend prompt".to_string())),
@@ -3085,6 +3312,7 @@ async fn delegated_turn_user_role_echo_does_not_redelegate_and_still_forwards_au
test.codex
.submit(Op::RealtimeConversationStart(ConversationStartParams {
architecture: None,
model: None,
output_modality: RealtimeOutputModality::Audio,
prompt: Some(Some("backend prompt".to_string())),
@@ -3237,6 +3465,7 @@ async fn inbound_handoff_request_does_not_block_realtime_event_forwarding() -> R
test.codex
.submit(Op::RealtimeConversationStart(ConversationStartParams {
architecture: None,
model: None,
output_modality: RealtimeOutputModality::Audio,
prompt: Some(Some("backend prompt".to_string())),
@@ -3373,6 +3602,7 @@ async fn inbound_handoff_request_steers_active_turn() -> Result<()> {
test.codex
.submit(Op::RealtimeConversationStart(ConversationStartParams {
architecture: None,
model: None,
output_modality: RealtimeOutputModality::Audio,
prompt: Some(Some("backend prompt".to_string())),
@@ -3525,6 +3755,7 @@ async fn inbound_handoff_request_starts_turn_and_does_not_block_realtime_audio()
test.codex
.submit(Op::RealtimeConversationStart(ConversationStartParams {
architecture: None,
model: None,
output_modality: RealtimeOutputModality::Audio,
prompt: Some(Some("backend prompt".to_string())),
codex-rs/protocol/src/protocol.rs
+11
View File
@@ -185,6 +185,8 @@ pub struct McpServerRefreshConfig {
#[derive(Debug, Clone, PartialEq)]
pub struct ConversationStartParams {
/// Overrides the configured realtime architecture for this session only.
pub architecture: Option<RealtimeConversationArchitecture>,
/// Overrides the configured realtime model for this session only.
pub model: Option<String>,
/// Selects whether the realtime session should produce text or audio output.
@@ -1495,6 +1497,15 @@ pub enum RealtimeConversationVersion {
V2,
}
#[derive(Debug, Clone, Copy, Default, Deserialize, Serialize, PartialEq, Eq, JsonSchema, TS)]
#[serde(rename_all = "snake_case")]
pub enum RealtimeConversationArchitecture {
#[default]
#[serde(rename = "realtimeapi")]
RealtimeApi,
Avas,
}
#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, JsonSchema, TS)]
pub struct RealtimeConversationStartedEvent {
pub realtime_session_id: Option<String>,
codex-rs/thread-manager-sample/src/main.rs
+1
View File
@@ -258,6 +258,7 @@ fn new_config(model: Option<String>, arg0_paths: Arg0DispatchPaths) -> anyhow::R
apps_mcp_product_sku: None,
realtime_audio: RealtimeAudioConfig::default(),
experimental_realtime_ws_base_url: None,
experimental_realtime_webrtc_call_base_url: None,
experimental_realtime_ws_model: None,
realtime: RealtimeConfig::default(),
experimental_realtime_ws_backend_prompt: None,