agent/codex
1
0
Fork 0
mirror of https://github.com/pchuan98/codex.git synced 2026-07-01 00:31:56 +08:00
Code Issues Packages Projects Releases Wiki Activity

Add WebRTC transport to realtime start (#16960)

Browse Source 
Adds WebRTC startup to the experimental app-server
`thread/realtime/start` method with an optional transport enum. The
websocket path remains the default; WebRTC offers create the realtime
session through the shared start flow and emit the answer SDP via
`thread/realtime/sdp`.

---------

Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
Ahmed Ibrahim
2026-04-07 15:43:38 -07:00
committed by GitHub
Unverified
parent 6c36e7d688
commit fb3dcfde1d
42 changed files with 1574 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files
codex-rs/app-server-protocol/schema/json/ClientRequest.json
+42
View File
@@ -2857,6 +2857,48 @@
],
"type": "object"
},
"ThreadRealtimeStartTransport": {
"description": "EXPERIMENTAL - transport used by thread realtime.",
"oneOf": [
{
"properties": {
"type": {
"enum": [
"websocket"
],
"title": "WebsocketThreadRealtimeStartTransportType",
"type": "string"
}
},
"required": [
"type"
],
"title": "WebsocketThreadRealtimeStartTransport",
"type": "object"
},
{
"properties": {
"sdp": {
"description": "SDP offer generated by a WebRTC RTCPeerConnection after configuring audio and the realtime events data channel.",
"type": "string"
},
"type": {
"enum": [
"webrtc"
],
"title": "WebrtcThreadRealtimeStartTransportType",
"type": "string"
}
},
"required": [
"sdp",
"type"
],
"title": "WebrtcThreadRealtimeStartTransport",
"type": "object"
}
]
},
"ThreadResumeParams": {
"description": "There are three ways to resume a thread: 1. By thread_id: load the thread from disk by thread_id and resume it. 2. By history: instantiate the thread from memory and resume it. 3. By path: load the thread from disk by path and resume it.\n\nThe precedence is: history > path > thread_id. If using history or path, the thread_id param will be ignored.\n\nPrefer using thread_id whenever possible.",
"properties": {
codex-rs/app-server-protocol/schema/json/ServerNotification.json
+36
View File
@@ -3304,6 +3304,22 @@
],
"type": "object"
},
"ThreadRealtimeSdpNotification": {
"description": "EXPERIMENTAL - emitted with the remote SDP for a WebRTC realtime session.",
"properties": {
"sdp": {
"type": "string"
},
"threadId": {
"type": "string"
}
},
"required": [
"sdp",
"threadId"
],
"type": "object"
},
"ThreadRealtimeStartedNotification": {
"description": "EXPERIMENTAL - emitted when thread realtime startup is accepted.",
"properties": {
@@ -4927,6 +4943,26 @@
"title": "Thread/realtime/outputAudio/deltaNotification",
"type": "object"
},
{
"properties": {
"method": {
"enum": [
"thread/realtime/sdp"
],
"title": "Thread/realtime/sdpNotificationMethod",
"type": "string"
},
"params": {
"$ref": "#/definitions/ThreadRealtimeSdpNotification"
}
},
"required": [
"method",
"params"
],
"title": "Thread/realtime/sdpNotification",
"type": "object"
},
{
"properties": {
"method": {
codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json
+80
View File
@@ -4335,6 +4335,26 @@
"title": "Thread/realtime/outputAudio/deltaNotification",
"type": "object"
},
{
"properties": {
"method": {
"enum": [
"thread/realtime/sdp"
],
"title": "Thread/realtime/sdpNotificationMethod",
"type": "string"
},
"params": {
"$ref": "#/definitions/v2/ThreadRealtimeSdpNotification"
}
},
"required": [
"method",
"params"
],
"title": "Thread/realtime/sdpNotification",
"type": "object"
},
{
"properties": {
"method": {
@@ -13655,6 +13675,66 @@
"title": "ThreadRealtimeOutputAudioDeltaNotification",
"type": "object"
},
"ThreadRealtimeSdpNotification": {
"$schema": "http://json-schema.org/draft-07/schema#",
"description": "EXPERIMENTAL - emitted with the remote SDP for a WebRTC realtime session.",
"properties": {
"sdp": {
"type": "string"
},
"threadId": {
"type": "string"
}
},
"required": [
"sdp",
"threadId"
],
"title": "ThreadRealtimeSdpNotification",
"type": "object"
},
"ThreadRealtimeStartTransport": {
"description": "EXPERIMENTAL - transport used by thread realtime.",
"oneOf": [
{
"properties": {
"type": {
"enum": [
"websocket"
],
"title": "WebsocketThreadRealtimeStartTransportType",
"type": "string"
}
},
"required": [
"type"
],
"title": "WebsocketThreadRealtimeStartTransport",
"type": "object"
},
{
"properties": {
"sdp": {
"description": "SDP offer generated by a WebRTC RTCPeerConnection after configuring audio and the realtime events data channel.",
"type": "string"
},
"type": {
"enum": [
"webrtc"
],
"title": "WebrtcThreadRealtimeStartTransportType",
"type": "string"
}
},
"required": [
"sdp",
"type"
],
"title": "WebrtcThreadRealtimeStartTransport",
"type": "object"
}
]
},
"ThreadRealtimeStartedNotification": {
"$schema": "http://json-schema.org/draft-07/schema#",
"description": "EXPERIMENTAL - emitted when thread realtime startup is accepted.",
codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json
+80
View File
@@ -9436,6 +9436,26 @@
"title": "Thread/realtime/outputAudio/deltaNotification",
"type": "object"
},
{
"properties": {
"method": {
"enum": [
"thread/realtime/sdp"
],
"title": "Thread/realtime/sdpNotificationMethod",
"type": "string"
},
"params": {
"$ref": "#/definitions/ThreadRealtimeSdpNotification"
}
},
"required": [
"method",
"params"
],
"title": "Thread/realtime/sdpNotification",
"type": "object"
},
{
"properties": {
"method": {
@@ -11510,6 +11530,66 @@
"title": "ThreadRealtimeOutputAudioDeltaNotification",
"type": "object"
},
"ThreadRealtimeSdpNotification": {
"$schema": "http://json-schema.org/draft-07/schema#",
"description": "EXPERIMENTAL - emitted with the remote SDP for a WebRTC realtime session.",
"properties": {
"sdp": {
"type": "string"
},
"threadId": {
"type": "string"
}
},
"required": [
"sdp",
"threadId"
],
"title": "ThreadRealtimeSdpNotification",
"type": "object"
},
"ThreadRealtimeStartTransport": {
"description": "EXPERIMENTAL - transport used by thread realtime.",
"oneOf": [
{
"properties": {
"type": {
"enum": [
"websocket"
],
"title": "WebsocketThreadRealtimeStartTransportType",
"type": "string"
}
},
"required": [
"type"
],
"title": "WebsocketThreadRealtimeStartTransport",
"type": "object"
},
{
"properties": {
"sdp": {
"description": "SDP offer generated by a WebRTC RTCPeerConnection after configuring audio and the realtime events data channel.",
"type": "string"
},
"type": {
"enum": [
"webrtc"
],
"title": "WebrtcThreadRealtimeStartTransportType",
"type": "string"
}
},
"required": [
"sdp",
"type"
],
"title": "WebrtcThreadRealtimeStartTransport",
"type": "object"
}
]
},
"ThreadRealtimeStartedNotification": {
"$schema": "http://json-schema.org/draft-07/schema#",
"description": "EXPERIMENTAL - emitted when thread realtime startup is accepted.",
codex-rs/app-server-protocol/schema/json/v2/ThreadRealtimeSdpNotification.json
+18
View File
@@ -0,0 +1,18 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"description": "EXPERIMENTAL - emitted with the remote SDP for a WebRTC realtime session.",
"properties": {
"sdp": {
"type": "string"
},
"threadId": {
"type": "string"
}
},
"required": [
"sdp",
"threadId"
],
"title": "ThreadRealtimeSdpNotification",
"type": "object"
}
codex-rs/app-server-protocol/schema/typescript/ServerNotification.ts
+2 -1
View File
@@ -41,6 +41,7 @@ import type { ThreadRealtimeClosedNotification } from "./v2/ThreadRealtimeClosed
import type { ThreadRealtimeErrorNotification } from "./v2/ThreadRealtimeErrorNotification";
import type { ThreadRealtimeItemAddedNotification } from "./v2/ThreadRealtimeItemAddedNotification";
import type { ThreadRealtimeOutputAudioDeltaNotification } from "./v2/ThreadRealtimeOutputAudioDeltaNotification";
import type { ThreadRealtimeSdpNotification } from "./v2/ThreadRealtimeSdpNotification";
import type { ThreadRealtimeStartedNotification } from "./v2/ThreadRealtimeStartedNotification";
import type { ThreadRealtimeTranscriptUpdatedNotification } from "./v2/ThreadRealtimeTranscriptUpdatedNotification";
import type { ThreadStartedNotification } from "./v2/ThreadStartedNotification";
@@ -57,4 +58,4 @@ import type { WindowsWorldWritableWarningNotification } from "./v2/WindowsWorldW
/**
* Notification sent from the server to the client.
*/
export type ServerNotification = { "method": "error", "params": ErrorNotification } | { "method": "thread/started", "params": ThreadStartedNotification } | { "method": "thread/status/changed", "params": ThreadStatusChangedNotification } | { "method": "thread/archived", "params": ThreadArchivedNotification } | { "method": "thread/unarchived", "params": ThreadUnarchivedNotification } | { "method": "thread/closed", "params": ThreadClosedNotification } | { "method": "skills/changed", "params": SkillsChangedNotification } | { "method": "thread/name/updated", "params": ThreadNameUpdatedNotification } | { "method": "thread/tokenUsage/updated", "params": ThreadTokenUsageUpdatedNotification } | { "method": "turn/started", "params": TurnStartedNotification } | { "method": "hook/started", "params": HookStartedNotification } | { "method": "turn/completed", "params": TurnCompletedNotification } | { "method": "hook/completed", "params": HookCompletedNotification } | { "method": "turn/diff/updated", "params": TurnDiffUpdatedNotification } | { "method": "turn/plan/updated", "params": TurnPlanUpdatedNotification } | { "method": "item/started", "params": ItemStartedNotification } | { "method": "item/autoApprovalReview/started", "params": ItemGuardianApprovalReviewStartedNotification } | { "method": "item/autoApprovalReview/completed", "params": ItemGuardianApprovalReviewCompletedNotification } | { "method": "item/completed", "params": ItemCompletedNotification } | { "method": "rawResponseItem/completed", "params": RawResponseItemCompletedNotification } | { "method": "item/agentMessage/delta", "params": AgentMessageDeltaNotification } | { "method": "item/plan/delta", "params": PlanDeltaNotification } | { "method": "command/exec/outputDelta", "params": CommandExecOutputDeltaNotification } | { "method": "item/commandExecution/outputDelta", "params": CommandExecutionOutputDeltaNotification } | { "method": "item/commandExecution/terminalInteraction", "params": TerminalInteractionNotification } | { "method": "item/fileChange/outputDelta", "params": FileChangeOutputDeltaNotification } | { "method": "serverRequest/resolved", "params": ServerRequestResolvedNotification } | { "method": "item/mcpToolCall/progress", "params": McpToolCallProgressNotification } | { "method": "mcpServer/oauthLogin/completed", "params": McpServerOauthLoginCompletedNotification } | { "method": "mcpServer/startupStatus/updated", "params": McpServerStatusUpdatedNotification } | { "method": "account/updated", "params": AccountUpdatedNotification } | { "method": "account/rateLimits/updated", "params": AccountRateLimitsUpdatedNotification } | { "method": "app/list/updated", "params": AppListUpdatedNotification } | { "method": "fs/changed", "params": FsChangedNotification } | { "method": "item/reasoning/summaryTextDelta", "params": ReasoningSummaryTextDeltaNotification } | { "method": "item/reasoning/summaryPartAdded", "params": ReasoningSummaryPartAddedNotification } | { "method": "item/reasoning/textDelta", "params": ReasoningTextDeltaNotification } | { "method": "thread/compacted", "params": ContextCompactedNotification } | { "method": "model/rerouted", "params": ModelReroutedNotification } | { "method": "deprecationNotice", "params": DeprecationNoticeNotification } | { "method": "configWarning", "params": ConfigWarningNotification } | { "method": "fuzzyFileSearch/sessionUpdated", "params": FuzzyFileSearchSessionUpdatedNotification } | { "method": "fuzzyFileSearch/sessionCompleted", "params": FuzzyFileSearchSessionCompletedNotification } | { "method": "thread/realtime/started", "params": ThreadRealtimeStartedNotification } | { "method": "thread/realtime/itemAdded", "params": ThreadRealtimeItemAddedNotification } | { "method": "thread/realtime/transcriptUpdated", "params": ThreadRealtimeTranscriptUpdatedNotification } | { "method": "thread/realtime/outputAudio/delta", "params": ThreadRealtimeOutputAudioDeltaNotification } | { "method": "thread/realtime/error", "params": ThreadRealtimeErrorNotification } | { "method": "thread/realtime/closed", "params": ThreadRealtimeClosedNotification } | { "method": "windows/worldWritableWarning", "params": WindowsWorldWritableWarningNotification } | { "method": "windowsSandbox/setupCompleted", "params": WindowsSandboxSetupCompletedNotification } | { "method": "account/login/completed", "params": AccountLoginCompletedNotification };
export type ServerNotification = { "method": "error", "params": ErrorNotification } | { "method": "thread/started", "params": ThreadStartedNotification } | { "method": "thread/status/changed", "params": ThreadStatusChangedNotification } | { "method": "thread/archived", "params": ThreadArchivedNotification } | { "method": "thread/unarchived", "params": ThreadUnarchivedNotification } | { "method": "thread/closed", "params": ThreadClosedNotification } | { "method": "skills/changed", "params": SkillsChangedNotification } | { "method": "thread/name/updated", "params": ThreadNameUpdatedNotification } | { "method": "thread/tokenUsage/updated", "params": ThreadTokenUsageUpdatedNotification } | { "method": "turn/started", "params": TurnStartedNotification } | { "method": "hook/started", "params": HookStartedNotification } | { "method": "turn/completed", "params": TurnCompletedNotification } | { "method": "hook/completed", "params": HookCompletedNotification } | { "method": "turn/diff/updated", "params": TurnDiffUpdatedNotification } | { "method": "turn/plan/updated", "params": TurnPlanUpdatedNotification } | { "method": "item/started", "params": ItemStartedNotification } | { "method": "item/autoApprovalReview/started", "params": ItemGuardianApprovalReviewStartedNotification } | { "method": "item/autoApprovalReview/completed", "params": ItemGuardianApprovalReviewCompletedNotification } | { "method": "item/completed", "params": ItemCompletedNotification } | { "method": "rawResponseItem/completed", "params": RawResponseItemCompletedNotification } | { "method": "item/agentMessage/delta", "params": AgentMessageDeltaNotification } | { "method": "item/plan/delta", "params": PlanDeltaNotification } | { "method": "command/exec/outputDelta", "params": CommandExecOutputDeltaNotification } | { "method": "item/commandExecution/outputDelta", "params": CommandExecutionOutputDeltaNotification } | { "method": "item/commandExecution/terminalInteraction", "params": TerminalInteractionNotification } | { "method": "item/fileChange/outputDelta", "params": FileChangeOutputDeltaNotification } | { "method": "serverRequest/resolved", "params": ServerRequestResolvedNotification } | { "method": "item/mcpToolCall/progress", "params": McpToolCallProgressNotification } | { "method": "mcpServer/oauthLogin/completed", "params": McpServerOauthLoginCompletedNotification } | { "method": "mcpServer/startupStatus/updated", "params": McpServerStatusUpdatedNotification } | { "method": "account/updated", "params": AccountUpdatedNotification } | { "method": "account/rateLimits/updated", "params": AccountRateLimitsUpdatedNotification } | { "method": "app/list/updated", "params": AppListUpdatedNotification } | { "method": "fs/changed", "params": FsChangedNotification } | { "method": "item/reasoning/summaryTextDelta", "params": ReasoningSummaryTextDeltaNotification } | { "method": "item/reasoning/summaryPartAdded", "params": ReasoningSummaryPartAddedNotification } | { "method": "item/reasoning/textDelta", "params": ReasoningTextDeltaNotification } | { "method": "thread/compacted", "params": ContextCompactedNotification } | { "method": "model/rerouted", "params": ModelReroutedNotification } | { "method": "deprecationNotice", "params": DeprecationNoticeNotification } | { "method": "configWarning", "params": ConfigWarningNotification } | { "method": "fuzzyFileSearch/sessionUpdated", "params": FuzzyFileSearchSessionUpdatedNotification } | { "method": "fuzzyFileSearch/sessionCompleted", "params": FuzzyFileSearchSessionCompletedNotification } | { "method": "thread/realtime/started", "params": ThreadRealtimeStartedNotification } | { "method": "thread/realtime/itemAdded", "params": ThreadRealtimeItemAddedNotification } | { "method": "thread/realtime/transcriptUpdated", "params": ThreadRealtimeTranscriptUpdatedNotification } | { "method": "thread/realtime/outputAudio/delta", "params": ThreadRealtimeOutputAudioDeltaNotification } | { "method": "thread/realtime/sdp", "params": ThreadRealtimeSdpNotification } | { "method": "thread/realtime/error", "params": ThreadRealtimeErrorNotification } | { "method": "thread/realtime/closed", "params": ThreadRealtimeClosedNotification } | { "method": "windows/worldWritableWarning", "params": WindowsWorldWritableWarningNotification } | { "method": "windowsSandbox/setupCompleted", "params": WindowsSandboxSetupCompletedNotification } | { "method": "account/login/completed", "params": AccountLoginCompletedNotification };
codex-rs/app-server-protocol/schema/typescript/v2/ThreadRealtimeSdpNotification.ts
+8
View File
@@ -0,0 +1,8 @@
// GENERATED CODE! DO NOT MODIFY BY HAND!
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
/**
* EXPERIMENTAL - emitted with the remote SDP for a WebRTC realtime session.
*/
export type ThreadRealtimeSdpNotification = { threadId: string, sdp: string, };
codex-rs/app-server-protocol/schema/typescript/v2/ThreadRealtimeStartTransport.ts
+13
View File
@@ -0,0 +1,13 @@
// GENERATED CODE! DO NOT MODIFY BY HAND!
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
/**
* EXPERIMENTAL - transport used by thread realtime.
*/
export type ThreadRealtimeStartTransport = { "type": "websocket" } | { "type": "webrtc",
/**
* SDP offer generated by a WebRTC RTCPeerConnection after configuring audio and the
* realtime events data channel.
*/
sdp: string, };
codex-rs/app-server-protocol/schema/typescript/v2/index.ts
+2
View File
@@ -294,6 +294,8 @@ export type { ThreadRealtimeClosedNotification } from "./ThreadRealtimeClosedNot
export type { ThreadRealtimeErrorNotification } from "./ThreadRealtimeErrorNotification";
export type { ThreadRealtimeItemAddedNotification } from "./ThreadRealtimeItemAddedNotification";
export type { ThreadRealtimeOutputAudioDeltaNotification } from "./ThreadRealtimeOutputAudioDeltaNotification";
export type { ThreadRealtimeSdpNotification } from "./ThreadRealtimeSdpNotification";
export type { ThreadRealtimeStartTransport } from "./ThreadRealtimeStartTransport";
export type { ThreadRealtimeStartedNotification } from "./ThreadRealtimeStartedNotification";
export type { ThreadRealtimeTranscriptUpdatedNotification } from "./ThreadRealtimeTranscriptUpdatedNotification";
export type { ThreadResumeParams } from "./ThreadResumeParams";
codex-rs/app-server-protocol/src/protocol/common.rs
+6 -1
View File
@@ -1005,6 +1005,8 @@ server_notification_definitions! {
ThreadRealtimeTranscriptUpdated => "thread/realtime/transcriptUpdated" (v2::ThreadRealtimeTranscriptUpdatedNotification),
#[experimental("thread/realtime/outputAudio/delta")]
ThreadRealtimeOutputAudioDelta => "thread/realtime/outputAudio/delta" (v2::ThreadRealtimeOutputAudioDeltaNotification),
#[experimental("thread/realtime/sdp")]
ThreadRealtimeSdp => "thread/realtime/sdp" (v2::ThreadRealtimeSdpNotification),
#[experimental("thread/realtime/error")]
ThreadRealtimeError => "thread/realtime/error" (v2::ThreadRealtimeErrorNotification),
#[experimental("thread/realtime/closed")]
@@ -1761,6 +1763,7 @@ mod tests {
thread_id: "thr_123".to_string(),
prompt: "You are on a call".to_string(),
session_id: Some("sess_456".to_string()),
transport: None,
},
};
assert_eq!(
@@ -1770,7 +1773,8 @@ mod tests {
"params": {
"threadId": "thr_123",
"prompt": "You are on a call",
"sessionId": "sess_456"
"sessionId": "sess_456",
"transport": null
}
}),
serde_json::to_value(&request)?,
@@ -1850,6 +1854,7 @@ mod tests {
thread_id: "thr_123".to_string(),
prompt: "You are on a call".to_string(),
session_id: None,
transport: None,
},
};
let reason = crate::experimental_api::ExperimentalApi::experimental_reason(&request);
codex-rs/app-server-protocol/src/protocol/v2.rs
+24
View File
@@ -3854,6 +3854,21 @@ pub struct ThreadRealtimeStartParams {
pub prompt: String,
#[ts(optional = nullable)]
pub session_id: Option<String>,
#[ts(optional = nullable)]
pub transport: Option<ThreadRealtimeStartTransport>,
}
/// EXPERIMENTAL - transport used by thread realtime.
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)]
#[serde(tag = "type", rename_all = "camelCase")]
#[ts(export_to = "v2/", tag = "type")]
pub enum ThreadRealtimeStartTransport {
Websocket,
Webrtc {
/// SDP offer generated by a WebRTC RTCPeerConnection after configuring audio and the
/// realtime events data channel.
sdp: String,
},
}
/// EXPERIMENTAL - response for starting thread realtime.
@@ -3945,6 +3960,15 @@ pub struct ThreadRealtimeOutputAudioDeltaNotification {
pub audio: ThreadRealtimeAudioChunk,
}
/// EXPERIMENTAL - emitted with the remote SDP for a WebRTC realtime session.
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)]
#[serde(rename_all = "camelCase")]
#[ts(export_to = "v2/")]
pub struct ThreadRealtimeSdpNotification {
pub thread_id: String,
pub sdp: String,
}
/// EXPERIMENTAL - emitted when thread realtime encounters an error.
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)]
#[serde(rename_all = "camelCase")]
codex-rs/app-server/README.md
+46 -1
View File
@@ -152,7 +152,7 @@ Example with notification opt-out:
- `turn/start` — add user input to a thread and begin Codex generation; responds with the initial `turn` object and streams `turn/started`, `item/*`, and `turn/completed` notifications. For `collaborationMode`, `settings.developer_instructions: null` means "use built-in instructions for the selected mode".
- `turn/steer` — add user input to an already in-flight regular turn without starting a new turn; returns the active `turnId` that accepted the input. Review and manual compaction turns reject `turn/steer`.
- `turn/interrupt` — request cancellation of an in-flight turn by `(thread_id, turn_id)`; success is an empty `{}` response and the turn finishes with `status: "interrupted"`.
- `thread/realtime/start` — start a thread-scoped realtime session (experimental); returns `{}` and streams `thread/realtime/*` notifications.
- `thread/realtime/start` — start a thread-scoped realtime session (experimental); returns `{}` and streams `thread/realtime/*` notifications. Omit `transport` for the websocket transport, or pass `{ "type": "webrtc", "sdp": "..." }` to create a WebRTC session from a browser-generated SDP offer; the remote answer SDP is emitted as `thread/realtime/sdp`.
- `thread/realtime/appendAudio` — append an input audio chunk to the active realtime session (experimental); returns `{}`.
- `thread/realtime/appendText` — append text input to the active realtime session (experimental); returns `{}`.
- `thread/realtime/stop` — stop the active realtime session for the thread (experimental); returns `{}`.
@@ -563,6 +563,51 @@ Invoke a plugin by including a UI mention token such as `@sample` in the text in
} } }
```
### Example: Start realtime with WebRTC
Use `thread/realtime/start` with `transport.type: "webrtc"` when a browser or webview owns the `RTCPeerConnection` and app-server should create the server-side realtime session. The transport `sdp` must be the offer SDP produced by `RTCPeerConnection.createOffer()`, not a hand-written or minimal SDP string.
The offer should include the media sections the client wants to negotiate. For the standard realtime UI flow, create the audio track/transceiver and the `oai-events` data channel before calling `createOffer()`:
```javascript
const pc = new RTCPeerConnection();
audioElement.autoplay = true;
pc.ontrack = (event) => {
audioElement.srcObject = event.streams[0];
};
const mediaStream = await navigator.mediaDevices.getUserMedia({ audio: true });
pc.addTrack(mediaStream.getAudioTracks()[0], mediaStream);
pc.createDataChannel("oai-events");
const offer = await pc.createOffer();
await pc.setLocalDescription(offer);
```
Then send `offer.sdp` to app-server. Core uses `experimental_realtime_ws_backend_prompt` for the backend instructions and the thread conversation id for the realtime session id. The start response is `{}`; the remote answer SDP arrives later as `thread/realtime/sdp` and should be passed to `setRemoteDescription()`:
```json
{ "method": "thread/realtime/start", "id": 40, "params": {
"threadId": "thr_123",
"prompt": "You are on a call.",
"sessionId": null,
"transport": { "type": "webrtc", "sdp": "v=0\r\no=..." }
} }
{ "id": 40, "result": {} }
{ "method": "thread/realtime/sdp", "params": {
"threadId": "thr_123",
"sdp": "v=0\r\no=..."
} }
```
```javascript
await pc.setRemoteDescription({
type: "answer",
sdp: notification.params.sdp,
});
```
### Example: Interrupt an active turn
You can cancel a running Turn with `turn/interrupt`.
codex-rs/app-server/src/bespoke_event_handling.rs
+12 -1
View File
@@ -80,6 +80,7 @@ use codex_app_server_protocol::ThreadRealtimeClosedNotification;
use codex_app_server_protocol::ThreadRealtimeErrorNotification;
use codex_app_server_protocol::ThreadRealtimeItemAddedNotification;
use codex_app_server_protocol::ThreadRealtimeOutputAudioDeltaNotification;
use codex_app_server_protocol::ThreadRealtimeSdpNotification;
use codex_app_server_protocol::ThreadRealtimeStartedNotification;
use codex_app_server_protocol::ThreadRealtimeTranscriptUpdatedNotification;
use codex_app_server_protocol::ThreadRollbackResponse;
@@ -357,6 +358,17 @@ pub(crate) async fn apply_bespoke_event_handling(
.await;
}
}
EventMsg::RealtimeConversationSdp(event) => {
if let ApiVersion::V2 = api_version {
let notification = ThreadRealtimeSdpNotification {
thread_id: conversation_id.to_string(),
sdp: event.sdp,
};
outgoing
.send_server_notification(ServerNotification::ThreadRealtimeSdp(notification))
.await;
}
}
EventMsg::RealtimeConversationRealtime(event) => {
if let ApiVersion::V2 = api_version {
match event.payload {
@@ -1343,7 +1355,6 @@ pub(crate) async fn apply_bespoke_event_handling(
let message = ev.message.clone();
let codex_error_info = ev.codex_error_info.clone();
// If this error belongs to an in-flight `thread/rollback` request, fail that request
// (and clear pending state) so subsequent rollbacks are unblocked.
//
codex-rs/app-server/src/codex_message_processor.rs
+10
View File
@@ -146,6 +146,7 @@ use codex_app_server_protocol::ThreadRealtimeAppendTextParams;
use codex_app_server_protocol::ThreadRealtimeAppendTextResponse;
use codex_app_server_protocol::ThreadRealtimeStartParams;
use codex_app_server_protocol::ThreadRealtimeStartResponse;
use codex_app_server_protocol::ThreadRealtimeStartTransport;
use codex_app_server_protocol::ThreadRealtimeStopParams;
use codex_app_server_protocol::ThreadRealtimeStopResponse;
use codex_app_server_protocol::ThreadResumeParams;
@@ -269,6 +270,7 @@ use codex_protocol::models::ResponseItem;
use codex_protocol::protocol::AgentStatus;
use codex_protocol::protocol::ConversationAudioParams;
use codex_protocol::protocol::ConversationStartParams;
use codex_protocol::protocol::ConversationStartTransport;
use codex_protocol::protocol::ConversationTextParams;
use codex_protocol::protocol::EventMsg;
use codex_protocol::protocol::GitInfo as CoreGitInfo;
@@ -6851,6 +6853,14 @@ impl CodexMessageProcessor {
Op::RealtimeConversationStart(ConversationStartParams {
prompt: params.prompt,
session_id: params.session_id,
transport: params.transport.map(|transport| match transport {
ThreadRealtimeStartTransport::Websocket => {
ConversationStartTransport::Websocket
}
ThreadRealtimeStartTransport::Webrtc { sdp } => {
ConversationStartTransport::Webrtc { sdp }
}
}),
}),
)
.await;
codex-rs/app-server/tests/suite/v2/experimental_api.rs
+39
View File
@@ -12,6 +12,7 @@ use codex_app_server_protocol::JSONRPCResponse;
use codex_app_server_protocol::MockExperimentalMethodParams;
use codex_app_server_protocol::RequestId;
use codex_app_server_protocol::ThreadRealtimeStartParams;
use codex_app_server_protocol::ThreadRealtimeStartTransport;
use codex_app_server_protocol::ThreadStartParams;
use codex_app_server_protocol::ThreadStartResponse;
use pretty_assertions::assert_eq;
@@ -75,6 +76,44 @@ async fn realtime_conversation_start_requires_experimental_api_capability() -> R
thread_id: "thr_123".to_string(),
prompt: "hello".to_string(),
session_id: None,
transport: None,
})
.await?;
let error = timeout(
DEFAULT_TIMEOUT,
mcp.read_stream_until_error_message(RequestId::Integer(request_id)),
)
.await??;
assert_experimental_capability_error(error, "thread/realtime/start");
Ok(())
}
#[tokio::test]
async fn realtime_webrtc_start_requires_experimental_api_capability() -> Result<()> {
let codex_home = TempDir::new()?;
let mut mcp = McpProcess::new(codex_home.path()).await?;
let init = mcp
.initialize_with_capabilities(
default_client_info(),
Some(InitializeCapabilities {
experimental_api: false,
opt_out_notification_methods: None,
}),
)
.await?;
let JSONRPCMessage::Response(_) = init else {
anyhow::bail!("expected initialize response, got {init:?}");
};
let request_id = mcp
.send_thread_realtime_start_request(ThreadRealtimeStartParams {
thread_id: "thr_123".to_string(),
prompt: "hello".to_string(),
session_id: None,
transport: Some(ThreadRealtimeStartTransport::Webrtc {
sdp: "v=offer\r\n".to_string(),
}),
})
.await?;
let error = timeout(
codex-rs/app-server/tests/suite/v2/realtime_conversation.rs
+226
View File
@@ -16,8 +16,10 @@ use codex_app_server_protocol::ThreadRealtimeClosedNotification;
use codex_app_server_protocol::ThreadRealtimeErrorNotification;
use codex_app_server_protocol::ThreadRealtimeItemAddedNotification;
use codex_app_server_protocol::ThreadRealtimeOutputAudioDeltaNotification;
use codex_app_server_protocol::ThreadRealtimeSdpNotification;
use codex_app_server_protocol::ThreadRealtimeStartParams;
use codex_app_server_protocol::ThreadRealtimeStartResponse;
use codex_app_server_protocol::ThreadRealtimeStartTransport;
use codex_app_server_protocol::ThreadRealtimeStartedNotification;
use codex_app_server_protocol::ThreadRealtimeStopParams;
use codex_app_server_protocol::ThreadRealtimeStopResponse;
@@ -33,13 +35,59 @@ use pretty_assertions::assert_eq;
use serde::de::DeserializeOwned;
use serde_json::json;
use std::path::Path;
use std::sync::Arc;
use std::sync::Mutex;
use std::time::Duration;
use tempfile::TempDir;
use tokio::time::timeout;
use wiremock::Match;
use wiremock::Mock;
use wiremock::Request as WiremockRequest;
use wiremock::ResponseTemplate;
use wiremock::matchers::method;
use wiremock::matchers::path;
const DEFAULT_TIMEOUT: Duration = Duration::from_secs(10);
const STARTUP_CONTEXT_HEADER: &str = "Startup context from Codex.";
#[derive(Debug, Clone, Copy)]
enum StartupContextConfig<'a> {
Generated,
Override(&'a str),
}
#[derive(Debug, Clone)]
struct RealtimeCallRequestCapture {
requests: Arc<Mutex<Vec<WiremockRequest>>>,
}
impl RealtimeCallRequestCapture {
fn new() -> Self {
Self {
requests: Arc::new(Mutex::new(Vec::new())),
}
}
fn single_request(&self) -> WiremockRequest {
let requests = self
.requests
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
assert_eq!(requests.len(), 1, "expected one realtime call request");
requests[0].clone()
}
}
impl Match for RealtimeCallRequestCapture {
fn matches(&self, request: &WiremockRequest) -> bool {
self.requests
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.push(request.clone());
true
}
}
#[tokio::test]
async fn realtime_conversation_streams_v2_notifications() -> Result<()> {
skip_if_no_network!(Ok(()));
@@ -100,6 +148,7 @@ async fn realtime_conversation_streams_v2_notifications() -> Result<()> {
&responses_server.uri(),
realtime_server.uri(),
/*realtime_enabled*/ true,
StartupContextConfig::Generated,
)?;
let mut mcp = McpProcess::new(codex_home.path()).await?;
@@ -121,6 +170,7 @@ async fn realtime_conversation_streams_v2_notifications() -> Result<()> {
thread_id: thread_start.thread.id.clone(),
prompt: "backend prompt".to_string(),
session_id: None,
transport: None,
})
.await?;
let start_response: JSONRPCResponse = timeout(
@@ -309,6 +359,7 @@ async fn realtime_conversation_stop_emits_closed_notification() -> Result<()> {
&responses_server.uri(),
realtime_server.uri(),
/*realtime_enabled*/ true,
StartupContextConfig::Generated,
)?;
let mut mcp = McpProcess::new(codex_home.path()).await?;
@@ -330,6 +381,7 @@ async fn realtime_conversation_stop_emits_closed_notification() -> Result<()> {
thread_id: thread_start.thread.id.clone(),
prompt: "backend prompt".to_string(),
session_id: None,
transport: None,
})
.await?;
let start_response: JSONRPCResponse = timeout(
@@ -368,6 +420,169 @@ async fn realtime_conversation_stop_emits_closed_notification() -> Result<()> {
Ok(())
}
#[tokio::test]
async fn realtime_webrtc_start_emits_sdp_notification() -> Result<()> {
skip_if_no_network!(Ok(()));
let responses_server = create_mock_responses_server_sequence_unchecked(Vec::new()).await;
let call_capture = RealtimeCallRequestCapture::new();
Mock::given(method("POST"))
.and(path("/v1/realtime/calls"))
.and(call_capture.clone())
.respond_with(ResponseTemplate::new(200).set_body_string("v=answer\r\n"))
.mount(&responses_server)
.await;
let realtime_server = start_websocket_server(vec![vec![]]).await;
let codex_home = TempDir::new()?;
create_config_toml(
codex_home.path(),
&responses_server.uri(),
realtime_server.uri(),
/*realtime_enabled*/ true,
StartupContextConfig::Override("startup context"),
)?;
let mut mcp = McpProcess::new(codex_home.path()).await?;
mcp.initialize().await?;
login_with_api_key(&mut mcp, "sk-test-key").await?;
let thread_start_request_id = mcp
.send_thread_start_request(ThreadStartParams::default())
.await?;
let thread_start_response: JSONRPCResponse = timeout(
DEFAULT_TIMEOUT,
mcp.read_stream_until_response_message(RequestId::Integer(thread_start_request_id)),
)
.await??;
let thread_start: ThreadStartResponse = to_response(thread_start_response)?;
let thread_id = thread_start.thread.id;
let start_request_id = mcp
.send_thread_realtime_start_request(ThreadRealtimeStartParams {
thread_id: thread_id.clone(),
prompt: "backend prompt".to_string(),
session_id: None,
transport: Some(ThreadRealtimeStartTransport::Webrtc {
sdp: "v=offer\r\n".to_string(),
}),
})
.await?;
let start_response: JSONRPCResponse = timeout(
DEFAULT_TIMEOUT,
mcp.read_stream_until_response_message(RequestId::Integer(start_request_id)),
)
.await??;
let _: ThreadRealtimeStartResponse = to_response(start_response)?;
let sdp_notification =
read_notification::<ThreadRealtimeSdpNotification>(&mut mcp, "thread/realtime/sdp").await?;
assert_eq!(
sdp_notification,
ThreadRealtimeSdpNotification {
thread_id: thread_id.clone(),
sdp: "v=answer\r\n".to_string()
}
);
let closed_notification =
read_notification::<ThreadRealtimeClosedNotification>(&mut mcp, "thread/realtime/closed")
.await?;
assert_eq!(
closed_notification,
ThreadRealtimeClosedNotification {
thread_id: thread_id.clone(),
reason: Some("transport_closed".to_string())
}
);
let request = call_capture.single_request();
assert_eq!(request.url.path(), "/v1/realtime/calls");
assert_eq!(request.url.query(), None);
let body = String::from_utf8(request.body).context("multipart body should be utf-8")?;
let session = r#"{"tool_choice":"auto","type":"realtime","instructions":"backend prompt\n\nstartup context","output_modalities":["audio"],"audio":{"input":{"format":{"type":"audio/pcm","rate":24000},"noise_reduction":{"type":"near_field"},"turn_detection":{"type":"server_vad","interrupt_response":true,"create_response":true}},"output":{"format":{"type":"audio/pcm","rate":24000},"voice":"marin"}},"tools":[{"type":"function","name":"codex","description":"Delegate a request to Codex and return the final result to the user. Use this as the default action. If the user asks to do something next, later, after this, or once current work finishes, call this tool so the work is actually queued instead of merely promising to do it later.","parameters":{"type":"object","properties":{"prompt":{"type":"string","description":"The user request to delegate to Codex."}},"required":["prompt"],"additionalProperties":false}}]}"#;
assert_eq!(
body,
format!(
"--codex-realtime-call-boundary\r\n\
Content-Disposition: form-data; name=\"sdp\"\r\n\
Content-Type: application/sdp\r\n\
\r\n\
v=offer\r\n\
\r\n\
--codex-realtime-call-boundary\r\n\
Content-Disposition: form-data; name=\"session\"\r\n\
Content-Type: application/json\r\n\
\r\n\
{session}\r\n\
--codex-realtime-call-boundary--\r\n"
)
);
realtime_server.shutdown().await;
Ok(())
}
#[tokio::test]
async fn realtime_webrtc_start_surfaces_backend_error() -> Result<()> {
skip_if_no_network!(Ok(()));
let responses_server = create_mock_responses_server_sequence_unchecked(Vec::new()).await;
Mock::given(method("POST"))
.and(path("/v1/realtime/calls"))
.respond_with(ResponseTemplate::new(500).set_body_string("boom"))
.mount(&responses_server)
.await;
let realtime_server = start_websocket_server(vec![vec![]]).await;
let codex_home = TempDir::new()?;
create_config_toml(
codex_home.path(),
&responses_server.uri(),
realtime_server.uri(),
/*realtime_enabled*/ true,
StartupContextConfig::Override("startup context"),
)?;
let mut mcp = McpProcess::new(codex_home.path()).await?;
mcp.initialize().await?;
login_with_api_key(&mut mcp, "sk-test-key").await?;
let thread_start_request_id = mcp
.send_thread_start_request(ThreadStartParams::default())
.await?;
let thread_start_response: JSONRPCResponse = timeout(
DEFAULT_TIMEOUT,
mcp.read_stream_until_response_message(RequestId::Integer(thread_start_request_id)),
)
.await??;
let thread_start: ThreadStartResponse = to_response(thread_start_response)?;
let start_request_id = mcp
.send_thread_realtime_start_request(ThreadRealtimeStartParams {
thread_id: thread_start.thread.id,
prompt: "backend prompt".to_string(),
session_id: None,
transport: Some(ThreadRealtimeStartTransport::Webrtc {
sdp: "v=offer\r\n".to_string(),
}),
})
.await?;
let start_response: JSONRPCResponse = timeout(
DEFAULT_TIMEOUT,
mcp.read_stream_until_response_message(RequestId::Integer(start_request_id)),
)
.await??;
let _: ThreadRealtimeStartResponse = to_response(start_response)?;
let error =
read_notification::<ThreadRealtimeErrorNotification>(&mut mcp, "thread/realtime/error")
.await?;
assert!(error.message.contains("currently experiencing high demand"));
realtime_server.shutdown().await;
Ok(())
}
#[tokio::test]
async fn realtime_conversation_requires_feature_flag() -> Result<()> {
skip_if_no_network!(Ok(()));
@@ -381,6 +596,7 @@ async fn realtime_conversation_requires_feature_flag() -> Result<()> {
&responses_server.uri(),
realtime_server.uri(),
/*realtime_enabled*/ false,
StartupContextConfig::Generated,
)?;
let mut mcp = McpProcess::new(codex_home.path()).await?;
@@ -401,6 +617,7 @@ async fn realtime_conversation_requires_feature_flag() -> Result<()> {
thread_id: thread_start.thread.id.clone(),
prompt: "backend prompt".to_string(),
session_id: None,
transport: None,
})
.await?;
let error = timeout(
@@ -450,12 +667,19 @@ fn create_config_toml(
responses_server_uri: &str,
realtime_server_uri: &str,
realtime_enabled: bool,
startup_context: StartupContextConfig<'_>,
) -> std::io::Result<()> {
let realtime_feature_key = FEATURES
.iter()
.find(|spec| spec.id == Feature::RealtimeConversation)
.map(|spec| spec.key)
.unwrap_or("realtime_conversation");
let startup_context = match startup_context {
StartupContextConfig::Generated => String::new(),
StartupContextConfig::Override(context) => {
format!("experimental_realtime_ws_startup_context = {context:?}\n")
}
};
std::fs::write(
codex_home.join("config.toml"),
@@ -466,6 +690,8 @@ approval_policy = "never"
sandbox_mode = "read-only"
model_provider = "mock_provider"
experimental_realtime_ws_base_url = "{realtime_server_uri}"
experimental_realtime_ws_backend_prompt = "backend prompt"
{startup_context}
[realtime]
version = "v2"
codex-rs/codex-api/src/endpoint/memories.rs
+6 -1
View File
@@ -72,6 +72,7 @@ mod tests {
use crate::provider::RetryConfig;
use async_trait::async_trait;
use codex_client::Request;
use codex_client::RequestBody;
use codex_client::Response;
use codex_client::StreamResponse;
use codex_client::TransportError;
@@ -213,7 +214,11 @@ mod tests {
request.url,
"https://example.com/api/codex/memories/trace_summarize"
);
let body = request.body.expect("request body should be present");
let body = request
.body
.as_ref()
.and_then(RequestBody::json)
.expect("request body should be JSON");
assert_eq!(body["model"], "gpt-test");
assert_eq!(body["traces"][0]["id"], "trace-1");
assert_eq!(
codex-rs/codex-api/src/endpoint/mod.rs
+4
View File
@@ -1,6 +1,7 @@
pub(crate) mod compact;
pub(crate) mod memories;
pub(crate) mod models;
pub(crate) mod realtime_call;
pub(crate) mod realtime_websocket;
pub(crate) mod responses;
pub(crate) mod responses_websocket;
@@ -9,6 +10,8 @@ mod session;
pub use compact::CompactClient;
pub use memories::MemoriesClient;
pub use models::ModelsClient;
pub use realtime_call::RealtimeCallClient;
pub use realtime_call::RealtimeCallResponse;
pub use realtime_websocket::RealtimeEventParser;
pub use realtime_websocket::RealtimeSessionConfig;
pub use realtime_websocket::RealtimeSessionMode;
@@ -16,6 +19,7 @@ pub use realtime_websocket::RealtimeWebsocketClient;
pub use realtime_websocket::RealtimeWebsocketConnection;
pub use realtime_websocket::RealtimeWebsocketEvents;
pub use realtime_websocket::RealtimeWebsocketWriter;
pub use realtime_websocket::session_update_session_json;
pub use responses::ResponsesClient;
pub use responses::ResponsesOptions;
pub use responses_websocket::ResponsesWebsocketClient;
codex-rs/codex-api/src/endpoint/realtime_call.rs
+415
View File
@@ -0,0 +1,415 @@
use crate::auth::AuthProvider;
use crate::endpoint::realtime_websocket::RealtimeSessionConfig;
use crate::endpoint::realtime_websocket::session_update_session_json;
use crate::endpoint::session::EndpointSession;
use crate::error::ApiError;
use crate::provider::Provider;
use bytes::Bytes;
use codex_client::HttpTransport;
use codex_client::RequestBody;
use codex_client::RequestTelemetry;
use http::HeaderMap;
use http::HeaderValue;
use http::Method;
use http::header::CONTENT_TYPE;
use serde::Serialize;
use serde_json::Value;
use serde_json::to_string;
use serde_json::to_value;
use std::sync::Arc;
use tracing::instrument;
const MULTIPART_BOUNDARY: &str = "codex-realtime-call-boundary";
const MULTIPART_CONTENT_TYPE: &str = "multipart/form-data; boundary=codex-realtime-call-boundary";
pub struct RealtimeCallClient<T: HttpTransport, A: AuthProvider> {
session: EndpointSession<T, A>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct RealtimeCallResponse {
pub sdp: String,
}
#[derive(Serialize)]
struct BackendRealtimeCallRequest<'a> {
sdp: &'a str,
session: &'a Value,
}
impl<T: HttpTransport, A: AuthProvider> RealtimeCallClient<T, A> {
pub fn new(transport: T, provider: Provider, auth: A) -> Self {
Self {
session: EndpointSession::new(transport, provider, auth),
}
}
pub fn with_telemetry(self, request: Option<Arc<dyn RequestTelemetry>>) -> Self {
Self {
session: self.session.with_request_telemetry(request),
}
}
fn path() -> &'static str {
"realtime/calls"
}
fn uses_backend_request_shape(&self) -> bool {
self.session.provider().base_url.contains("/backend-api")
}
#[instrument(
name = "realtime_call.create",
level = "info",
skip_all,
fields(
http.method = "POST",
api.path = "realtime/calls"
)
)]
pub async fn create(&self, sdp: String) -> Result<RealtimeCallResponse, ApiError> {
self.create_with_headers(sdp, HeaderMap::new()).await
}
pub async fn create_with_session(
&self,
sdp: String,
session_config: RealtimeSessionConfig,
) -> Result<RealtimeCallResponse, ApiError> {
self.create_with_session_and_headers(sdp, session_config, HeaderMap::new())
.await
}
pub async fn create_with_headers(
&self,
sdp: String,
extra_headers: HeaderMap,
) -> Result<RealtimeCallResponse, ApiError> {
let resp = self
.session
.execute_with(
Method::POST,
Self::path(),
extra_headers,
/*body*/ None,
|req| {
req.headers
.insert(CONTENT_TYPE, HeaderValue::from_static("application/sdp"));
req.body = Some(RequestBody::Raw(Bytes::from(sdp.clone())));
},
)
.await?;
let sdp = decode_sdp_response(resp.body.as_ref())?;
Ok(RealtimeCallResponse { sdp })
}
pub async fn create_with_session_and_headers(
&self,
sdp: String,
session_config: RealtimeSessionConfig,
extra_headers: HeaderMap,
) -> Result<RealtimeCallResponse, ApiError> {
let mut session = realtime_session_json(session_config)?;
if let Some(session) = session.as_object_mut() {
session.remove("id");
}
// TODO(aibrahim): Align the SIWC route with the API multipart shape and remove this branch.
if self.uses_backend_request_shape() {
let body = to_value(BackendRealtimeCallRequest {
sdp: &sdp,
session: &session,
})
.map_err(|err| ApiError::Stream(format!("failed to encode realtime call: {err}")))?;
let resp = self
.session
.execute(Method::POST, Self::path(), extra_headers, Some(body))
.await?;
let sdp = decode_sdp_response(resp.body.as_ref())?;
return Ok(RealtimeCallResponse { sdp });
}
let session = to_string(&session).map_err(|err| ApiError::InvalidRequest {
message: err.to_string(),
})?;
let mut body = Vec::new();
body.extend_from_slice(format!("--{MULTIPART_BOUNDARY}\r\n").as_bytes());
body.extend_from_slice(b"Content-Disposition: form-data; name=\"sdp\"\r\n");
body.extend_from_slice(b"Content-Type: application/sdp\r\n\r\n");
body.extend_from_slice(sdp.as_bytes());
body.extend_from_slice(b"\r\n");
body.extend_from_slice(format!("--{MULTIPART_BOUNDARY}\r\n").as_bytes());
body.extend_from_slice(b"Content-Disposition: form-data; name=\"session\"\r\n");
body.extend_from_slice(b"Content-Type: application/json\r\n\r\n");
body.extend_from_slice(session.as_bytes());
body.extend_from_slice(b"\r\n");
body.extend_from_slice(format!("--{MULTIPART_BOUNDARY}--\r\n").as_bytes());
let resp = self
.session
.execute_with(
Method::POST,
Self::path(),
extra_headers,
/*body*/ None,
|req| {
req.headers.insert(
CONTENT_TYPE,
HeaderValue::from_static(MULTIPART_CONTENT_TYPE),
);
req.body = Some(RequestBody::Raw(Bytes::from(body.clone())));
},
)
.await?;
let sdp = decode_sdp_response(resp.body.as_ref())?;
Ok(RealtimeCallResponse { sdp })
}
}
fn realtime_session_json(session_config: RealtimeSessionConfig) -> Result<Value, ApiError> {
session_update_session_json(session_config)
.map_err(|err| ApiError::Stream(format!("failed to encode realtime call session: {err}")))
}
fn decode_sdp_response(body: &[u8]) -> Result<String, ApiError> {
String::from_utf8(body.to_vec()).map_err(|err| {
ApiError::Stream(format!(
"failed to decode realtime call SDP response: {err}"
))
})
}
#[cfg(test)]
mod tests {
use super::*;
use crate::endpoint::realtime_websocket::RealtimeEventParser;
use crate::endpoint::realtime_websocket::RealtimeSessionMode;
use crate::provider::RetryConfig;
use async_trait::async_trait;
use codex_client::Request;
use codex_client::Response;
use codex_client::StreamResponse;
use codex_client::TransportError;
use http::StatusCode;
use pretty_assertions::assert_eq;
use std::sync::Mutex;
use std::time::Duration;
#[derive(Clone)]
struct CapturingTransport {
last_request: Arc<Mutex<Option<Request>>>,
}
impl CapturingTransport {
fn new() -> Self {
Self {
last_request: Arc::new(Mutex::new(None)),
}
}
}
#[async_trait]
impl HttpTransport for CapturingTransport {
async fn execute(&self, req: Request) -> Result<Response, TransportError> {
*self.last_request.lock().unwrap() = Some(req);
Ok(Response {
status: StatusCode::OK,
headers: HeaderMap::new(),
body: Bytes::from_static(b"v=0\r\n"),
})
}
async fn stream(&self, _req: Request) -> Result<StreamResponse, TransportError> {
Err(TransportError::Build("stream should not run".to_string()))
}
}
#[derive(Clone, Default)]
struct DummyAuth;
impl AuthProvider for DummyAuth {
fn bearer_token(&self) -> Option<String> {
Some("test-token".to_string())
}
}
fn provider(base_url: &str) -> Provider {
Provider {
name: "test".to_string(),
base_url: base_url.to_string(),
query_params: None,
headers: HeaderMap::new(),
retry: RetryConfig {
max_attempts: 1,
base_delay: Duration::from_millis(1),
retry_429: false,
retry_5xx: true,
retry_transport: true,
},
stream_idle_timeout: Duration::from_secs(1),
}
}
fn realtime_session_config(session_id: &str) -> RealtimeSessionConfig {
RealtimeSessionConfig {
instructions: "hi".to_string(),
model: Some("gpt-realtime".to_string()),
session_id: Some(session_id.to_string()),
event_parser: RealtimeEventParser::RealtimeV2,
session_mode: RealtimeSessionMode::Conversational,
}
}
#[tokio::test]
async fn sends_sdp_offer_as_raw_body() {
let transport = CapturingTransport::new();
let client = RealtimeCallClient::new(
transport.clone(),
provider("https://api.openai.com/v1"),
DummyAuth,
);
let response = client
.create("v=offer\r\n".to_string())
.await
.expect("request should succeed");
assert_eq!(
response,
RealtimeCallResponse {
sdp: "v=0\r\n".to_string()
}
);
let request = transport.last_request.lock().unwrap().clone().unwrap();
assert_eq!(request.method, Method::POST);
assert_eq!(request.url, "https://api.openai.com/v1/realtime/calls");
assert_eq!(
request.headers.get(CONTENT_TYPE).unwrap(),
HeaderValue::from_static("application/sdp")
);
assert_eq!(
request
.headers
.get(http::header::AUTHORIZATION)
.and_then(|value| value.to_str().ok()),
Some("Bearer test-token")
);
assert_eq!(
request.body,
Some(RequestBody::Raw(Bytes::from_static(b"v=offer\r\n")))
);
}
#[tokio::test]
async fn sends_api_session_call_as_multipart_body() {
let transport = CapturingTransport::new();
let client = RealtimeCallClient::new(
transport.clone(),
provider("https://api.openai.com/v1"),
DummyAuth,
);
let response = client
.create_with_session(
"v=offer\r\n".to_string(),
realtime_session_config("sess-api"),
)
.await
.expect("request should succeed");
assert_eq!(
response,
RealtimeCallResponse {
sdp: "v=0\r\n".to_string()
}
);
let request = transport.last_request.lock().unwrap().clone().unwrap();
assert_eq!(request.method, Method::POST);
assert_eq!(request.url, "https://api.openai.com/v1/realtime/calls");
assert_eq!(
request.headers.get(CONTENT_TYPE).unwrap(),
HeaderValue::from_static(MULTIPART_CONTENT_TYPE)
);
let Some(RequestBody::Raw(body)) = request.body else {
panic!("multipart body should be raw");
};
let body = std::str::from_utf8(&body).expect("multipart body should be utf-8");
let mut session = realtime_session_json(realtime_session_config("sess-api"))
.expect("session should encode");
session
.as_object_mut()
.expect("session should be an object")
.remove("id");
let session = to_string(&session).expect("session should serialize");
assert_eq!(
body,
format!(
"--codex-realtime-call-boundary\r\n\
Content-Disposition: form-data; name=\"sdp\"\r\n\
Content-Type: application/sdp\r\n\
\r\n\
v=offer\r\n\
\r\n\
--codex-realtime-call-boundary\r\n\
Content-Disposition: form-data; name=\"session\"\r\n\
Content-Type: application/json\r\n\
\r\n\
{session}\r\n\
--codex-realtime-call-boundary--\r\n"
)
);
}
#[tokio::test]
async fn sends_backend_session_call_as_json_body() {
let transport = CapturingTransport::new();
let client = RealtimeCallClient::new(
transport.clone(),
provider("https://chatgpt.com/backend-api/codex"),
DummyAuth,
);
let response = client
.create_with_session(
"v=offer\r\n".to_string(),
realtime_session_config("sess-backend"),
)
.await
.expect("request should succeed");
assert_eq!(
response,
RealtimeCallResponse {
sdp: "v=0\r\n".to_string()
}
);
let request = transport.last_request.lock().unwrap().clone().unwrap();
assert_eq!(request.method, Method::POST);
assert_eq!(
request.url,
"https://chatgpt.com/backend-api/codex/realtime/calls"
);
let mut expected_session = realtime_session_json(realtime_session_config("sess-backend"))
.expect("session should encode");
expected_session
.as_object_mut()
.expect("session should be an object")
.remove("id");
assert_eq!(
request.body,
Some(RequestBody::Json(
to_value(BackendRealtimeCallRequest {
sdp: "v=offer\r\n",
session: &expected_session,
})
.expect("request should encode")
))
);
}
}
codex-rs/codex-api/src/endpoint/realtime_websocket/methods_common.rs
+15
View File
@@ -8,8 +8,12 @@ use crate::endpoint::realtime_websocket::methods_v2::session_update_session as v
use crate::endpoint::realtime_websocket::methods_v2::websocket_intent as v2_websocket_intent;
use crate::endpoint::realtime_websocket::protocol::RealtimeEventParser;
use crate::endpoint::realtime_websocket::protocol::RealtimeOutboundMessage;
use crate::endpoint::realtime_websocket::protocol::RealtimeSessionConfig;
use crate::endpoint::realtime_websocket::protocol::RealtimeSessionMode;
use crate::endpoint::realtime_websocket::protocol::SessionUpdateSession;
use serde_json::Result as JsonResult;
use serde_json::Value;
use serde_json::to_value;
pub(super) const REALTIME_AUDIO_SAMPLE_RATE: u32 = 24_000;
const AGENT_FINAL_MESSAGE_PREFIX: &str = "\"Agent Final Message\":\n\n";
@@ -60,6 +64,17 @@ pub(super) fn session_update_session(
}
}
pub fn session_update_session_json(config: RealtimeSessionConfig) -> JsonResult<Value> {
let mut session = session_update_session(
config.event_parser,
config.instructions,
config.session_mode,
);
session.id = config.session_id;
session.model = config.model;
to_value(session)
}
pub(super) fn websocket_intent(event_parser: RealtimeEventParser) -> Option<&'static str> {
match event_parser {
RealtimeEventParser::V1 => v1_websocket_intent(),
codex-rs/codex-api/src/endpoint/realtime_websocket/methods_v1.rs
+2
View File
@@ -40,7 +40,9 @@ pub(super) fn conversation_handoff_append_message(
pub(super) fn session_update_session(instructions: String) -> SessionUpdateSession {
SessionUpdateSession {
id: None,
r#type: SessionType::Quicksilver,
model: None,
instructions: Some(instructions),
output_modalities: None,
audio: SessionAudio {
codex-rs/codex-api/src/endpoint/realtime_websocket/methods_v2.rs
+4
View File
@@ -62,7 +62,9 @@ pub(super) fn session_update_session(
) -> SessionUpdateSession {
match session_mode {
RealtimeSessionMode::Conversational => SessionUpdateSession {
id: None,
r#type: SessionType::Realtime,
model: None,
instructions: Some(instructions),
output_modalities: Some(vec![REALTIME_V2_OUTPUT_MODALITY_AUDIO.to_string()]),
audio: SessionAudio {
@@ -107,7 +109,9 @@ pub(super) fn session_update_session(
tool_choice: Some(REALTIME_V2_TOOL_CHOICE.to_string()),
},
RealtimeSessionMode::Transcription => SessionUpdateSession {
id: None,
r#type: SessionType::Transcription,
model: None,
instructions: None,
output_modalities: None,
audio: SessionAudio {
codex-rs/codex-api/src/endpoint/realtime_websocket/mod.rs
+1
View File
@@ -11,6 +11,7 @@ pub use methods::RealtimeWebsocketClient;
pub use methods::RealtimeWebsocketConnection;
pub use methods::RealtimeWebsocketEvents;
pub use methods::RealtimeWebsocketWriter;
pub use methods_common::session_update_session_json;
pub use protocol::RealtimeEventParser;
pub use protocol::RealtimeSessionConfig;
pub use protocol::RealtimeSessionMode;
codex-rs/codex-api/src/endpoint/realtime_websocket/protocol.rs
+4
View File
@@ -48,9 +48,13 @@ pub(super) enum RealtimeOutboundMessage {
#[derive(Debug, Clone, Serialize)]
pub(super) struct SessionUpdateSession {
#[serde(skip_serializing_if = "Option::is_none")]
pub(super) id: Option<String>,
#[serde(rename = "type")]
pub(super) r#type: SessionType,
#[serde(skip_serializing_if = "Option::is_none")]
pub(super) model: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub(super) instructions: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub(super) output_modalities: Option<Vec<String>>,
codex-rs/codex-api/src/endpoint/session.rs
+2 -1
View File
@@ -5,6 +5,7 @@ use crate::provider::Provider;
use crate::telemetry::run_with_request_telemetry;
use codex_client::HttpTransport;
use codex_client::Request;
use codex_client::RequestBody;
use codex_client::RequestTelemetry;
use codex_client::Response;
use codex_client::StreamResponse;
@@ -53,7 +54,7 @@ impl<T: HttpTransport, A: AuthProvider> EndpointSession<T, A> {
let mut req = self.provider.build_request(method.clone(), path);
req.headers.extend(extra_headers.clone());
if let Some(body) = body {
req.body = Some(body.clone());
req.body = Some(RequestBody::Json(body.clone()));
}
add_auth_headers(&self.auth, req)
}
codex-rs/codex-api/src/lib.rs
+3
View File
@@ -37,6 +37,8 @@ pub use crate::common::response_create_client_metadata;
pub use crate::endpoint::CompactClient;
pub use crate::endpoint::MemoriesClient;
pub use crate::endpoint::ModelsClient;
pub use crate::endpoint::RealtimeCallClient;
pub use crate::endpoint::RealtimeCallResponse;
pub use crate::endpoint::RealtimeEventParser;
pub use crate::endpoint::RealtimeSessionConfig;
pub use crate::endpoint::RealtimeSessionMode;
@@ -48,6 +50,7 @@ pub use crate::endpoint::ResponsesClient;
pub use crate::endpoint::ResponsesOptions;
pub use crate::endpoint::ResponsesWebsocketClient;
pub use crate::endpoint::ResponsesWebsocketConnection;
pub use crate::endpoint::session_update_session_json;
pub use crate::error::ApiError;
pub use crate::provider::Provider;
pub use crate::provider::RetryConfig;
codex-rs/codex-api/tests/clients.rs
+2
View File
@@ -13,6 +13,7 @@ use codex_api::ResponsesClient;
use codex_api::ResponsesOptions;
use codex_client::HttpTransport;
use codex_client::Request;
use codex_client::RequestBody;
use codex_client::Response;
use codex_client::StreamResponse;
use codex_client::TransportError;
@@ -363,6 +364,7 @@ async fn azure_default_store_attaches_ids_and_headers() -> Result<()> {
let input_id = req
.body
.as_ref()
.and_then(RequestBody::json)
.and_then(|body| body.get("input"))
.and_then(|input| input.get(0))
.and_then(|item| item.get("id"))
codex-rs/codex-client/src/lib.rs
+1
View File
@@ -22,6 +22,7 @@ pub use crate::default_client::CodexRequestBuilder;
pub use crate::error::StreamError;
pub use crate::error::TransportError;
pub use crate::request::Request;
pub use crate::request::RequestBody;
pub use crate::request::RequestCompression;
pub use crate::request::Response;
pub use crate::retry::RetryOn;
codex-rs/codex-client/src/request.rs
+22 -2
View File
@@ -12,12 +12,27 @@ pub enum RequestCompression {
Zstd,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum RequestBody {
Json(Value),
Raw(Bytes),
}
impl RequestBody {
pub fn json(&self) -> Option<&Value> {
match self {
Self::Json(value) => Some(value),
Self::Raw(_) => None,
}
}
}
#[derive(Debug, Clone)]
pub struct Request {
pub method: Method,
pub url: String,
pub headers: HeaderMap,
pub body: Option<Value>,
pub body: Option<RequestBody>,
pub compression: RequestCompression,
pub timeout: Option<Duration>,
}
@@ -35,7 +50,12 @@ impl Request {
}
pub fn with_json<T: Serialize>(mut self, body: &T) -> Self {
self.body = serde_json::to_value(body).ok();
self.body = serde_json::to_value(body).ok().map(RequestBody::Json);
self
}
pub fn with_raw_body(mut self, body: impl Into<Bytes>) -> Self {
self.body = Some(RequestBody::Raw(body.into()));
self
}
codex-rs/codex-client/src/transport.rs
+64 -44
View File
@@ -2,6 +2,7 @@ use crate::default_client::CodexHttpClient;
use crate::default_client::CodexRequestBuilder;
use crate::error::TransportError;
use crate::request::Request;
use crate::request::RequestBody;
use crate::request::RequestCompression;
use crate::request::Response;
use async_trait::async_trait;
@@ -60,52 +61,63 @@ impl ReqwestTransport {
builder = builder.timeout(timeout);
}
if let Some(body) = body {
if compression != RequestCompression::None {
if headers.contains_key(http::header::CONTENT_ENCODING) {
match body {
Some(RequestBody::Raw(raw_body)) => {
if compression != RequestCompression::None {
return Err(TransportError::Build(
"request compression was requested but content-encoding is already set"
.to_string(),
"request compression cannot be used with raw bodies".to_string(),
));
}
let json = serde_json::to_vec(&body)
.map_err(|err| TransportError::Build(err.to_string()))?;
let pre_compression_bytes = json.len();
let compression_start = std::time::Instant::now();
let (compressed, content_encoding) = match compression {
RequestCompression::None => unreachable!("guarded by compression != None"),
RequestCompression::Zstd => (
zstd::stream::encode_all(std::io::Cursor::new(json), 3)
.map_err(|err| TransportError::Build(err.to_string()))?,
http::HeaderValue::from_static("zstd"),
),
};
let post_compression_bytes = compressed.len();
let compression_duration = compression_start.elapsed();
// Ensure the server knows to unpack the request body.
headers.insert(http::header::CONTENT_ENCODING, content_encoding);
if !headers.contains_key(http::header::CONTENT_TYPE) {
headers.insert(
http::header::CONTENT_TYPE,
http::HeaderValue::from_static("application/json"),
);
}
tracing::info!(
pre_compression_bytes,
post_compression_bytes,
compression_duration_ms = compression_duration.as_millis(),
"Compressed request body with zstd"
);
builder = builder.headers(headers).body(compressed);
} else {
builder = builder.headers(headers).json(&body);
builder = builder.headers(headers).body(raw_body);
}
Some(RequestBody::Json(body)) => {
if compression != RequestCompression::None {
if headers.contains_key(http::header::CONTENT_ENCODING) {
return Err(TransportError::Build(
"request compression was requested but content-encoding is already set"
.to_string(),
));
}
let json = serde_json::to_vec(&body)
.map_err(|err| TransportError::Build(err.to_string()))?;
let pre_compression_bytes = json.len();
let compression_start = std::time::Instant::now();
let (compressed, content_encoding) = match compression {
RequestCompression::None => unreachable!("guarded by compression != None"),
RequestCompression::Zstd => (
zstd::stream::encode_all(std::io::Cursor::new(json), 3)
.map_err(|err| TransportError::Build(err.to_string()))?,
http::HeaderValue::from_static("zstd"),
),
};
let post_compression_bytes = compressed.len();
let compression_duration = compression_start.elapsed();
// Ensure the server knows to unpack the request body.
headers.insert(http::header::CONTENT_ENCODING, content_encoding);
if !headers.contains_key(http::header::CONTENT_TYPE) {
headers.insert(
http::header::CONTENT_TYPE,
http::HeaderValue::from_static("application/json"),
);
}
tracing::info!(
pre_compression_bytes,
post_compression_bytes,
compression_duration_ms = compression_duration.as_millis(),
"Compressed request body with zstd"
);
builder = builder.headers(headers).body(compressed);
} else {
builder = builder.headers(headers).json(&body);
}
}
None => {
builder = builder.headers(headers);
}
} else {
builder = builder.headers(headers);
}
Ok(builder)
}
@@ -119,6 +131,14 @@ impl ReqwestTransport {
}
}
fn request_body_for_trace(req: &Request) -> String {
match req.body.as_ref() {
Some(RequestBody::Json(body)) => body.to_string(),
Some(RequestBody::Raw(body)) => format!("<raw body: {} bytes>", body.len()),
None => String::new(),
}
}
#[async_trait]
impl HttpTransport for ReqwestTransport {
async fn execute(&self, req: Request) -> Result<Response, TransportError> {
@@ -127,7 +147,7 @@ impl HttpTransport for ReqwestTransport {
"{} to {}: {}",
req.method,
req.url,
req.body.as_ref().unwrap_or_default()
request_body_for_trace(&req)
);
}
@@ -159,7 +179,7 @@ impl HttpTransport for ReqwestTransport {
"{} to {}: {}",
req.method,
req.url,
req.body.as_ref().unwrap_or_default()
request_body_for_trace(&req)
);
}
codex-rs/core/src/client.rs
+26
View File
@@ -39,6 +39,8 @@ use codex_api::MemoriesClient as ApiMemoriesClient;
use codex_api::MemorySummarizeInput as ApiMemorySummarizeInput;
use codex_api::MemorySummarizeOutput as ApiMemorySummarizeOutput;
use codex_api::RawMemory as ApiRawMemory;
use codex_api::RealtimeCallClient as ApiRealtimeCallClient;
use codex_api::RealtimeSessionConfig;
use codex_api::Reasoning;
use codex_api::RequestTelemetry;
use codex_api::ReqwestTransport;
@@ -443,6 +445,30 @@ impl ModelClient {
.map_err(map_api_error)
}
pub async fn create_realtime_call(
&self,
sdp: String,
session_config: RealtimeSessionConfig,
) -> Result<String> {
self.create_realtime_call_with_headers(sdp, session_config, ApiHeaderMap::new())
.await
}
pub async fn create_realtime_call_with_headers(
&self,
sdp: String,
session_config: RealtimeSessionConfig,
extra_headers: ApiHeaderMap,
) -> Result<String> {
let client_setup = self.current_client_setup().await?;
let transport = ReqwestTransport::new(build_reqwest_client());
ApiRealtimeCallClient::new(transport, client_setup.api_provider, client_setup.api_auth)
.create_with_session_and_headers(sdp, session_config, extra_headers)
.await
.map(|response| response.sdp)
.map_err(map_api_error)
}
/// Builds memory summaries for each provided normalized raw memory.
///
/// This is a unary call (no streaming) to `/v1/memories/trace_summarize`.
codex-rs/core/src/codex.rs
+1
View File
@@ -7037,6 +7037,7 @@ fn realtime_text_for_event(msg: &EventMsg) -> Option<String> {
EventMsg::Error(_)
| EventMsg::Warning(_)
| EventMsg::RealtimeConversationStarted(_)
| EventMsg::RealtimeConversationSdp(_)
| EventMsg::RealtimeConversationRealtime(_)
| EventMsg::RealtimeConversationClosed(_)
| EventMsg::ModelReroute(_)
codex-rs/core/src/realtime_conversation.rs
+141 -33
View File
@@ -1,3 +1,4 @@
use crate::client::ModelClient;
use crate::codex::Session;
use crate::realtime_context::build_realtime_startup_context;
use async_channel::Receiver;
@@ -27,12 +28,14 @@ use codex_protocol::error::Result as CodexResult;
use codex_protocol::protocol::CodexErrorInfo;
use codex_protocol::protocol::ConversationAudioParams;
use codex_protocol::protocol::ConversationStartParams;
use codex_protocol::protocol::ConversationStartTransport;
use codex_protocol::protocol::ConversationTextParams;
use codex_protocol::protocol::ErrorEvent;
use codex_protocol::protocol::Event;
use codex_protocol::protocol::EventMsg;
use codex_protocol::protocol::RealtimeConversationClosedEvent;
use codex_protocol::protocol::RealtimeConversationRealtimeEvent;
use codex_protocol::protocol::RealtimeConversationSdpEvent;
use codex_protocol::protocol::RealtimeConversationStartedEvent;
use codex_protocol::protocol::RealtimeHandoffRequested;
use http::HeaderMap;
@@ -139,6 +142,24 @@ struct ConversationState {
realtime_active: Arc<AtomicBool>,
}
struct RealtimeStart {
api_provider: ApiProvider,
extra_headers: Option<HeaderMap>,
session_config: RealtimeSessionConfig,
model_client: ModelClient,
sdp: Option<String>,
}
struct RealtimeStartOutput {
realtime_active: Arc<AtomicBool>,
connection: RealtimeStartConnection,
}
enum RealtimeStartConnection {
Websocket { events_rx: Receiver<RealtimeEvent> },
Webrtc { sdp: String },
}
#[allow(dead_code)]
impl RealtimeConversationManager {
pub(crate) fn new() -> Self {
@@ -154,12 +175,7 @@ impl RealtimeConversationManager {
.and_then(|state| state.realtime_active.load(Ordering::Relaxed).then_some(()))
}
pub(crate) async fn start(
&self,
api_provider: ApiProvider,
extra_headers: Option<HeaderMap>,
session_config: RealtimeSessionConfig,
) -> CodexResult<(Receiver<RealtimeEvent>, Arc<AtomicBool>)> {
async fn start(&self, start: RealtimeStart) -> CodexResult<RealtimeStartOutput> {
let previous_state = {
let mut guard = self.state.lock().await;
guard.take()
@@ -167,10 +183,37 @@ impl RealtimeConversationManager {
if let Some(state) = previous_state {
stop_conversation_state(state, RealtimeFanoutTaskStop::Abort).await;
}
self.start_inner(start).await
}
async fn start_inner(&self, start: RealtimeStart) -> CodexResult<RealtimeStartOutput> {
let RealtimeStart {
api_provider,
extra_headers,
session_config,
model_client,
sdp,
} = start;
let session_kind = match session_config.event_parser {
RealtimeEventParser::V1 => RealtimeSessionKind::V1,
RealtimeEventParser::RealtimeV2 => RealtimeSessionKind::V2,
};
let realtime_active = Arc::new(AtomicBool::new(true));
if let Some(sdp) = sdp {
let sdp = model_client
.create_realtime_call_with_headers(
sdp,
session_config,
extra_headers.unwrap_or_default(),
)
.await?;
return Ok(RealtimeStartOutput {
realtime_active,
connection: RealtimeStartConnection::Webrtc { sdp },
});
}
let client = RealtimeWebsocketClient::new(api_provider);
let connection = client
@@ -216,7 +259,10 @@ impl RealtimeConversationManager {
fanout_task: None,
realtime_active: Arc::clone(&realtime_active),
});
Ok((events_rx, realtime_active))
Ok(RealtimeStartOutput {
realtime_active,
connection: RealtimeStartConnection::Websocket { events_rx },
})
}
pub(crate) async fn register_fanout_task(
@@ -447,6 +493,7 @@ struct PreparedRealtimeConversationStart {
requested_session_id: Option<String>,
version: RealtimeWsVersion,
session_config: RealtimeSessionConfig,
transport: ConversationStartTransport,
}
async fn prepare_realtime_start(
@@ -460,16 +507,54 @@ async fn prepare_realtime_start(
.auth_manager()
.unwrap_or_else(|| Arc::clone(&sess.services.auth_manager));
let auth = auth_manager.auth().await;
let realtime_api_key = realtime_api_key(auth.as_ref(), &provider)?;
let mut api_provider = provider.to_api_provider(Some(AuthMode::ApiKey))?;
let config = sess.get_config().await;
let transport = params
.transport
.unwrap_or(ConversationStartTransport::Websocket);
let mut api_provider = if matches!(transport, ConversationStartTransport::Websocket) {
provider.to_api_provider(Some(AuthMode::ApiKey))?
} else {
provider.to_api_provider(auth.as_ref().map(CodexAuth::auth_mode))?
};
if let Some(realtime_ws_base_url) = &config.experimental_realtime_ws_base_url {
api_provider.base_url = realtime_ws_base_url.clone();
}
let version = config.realtime.version;
let session_config =
build_realtime_session_config(sess, params.prompt, params.session_id).await?;
let requested_session_id = session_config.session_id.clone();
let extra_headers = match transport {
ConversationStartTransport::Websocket => {
let realtime_api_key = realtime_api_key(auth.as_ref(), &provider)?;
realtime_request_headers(
requested_session_id.as_deref(),
Some(realtime_api_key.as_str()),
)?
}
ConversationStartTransport::Webrtc { .. } => {
realtime_request_headers(requested_session_id.as_deref(), /*api_key*/ None)?
}
};
Ok(PreparedRealtimeConversationStart {
api_provider,
extra_headers,
requested_session_id,
version,
session_config,
transport,
})
}
pub(crate) async fn build_realtime_session_config(
sess: &Arc<Session>,
prompt: String,
session_id: Option<String>,
) -> CodexResult<RealtimeSessionConfig> {
let config = sess.get_config().await;
let prompt = config
.experimental_realtime_ws_backend_prompt
.clone()
.unwrap_or(params.prompt);
.unwrap_or(prompt);
let startup_context = match config.experimental_realtime_ws_startup_context.clone() {
Some(startup_context) => startup_context,
None => {
@@ -484,8 +569,7 @@ async fn prepare_realtime_start(
format!("{prompt}\n\n{startup_context}")
};
let model = config.experimental_realtime_ws_model.clone();
let version = config.realtime.version;
let event_parser = match version {
let event_parser = match config.realtime.version {
RealtimeWsVersion::V1 => RealtimeEventParser::V1,
RealtimeWsVersion::V2 => RealtimeEventParser::RealtimeV2,
};
@@ -493,22 +577,12 @@ async fn prepare_realtime_start(
RealtimeWsMode::Conversational => RealtimeSessionMode::Conversational,
RealtimeWsMode::Transcription => RealtimeSessionMode::Transcription,
};
let requested_session_id = params.session_id.or(Some(sess.conversation_id.to_string()));
let session_config = RealtimeSessionConfig {
Ok(RealtimeSessionConfig {
instructions: prompt,
model,
session_id: requested_session_id.clone(),
session_id: Some(session_id.unwrap_or_else(|| sess.conversation_id.to_string())),
event_parser,
session_mode,
};
let extra_headers =
realtime_request_headers(requested_session_id.as_deref(), realtime_api_key.as_str())?;
Ok(PreparedRealtimeConversationStart {
api_provider,
extra_headers,
requested_session_id,
version,
session_config,
})
}
@@ -523,12 +597,21 @@ async fn handle_start_inner(
requested_session_id,
version,
session_config,
transport,
} = prepared_start;
info!("starting realtime conversation");
let (events_rx, realtime_active) = sess
.conversation
.start(api_provider, extra_headers, session_config)
.await?;
let sdp = match transport {
ConversationStartTransport::Websocket => None,
ConversationStartTransport::Webrtc { sdp } => Some(sdp),
};
let start = RealtimeStart {
api_provider,
extra_headers,
session_config,
model_client: sess.services.model_client.clone(),
sdp,
};
let start_output = sess.conversation.start(start).await?;
info!("realtime conversation started");
@@ -541,6 +624,29 @@ async fn handle_start_inner(
})
.await;
let RealtimeStartOutput {
realtime_active,
connection,
} = start_output;
let events_rx = match connection {
RealtimeStartConnection::Websocket { events_rx } => events_rx,
RealtimeStartConnection::Webrtc { sdp } => {
sess.send_event_raw(Event {
id: sub_id.to_string(),
msg: EventMsg::RealtimeConversationSdp(RealtimeConversationSdpEvent { sdp }),
})
.await;
sess.conversation.finish_if_active(&realtime_active).await;
send_realtime_conversation_closed(
sess,
sub_id.to_string(),
RealtimeConversationEnd::TransportClosed,
)
.await;
return Ok(());
}
};
let sess_clone = Arc::clone(sess);
let sub_id = sub_id.to_string();
let fanout_realtime_active = Arc::clone(&realtime_active);
@@ -660,7 +766,7 @@ fn realtime_api_key(auth: Option<&CodexAuth>, provider: &ModelProviderInfo) -> C
fn realtime_request_headers(
session_id: Option<&str>,
api_key: &str,
api_key: Option<&str>,
) -> CodexResult<Option<HeaderMap>> {
let mut headers = HeaderMap::new();
@@ -670,10 +776,12 @@ fn realtime_request_headers(
headers.insert("x-session-id", session_id);
}
let auth_value = HeaderValue::from_str(&format!("Bearer {api_key}")).map_err(|err| {
CodexErr::InvalidRequest(format!("invalid realtime api key header: {err}"))
})?;
headers.insert(AUTHORIZATION, auth_value);
if let Some(api_key) = api_key {
let auth_value = HeaderValue::from_str(&format!("Bearer {api_key}")).map_err(|err| {
CodexErr::InvalidRequest(format!("invalid realtime api key header: {err}"))
})?;
headers.insert(AUTHORIZATION, auth_value);
}
Ok(Some(headers))
}
codex-rs/core/tests/suite/compact_remote.rs
+1
View File
@@ -118,6 +118,7 @@ async fn start_realtime_conversation(codex: &codex_core::CodexThread) -> Result<
.submit(Op::RealtimeConversationStart(ConversationStartParams {
prompt: "backend prompt".to_string(),
session_id: None,
transport: None,
}))
.await?;
codex-rs/core/tests/suite/realtime_conversation.rs
+149
View File
@@ -7,6 +7,7 @@ use codex_protocol::ThreadId;
use codex_protocol::protocol::CodexErrorInfo;
use codex_protocol::protocol::ConversationAudioParams;
use codex_protocol::protocol::ConversationStartParams;
use codex_protocol::protocol::ConversationStartTransport;
use codex_protocol::protocol::ConversationTextParams;
use codex_protocol::protocol::ErrorEvent;
use codex_protocol::protocol::EventMsg;
@@ -32,15 +33,56 @@ use serde_json::Value;
use serde_json::json;
use std::fs;
use std::process::Command;
use std::sync::Arc;
use std::sync::Mutex;
use std::time::Duration;
use tokio::sync::oneshot;
use tokio::time::timeout;
use wiremock::Match;
use wiremock::Mock;
use wiremock::Request as WiremockRequest;
use wiremock::ResponseTemplate;
use wiremock::matchers::method;
use wiremock::matchers::path_regex;
const STARTUP_CONTEXT_HEADER: &str = "Startup context from Codex.";
const MEMORY_PROMPT_PHRASE: &str =
"You have access to a memory folder with guidance from prior runs.";
const REALTIME_CONVERSATION_TEST_SUBPROCESS_ENV_VAR: &str =
"CODEX_REALTIME_CONVERSATION_TEST_SUBPROCESS";
#[derive(Debug, Clone)]
struct RealtimeCallRequestCapture {
requests: Arc<Mutex<Vec<WiremockRequest>>>,
}
impl RealtimeCallRequestCapture {
fn new() -> Self {
Self {
requests: Arc::new(Mutex::new(Vec::new())),
}
}
fn single_request(&self) -> WiremockRequest {
let requests = self
.requests
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
assert_eq!(requests.len(), 1, "expected one realtime call request");
requests[0].clone()
}
}
impl Match for RealtimeCallRequestCapture {
fn matches(&self, request: &WiremockRequest) -> bool {
self.requests
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.push(request.clone());
true
}
}
fn websocket_request_text(
request: &core_test_support::responses::WebSocketRequest,
) -> Option<String> {
@@ -182,6 +224,7 @@ async fn conversation_start_audio_text_close_round_trip() -> Result<()> {
.submit(Op::RealtimeConversationStart(ConversationStartParams {
prompt: "backend prompt".to_string(),
session_id: None,
transport: None,
}))
.await?;
@@ -292,6 +335,89 @@ async fn conversation_start_audio_text_close_round_trip() -> Result<()> {
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn conversation_webrtc_start_posts_generated_session() -> Result<()> {
let server = start_mock_server().await;
let capture = RealtimeCallRequestCapture::new();
Mock::given(method("POST"))
.and(path_regex(".*/realtime/calls$"))
.and(capture.clone())
.respond_with(ResponseTemplate::new(200).set_body_string("v=answer\r\n"))
.mount(&server)
.await;
let mut builder = test_codex().with_config(|config| {
config.experimental_realtime_ws_backend_prompt = Some("backend prompt".to_string());
config.experimental_realtime_ws_model = Some("realtime-test-model".to_string());
config.experimental_realtime_ws_startup_context = Some("startup context".to_string());
});
let test = builder.build(&server).await?;
test.codex
.submit(Op::RealtimeConversationStart(ConversationStartParams {
prompt: "backend prompt".to_string(),
session_id: None,
transport: Some(ConversationStartTransport::Webrtc {
sdp: "v=offer\r\n".to_string(),
}),
}))
.await?;
let created = wait_for_event_match(&test.codex, |msg| match msg {
EventMsg::RealtimeConversationSdp(created) => Some(Ok(created.clone())),
EventMsg::Error(err) => Some(Err(err.clone())),
_ => None,
})
.await
.unwrap_or_else(|err: ErrorEvent| panic!("conversation call create failed: {err:?}"));
assert_eq!(created.sdp, "v=answer\r\n");
let closed = wait_for_event_match(&test.codex, |msg| match msg {
EventMsg::RealtimeConversationClosed(closed) => Some(closed.clone()),
_ => None,
})
.await;
assert_eq!(closed.reason.as_deref(), Some("transport_closed"));
let request = capture.single_request();
assert_eq!(request.url.path(), "/v1/realtime/calls");
assert_eq!(request.url.query(), None);
assert_eq!(
request
.headers
.get("authorization")
.and_then(|value| value.to_str().ok()),
Some("Bearer dummy")
);
assert_eq!(
request
.headers
.get("content-type")
.and_then(|value| value.to_str().ok()),
Some("multipart/form-data; boundary=codex-realtime-call-boundary")
);
let body = String::from_utf8(request.body).context("multipart body should be utf-8")?;
let session = r#"{"audio":{"input":{"format":{"type":"audio/pcm","rate":24000}},"output":{"voice":"fathom"}},"type":"quicksilver","model":"realtime-test-model","instructions":"backend prompt\n\nstartup context"}"#;
assert_eq!(
body,
format!(
"--codex-realtime-call-boundary\r\n\
Content-Disposition: form-data; name=\"sdp\"\r\n\
Content-Type: application/sdp\r\n\
\r\n\
v=offer\r\n\
\r\n\
--codex-realtime-call-boundary\r\n\
Content-Disposition: form-data; name=\"session\"\r\n\
Content-Type: application/json\r\n\
\r\n\
{session}\r\n\
--codex-realtime-call-boundary--\r\n"
)
);
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn conversation_start_uses_openai_env_key_fallback_with_chatgpt_auth() -> Result<()> {
if std::env::var_os(REALTIME_CONVERSATION_TEST_SUBPROCESS_ENV_VAR).is_none() {
@@ -324,6 +450,7 @@ async fn conversation_start_uses_openai_env_key_fallback_with_chatgpt_auth() ->
.submit(Op::RealtimeConversationStart(ConversationStartParams {
prompt: "backend prompt".to_string(),
session_id: None,
transport: None,
}))
.await?;
@@ -383,6 +510,7 @@ async fn conversation_transport_close_emits_closed_event() -> Result<()> {
.submit(Op::RealtimeConversationStart(ConversationStartParams {
prompt: "backend prompt".to_string(),
session_id: None,
transport: None,
}))
.await?;
@@ -466,6 +594,7 @@ async fn conversation_start_preflight_failure_emits_realtime_error_only() -> Res
.submit(Op::RealtimeConversationStart(ConversationStartParams {
prompt: "backend prompt".to_string(),
session_id: None,
transport: None,
}))
.await?;
@@ -506,6 +635,7 @@ async fn conversation_start_connect_failure_emits_realtime_error_only() -> Resul
.submit(Op::RealtimeConversationStart(ConversationStartParams {
prompt: "backend prompt".to_string(),
session_id: None,
transport: None,
}))
.await?;
@@ -594,6 +724,7 @@ async fn conversation_second_start_replaces_runtime() -> Result<()> {
.submit(Op::RealtimeConversationStart(ConversationStartParams {
prompt: "old".to_string(),
session_id: Some("conv_old".to_string()),
transport: None,
}))
.await?;
wait_for_event_match(&test.codex, |msg| match msg {
@@ -610,6 +741,7 @@ async fn conversation_second_start_replaces_runtime() -> Result<()> {
.submit(Op::RealtimeConversationStart(ConversationStartParams {
prompt: "new".to_string(),
session_id: Some("conv_new".to_string()),
transport: None,
}))
.await?;
wait_for_event_match(&test.codex, |msg| match msg {
@@ -696,6 +828,7 @@ async fn conversation_uses_experimental_realtime_ws_base_url_override() -> Resul
.submit(Op::RealtimeConversationStart(ConversationStartParams {
prompt: "backend prompt".to_string(),
session_id: None,
transport: None,
}))
.await?;
@@ -750,6 +883,7 @@ async fn conversation_uses_experimental_realtime_ws_backend_prompt_override() ->
.submit(Op::RealtimeConversationStart(ConversationStartParams {
prompt: "prompt from op".to_string(),
session_id: None,
transport: None,
}))
.await?;
@@ -812,6 +946,7 @@ async fn conversation_uses_experimental_realtime_ws_startup_context_override() -
.submit(Op::RealtimeConversationStart(ConversationStartParams {
prompt: "prompt from op".to_string(),
session_id: None,
transport: None,
}))
.await?;
@@ -872,6 +1007,7 @@ async fn conversation_disables_realtime_startup_context_with_empty_override() ->
.submit(Op::RealtimeConversationStart(ConversationStartParams {
prompt: "prompt from op".to_string(),
session_id: None,
transport: None,
}))
.await?;
@@ -925,6 +1061,7 @@ async fn conversation_start_injects_startup_context_from_thread_history() -> Res
.submit(Op::RealtimeConversationStart(ConversationStartParams {
prompt: "backend prompt".to_string(),
session_id: None,
transport: None,
}))
.await?;
@@ -978,6 +1115,7 @@ async fn conversation_startup_context_falls_back_to_workspace_map() -> Result<()
.submit(Op::RealtimeConversationStart(ConversationStartParams {
prompt: "backend prompt".to_string(),
session_id: None,
transport: None,
}))
.await?;
@@ -1029,6 +1167,7 @@ async fn conversation_startup_context_is_truncated_and_sent_once_per_start() ->
.submit(Op::RealtimeConversationStart(ConversationStartParams {
prompt: "backend prompt".to_string(),
session_id: None,
transport: None,
}))
.await?;
@@ -1113,6 +1252,7 @@ async fn conversation_mirrors_assistant_message_text_to_realtime_handoff() -> Re
.submit(Op::RealtimeConversationStart(ConversationStartParams {
prompt: "backend prompt".to_string(),
session_id: None,
transport: None,
}))
.await?;
@@ -1239,6 +1379,7 @@ async fn conversation_handoff_persists_across_item_done_until_turn_complete() ->
.submit(Op::RealtimeConversationStart(ConversationStartParams {
prompt: "backend prompt".to_string(),
session_id: None,
transport: None,
}))
.await?;
@@ -1380,6 +1521,7 @@ async fn inbound_handoff_request_starts_turn() -> Result<()> {
.submit(Op::RealtimeConversationStart(ConversationStartParams {
prompt: "backend prompt".to_string(),
session_id: None,
transport: None,
}))
.await?;
@@ -1474,6 +1616,7 @@ async fn inbound_handoff_request_uses_active_transcript() -> Result<()> {
.submit(Op::RealtimeConversationStart(ConversationStartParams {
prompt: "backend prompt".to_string(),
session_id: None,
transport: None,
}))
.await?;
@@ -1566,6 +1709,7 @@ async fn inbound_handoff_request_clears_active_transcript_after_each_handoff() -
.submit(Op::RealtimeConversationStart(ConversationStartParams {
prompt: "backend prompt".to_string(),
session_id: None,
transport: None,
}))
.await?;
@@ -1665,6 +1809,7 @@ async fn inbound_conversation_item_does_not_start_turn_and_still_forwards_audio(
.submit(Op::RealtimeConversationStart(ConversationStartParams {
prompt: "backend prompt".to_string(),
session_id: None,
transport: None,
}))
.await?;
@@ -1777,6 +1922,7 @@ async fn delegated_turn_user_role_echo_does_not_redelegate_and_still_forwards_au
.submit(Op::RealtimeConversationStart(ConversationStartParams {
prompt: "backend prompt".to_string(),
session_id: None,
transport: None,
}))
.await?;
@@ -1919,6 +2065,7 @@ async fn inbound_handoff_request_does_not_block_realtime_event_forwarding() -> R
.submit(Op::RealtimeConversationStart(ConversationStartParams {
prompt: "backend prompt".to_string(),
session_id: None,
transport: None,
}))
.await?;
@@ -2045,6 +2192,7 @@ async fn inbound_handoff_request_steers_active_turn() -> Result<()> {
.submit(Op::RealtimeConversationStart(ConversationStartParams {
prompt: "backend prompt".to_string(),
session_id: None,
transport: None,
}))
.await?;
let _ = wait_for_event_match(&test.codex, |msg| match msg {
@@ -2186,6 +2334,7 @@ async fn inbound_handoff_request_starts_turn_and_does_not_block_realtime_audio()
.submit(Op::RealtimeConversationStart(ConversationStartParams {
prompt: "backend prompt".to_string(),
session_id: None,
transport: None,
}))
.await?;
codex-rs/mcp-server/src/codex_tool_runner.rs
+1
View File
@@ -387,6 +387,7 @@ async fn run_codex_tool_session_inner(
| EventMsg::CollabResumeBegin(_)
| EventMsg::CollabResumeEnd(_)
| EventMsg::RealtimeConversationStarted(_)
| EventMsg::RealtimeConversationSdp(_)
| EventMsg::RealtimeConversationRealtime(_)
| EventMsg::RealtimeConversationClosed(_)
| EventMsg::DeprecationNotice(_) => {
codex-rs/protocol/src/protocol.rs
+38
View File
@@ -135,6 +135,16 @@ pub struct ConversationStartParams {
pub prompt: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub session_id: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub transport: Option<ConversationStartTransport>,
}
#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, JsonSchema, TS)]
#[serde(tag = "type", rename_all = "snake_case")]
#[ts(tag = "type")]
pub enum ConversationStartTransport {
Websocket,
Webrtc { sdp: String },
}
#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq, JsonSchema, TS)]
@@ -1223,6 +1233,9 @@ pub enum EventMsg {
/// Realtime conversation lifecycle close event.
RealtimeConversationClosed(RealtimeConversationClosedEvent),
/// Realtime session description protocol payload.
RealtimeConversationSdp(RealtimeConversationSdpEvent),
/// Model routing changed from the requested model to a different model.
ModelReroute(ModelRerouteEvent),
@@ -1525,6 +1538,11 @@ pub struct RealtimeConversationClosedEvent {
pub reason: Option<String>,
}
#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, JsonSchema, TS)]
pub struct RealtimeConversationSdpEvent {
pub sdp: String,
}
impl From<CollabAgentSpawnBeginEvent> for EventMsg {
fn from(event: CollabAgentSpawnBeginEvent) -> Self {
EventMsg::CollabAgentSpawnBegin(event)
@@ -4393,6 +4411,14 @@ mod tests {
let start = Op::RealtimeConversationStart(ConversationStartParams {
prompt: "be helpful".to_string(),
session_id: Some("conv_1".to_string()),
transport: None,
});
let webrtc_start = Op::RealtimeConversationStart(ConversationStartParams {
prompt: "be helpful".to_string(),
session_id: Some("conv_1".to_string()),
transport: Some(ConversationStartTransport::Webrtc {
sdp: "v=offer\r\n".to_string(),
}),
});
let text = Op::RealtimeConversationText(ConversationTextParams {
text: "hello".to_string(),
@@ -4433,6 +4459,18 @@ mod tests {
serde_json::from_value::<Op>(serde_json::to_value(&close).unwrap()).unwrap(),
close
);
assert_eq!(
serde_json::to_value(&webrtc_start).unwrap(),
json!({
"type": "realtime_conversation_start",
"prompt": "be helpful",
"session_id": "conv_1",
"transport": {
"type": "webrtc",
"sdp": "v=offer\r\n"
}
})
);
}
#[test]
codex-rs/rollout/src/policy.rs
+1
View File
@@ -132,6 +132,7 @@ fn event_msg_persistence_mode(ev: &EventMsg) -> Option<EventPersistenceMode> {
| EventMsg::DynamicToolCallResponse(_) => Some(EventPersistenceMode::Extended),
EventMsg::Warning(_)
| EventMsg::RealtimeConversationStarted(_)
| EventMsg::RealtimeConversationSdp(_)
| EventMsg::RealtimeConversationRealtime(_)
| EventMsg::RealtimeConversationClosed(_)
| EventMsg::ModelReroute(_)
codex-rs/tui/src/app/app_server_adapter.rs
+14
View File
@@ -391,6 +391,9 @@ fn server_notification_thread_target(
ServerNotification::ThreadRealtimeOutputAudioDelta(notification) => {
Some(notification.thread_id.as_str())
}
ServerNotification::ThreadRealtimeSdp(notification) => {
Some(notification.thread_id.as_str())
}
ServerNotification::ThreadRealtimeError(notification) => {
Some(notification.thread_id.as_str())
}
@@ -625,6 +628,17 @@ fn server_notification_thread_events(
}),
}],
)),
ServerNotification::ThreadRealtimeSdp(notification) => Some((
ThreadId::from_string(&notification.thread_id).ok()?,
vec![Event {
id: String::new(),
msg: EventMsg::RealtimeConversationSdp(
codex_protocol::protocol::RealtimeConversationSdpEvent {
sdp: notification.sdp,
},
),
}],
)),
ServerNotification::ThreadRealtimeError(notification) => Some((
ThreadId::from_string(&notification.thread_id).ok()?,
vec![Event {
codex-rs/tui/src/app_server_session.rs
+10
View File
@@ -42,6 +42,7 @@ use codex_app_server_protocol::ThreadRealtimeAppendTextParams;
use codex_app_server_protocol::ThreadRealtimeAppendTextResponse;
use codex_app_server_protocol::ThreadRealtimeStartParams;
use codex_app_server_protocol::ThreadRealtimeStartResponse;
use codex_app_server_protocol::ThreadRealtimeStartTransport;
use codex_app_server_protocol::ThreadRealtimeStopParams;
use codex_app_server_protocol::ThreadRealtimeStopResponse;
use codex_app_server_protocol::ThreadResumeParams;
@@ -76,6 +77,7 @@ use codex_protocol::openai_models::ReasoningEffortPreset;
use codex_protocol::protocol::AskForApproval;
use codex_protocol::protocol::ConversationAudioParams;
use codex_protocol::protocol::ConversationStartParams;
use codex_protocol::protocol::ConversationStartTransport;
use codex_protocol::protocol::ConversationTextParams;
use codex_protocol::protocol::CreditsSnapshot;
use codex_protocol::protocol::RateLimitSnapshot;
@@ -660,6 +662,14 @@ impl AppServerSession {
thread_id: thread_id.to_string(),
prompt: params.prompt,
session_id: params.session_id,
transport: params.transport.map(|transport| match transport {
ConversationStartTransport::Websocket => {
ThreadRealtimeStartTransport::Websocket
}
ConversationStartTransport::Webrtc { sdp } => {
ThreadRealtimeStartTransport::Webrtc { sdp }
}
}),
},
})
.await
codex-rs/tui/src/chatwidget.rs
+2
View File
@@ -6501,6 +6501,7 @@ impl ChatWidget {
);
}
}
ServerNotification::ThreadRealtimeSdp(_) => {}
ServerNotification::ServerRequestResolved(_)
| ServerNotification::AccountUpdated(_)
| ServerNotification::AccountRateLimitsUpdated(_)
@@ -7020,6 +7021,7 @@ impl ChatWidget {
self.on_realtime_conversation_started(ev);
}
}
EventMsg::RealtimeConversationSdp(_) => {}
EventMsg::RealtimeConversationRealtime(ev) => {
if !from_replay {
self.on_realtime_conversation_realtime(ev);
codex-rs/tui/src/chatwidget/realtime.rs
+1
View File
@@ -229,6 +229,7 @@ impl ChatWidget {
ConversationStartParams {
prompt: REALTIME_CONVERSATION_PROMPT.to_string(),
session_id: None,
transport: None,
},
));
self.request_redraw();