mirror of
https://github.com/pchuan98/codex.git
synced 2026-07-01 00:31:56 +08:00
[codex] Support assistant realtime append text (#28836)
## Why
Frontend realtime voice continuity needs to replay a tiny
previous-session overlap as actual conversation items, including
assistant text. The app-server `thread/realtime/appendText` API already
carries a role through to the Rust realtime websocket layer, but the
shared role enum only accepted `user` and `developer`.
## What Changed
- Added `assistant` to `ConversationTextRole` and regenerated the
app-server schema/type fixtures.
- Added `output_text` as a realtime conversation content type.
- Updated realtime websocket item creation so assistant appendText emits
`content: [{ type: "output_text", text }]`, while user and developer
continue to emit `input_text`.
- Updated app-server docs and tests to cover assistant appendText
alongside the existing developer role behavior.
## Validation
- `just write-app-server-schema`
- `just fmt` (first sandboxed attempt failed because `uv` could not
access `~/.cache/uv`; reran with filesystem access and passed)
- `just test -p codex-api` passed: 126/126
- `just test -p codex-app-server-protocol` passed: 239/239, including
generated JSON/TypeScript fixture checks
- `just test -p codex-app-server` was started locally but stopped per
request after unrelated local sandbox/Seatbelt failures (`sandbox-exec:
sandbox_apply: Operation not permitted`) and one missing local `codex`
binary failure; CI should be faster and more authoritative for the full
suite.
This commit is contained in:
committed by
GitHub
Unverified
parent
683bd170dc
commit
e922f46a0f
+2
-1
@@ -643,7 +643,8 @@
|
||||
"ConversationTextRole": {
|
||||
"enum": [
|
||||
"user",
|
||||
"developer"
|
||||
"developer",
|
||||
"assistant"
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
|
||||
+2
-1
@@ -8707,7 +8707,8 @@
|
||||
"ConversationTextRole": {
|
||||
"enum": [
|
||||
"user",
|
||||
"developer"
|
||||
"developer",
|
||||
"assistant"
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
|
||||
+2
-1
@@ -4992,7 +4992,8 @@
|
||||
"ConversationTextRole": {
|
||||
"enum": [
|
||||
"user",
|
||||
"developer"
|
||||
"developer",
|
||||
"assistant"
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
|
||||
@@ -2,4 +2,4 @@
|
||||
|
||||
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
|
||||
|
||||
export type ConversationTextRole = "user" | "developer";
|
||||
export type ConversationTextRole = "user" | "developer" | "assistant";
|
||||
|
||||
@@ -167,7 +167,7 @@ Example with notification opt-out:
|
||||
- `turn/interrupt` — request cancellation of an in-flight turn by `(thread_id, turn_id)`; success is an empty `{}` response and the turn finishes with `status: "interrupted"`.
|
||||
- `thread/realtime/start` — start a thread-scoped realtime session (experimental); pass `outputModality: "text"` or `outputModality: "audio"` to choose model output, optionally pass `model` and `version` to override configured realtime selection for this session only, and pass `includeStartupContext: false` to omit Codex's generated startup context. By default, automatic Codex text follows the protocol's speakable output path. Pass `clientManagedHandoffs: true` to disable automatic Codex response delivery so only the client's explicit append calls produce handoffs. Pass `codexResponsesAsItems: true` to send automatic Codex responses as realtime conversation items instead, and optionally pass `codexResponseItemPrefix` to prepend experiment instructions to those items. For V1 sessions, pass `codexResponseHandoffPrefix` while item mode is disabled to route automatic Codex commentary through `conversation.handoff.append` with that prefix; final answers remain unprefixed. Returns `{}` and streams `thread/realtime/*` notifications. Omit `transport` for the websocket transport, or pass `{ "type": "webrtc", "sdp": "..." }` to create a WebRTC session from a browser-generated SDP offer; the remote answer SDP is emitted as `thread/realtime/sdp`.
|
||||
- `thread/realtime/appendAudio` — append an input audio chunk to the active realtime session (experimental); returns `{}`.
|
||||
- `thread/realtime/appendText` — append text input to the active realtime session with a required `role` of `user` or `developer` (experimental); returns `{}`. Older clients that omit `role` default to `user`.
|
||||
- `thread/realtime/appendText` — append text input to the active realtime session with a required `role` of `user`, `developer`, or `assistant` (experimental); returns `{}`. Older clients that omit `role` default to `user`.
|
||||
- `thread/realtime/appendSpeech` — append text that the realtime model should speak to the user (experimental); returns `{}`.
|
||||
- `thread/realtime/stop` — stop the active realtime session for the thread (experimental); returns `{}`.
|
||||
- `review/start` — kick off Codex’s automated reviewer for a thread; responds like `turn/start` and emits `item/started`/`item/completed` notifications with `enteredReviewMode` and `exitedReviewMode` items, plus a final assistant `agentMessage` containing the review.
|
||||
|
||||
@@ -709,6 +709,20 @@ async fn realtime_conversation_streams_v2_notifications() -> Result<()> {
|
||||
.await??;
|
||||
let _: ThreadRealtimeAppendTextResponse = to_response(text_append_response)?;
|
||||
|
||||
let assistant_append_request_id = mcp
|
||||
.send_thread_realtime_append_text_request(ThreadRealtimeAppendTextParams {
|
||||
thread_id: started.thread_id.clone(),
|
||||
text: "welcome back".to_string(),
|
||||
role: ConversationTextRole::Assistant,
|
||||
})
|
||||
.await?;
|
||||
let assistant_append_response: JSONRPCResponse = timeout(
|
||||
DEFAULT_TIMEOUT,
|
||||
mcp.read_stream_until_response_message(RequestId::Integer(assistant_append_request_id)),
|
||||
)
|
||||
.await??;
|
||||
let _: ThreadRealtimeAppendTextResponse = to_response(assistant_append_response)?;
|
||||
|
||||
let output_audio = read_notification::<ThreadRealtimeOutputAudioDeltaNotification>(
|
||||
&mut mcp,
|
||||
"thread/realtime/outputAudio/delta",
|
||||
@@ -790,7 +804,7 @@ async fn realtime_conversation_streams_v2_notifications() -> Result<()> {
|
||||
let connections = realtime_server.connections();
|
||||
assert_eq!(connections.len(), 1);
|
||||
let connection = &connections[0];
|
||||
assert_eq!(connection.len(), 3);
|
||||
assert_eq!(connection.len(), 4);
|
||||
assert_eq!(
|
||||
connection[0].body_json()["type"].as_str(),
|
||||
Some("session.update")
|
||||
@@ -799,13 +813,14 @@ async fn realtime_conversation_streams_v2_notifications() -> Result<()> {
|
||||
connection[0].body_json()["session"]["instructions"].as_str(),
|
||||
Some(startup_context_instructions.as_str()),
|
||||
);
|
||||
let text_request = connection
|
||||
let text_requests = connection
|
||||
.iter()
|
||||
.map(WebSocketRequest::body_json)
|
||||
.find(|request| request["type"] == "conversation.item.create")
|
||||
.context("expected conversation item request")?;
|
||||
.filter(|request| request["type"] == "conversation.item.create")
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(text_requests.len(), 2);
|
||||
assert_eq!(
|
||||
text_request,
|
||||
text_requests[0],
|
||||
json!({
|
||||
"type": "conversation.item.create",
|
||||
"item": {
|
||||
@@ -818,6 +833,20 @@ async fn realtime_conversation_streams_v2_notifications() -> Result<()> {
|
||||
},
|
||||
})
|
||||
);
|
||||
assert_eq!(
|
||||
text_requests[1],
|
||||
json!({
|
||||
"type": "conversation.item.create",
|
||||
"item": {
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"content": [{
|
||||
"type": "output_text",
|
||||
"text": "welcome back",
|
||||
}],
|
||||
},
|
||||
})
|
||||
);
|
||||
let mut request_types = [
|
||||
connection[1].body_json()["type"]
|
||||
.as_str()
|
||||
@@ -827,11 +856,16 @@ async fn realtime_conversation_streams_v2_notifications() -> Result<()> {
|
||||
.as_str()
|
||||
.context("expected websocket request type")?
|
||||
.to_string(),
|
||||
connection[3].body_json()["type"]
|
||||
.as_str()
|
||||
.context("expected websocket request type")?
|
||||
.to_string(),
|
||||
];
|
||||
request_types.sort();
|
||||
assert_eq!(
|
||||
request_types,
|
||||
[
|
||||
"conversation.item.create".to_string(),
|
||||
"conversation.item.create".to_string(),
|
||||
"input_audio_buffer.append".to_string(),
|
||||
]
|
||||
|
||||
@@ -1637,10 +1637,29 @@ mod tests {
|
||||
.into_text()
|
||||
.expect("text");
|
||||
let fourth_json: Value = serde_json::from_str(&fourth).expect("json");
|
||||
assert_eq!(fourth_json["type"], "conversation.handoff.append");
|
||||
assert_eq!(fourth_json["handoff_id"], "handoff_1");
|
||||
assert_eq!(fourth_json["type"], "conversation.item.create");
|
||||
assert_eq!(fourth_json["item"]["role"], "assistant");
|
||||
assert_eq!(
|
||||
fourth_json["output_text"],
|
||||
fourth_json["item"]["content"][0]["type"],
|
||||
Value::String("output_text".to_string())
|
||||
);
|
||||
assert_eq!(
|
||||
fourth_json["item"]["content"][0]["text"],
|
||||
Value::String("assistant context".to_string())
|
||||
);
|
||||
|
||||
let fifth = ws
|
||||
.next()
|
||||
.await
|
||||
.expect("fifth msg")
|
||||
.expect("fifth msg ok")
|
||||
.into_text()
|
||||
.expect("text");
|
||||
let fifth_json: Value = serde_json::from_str(&fifth).expect("json");
|
||||
assert_eq!(fifth_json["type"], "conversation.handoff.append");
|
||||
assert_eq!(fifth_json["handoff_id"], "handoff_1");
|
||||
assert_eq!(
|
||||
fifth_json["output_text"],
|
||||
"\"Agent Final Message\":\n\nhello from background agent"
|
||||
);
|
||||
|
||||
@@ -1766,6 +1785,13 @@ mod tests {
|
||||
)
|
||||
.await
|
||||
.expect("send item");
|
||||
connection
|
||||
.send_conversation_item_create(
|
||||
"assistant context".to_string(),
|
||||
ConversationTextRole::Assistant,
|
||||
)
|
||||
.await
|
||||
.expect("send assistant item");
|
||||
connection
|
||||
.send_conversation_function_call_output(
|
||||
"handoff_1".to_string(),
|
||||
@@ -1988,16 +2014,35 @@ mod tests {
|
||||
.expect("text");
|
||||
let third_json: Value = serde_json::from_str(&third).expect("json");
|
||||
assert_eq!(third_json["type"], "conversation.item.create");
|
||||
assert_eq!(third_json["item"]["role"], "assistant");
|
||||
assert_eq!(
|
||||
third_json["item"]["type"],
|
||||
third_json["item"]["content"][0]["type"],
|
||||
Value::String("output_text".to_string())
|
||||
);
|
||||
assert_eq!(
|
||||
third_json["item"]["content"][0]["text"],
|
||||
Value::String("assistant context".to_string())
|
||||
);
|
||||
|
||||
let fourth = ws
|
||||
.next()
|
||||
.await
|
||||
.expect("fourth msg")
|
||||
.expect("fourth msg ok")
|
||||
.into_text()
|
||||
.expect("text");
|
||||
let fourth_json: Value = serde_json::from_str(&fourth).expect("json");
|
||||
assert_eq!(fourth_json["type"], "conversation.item.create");
|
||||
assert_eq!(
|
||||
fourth_json["item"]["type"],
|
||||
Value::String("function_call_output".to_string())
|
||||
);
|
||||
assert_eq!(
|
||||
third_json["item"]["call_id"],
|
||||
fourth_json["item"]["call_id"],
|
||||
Value::String("call_1".to_string())
|
||||
);
|
||||
assert_eq!(
|
||||
third_json["item"]["output"],
|
||||
fourth_json["item"]["output"],
|
||||
Value::String("delegated result".to_string())
|
||||
);
|
||||
});
|
||||
@@ -2054,6 +2099,13 @@ mod tests {
|
||||
)
|
||||
.await
|
||||
.expect("send text item");
|
||||
connection
|
||||
.send_conversation_item_create(
|
||||
"assistant context".to_string(),
|
||||
ConversationTextRole::Assistant,
|
||||
)
|
||||
.await
|
||||
.expect("send assistant item");
|
||||
connection
|
||||
.send_conversation_function_call_output(
|
||||
"call_1".to_string(),
|
||||
|
||||
@@ -19,12 +19,19 @@ pub(super) fn conversation_item_create_message(
|
||||
text: String,
|
||||
role: ConversationTextRole,
|
||||
) -> RealtimeOutboundMessage {
|
||||
let content_type = match role {
|
||||
ConversationTextRole::Assistant => ConversationContentType::OutputText,
|
||||
ConversationTextRole::User | ConversationTextRole::Developer => {
|
||||
ConversationContentType::InputText
|
||||
}
|
||||
};
|
||||
|
||||
RealtimeOutboundMessage::ConversationItemCreate {
|
||||
item: ConversationItemPayload::Message(ConversationMessageItem {
|
||||
r#type: ConversationItemType::Message,
|
||||
role,
|
||||
content: vec![ConversationItemContent {
|
||||
r#type: ConversationContentType::InputText,
|
||||
r#type: content_type,
|
||||
text,
|
||||
}],
|
||||
}),
|
||||
|
||||
@@ -40,12 +40,19 @@ pub(super) fn conversation_item_create_message(
|
||||
text: String,
|
||||
role: ConversationTextRole,
|
||||
) -> RealtimeOutboundMessage {
|
||||
let content_type = match role {
|
||||
ConversationTextRole::Assistant => ConversationContentType::OutputText,
|
||||
ConversationTextRole::User | ConversationTextRole::Developer => {
|
||||
ConversationContentType::InputText
|
||||
}
|
||||
};
|
||||
|
||||
RealtimeOutboundMessage::ConversationItemCreate {
|
||||
item: ConversationItemPayload::Message(ConversationMessageItem {
|
||||
r#type: ConversationItemType::Message,
|
||||
role,
|
||||
content: vec![ConversationItemContent {
|
||||
r#type: ConversationContentType::InputText,
|
||||
r#type: content_type,
|
||||
text,
|
||||
}],
|
||||
}),
|
||||
|
||||
@@ -195,6 +195,7 @@ pub(super) struct ConversationItemContent {
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub(super) enum ConversationContentType {
|
||||
InputText,
|
||||
OutputText,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
|
||||
@@ -417,6 +417,7 @@ pub enum ConversationTextRole {
|
||||
#[default]
|
||||
User,
|
||||
Developer,
|
||||
Assistant,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
|
||||
Reference in New Issue
Block a user