Add text element metadata to protocol, app server, and core (#9331)

The second part of breaking up PR
https://github.com/openai/codex/pull/9116

Summary:

- Add `TextElement` / `ByteRange` to protocol user inputs and user
message events with defaults.
- Thread `text_elements` through app-server v1/v2 request handling and
history rebuild.
- Preserve UI metadata only in user input/events (not `ContentItem`)
while keeping local image attachments in user events for rehydration.

Details:

- Protocol: `UserInput::Text` carries `text_elements`;
`UserMessageEvent` carries `text_elements` + `local_images`.
Serialization includes empty vectors for backward compatibility.
- app-server-protocol: v1 defines `V1TextElement` / `V1ByteRange` in
camelCase with conversions; v2 uses its own camelCase wrapper.
- app-server: v1/v2 input mapping includes `text_elements`; thread
history rebuilds include them.
- Core: user event emission preserves UI metadata while model history
stays clean; history replay round-trips the metadata.

codex-rs/core/src/codex.rs

@@ -36,6 +36,7 @@ use codex_protocol::ThreadId;
 use codex_protocol::approvals::ExecPolicyAmendment;
 use codex_protocol::config_types::WebSearchMode;
 use codex_protocol::items::TurnItem;
+use codex_protocol::items::UserMessageItem;
 use codex_protocol::openai_models::ModelInfo;
 use codex_protocol::protocol::FileChange;
 use codex_protocol::protocol::HasLegacyEvent;
@@ -1525,6 +1526,22 @@ impl Session {
         }
     }
 
+    pub(crate) async fn record_user_prompt_and_emit_turn_item(
+        &self,
+        turn_context: &TurnContext,
+        input: &[UserInput],
+        response_item: ResponseItem,
+    ) {
+        // Persist the user message to history, but emit the turn item from `UserInput` so
+        // UI-only `text_elements` are preserved. `ResponseItem::Message` does not carry
+        // those spans, and `record_response_item_and_emit_turn_item` would drop them.
+        self.record_conversation_items(turn_context, std::slice::from_ref(&response_item))
+            .await;
+        let turn_item = TurnItem::UserMessage(UserMessageItem::new(input));
+        self.emit_turn_item_started(turn_context, &turn_item).await;
+        self.emit_turn_item_completed(turn_context, turn_item).await;
+    }
+
     pub(crate) async fn notify_background_event(
         &self,
         turn_context: &TurnContext,
@@ -2563,9 +2580,9 @@ pub(crate) async fn run_turn(
             .await;
     }
 
-    let initial_input_for_turn: ResponseInputItem = ResponseInputItem::from(input);
+    let initial_input_for_turn: ResponseInputItem = ResponseInputItem::from(input.clone());
     let response_item: ResponseItem = initial_input_for_turn.clone().into();
-    sess.record_response_item_and_emit_turn_item(turn_context.as_ref(), response_item)
+    sess.record_user_prompt_and_emit_turn_item(turn_context.as_ref(), &input, response_item)
         .await;
 
     if !skill_items.is_empty() {

codex-rs/core/tests/suite/items.rs

@@ -6,6 +6,8 @@ use codex_core::protocol::ItemCompletedEvent;
 use codex_core::protocol::ItemStartedEvent;
 use codex_core::protocol::Op;
 use codex_protocol::items::TurnItem;
+use codex_protocol::user_input::ByteRange;
+use codex_protocol::user_input::TextElement;
 use codex_protocol::user_input::UserInput;
 use core_test_support::responses::ev_assistant_message;
 use core_test_support::responses::ev_completed;
@@ -38,12 +40,18 @@ async fn user_message_item_is_emitted() -> anyhow::Result<()> {
     let first_response = sse(vec![ev_response_created("resp-1"), ev_completed("resp-1")]);
     mount_sse_once(&server, first_response).await;
 
+    let text_elements = vec![TextElement {
+        byte_range: ByteRange { start: 0, end: 6 },
+        placeholder: Some("<file>".into()),
+    }];
+    let expected_input = UserInput::Text {
+        text: "please inspect sample.txt".into(),
+        text_elements: text_elements.clone(),
+    };
+
     codex
         .submit(Op::UserInput {
-            items: (vec![UserInput::Text {
-                text: "please inspect sample.txt".into(),
-                text_elements: Vec::new(),
-            }]),
+            items: vec![expected_input.clone()],
             final_output_json_schema: None,
         })
         .await?;
@@ -66,20 +74,16 @@ async fn user_message_item_is_emitted() -> anyhow::Result<()> {
     .await;
 
     assert_eq!(started_item.id, completed_item.id);
-    assert_eq!(
-        started_item.content,
-        vec![UserInput::Text {
-            text: "please inspect sample.txt".into(),
-            text_elements: Vec::new(),
-        }]
-    );
-    assert_eq!(
-        completed_item.content,
-        vec![UserInput::Text {
-            text: "please inspect sample.txt".into(),
-            text_elements: Vec::new(),
-        }]
-    );
+    assert_eq!(started_item.content, vec![expected_input.clone()]);
+    assert_eq!(completed_item.content, vec![expected_input]);
+
+    let legacy_message = wait_for_event_match(&codex, |ev| match ev {
+        EventMsg::UserMessage(event) => Some(event.clone()),
+        _ => None,
+    })
+    .await;
+    assert_eq!(legacy_message.message, "please inspect sample.txt");
+    assert_eq!(legacy_message.text_elements, text_elements);
     Ok(())
 }
 

codex-rs/core/tests/suite/resume.rs

@@ -1,6 +1,8 @@
 use anyhow::Result;
 use codex_core::protocol::EventMsg;
 use codex_core::protocol::Op;
+use codex_protocol::user_input::ByteRange;
+use codex_protocol::user_input::TextElement;
 use codex_protocol::user_input::UserInput;
 use core_test_support::responses::ev_assistant_message;
 use core_test_support::responses::ev_completed;
@@ -12,6 +14,7 @@ use core_test_support::responses::start_mock_server;
 use core_test_support::skip_if_no_network;
 use core_test_support::test_codex::test_codex;
 use core_test_support::wait_for_event;
+use pretty_assertions::assert_eq;
 use std::sync::Arc;
 
 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
@@ -32,11 +35,16 @@ async fn resume_includes_initial_messages_from_rollout_events() -> Result<()> {
     ]);
     mount_sse_once(&server, initial_sse).await;
 
+    let text_elements = vec![TextElement {
+        byte_range: ByteRange { start: 0, end: 6 },
+        placeholder: Some("<note>".into()),
+    }];
+
     codex
         .submit(Op::UserInput {
             items: vec![UserInput::Text {
                 text: "Record some messages".into(),
-                text_elements: Vec::new(),
+                text_elements: text_elements.clone(),
             }],
             final_output_json_schema: None,
         })
@@ -57,6 +65,7 @@ async fn resume_includes_initial_messages_from_rollout_events() -> Result<()> {
             EventMsg::TokenCount(_),
         ] => {
             assert_eq!(first_user.message, "Record some messages");
+            assert_eq!(first_user.text_elements, text_elements);
             assert_eq!(assistant_message.message, "Completed first turn");
         }
         other => panic!("unexpected initial messages after resume: {other:#?}"),