[codex] define code mode host handshake protocol (#29515)

## Summary

- add validated protocol-version, capability, and session identifier
types
- define explicit `ClientToHost` and `HostToClient` JSON envelopes for
connection negotiation and session open/close acknowledgements
- reject invalid states and unknown fields during decoding, with
explicit wire-format and round-trip coverage

## Why

This establishes the transport-neutral encoding shape needed to build
and test the new code-mode host incrementally. Cell, tool callback, and
failure-domain messages are intentionally deferred until their actors
and behavior tests establish the required semantics.

This is additive protocol scaffolding and does not change the current
production code-mode implementation.

## Validation

codex-rs/code-mode-protocol/src/host/error.rs

@@ -0,0 +1,19 @@
+use serde::Deserialize;
+use serde::Serialize;
+
+use super::Capability;
+use super::SupportedProtocolVersions;
+
+/// Explains why connection negotiation was rejected before any session opened.
+#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
+#[serde(deny_unknown_fields, tag = "type", rename_all_fields = "camelCase")]
+pub enum HandshakeRejectReason {
+    #[serde(rename = "noCompatibleVersion")]
+    NoCompatibleVersion {
+        supported_versions: SupportedProtocolVersions,
+    },
+    #[serde(rename = "missingRequiredCapability")]
+    MissingRequiredCapability { capability: Capability },
+    #[serde(rename = "invalidHello")]
+    InvalidHello { message: String },
+}

codex-rs/code-mode-protocol/src/host/host_tests.rs

@@ -0,0 +1,207 @@
+use pretty_assertions::assert_eq;
+use serde_json::json;
+
+use super::Capability;
+use super::CapabilitySet;
+use super::ClientHello;
+use super::ClientToHost;
+use super::HandshakeRejectReason;
+use super::HostHello;
+use super::HostToClient;
+use super::ProtocolVersion;
+use super::SessionId;
+use super::SupportedProtocolVersions;
+
+fn session_id() -> SessionId {
+    SessionId::new("session-1").expect("valid session ID")
+}
+
+fn capability(value: &str) -> Capability {
+    Capability::new(value).expect("valid capability")
+}
+
+fn supported_versions() -> SupportedProtocolVersions {
+    SupportedProtocolVersions::try_new([ProtocolVersion::V1])
+        .expect("nonempty unique protocol versions")
+}
+
+#[test]
+fn handshake_wire_contract_is_explicit_and_round_trips() {
+    let client_hello = ClientToHost::ClientHello(
+        ClientHello::new(
+            supported_versions(),
+            CapabilitySet::try_new([capability("required")]).expect("valid required set"),
+            CapabilitySet::try_new([capability("optional")]).expect("valid optional set"),
+        )
+        .expect("disjoint capabilities"),
+    );
+    let client_hello_json = json!({
+        "type": "connection/hello",
+        "supportedVersions": [1],
+        "requiredCapabilities": ["required"],
+        "optionalCapabilities": ["optional"],
+    });
+    assert_eq!(
+        serde_json::to_value(&client_hello).expect("serialize"),
+        client_hello_json
+    );
+    assert_eq!(
+        serde_json::from_value::<ClientToHost>(client_hello_json).expect("deserialize"),
+        client_hello
+    );
+
+    let host_hello = HostToClient::HostHello(HostHello::new(
+        ProtocolVersion::V1,
+        CapabilitySet::try_new([capability("required")]).expect("valid capabilities"),
+    ));
+    let host_hello_json = json!({
+        "type": "connection/ready",
+        "selectedVersion": 1,
+        "capabilities": ["required"],
+    });
+    assert_eq!(
+        serde_json::to_value(&host_hello).expect("serialize"),
+        host_hello_json
+    );
+    assert_eq!(
+        serde_json::from_value::<HostToClient>(host_hello_json).expect("deserialize"),
+        host_hello
+    );
+
+    let rejected = HostToClient::HandshakeRejected {
+        reason: HandshakeRejectReason::NoCompatibleVersion {
+            supported_versions: supported_versions(),
+        },
+    };
+    let rejected_json = json!({
+        "type": "connection/rejected",
+        "reason": {
+            "type": "noCompatibleVersion",
+            "supportedVersions": [1],
+        },
+    });
+    assert_eq!(
+        serde_json::to_value(&rejected).expect("serialize"),
+        rejected_json
+    );
+    assert_eq!(
+        serde_json::from_value::<HostToClient>(rejected_json).expect("deserialize"),
+        rejected
+    );
+}
+
+#[test]
+fn session_lifecycle_wire_contract_is_explicit_and_round_trips() {
+    let client_messages = [
+        (
+            ClientToHost::OpenSession {
+                session_id: session_id(),
+            },
+            json!({ "type": "session/open", "sessionId": "session-1" }),
+        ),
+        (
+            ClientToHost::CloseSession {
+                session_id: session_id(),
+            },
+            json!({ "type": "session/close", "sessionId": "session-1" }),
+        ),
+    ];
+    for (message, encoded) in client_messages {
+        assert_eq!(serde_json::to_value(&message).expect("serialize"), encoded);
+        assert_eq!(
+            serde_json::from_value::<ClientToHost>(encoded).expect("deserialize"),
+            message
+        );
+    }
+
+    let host_messages = [
+        (
+            HostToClient::SessionReady {
+                session_id: session_id(),
+            },
+            json!({ "type": "session/ready", "sessionId": "session-1" }),
+        ),
+        (
+            HostToClient::SessionClosed {
+                session_id: session_id(),
+            },
+            json!({ "type": "session/closed", "sessionId": "session-1" }),
+        ),
+    ];
+    for (message, encoded) in host_messages {
+        assert_eq!(serde_json::to_value(&message).expect("serialize"), encoded);
+        assert_eq!(
+            serde_json::from_value::<HostToClient>(encoded).expect("deserialize"),
+            message
+        );
+    }
+}
+
+#[test]
+fn invalid_protocol_states_cannot_be_constructed_or_decoded() {
+    assert!(SessionId::new("").is_err());
+    assert!(Capability::new("   ").is_err());
+    assert!(ProtocolVersion::new(/*value*/ 0).is_none());
+    assert!(SupportedProtocolVersions::try_new([]).is_err());
+    assert!(
+        SupportedProtocolVersions::try_new([ProtocolVersion::V1, ProtocolVersion::V1]).is_err()
+    );
+    assert!(CapabilitySet::try_new([capability("same"), capability("same")]).is_err());
+
+    let version_two = ProtocolVersion::new(/*value*/ 2).expect("valid protocol version");
+    let versions = SupportedProtocolVersions::try_new([ProtocolVersion::V1, version_two])
+        .expect("valid versions");
+    assert!(versions.contains(ProtocolVersion::V1));
+    assert_eq!(
+        versions.iter().collect::<Vec<_>>(),
+        vec![ProtocolVersion::V1, version_two]
+    );
+
+    let overlapping = capability("overlapping");
+    assert!(
+        ClientHello::new(
+            supported_versions(),
+            CapabilitySet::try_new([overlapping.clone()]).expect("valid required set"),
+            CapabilitySet::try_new([overlapping]).expect("valid optional set"),
+        )
+        .is_err()
+    );
+
+    for invalid in [
+        json!({ "type": "session/open", "sessionId": "" }),
+        json!({
+            "type": "connection/hello",
+            "supportedVersions": [],
+            "requiredCapabilities": [],
+            "optionalCapabilities": [],
+        }),
+        json!({
+            "type": "connection/hello",
+            "supportedVersions": [1],
+            "requiredCapabilities": ["overlapping"],
+            "optionalCapabilities": ["overlapping"],
+        }),
+    ] {
+        assert!(serde_json::from_value::<ClientToHost>(invalid).is_err());
+    }
+}
+
+#[test]
+fn unknown_fields_are_rejected() {
+    assert!(
+        serde_json::from_value::<ClientToHost>(json!({
+            "type": "session/open",
+            "sessionId": "session-1",
+            "unexpected": true,
+        }))
+        .is_err()
+    );
+    assert!(
+        serde_json::from_value::<HostToClient>(json!({
+            "type": "session/ready",
+            "sessionId": "session-1",
+            "unexpected": true,
+        }))
+        .is_err()
+    );
+}

codex-rs/code-mode-protocol/src/host/message.rs

@@ -0,0 +1,142 @@
+use std::fmt;
+
+use serde::Deserialize;
+use serde::Serialize;
+
+use super::Capability;
+use super::CapabilitySet;
+use super::HandshakeRejectReason;
+use super::ProtocolVersion;
+use super::SessionId;
+use super::SupportedProtocolVersions;
+
+#[derive(Clone, Debug, PartialEq, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct ClientHello {
+    supported_versions: SupportedProtocolVersions,
+    required_capabilities: CapabilitySet,
+    optional_capabilities: CapabilitySet,
+}
+
+impl ClientHello {
+    pub fn new(
+        supported_versions: SupportedProtocolVersions,
+        required_capabilities: CapabilitySet,
+        optional_capabilities: CapabilitySet,
+    ) -> Result<Self, ClientHelloError> {
+        if let Some(capability) = required_capabilities
+            .iter()
+            .find(|capability| optional_capabilities.contains(capability))
+        {
+            return Err(ClientHelloError::OverlappingCapability(capability.clone()));
+        }
+        Ok(Self {
+            supported_versions,
+            required_capabilities,
+            optional_capabilities,
+        })
+    }
+
+    pub fn supported_versions(&self) -> &SupportedProtocolVersions {
+        &self.supported_versions
+    }
+
+    pub fn required_capabilities(&self) -> &CapabilitySet {
+        &self.required_capabilities
+    }
+
+    pub fn optional_capabilities(&self) -> &CapabilitySet {
+        &self.optional_capabilities
+    }
+}
+
+#[derive(Deserialize)]
+#[serde(deny_unknown_fields, rename_all = "camelCase")]
+struct ClientHelloWire {
+    supported_versions: SupportedProtocolVersions,
+    required_capabilities: CapabilitySet,
+    optional_capabilities: CapabilitySet,
+}
+
+impl<'de> Deserialize<'de> for ClientHello {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: serde::Deserializer<'de>,
+    {
+        let wire = ClientHelloWire::deserialize(deserializer)?;
+        Self::new(
+            wire.supported_versions,
+            wire.required_capabilities,
+            wire.optional_capabilities,
+        )
+        .map_err(serde::de::Error::custom)
+    }
+}
+
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub enum ClientHelloError {
+    OverlappingCapability(Capability),
+}
+
+impl fmt::Display for ClientHelloError {
+    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            Self::OverlappingCapability(capability) => write!(
+                formatter,
+                "capability `{capability}` cannot be both required and optional"
+            ),
+        }
+    }
+}
+
+impl std::error::Error for ClientHelloError {}
+
+#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
+#[serde(deny_unknown_fields, rename_all = "camelCase")]
+pub struct HostHello {
+    selected_version: ProtocolVersion,
+    capabilities: CapabilitySet,
+}
+
+impl HostHello {
+    pub fn new(selected_version: ProtocolVersion, capabilities: CapabilitySet) -> Self {
+        Self {
+            selected_version,
+            capabilities,
+        }
+    }
+
+    pub fn selected_version(&self) -> ProtocolVersion {
+        self.selected_version
+    }
+
+    pub fn capabilities(&self) -> &CapabilitySet {
+        &self.capabilities
+    }
+}
+
+/// Messages sent from a client to the code-mode host.
+#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
+#[serde(deny_unknown_fields, tag = "type", rename_all_fields = "camelCase")]
+pub enum ClientToHost {
+    #[serde(rename = "connection/hello")]
+    ClientHello(ClientHello),
+    #[serde(rename = "session/open")]
+    OpenSession { session_id: SessionId },
+    #[serde(rename = "session/close")]
+    CloseSession { session_id: SessionId },
+}
+
+/// Messages sent from the code-mode host to a client.
+#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
+#[serde(deny_unknown_fields, tag = "type", rename_all_fields = "camelCase")]
+pub enum HostToClient {
+    #[serde(rename = "connection/ready")]
+    HostHello(HostHello),
+    #[serde(rename = "connection/rejected")]
+    HandshakeRejected { reason: HandshakeRejectReason },
+    #[serde(rename = "session/ready")]
+    SessionReady { session_id: SessionId },
+    #[serde(rename = "session/closed")]
+    SessionClosed { session_id: SessionId },
+}

codex-rs/code-mode-protocol/src/host/mod.rs

@@ -0,0 +1,29 @@
+//! Transport-neutral messages for the callback-only code-mode host boundary.
+//!
+//! Protocol version 1 relies on ordered framing and connection-scoped
+//! fail-stop behavior rather than message sequence numbers. It defines no
+//! optional capabilities yet; capability names provide an extension point for
+//! later versions without weakening the v1 decoder.
+
+mod error;
+mod message;
+mod types;
+
+pub use error::HandshakeRejectReason;
+pub use message::ClientHello;
+pub use message::ClientHelloError;
+pub use message::ClientToHost;
+pub use message::HostHello;
+pub use message::HostToClient;
+pub use types::Capability;
+pub use types::CapabilitySet;
+pub use types::DuplicateCapability;
+pub use types::InvalidIdentifier;
+pub use types::InvalidSupportedProtocolVersions;
+pub use types::ProtocolVersion;
+pub use types::SessionId;
+pub use types::SupportedProtocolVersions;
+
+#[cfg(test)]
+#[path = "host_tests.rs"]
+mod tests;

codex-rs/code-mode-protocol/src/host/types.rs

@@ -0,0 +1,226 @@
+use std::collections::BTreeSet;
+use std::fmt;
+use std::num::NonZeroU32;
+
+use serde::Deserialize;
+use serde::Deserializer;
+use serde::Serialize;
+use serde::Serializer;
+use serde::de::Error as _;
+
+#[derive(Clone, Copy, Debug, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)]
+#[serde(transparent)]
+pub struct ProtocolVersion(NonZeroU32);
+
+impl ProtocolVersion {
+    pub const V1: Self = Self(NonZeroU32::MIN);
+
+    pub const fn new(value: u32) -> Option<Self> {
+        match NonZeroU32::new(value) {
+            Some(value) => Some(Self(value)),
+            None => None,
+        }
+    }
+
+    pub const fn get(self) -> u32 {
+        self.0.get()
+    }
+}
+
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub struct InvalidIdentifier;
+
+impl fmt::Display for InvalidIdentifier {
+    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
+        formatter.write_str("identifier must not be empty")
+    }
+}
+
+impl std::error::Error for InvalidIdentifier {}
+
+#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
+struct NonEmptyString(String);
+
+impl NonEmptyString {
+    fn new(value: impl Into<String>) -> Result<Self, InvalidIdentifier> {
+        let value = value.into();
+        if value.trim().is_empty() {
+            Err(InvalidIdentifier)
+        } else {
+            Ok(Self(value))
+        }
+    }
+
+    fn as_str(&self) -> &str {
+        &self.0
+    }
+}
+
+impl Serialize for NonEmptyString {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        self.0.serialize(serializer)
+    }
+}
+
+impl<'de> Deserialize<'de> for NonEmptyString {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        Self::new(String::deserialize(deserializer)?).map_err(D::Error::custom)
+    }
+}
+
+/// A named protocol feature advertised during connection negotiation.
+#[derive(Clone, Debug, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)]
+#[serde(transparent)]
+pub struct Capability(NonEmptyString);
+
+impl Capability {
+    pub fn new(value: impl Into<String>) -> Result<Self, InvalidIdentifier> {
+        NonEmptyString::new(value).map(Self)
+    }
+
+    pub fn as_str(&self) -> &str {
+        self.0.as_str()
+    }
+}
+
+impl fmt::Display for Capability {
+    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
+        formatter.write_str(self.as_str())
+    }
+}
+
+/// Identifies one logical code-mode session on a connection.
+#[derive(Clone, Debug, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)]
+#[serde(transparent)]
+pub struct SessionId(NonEmptyString);
+
+impl SessionId {
+    pub fn new(value: impl Into<String>) -> Result<Self, InvalidIdentifier> {
+        NonEmptyString::new(value).map(Self)
+    }
+
+    pub fn as_str(&self) -> &str {
+        self.0.as_str()
+    }
+}
+
+impl fmt::Display for SessionId {
+    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
+        formatter.write_str(self.as_str())
+    }
+}
+
+#[derive(Clone, Debug, Default, Eq, PartialEq, Serialize)]
+#[serde(transparent)]
+pub struct CapabilitySet(BTreeSet<Capability>);
+
+impl CapabilitySet {
+    pub fn empty() -> Self {
+        Self::default()
+    }
+
+    pub fn try_new(
+        capabilities: impl IntoIterator<Item = Capability>,
+    ) -> Result<Self, DuplicateCapability> {
+        let mut unique = BTreeSet::new();
+        for capability in capabilities {
+            if !unique.insert(capability.clone()) {
+                return Err(DuplicateCapability { capability });
+            }
+        }
+        Ok(Self(unique))
+    }
+
+    pub fn contains(&self, capability: &Capability) -> bool {
+        self.0.contains(capability)
+    }
+
+    pub fn iter(&self) -> impl Iterator<Item = &Capability> {
+        self.0.iter()
+    }
+}
+
+impl<'de> Deserialize<'de> for CapabilitySet {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        Self::try_new(Vec::<Capability>::deserialize(deserializer)?).map_err(D::Error::custom)
+    }
+}
+
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub struct DuplicateCapability {
+    capability: Capability,
+}
+
+impl fmt::Display for DuplicateCapability {
+    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(formatter, "duplicate capability `{}`", self.capability)
+    }
+}
+
+impl std::error::Error for DuplicateCapability {}
+
+#[derive(Clone, Debug, Eq, PartialEq, Serialize)]
+#[serde(transparent)]
+pub struct SupportedProtocolVersions(BTreeSet<ProtocolVersion>);
+
+impl SupportedProtocolVersions {
+    pub fn try_new(
+        versions: impl IntoIterator<Item = ProtocolVersion>,
+    ) -> Result<Self, InvalidSupportedProtocolVersions> {
+        let mut unique = BTreeSet::new();
+        for version in versions {
+            if !unique.insert(version) {
+                return Err(InvalidSupportedProtocolVersions::Duplicate(version));
+            }
+        }
+        if unique.is_empty() {
+            return Err(InvalidSupportedProtocolVersions::Empty);
+        }
+        Ok(Self(unique))
+    }
+
+    pub fn contains(&self, version: ProtocolVersion) -> bool {
+        self.0.contains(&version)
+    }
+
+    pub fn iter(&self) -> impl Iterator<Item = ProtocolVersion> + '_ {
+        self.0.iter().copied()
+    }
+}
+
+impl<'de> Deserialize<'de> for SupportedProtocolVersions {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        Self::try_new(Vec::<ProtocolVersion>::deserialize(deserializer)?).map_err(D::Error::custom)
+    }
+}
+
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub enum InvalidSupportedProtocolVersions {
+    Empty,
+    Duplicate(ProtocolVersion),
+}
+
+impl fmt::Display for InvalidSupportedProtocolVersions {
+    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            Self::Empty => formatter.write_str("at least one protocol version is required"),
+            Self::Duplicate(version) => {
+                write!(formatter, "duplicate protocol version {}", version.get())
+            }
+        }
+    }
+}
+
+impl std::error::Error for InvalidSupportedProtocolVersions {}

codex-rs/code-mode-protocol/src/lib.rs

@@ -1,4 +1,5 @@
 mod description;
+pub mod host;
 mod response;
 mod runtime;
 mod session;