From f190a95a4f0d7195c1c5cff48157b6c451be7c27 Mon Sep 17 00:00:00 2001 From: jif-oai Date: Wed, 25 Mar 2026 16:07:04 +0000 Subject: [PATCH] feat: rendering library v1 (#15778) The goal will be to replace askama --- codex-rs/Cargo.lock | 7 + codex-rs/Cargo.toml | 4 +- codex-rs/utils/template/BUILD.bazel | 6 + codex-rs/utils/template/Cargo.toml | 11 + codex-rs/utils/template/README.md | 41 +++ codex-rs/utils/template/src/lib.rs | 442 ++++++++++++++++++++++++++++ 6 files changed, 510 insertions(+), 1 deletion(-) create mode 100644 codex-rs/utils/template/BUILD.bazel create mode 100644 codex-rs/utils/template/Cargo.toml create mode 100644 codex-rs/utils/template/README.md create mode 100644 codex-rs/utils/template/src/lib.rs diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index 9ecf9b1de..33244c118 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -3006,6 +3006,13 @@ dependencies = [ "regex-lite", ] +[[package]] +name = "codex-utils-template" +version = "0.0.0" +dependencies = [ + "pretty_assertions", +] + [[package]] name = "codex-v8-poc" version = "0.0.0" diff --git a/codex-rs/Cargo.toml b/codex-rs/Cargo.toml index 5854c1c46..3d2ee9948 100644 --- a/codex-rs/Cargo.toml +++ b/codex-rs/Cargo.toml @@ -70,6 +70,7 @@ members = [ "utils/path-utils", "utils/fuzzy-match", "utils/stream-parser", + "utils/template", "codex-client", "codex-api", "state", @@ -166,6 +167,7 @@ codex-utils-rustls-provider = { path = "utils/rustls-provider" } codex-utils-sandbox-summary = { path = "utils/sandbox-summary" } codex-utils-sleep-inhibitor = { path = "utils/sleep-inhibitor" } codex-utils-stream-parser = { path = "utils/stream-parser" } +codex-utils-template = { path = "utils/template" } codex-utils-string = { path = "utils/string" } codex-windows-sandbox = { path = "windows-sandbox-rs" } core_test_support = { path = "core/tests/common" } @@ -383,7 +385,7 @@ ignored = [ "icu_provider", "openssl-sys", "codex-utils-readiness", - "codex-secrets", + "codex-utils-template", "codex-v8-poc", ] diff --git a/codex-rs/utils/template/BUILD.bazel b/codex-rs/utils/template/BUILD.bazel new file mode 100644 index 000000000..4b3376de4 --- /dev/null +++ b/codex-rs/utils/template/BUILD.bazel @@ -0,0 +1,6 @@ +load("//:defs.bzl", "codex_rust_crate") + +codex_rust_crate( + name = "template", + crate_name = "codex_utils_template", +) diff --git a/codex-rs/utils/template/Cargo.toml b/codex-rs/utils/template/Cargo.toml new file mode 100644 index 000000000..60a0c5b98 --- /dev/null +++ b/codex-rs/utils/template/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "codex-utils-template" +version.workspace = true +edition.workspace = true +license.workspace = true + +[lints] +workspace = true + +[dev-dependencies] +pretty_assertions = { workspace = true } diff --git a/codex-rs/utils/template/README.md b/codex-rs/utils/template/README.md new file mode 100644 index 000000000..3197e115e --- /dev/null +++ b/codex-rs/utils/template/README.md @@ -0,0 +1,41 @@ +# codex-utils-template + +Small, strict string templating for prompt and text assets. + +Supported syntax: + +- `{{ name }}` placeholder interpolation +- `{{{{` for a literal `{{` +- `}}}}` for a literal `}}` + +The library is intentionally strict: + +- parsing fails on malformed placeholders +- rendering fails on missing values +- rendering fails on duplicate values +- rendering fails on extra values not used by the template + +## Example + +```rust +use codex_utils_template::Template; +use codex_utils_template::render; + +let template = Template::parse( + "Hello, {{ name }}.\nLiteral braces: {{{{ and }}}}.\nMode: {{ mode }}", +)?; + +let rendered = template.render([ + ("name", "Codex"), + ("mode", "strict"), +])?; + +assert_eq!( + rendered, + "Hello, Codex.\nLiteral braces: {{ and }}.\nMode: strict" +); + +let one_shot = render("Hi {{ who }}!", [("who", "there")])?; +assert_eq!(one_shot, "Hi there!"); +# Ok::<(), Box>(()) +``` diff --git a/codex-rs/utils/template/src/lib.rs b/codex-rs/utils/template/src/lib.rs new file mode 100644 index 000000000..62d2314ff --- /dev/null +++ b/codex-rs/utils/template/src/lib.rs @@ -0,0 +1,442 @@ +//! Minimal strict templating for prompt and text assets. +//! +//! Supported syntax: +//! - `{{ name }}` placeholder interpolation +//! - `{{{{` for a literal `{{` +//! - `}}}}` for a literal `}}` + +use std::collections::BTreeMap; +use std::collections::BTreeSet; +use std::error::Error; +use std::fmt; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum TemplateParseError { + EmptyPlaceholder { start: usize }, + NestedPlaceholder { start: usize }, + UnmatchedClosingDelimiter { start: usize }, + UnterminatedPlaceholder { start: usize }, +} + +impl fmt::Display for TemplateParseError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::EmptyPlaceholder { start } => { + write!(f, "template placeholder at byte {start} is empty") + } + Self::NestedPlaceholder { start } => { + write!( + f, + "template placeholder starting at byte {start} contains a nested `{{`" + ) + } + Self::UnmatchedClosingDelimiter { start } => { + write!(f, "template contains an unmatched `}}` at byte {start}") + } + Self::UnterminatedPlaceholder { start } => { + write!( + f, + "template placeholder starting at byte {start} is missing `}}`" + ) + } + } + } +} + +impl Error for TemplateParseError {} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum TemplateRenderError { + DuplicateValue { name: String }, + ExtraValue { name: String }, + MissingValue { name: String }, +} + +impl fmt::Display for TemplateRenderError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::DuplicateValue { name } => { + write!(f, "template value `{name}` was provided more than once") + } + Self::ExtraValue { name } => { + write!(f, "template value `{name}` is not used by this template") + } + Self::MissingValue { name } => { + write!(f, "template placeholder `{name}` is missing a value") + } + } + } +} + +impl Error for TemplateRenderError {} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum TemplateError { + Parse(TemplateParseError), + Render(TemplateRenderError), +} + +impl fmt::Display for TemplateError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Parse(err) => err.fmt(f), + Self::Render(err) => err.fmt(f), + } + } +} + +impl Error for TemplateError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::Parse(err) => Some(err), + Self::Render(err) => Some(err), + } + } +} + +impl From for TemplateError { + fn from(value: TemplateParseError) -> Self { + Self::Parse(value) + } +} + +impl From for TemplateError { + fn from(value: TemplateRenderError) -> Self { + Self::Render(value) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +enum Segment { + Literal(String), + Placeholder(String), +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Template { + placeholders: BTreeSet, + segments: Vec, +} + +impl Template { + pub fn parse(source: &str) -> Result { + let mut placeholders = BTreeSet::new(); + let mut segments = Vec::new(); + let mut literal_start = 0usize; + let mut cursor = 0usize; + + while cursor < source.len() { + let rest = &source[cursor..]; + if rest.starts_with("{{{{") { + push_literal(&mut segments, &source[literal_start..cursor]); + push_literal(&mut segments, "{{"); + cursor += "{{{{".len(); + literal_start = cursor; + continue; + } + if rest.starts_with("}}}}") { + push_literal(&mut segments, &source[literal_start..cursor]); + push_literal(&mut segments, "}}"); + cursor += "}}}}".len(); + literal_start = cursor; + continue; + } + if rest.starts_with("{{") { + push_literal(&mut segments, &source[literal_start..cursor]); + let (placeholder, next_cursor) = parse_placeholder(source, cursor)?; + placeholders.insert(placeholder.clone()); + segments.push(Segment::Placeholder(placeholder)); + cursor = next_cursor; + literal_start = cursor; + continue; + } + if rest.starts_with("}}") { + return Err(TemplateParseError::UnmatchedClosingDelimiter { start: cursor }); + } + + let Some(ch) = rest.chars().next() else { + break; + }; + cursor += ch.len_utf8(); + } + + push_literal(&mut segments, &source[literal_start..]); + Ok(Self { + placeholders, + segments, + }) + } + + pub fn placeholders(&self) -> impl ExactSizeIterator { + self.placeholders.iter().map(String::as_str) + } + + pub fn render(&self, variables: I) -> Result + where + I: IntoIterator, + K: AsRef, + V: AsRef, + { + let variables = build_variable_map(variables)?; + + for placeholder in &self.placeholders { + if !variables.contains_key(placeholder.as_str()) { + return Err(TemplateRenderError::MissingValue { + name: placeholder.clone(), + }); + } + } + + for name in variables.keys() { + if !self.placeholders.contains(name.as_str()) { + return Err(TemplateRenderError::ExtraValue { name: name.clone() }); + } + } + + let mut rendered = String::new(); + for segment in &self.segments { + match segment { + Segment::Literal(literal) => rendered.push_str(literal), + Segment::Placeholder(name) => { + let Some(value) = variables.get(name.as_str()) else { + return Err(TemplateRenderError::MissingValue { name: name.clone() }); + }; + rendered.push_str(value); + } + } + } + Ok(rendered) + } +} + +pub fn render(template: &str, variables: I) -> Result +where + I: IntoIterator, + K: AsRef, + V: AsRef, +{ + Template::parse(template)? + .render(variables) + .map_err(Into::into) +} + +fn push_literal(segments: &mut Vec, literal: &str) { + if literal.is_empty() { + return; + } + + if let Some(Segment::Literal(existing)) = segments.last_mut() { + existing.push_str(literal); + } else { + segments.push(Segment::Literal(literal.to_string())); + } +} + +fn parse_placeholder(source: &str, start: usize) -> Result<(String, usize), TemplateParseError> { + let placeholder_start = start + "{{".len(); + let mut cursor = placeholder_start; + + while cursor < source.len() { + let rest = &source[cursor..]; + if rest.starts_with("{{") { + return Err(TemplateParseError::NestedPlaceholder { start }); + } + if rest.starts_with("}}") { + let placeholder = source[placeholder_start..cursor].trim(); + if placeholder.is_empty() { + return Err(TemplateParseError::EmptyPlaceholder { start }); + } + return Ok((placeholder.to_string(), cursor + "}}".len())); + } + + let Some(ch) = rest.chars().next() else { + break; + }; + cursor += ch.len_utf8(); + } + + Err(TemplateParseError::UnterminatedPlaceholder { start }) +} + +fn build_variable_map( + variables: I, +) -> Result, TemplateRenderError> +where + I: IntoIterator, + K: AsRef, + V: AsRef, +{ + let mut map = BTreeMap::new(); + for (name, value) in variables { + let name = name.as_ref().to_string(); + if map + .insert(name.clone(), value.as_ref().to_string()) + .is_some() + { + return Err(TemplateRenderError::DuplicateValue { name }); + } + } + Ok(map) +} + +#[cfg(test)] +mod tests { + use super::Template; + use super::TemplateError; + use super::TemplateParseError; + use super::TemplateRenderError; + use super::render; + use pretty_assertions::assert_eq; + + #[test] + fn render_replaces_placeholders_with_and_without_whitespace() { + let rendered = render( + "Hello, {{ name }}. You are in {{place}}. {{ name }} is repeated.", + [("name", "Codex"), ("place", "codex-rs")], + ) + .unwrap(); + + assert_eq!( + rendered, + "Hello, Codex. You are in codex-rs. Codex is repeated." + ); + } + + #[test] + fn parsed_templates_can_be_reused() { + let template = Template::parse("{{greeting}}, {{ name }}!").unwrap(); + + assert_eq!( + template.render([("greeting", "Hello"), ("name", "Codex")]), + Ok("Hello, Codex!".to_string()) + ); + assert_eq!( + template.render([("greeting", "Hi"), ("name", "builder")]), + Ok("Hi, builder!".to_string()) + ); + } + + #[test] + fn placeholders_are_sorted_and_unique() { + let template = Template::parse("{{ b }} {{ a }} {{ b }}").unwrap(); + + assert_eq!(template.placeholders().collect::>(), vec!["a", "b"]); + } + + #[test] + fn render_supports_multiline_templates_and_adjacent_placeholders() { + let rendered = render( + "Line 1: {{first}}{{second}}\nLine 2: {{ third }}", + [("first", "A"), ("second", "B"), ("third", "C")], + ) + .unwrap(); + + assert_eq!(rendered, "Line 1: AB\nLine 2: C"); + } + + #[test] + fn render_supports_literal_delimiter_escapes() { + let rendered = render( + "literal open: {{{{, literal close: }}}}, value: {{ name }}", + [("name", "Codex")], + ) + .unwrap(); + + assert_eq!( + rendered, + "literal open: {{, literal close: }}, value: Codex" + ); + } + + #[test] + fn parse_errors_when_placeholder_is_empty() { + let err = Template::parse("Hello, {{ }}.").unwrap_err(); + + assert_eq!(err, TemplateParseError::EmptyPlaceholder { start: 7 }); + } + + #[test] + fn parse_errors_when_placeholder_is_unterminated() { + let err = Template::parse("Hello, {{ name.").unwrap_err(); + + assert_eq!( + err, + TemplateParseError::UnterminatedPlaceholder { start: 7 } + ); + } + + #[test] + fn parse_errors_when_placeholder_is_nested() { + let err = Template::parse("Hello, {{ outer {{ inner }} }}.").unwrap_err(); + + assert_eq!(err, TemplateParseError::NestedPlaceholder { start: 7 }); + } + + #[test] + fn parse_errors_when_closing_delimiter_is_unmatched() { + let err = Template::parse("Hello, }} world.").unwrap_err(); + + assert_eq!( + err, + TemplateParseError::UnmatchedClosingDelimiter { start: 7 } + ); + } + + #[test] + fn render_errors_when_placeholder_is_missing() { + let template = Template::parse("Hello, {{ name }}.").unwrap(); + + assert_eq!( + template.render(Vec::<(&str, &str)>::new()), + Err(TemplateRenderError::MissingValue { + name: "name".to_string() + }) + ); + } + + #[test] + fn render_errors_when_extra_value_is_provided() { + let template = Template::parse("Hello, {{ name }}.").unwrap(); + + assert_eq!( + template.render([("name", "Codex"), ("unused", "extra")]), + Err(TemplateRenderError::ExtraValue { + name: "unused".to_string() + }) + ); + } + + #[test] + fn render_errors_when_duplicate_value_is_provided() { + let template = Template::parse("Hello, {{ name }}.").unwrap(); + + assert_eq!( + template.render([("name", "Codex"), ("name", "other")]), + Err(TemplateRenderError::DuplicateValue { + name: "name".to_string() + }) + ); + } + + #[test] + fn render_function_wraps_parse_errors() { + let err = render("Hello, }} world.", [("name", "Codex")]).unwrap_err(); + + assert_eq!( + err, + TemplateError::Parse(TemplateParseError::UnmatchedClosingDelimiter { start: 7 }) + ); + } + + #[test] + fn render_function_wraps_render_errors() { + let err = render("Hello, {{ name }}.", [("extra", "Codex")]).unwrap_err(); + + assert_eq!( + err, + TemplateError::Render(TemplateRenderError::MissingValue { + name: "name".to_string() + }) + ); + } +}