agent/codex
2
0
Fork 0
mirror of https://github.com/pchuan98/codex.git synced 2026-07-01 00:31:56 +08:00
Code Issues Packages Projects Releases Wiki Activity

tui2: stop baking streaming wraps; reflow agent markdown (#8761)

Browse Source 
Background
Streaming assistant prose in tui2 was being rendered with viewport-width
wrapping during streaming, then stored in history cells as already split
`Line`s. Those width-derived breaks became indistinguishable from hard
newlines, so the transcript could not "un-split" on resize. This also
degraded copy/paste, since soft wraps looked like hard breaks.

What changed
- Introduce width-agnostic `MarkdownLogicalLine` output in
`tui2/src/markdown_render.rs`, preserving markdown wrap semantics:
initial/subsequent indents, per-line style, and a preformatted flag.
- Update the streaming collector (`tui2/src/markdown_stream.rs`) to emit
logical lines (newline-gated) and remove any captured viewport width.
- Update streaming orchestration (`tui2/src/streaming/*`) to queue and
emit logical lines, producing `AgentMessageCell::new_logical(...)`.
- Make `AgentMessageCell` store logical lines and wrap at render time in
`HistoryCell::transcript_lines_with_joiners(width)`, emitting joiners so
copy/paste can join soft-wrap continuations correctly.

Overlay deferral
When an overlay is active, defer *cells* (not rendered `Vec<Line>`) and
render them at overlay close time. This avoids baking width-derived
wraps based on a stale width.

Tests + docs
- Add resize/reflow regression tests + snapshots for streamed agent
output.
- Expand module/API docs for the new logical-line streaming pipeline and
clarify joiner semantics.
- Align scrollback-related docs/comments with current tui2 behavior
(main draw loop does not flush queued "history lines" to the terminal).

More details
See `codex-rs/tui2/docs/streaming_wrapping_design.md` for the full
problem statement and solution approach, and
`codex-rs/tui2/docs/tui_viewport_and_history.md` for viewport vs printed
output behavior.
This commit is contained in:
Josh McKinney
2026-01-05 18:37:58 -08:00
committed by GitHub
Unverified
parent 771f1ca6ab
commit c92dbea7c1
13 changed files with 868 additions and 325 deletions
Download Patch File Download Diff File Expand all files Collapse all files
codex-rs/tui2/src/app.rs
+20 -21
View File
@@ -344,10 +344,22 @@ pub(crate) struct App {
transcript_copy_action: TranscriptCopyAction,
transcript_scrollbar_ui: TranscriptScrollbarUi,
// Pager overlay state (Transcript or Static like Diff)
// Pager overlay state (Transcript or Static like Diff).
pub(crate) overlay: Option<Overlay>,
pub(crate) deferred_history_lines: Vec<Line<'static>>,
has_emitted_history_lines: bool,
/// History cells received while an overlay is active.
///
/// While in an alt-screen overlay, the normal terminal buffer is not visible.
/// Instead we queue the incoming cells here and, on overlay close, render them at the *current*
/// width and queue them in one batch via `Tui::insert_history_lines`.
///
/// This matters for correctness if/when scrollback printing is enabled: if we deferred
/// already-rendered `Vec<Line>`, we'd bake viewport-width wrapping based on the width at the
/// time the cell arrived (which may differ from the width when the overlay closes).
pub(crate) deferred_history_cells: Vec<Arc<dyn HistoryCell>>,
/// True once at least one history cell has been inserted into terminal scrollback.
///
/// Used to decide whether to insert an extra blank separator line when flushing deferred cells.
pub(crate) has_emitted_history_lines: bool,
pub(crate) enhanced_keys_supported: bool,
@@ -511,7 +523,7 @@ impl App {
transcript_copy_action: TranscriptCopyAction::default(),
transcript_scrollbar_ui: TranscriptScrollbarUi::default(),
overlay: None,
deferred_history_lines: Vec::new(),
deferred_history_cells: Vec::new(),
has_emitted_history_lines: false,
commit_anim_running: Arc::new(AtomicBool::new(false)),
scroll_config,
@@ -1449,21 +1461,8 @@ impl App {
tui.frame_requester().schedule_frame();
}
self.transcript_cells.push(cell.clone());
let mut display = cell.display_lines(tui.terminal.last_known_screen_size.width);
if !display.is_empty() {
// Only insert a separating blank line for new cells that are not
// part of an ongoing stream. Streaming continuations should not
// accrue extra blank lines between chunks.
if !cell.is_stream_continuation() {
if self.has_emitted_history_lines {
display.insert(0, Line::from(""));
} else {
self.has_emitted_history_lines = true;
}
}
if self.overlay.is_some() {
self.deferred_history_lines.extend(display);
}
if self.overlay.is_some() {
self.deferred_history_cells.push(cell);
}
}
AppEvent::StartCommitAnimation => {
@@ -2135,7 +2134,7 @@ mod tests {
transcript_copy_action: TranscriptCopyAction::default(),
transcript_scrollbar_ui: TranscriptScrollbarUi::default(),
overlay: None,
deferred_history_lines: Vec::new(),
deferred_history_cells: Vec::new(),
has_emitted_history_lines: false,
enhanced_keys_supported: false,
commit_anim_running: Arc::new(AtomicBool::new(false)),
@@ -2188,7 +2187,7 @@ mod tests {
transcript_copy_action: TranscriptCopyAction::default(),
transcript_scrollbar_ui: TranscriptScrollbarUi::default(),
overlay: None,
deferred_history_lines: Vec::new(),
deferred_history_cells: Vec::new(),
has_emitted_history_lines: false,
enhanced_keys_supported: false,
commit_anim_running: Arc::new(AtomicBool::new(false)),
codex-rs/tui2/src/app_backtrack.rs
+33 -3
View File
@@ -123,12 +123,42 @@ impl App {
}
/// Close transcript overlay and restore normal UI.
///
/// Any history emitted while the overlay was open is flushed to the normal-buffer queue here.
///
/// Importantly, we defer *cells* (not rendered lines) so we can render them against the current
/// width on close and avoid baking width-derived wrapping based on an earlier viewport size.
/// (This matters if/when scrollback printing is enabled; `Tui::insert_history_lines` currently
/// queues lines without printing them during the main draw loop.)
pub(crate) fn close_transcript_overlay(&mut self, tui: &mut tui::Tui) {
let _ = tui.leave_alt_screen();
let was_backtrack = self.backtrack.overlay_preview_active;
if !self.deferred_history_lines.is_empty() {
let lines = std::mem::take(&mut self.deferred_history_lines);
tui.insert_history_lines(lines);
if !self.deferred_history_cells.is_empty() {
let cells = std::mem::take(&mut self.deferred_history_cells);
let width = tui.terminal.last_known_screen_size.width;
let mut lines: Vec<ratatui::text::Line<'static>> = Vec::new();
for cell in cells {
let mut display = cell.display_lines(width);
if display.is_empty() {
continue;
}
// Only insert a separating blank line for new cells that are not part of an
// ongoing stream. Streaming continuations should not accrue extra blank lines
// between chunks.
if !cell.is_stream_continuation() {
if self.has_emitted_history_lines {
display.insert(0, ratatui::text::Line::from(""));
} else {
self.has_emitted_history_lines = true;
}
}
lines.extend(display);
}
if !lines.is_empty() {
tui.insert_history_lines(lines);
}
}
self.overlay = None;
self.backtrack.overlay_preview_active = false;
codex-rs/tui2/src/chatwidget.rs
+3 -3
View File
@@ -1023,9 +1023,9 @@ impl ChatWidget {
self.needs_final_message_separator = false;
needs_redraw = true;
}
self.stream_controller = Some(StreamController::new(
self.last_rendered_width.get().map(|w| w.saturating_sub(2)),
));
// Streaming must not capture the current viewport width: width-derived wraps are
// applied later, at render time, so the transcript can reflow on resize.
self.stream_controller = Some(StreamController::new());
}
if let Some(controller) = self.stream_controller.as_mut()
&& controller.push(&delta)
codex-rs/tui2/src/history_cell.rs
+263 -20
View File
@@ -119,6 +119,19 @@ pub(crate) trait HistoryCell: std::fmt::Debug + Send + Sync + Any {
/// Most cells can use the default implementation (no joiners), but cells that apply wrapping
/// should override this and return joiners derived from the same wrapping operation so
/// clipboard reconstruction can distinguish hard breaks from soft wraps.
///
/// `joiner_before[i]` describes the boundary *between* `lines[i - 1]` and `lines[i]`:
///
/// - `None` means "hard break": copy inserts a newline between the two lines.
/// - `Some(joiner)` means "soft wrap continuation": copy inserts `joiner` and continues on the
/// same logical line.
///
/// Example (one logical line wrapped across two visual lines):
///
/// - `lines = ["• Hello", " world"]`
/// - `joiner_before = [None, Some(\" \")]`
///
/// Copy should produce `"Hello world"` (no hard newline).
fn transcript_lines_with_joiners(&self, width: u16) -> TranscriptLinesWithJoiners {
let lines = self.transcript_lines(width);
TranscriptLinesWithJoiners {
@@ -313,14 +326,64 @@ impl HistoryCell for ReasoningSummaryCell {
#[derive(Debug)]
pub(crate) struct AgentMessageCell {
lines: Vec<Line<'static>>,
/// Width-agnostic logical markdown lines for this chunk.
///
/// These are produced either:
/// - by streaming (`markdown_stream` → `markdown_render::render_markdown_logical_lines`), or
/// - by legacy/non-streaming callers that pass pre-rendered `Vec<Line>` via [`Self::new`].
///
/// Importantly, this stores *logical* lines, not already-wrapped visual lines, so the transcript
/// can reflow on resize.
logical_lines: Vec<crate::markdown_render::MarkdownLogicalLine>,
/// Whether this cell should render the leading transcript bullet (`• `).
///
/// Streaming emits multiple immutable `AgentMessageCell`s per assistant message; only the first
/// chunk shows the bullet. Continuations use a two-space gutter.
is_first_line: bool,
}
impl AgentMessageCell {
/// Construct an agent message cell from already-rendered `Line`s.
///
/// This is primarily used by non-streaming paths. The lines are treated as already "logical"
/// lines (no additional markdown indentation metadata is available), and wrapping is still
/// performed at render time so the transcript can reflow on resize.
pub(crate) fn new(lines: Vec<Line<'static>>, is_first_line: bool) -> Self {
Self {
lines,
logical_lines: lines
.into_iter()
.map(|line| {
let is_preformatted = line.style.fg == Some(ratatui::style::Color::Cyan);
let line_style = line.style;
let content = Line {
style: Style::default(),
alignment: line.alignment,
spans: line.spans,
};
crate::markdown_render::MarkdownLogicalLine {
content,
initial_indent: Line::default(),
subsequent_indent: Line::default(),
line_style,
is_preformatted,
}
})
.collect(),
is_first_line,
}
}
/// Construct an agent message cell from markdown logical lines.
///
/// This is the preferred streaming constructor: it preserves markdown indentation rules (list
/// markers, nested list continuation indent, blockquote prefix, etc.) so wrapping can be
/// performed correctly at render time for the current viewport width.
pub(crate) fn new_logical(
logical_lines: Vec<crate::markdown_render::MarkdownLogicalLine>,
is_first_line: bool,
) -> Self {
Self {
logical_lines,
is_first_line,
}
}
@@ -331,43 +394,98 @@ impl HistoryCell for AgentMessageCell {
self.transcript_lines_with_joiners(width).lines
}
/// Render wrapped transcript lines plus soft-wrap joiners.
///
/// This is where width-dependent wrapping happens for streaming agent output. The cell composes
/// indentation as:
///
/// - transcript gutter (`• ` or ` `), plus
/// - markdown-provided indent/prefix spans (`initial_indent` / `subsequent_indent`)
///
/// The wrapping algorithm returns a `joiner_before` vector so copy/paste can treat soft wraps
/// as joinable (no hard newline) while preserving exact whitespace at wrap boundaries.
fn transcript_lines_with_joiners(&self, width: u16) -> TranscriptLinesWithJoiners {
use ratatui::style::Color;
if width == 0 {
return TranscriptLinesWithJoiners {
lines: Vec::new(),
joiner_before: Vec::new(),
};
}
let mut out_lines: Vec<Line<'static>> = Vec::new();
let mut joiner_before: Vec<Option<String>> = Vec::new();
let mut is_first_output_line = true;
for line in &self.lines {
let is_code_block_line = line.style.fg == Some(Color::Cyan);
let initial_indent: Line<'static> = if is_first_output_line && self.is_first_line {
// `at_cell_start` tracks whether we're about to emit the first *visual* line of this cell.
// Only the first chunk of a streamed message gets the `• ` gutter; continuations use ` `.
let mut at_cell_start = true;
for logical in &self.logical_lines {
let gutter_first_visual_line: Line<'static> = if at_cell_start && self.is_first_line {
"• ".dim().into()
} else {
" ".into()
};
let subsequent_indent: Line<'static> = " ".into();
let gutter_continuation: Line<'static> = " ".into();
if is_code_block_line {
let mut spans = initial_indent.spans;
spans.extend(line.spans.iter().cloned());
out_lines.push(Line::from(spans).style(line.style));
// Compose the transcript gutter with markdown-provided indentation:
//
// - `gutter_*` is the transcript-level prefix (`• ` / ` `).
// - `initial_indent` / `subsequent_indent` come from markdown structure (blockquote
// prefix, list marker indentation, nested list continuation indentation, etc.).
//
// We apply these indents during wrapping so:
// - the UI renders with correct continuation indentation, and
// - soft-wrap joiners stay aligned with the exact whitespace the wrapper skipped.
let compose_indent =
|gutter: &Line<'static>, md_indent: &Line<'static>| -> Line<'static> {
let mut spans = gutter.spans.clone();
spans.extend(md_indent.spans.iter().cloned());
Line::from(spans)
};
// Preformatted lines are rendered as a single visual line (no wrapping).
// This preserves code-block whitespace and keeps code copy behavior stable.
if logical.is_preformatted {
let mut spans = gutter_first_visual_line.spans.clone();
spans.extend(logical.initial_indent.spans.iter().cloned());
spans.extend(logical.content.spans.iter().cloned());
out_lines.push(Line::from(spans).style(logical.line_style));
joiner_before.push(None);
is_first_output_line = false;
at_cell_start = false;
continue;
}
// Prose path: wrap to current width and capture joiners.
//
// `word_wrap_line_with_joiners` guarantees:
// - `wrapped.len() == wrapped_joiners.len()`
// - `wrapped_joiners[0] == None` (first visual segment of a logical line is a hard break)
// - subsequent entries are `Some(joiner)` (soft-wrap continuations).
let opts = RtOptions::new(width as usize)
.initial_indent(initial_indent)
.subsequent_indent(subsequent_indent.clone());
.initial_indent(compose_indent(
&gutter_first_visual_line,
&logical.initial_indent,
))
.subsequent_indent(compose_indent(
&gutter_continuation,
&logical.subsequent_indent,
));
let (wrapped, wrapped_joiners) =
crate::wrapping::word_wrap_line_with_joiners(line, opts);
for (l, j) in wrapped.into_iter().zip(wrapped_joiners) {
out_lines.push(line_to_static(&l));
joiner_before.push(j);
is_first_output_line = false;
crate::wrapping::word_wrap_line_with_joiners(&logical.content, opts);
for (visual, joiner) in wrapped.into_iter().zip(wrapped_joiners) {
out_lines.push(line_to_static(&visual).style(logical.line_style));
joiner_before.push(joiner);
at_cell_start = false;
}
}
debug_assert_eq!(out_lines.len(), joiner_before.len());
debug_assert!(
joiner_before
.first()
.is_none_or(std::option::Option::is_none)
);
TranscriptLinesWithJoiners {
lines: out_lines,
joiner_before,
@@ -1674,6 +1792,131 @@ mod tests {
render_lines(&cell.transcript_lines(u16::MAX))
}
/// Remove a single leading markdown blockquote marker (`> `) from `line`.
///
/// This is a test-only normalization helper.
///
/// In the rendered transcript, blockquote indentation is represented as literal `> ` spans in
/// the line prefix. For wrapped blockquote prose, those prefix spans can appear on every visual
/// line (including soft-wrap continuations). When we want to compare the *logical* joined text
/// across different widths, we strip the repeated marker on continuation lines so the
/// comparison doesn't fail due to prefix duplication.
fn strip_leading_blockquote_marker(line: &str) -> String {
let mut out = String::with_capacity(line.len());
let mut seen_non_space = false;
let mut removed = false;
let mut chars = line.chars().peekable();
while let Some(ch) = chars.next() {
if !seen_non_space {
if ch == ' ' {
out.push(ch);
continue;
}
seen_non_space = true;
if ch == '>' && !removed {
removed = true;
if matches!(chars.peek(), Some(' ')) {
chars.next();
}
continue;
}
}
out.push(ch);
}
out
}
/// Normalize rendered transcript output into a width-insensitive "logical text" string.
///
/// This is used by resize/reflow tests:
///
/// - Joiners tell us which visual line breaks are soft wraps (`Some(joiner)`) vs hard breaks
/// (`None`).
/// - For soft-wrap continuation lines, we strip repeated blockquote markers so we can compare
/// the underlying prose independent of prefix repetition.
/// - Finally, we collapse whitespace so wrapping differences (line breaks vs spaces) do not
/// affect equality.
fn normalize_rendered_text_with_joiners(tr: &TranscriptLinesWithJoiners) -> String {
let mut rendered = render_lines(&tr.lines);
for (line, joiner) in rendered.iter_mut().zip(&tr.joiner_before) {
if joiner.is_some() {
*line = strip_leading_blockquote_marker(line);
}
}
rendered
.join("\n")
.split_whitespace()
.collect::<Vec<_>>()
.join(" ")
}
#[test]
fn agent_message_cell_reflows_streamed_prose_on_resize() {
let md = concat!(
"- This is a long list item that should reflow when the viewport width changes. ",
"The old streaming implementation used to bake soft wraps into hard line breaks.\n",
"> A blockquote line that is also long enough to wrap and should reflow cleanly.\n",
);
let logical_lines = crate::markdown_stream::simulate_stream_markdown_for_tests(&[md], true);
let cell = AgentMessageCell::new_logical(logical_lines, true);
let narrow = cell.transcript_lines_with_joiners(28);
let wide = cell.transcript_lines_with_joiners(80);
assert!(
narrow.lines.len() > wide.lines.len(),
"expected fewer visual lines at wider width; narrow={} wide={}",
narrow.lines.len(),
wide.lines.len()
);
assert_eq!(
normalize_rendered_text_with_joiners(&narrow),
normalize_rendered_text_with_joiners(&wide)
);
let snapshot = format!(
"narrow:\n{}\n\nwide:\n{}",
render_lines(&narrow.lines).join("\n"),
render_lines(&wide.lines).join("\n")
);
insta::assert_snapshot!("agent_message_cell_reflow_on_resize", snapshot);
}
#[test]
fn agent_message_cell_reflows_streamed_prose_vt100_snapshot() {
use crate::test_backend::VT100Backend;
let md = concat!(
"- This is a long list item that should reflow when the viewport width changes.\n",
"> A blockquote that also reflows across widths.\n",
);
let logical_lines = crate::markdown_stream::simulate_stream_markdown_for_tests(&[md], true);
let cell = AgentMessageCell::new_logical(logical_lines, true);
let render = |width, height| -> String {
let backend = VT100Backend::new(width, height);
let mut terminal = ratatui::Terminal::new(backend).expect("terminal");
terminal
.draw(|f| {
let area = f.area();
let lines = cell.display_lines(area.width);
Paragraph::new(Text::from(lines))
.wrap(Wrap { trim: false })
.render(area, f.buffer_mut());
})
.expect("draw");
terminal.backend().vt100().screen().contents()
};
let narrow = render(30, 12);
let wide = render(70, 12);
insta::assert_snapshot!(
"agent_message_cell_reflow_on_resize_vt100",
format!("narrow:\n{narrow}\n\nwide:\n{wide}")
);
}
#[tokio::test]
async fn mcp_tools_output_masks_sensitive_values() {
let mut config = test_config().await;
codex-rs/tui2/src/insert_history.rs
+5 -1
View File
@@ -1,9 +1,13 @@
//! Render `ratatui` transcript lines into terminal scrollback.
//! Render `ratatui` transcript lines into terminal output (scrollback) and/or deterministic ANSI.
//!
//! `insert_history_lines` is responsible for inserting rendered transcript lines
//! *above* the TUI viewport by emitting ANSI control sequences through the
//! terminal backend writer.
//!
//! Note: the current `tui2` main draw loop does not call `insert_history_lines` (see
//! `codex-rs/tui2/src/tui.rs`). This module is still used for deterministic ANSI emission via
//! `write_spans` (e.g., "print after exit" flows) and for tests.
//!
//! ## Why we use crossterm style commands
//!
//! `write_spans` is also used by non-terminal callers (e.g.
codex-rs/tui2/src/markdown_render.rs
+170 -18
View File
@@ -1,3 +1,37 @@
//! Markdown rendering for `tui2`.
//!
//! This module has two related but intentionally distinct responsibilities:
//!
//! 1. **Parse Markdown into styled text** (for display).
//! 2. **Preserve width-agnostic structure for reflow** (for streaming + resize).
//!
//! ## Why logical lines exist
//!
//! TUI2 supports viewport resize reflow and copy/paste that treats soft-wrapped prose as a single
//! logical line. If we apply wrapping while rendering and store the resulting `Vec<Line>`, those
//! width-derived breaks become indistinguishable from hard newlines and cannot be "unwrapped" when
//! the viewport gets wider.
//!
//! To avoid baking width, streaming uses [`MarkdownLogicalLine`] output:
//!
//! - `content` holds the styled spans for a single *logical* line (a hard break boundary).
//! - `initial_indent` / `subsequent_indent` encode markdown-aware indentation rules for wraps
//! (list markers, nested lists, blockquotes, etc.).
//! - `line_style` captures line-level styling (e.g., blockquote green) that must apply to all
//! wrapped segments.
//! - `is_preformatted` marks runs that should not be wrapped like prose (e.g., fenced code).
//!
//! History cells can then wrap `content` at the *current* width, applying indents appropriately and
//! returning soft-wrap joiners for correct copy/paste.
//!
//! ## Outputs
//!
//! - [`render_markdown_text_with_width`]: emits a `Text` suitable for immediate display and may
//! apply wrapping if a width is provided.
//! - [`render_markdown_logical_lines`]: emits width-agnostic logical lines (no wrapping).
//!
//! The underlying `Writer` can emit either (or both) depending on call site needs.
use crate::render::line_utils::line_to_static;
use crate::wrapping::RtOptions;
use crate::wrapping::word_wrap_line;
@@ -14,6 +48,31 @@ use ratatui::text::Line;
use ratatui::text::Span;
use ratatui::text::Text;
/// A single width-agnostic markdown "logical line" plus the metadata required to wrap it later.
///
/// A logical line is a hard-break boundary produced by markdown parsing (explicit newlines,
/// paragraph boundaries, list item boundaries, etc.). It is not a viewport-derived wrap segment.
///
/// Wrapping is performed later (typically in `HistoryCell::transcript_lines_with_joiners(width)`),
/// where a cell can:
///
/// - prepend a transcript gutter prefix (`• ` / ` `),
/// - prepend markdown-specific indents (`initial_indent` / `subsequent_indent`), and
/// - wrap `content` to the current width while producing joiners for copy/paste.
#[derive(Clone, Debug)]
pub(crate) struct MarkdownLogicalLine {
/// The raw content for this logical line (does not include markdown prefix/indent spans).
pub(crate) content: Line<'static>,
/// Prefix/indent spans to apply to the first visual line when wrapping.
pub(crate) initial_indent: Line<'static>,
/// Prefix/indent spans to apply to wrapped continuation lines.
pub(crate) subsequent_indent: Line<'static>,
/// Line-level style to apply to all wrapped segments.
pub(crate) line_style: Style,
/// True when this line is preformatted and should not be wrapped like prose.
pub(crate) is_preformatted: bool,
}
struct MarkdownStyles {
h1: Style,
h2: Style,
@@ -56,8 +115,12 @@ impl Default for MarkdownStyles {
#[derive(Clone, Debug)]
struct IndentContext {
/// Prefix spans to apply for this nesting level (e.g., blockquote `> `, list indentation).
prefix: Vec<Span<'static>>,
/// Optional list marker spans (e.g., `- ` or `1. `) that apply only to the first visual line of
/// a list item.
marker: Option<Vec<Span<'static>>>,
/// True if this context represents a list indentation level.
is_list: bool,
}
@@ -75,21 +138,45 @@ pub fn render_markdown_text(input: &str) -> Text<'static> {
render_markdown_text_with_width(input, None)
}
/// Render markdown into a ratatui `Text`, optionally wrapping to a specific width.
///
/// This is primarily used for non-streaming rendering where storing width-derived wrapping is
/// acceptable or where the caller immediately consumes the output.
pub(crate) fn render_markdown_text_with_width(input: &str, width: Option<usize>) -> Text<'static> {
let mut options = Options::empty();
options.insert(Options::ENABLE_STRIKETHROUGH);
let parser = Parser::new_ext(input, options);
let mut w = Writer::new(parser, width);
let mut w = Writer::new(parser, width, true, false);
w.run();
w.text
}
/// Render markdown into width-agnostic logical lines (no wrapping).
///
/// This is used by streaming so that the transcript can reflow on resize: wrapping is deferred to
/// the history cell at render time.
pub(crate) fn render_markdown_logical_lines(input: &str) -> Vec<MarkdownLogicalLine> {
let mut options = Options::empty();
options.insert(Options::ENABLE_STRIKETHROUGH);
let parser = Parser::new_ext(input, options);
let mut w = Writer::new(parser, None, false, true);
w.run();
w.logical_lines
}
/// A markdown event sink that builds either:
/// - a wrapped `Text` (`emit_text = true`), and/or
/// - width-agnostic [`MarkdownLogicalLine`]s (`emit_logical_lines = true`).
///
/// The writer tracks markdown structure (paragraphs, lists, blockquotes, code blocks) and builds up
/// a "current logical line". `flush_current_line` commits it to the selected output(s).
struct Writer<'a, I>
where
I: Iterator<Item = Event<'a>>,
{
iter: I,
text: Text<'static>,
logical_lines: Vec<MarkdownLogicalLine>,
styles: MarkdownStyles,
inline_styles: Vec<Style>,
indent_stack: Vec<IndentContext>,
@@ -105,16 +192,21 @@ where
current_subsequent_indent: Vec<Span<'static>>,
current_line_style: Style,
current_line_in_code_block: bool,
emit_text: bool,
emit_logical_lines: bool,
has_output_lines: bool,
}
impl<'a, I> Writer<'a, I>
where
I: Iterator<Item = Event<'a>>,
{
fn new(iter: I, wrap_width: Option<usize>) -> Self {
fn new(iter: I, wrap_width: Option<usize>, emit_text: bool, emit_logical_lines: bool) -> Self {
Self {
iter,
text: Text::default(),
logical_lines: Vec::new(),
styles: MarkdownStyles::default(),
inline_styles: Vec::new(),
indent_stack: Vec::new(),
@@ -130,6 +222,9 @@ where
current_subsequent_indent: Vec::new(),
current_line_style: Style::default(),
current_line_in_code_block: false,
emit_text,
emit_logical_lines,
has_output_lines: false,
}
}
@@ -150,7 +245,7 @@ where
Event::HardBreak => self.hard_break(),
Event::Rule => {
self.flush_current_line();
if !self.text.lines.is_empty() {
if self.has_output_lines {
self.push_blank_line();
}
self.push_line(Line::from("———"));
@@ -396,7 +491,7 @@ where
fn start_codeblock(&mut self, _lang: Option<String>, indent: Option<Span<'static>>) {
self.flush_current_line();
if !self.text.lines.is_empty() {
if self.has_output_lines {
self.push_blank_line();
}
self.in_code_block = true;
@@ -436,30 +531,69 @@ where
}
}
/// Commit the current logical line to configured outputs.
///
/// - When emitting logical lines, this records `content` plus indent metadata so callers can
/// wrap later at the current viewport width.
/// - When emitting `Text`, wrapping may be applied immediately if `wrap_width` is set.
fn flush_current_line(&mut self) {
if let Some(line) = self.current_line_content.take() {
let style = self.current_line_style;
let Some(line) = self.current_line_content.take() else {
return;
};
let initial_indent: Line<'static> =
Line::from(std::mem::take(&mut self.current_initial_indent));
let subsequent_indent: Line<'static> =
Line::from(std::mem::take(&mut self.current_subsequent_indent));
let line_style = self.current_line_style;
let is_preformatted = self.current_line_in_code_block;
if self.emit_logical_lines {
if self.emit_text {
self.logical_lines.push(MarkdownLogicalLine {
content: line.clone(),
initial_indent: initial_indent.clone(),
subsequent_indent: subsequent_indent.clone(),
line_style,
is_preformatted,
});
} else {
self.logical_lines.push(MarkdownLogicalLine {
content: line,
initial_indent,
subsequent_indent,
line_style,
is_preformatted,
});
self.has_output_lines = true;
self.current_line_in_code_block = false;
return;
}
self.has_output_lines = true;
}
if self.emit_text {
// NB we don't wrap code in code blocks, in order to preserve whitespace for copy/paste.
if !self.current_line_in_code_block
&& let Some(width) = self.wrap_width
{
if !is_preformatted && let Some(width) = self.wrap_width {
let opts = RtOptions::new(width)
.initial_indent(self.current_initial_indent.clone().into())
.subsequent_indent(self.current_subsequent_indent.clone().into());
.initial_indent(initial_indent)
.subsequent_indent(subsequent_indent);
for wrapped in word_wrap_line(&line, opts) {
let owned = line_to_static(&wrapped).style(style);
let owned = line_to_static(&wrapped).style(line_style);
self.text.lines.push(owned);
}
} else {
let mut spans = self.current_initial_indent.clone();
let mut spans = initial_indent.spans;
let mut line = line;
spans.append(&mut line.spans);
self.text.lines.push(Line::from_iter(spans).style(style));
self.text
.lines
.push(Line::from_iter(spans).style(line_style));
}
self.current_initial_indent.clear();
self.current_subsequent_indent.clear();
self.current_line_in_code_block = false;
self.has_output_lines = true;
}
self.current_line_in_code_block = false;
}
fn push_line(&mut self, line: Line<'static>) {
@@ -503,13 +637,31 @@ where
fn push_blank_line(&mut self) {
self.flush_current_line();
if self.indent_stack.iter().all(|ctx| ctx.is_list) {
self.text.lines.push(Line::default());
if self.emit_text {
self.text.lines.push(Line::default());
self.has_output_lines = true;
}
if self.emit_logical_lines {
self.logical_lines.push(MarkdownLogicalLine {
content: Line::default(),
initial_indent: Line::default(),
subsequent_indent: Line::default(),
line_style: Style::default(),
is_preformatted: false,
});
self.has_output_lines = true;
}
} else {
self.push_line(Line::default());
self.flush_current_line();
}
}
/// Compute the indentation spans for the current nesting stack.
///
/// `pending_marker_line` controls whether we are about to emit a list item's marker line
/// (e.g., `- ` or `1. `). For marker lines, we include exactly one marker (the most recent) and
/// suppress earlier list-level prefixes so nested list markers align correctly.
fn prefix_spans(&self, pending_marker_line: bool) -> Vec<Span<'static>> {
let mut prefix: Vec<Span<'static>> = Vec::new();
let last_marker_index = if pending_marker_line {
codex-rs/tui2/src/markdown_stream.rs
+102 -169
View File
@@ -1,21 +1,45 @@
use ratatui::text::Line;
//! Streaming markdown accumulator for `tui2`.
//!
//! Streaming assistant output arrives as small text deltas. The UI wants to render "stable"
//! transcript chunks during streaming without:
//!
//! - duplicating or reordering content when deltas split UTF-8 boundaries, and
//! - baking viewport-width wrapping into the persisted transcript model.
//!
//! This module provides [`MarkdownStreamCollector`], which implements a deliberately simple model:
//!
//! - The collector buffers raw deltas in a `String`.
//! - It only **commits** output when the buffered source contains a hard newline (`'\n'`).
//! This avoids showing partial final lines that may still change as the model continues to emit.
//! - When committing, it re-renders the markdown for the *completed* prefix of the buffer and
//! returns only the newly completed logical lines since the last commit.
//!
//! ## Width-agnostic output
//!
//! The committed output is `Vec<MarkdownLogicalLine>`, produced by
//! [`crate::markdown_render::render_markdown_logical_lines`]. These logical lines intentionally do
//! not include viewport-derived wraps, which allows the transcript to reflow on resize (wrapping is
//! performed later by the history cell at render time).
use crate::markdown;
use crate::markdown_render::MarkdownLogicalLine;
/// Newline-gated accumulator that renders markdown and commits only fully
/// completed logical lines.
pub(crate) struct MarkdownStreamCollector {
/// Accumulated raw markdown source (concatenated streaming deltas).
buffer: String,
/// Number of logical lines already emitted from the latest rendered prefix.
///
/// This is an index into the vector returned by `render_markdown_logical_lines` when applied
/// to the committed prefix of `buffer`.
committed_line_count: usize,
width: Option<usize>,
}
impl MarkdownStreamCollector {
pub fn new(width: Option<usize>) -> Self {
pub fn new() -> Self {
Self {
buffer: String::new(),
committed_line_count: 0,
width,
}
}
@@ -24,6 +48,7 @@ impl MarkdownStreamCollector {
self.committed_line_count = 0;
}
/// Append a streaming delta to the internal buffer.
pub fn push_delta(&mut self, delta: &str) {
tracing::trace!("push_delta: {delta:?}");
self.buffer.push_str(delta);
@@ -32,7 +57,7 @@ impl MarkdownStreamCollector {
/// Render the full buffer and return only the newly completed logical lines
/// since the last commit. When the buffer does not end with a newline, the
/// final rendered line is considered incomplete and is not emitted.
pub fn commit_complete_lines(&mut self) -> Vec<Line<'static>> {
pub fn commit_complete_lines(&mut self) -> Vec<MarkdownLogicalLine> {
let source = self.buffer.clone();
let last_newline_idx = source.rfind('\n');
let source = if let Some(last_newline_idx) = last_newline_idx {
@@ -40,14 +65,9 @@ impl MarkdownStreamCollector {
} else {
return Vec::new();
};
let mut rendered: Vec<Line<'static>> = Vec::new();
markdown::append_markdown(&source, self.width, &mut rendered);
let rendered = crate::markdown_render::render_markdown_logical_lines(&source);
let mut complete_line_count = rendered.len();
if complete_line_count > 0
&& crate::render::line_utils::is_blank_line_spaces_only(
&rendered[complete_line_count - 1],
)
{
if complete_line_count > 0 && is_blank_logical_line(&rendered[complete_line_count - 1]) {
complete_line_count -= 1;
}
@@ -66,7 +86,7 @@ impl MarkdownStreamCollector {
/// If the buffer does not end with a newline, a temporary one is appended
/// for rendering. Optionally unwraps ```markdown language fences in
/// non-test builds.
pub fn finalize_and_drain(&mut self) -> Vec<Line<'static>> {
pub fn finalize_and_drain(&mut self) -> Vec<MarkdownLogicalLine> {
let raw_buffer = self.buffer.clone();
let mut source: String = raw_buffer.clone();
if !source.ends_with('\n') {
@@ -81,8 +101,7 @@ impl MarkdownStreamCollector {
);
tracing::trace!("markdown finalize (raw source):\n---\n{source}\n---");
let mut rendered: Vec<Line<'static>> = Vec::new();
markdown::append_markdown(&source, self.width, &mut rendered);
let rendered = crate::markdown_render::render_markdown_logical_lines(&source);
let out = if self.committed_line_count >= rendered.len() {
Vec::new()
@@ -96,12 +115,18 @@ impl MarkdownStreamCollector {
}
}
fn is_blank_logical_line(line: &MarkdownLogicalLine) -> bool {
crate::render::line_utils::is_blank_line_spaces_only(&line.content)
&& crate::render::line_utils::is_blank_line_spaces_only(&line.initial_indent)
&& crate::render::line_utils::is_blank_line_spaces_only(&line.subsequent_indent)
}
#[cfg(test)]
pub(crate) fn simulate_stream_markdown_for_tests(
deltas: &[&str],
finalize: bool,
) -> Vec<Line<'static>> {
let mut collector = MarkdownStreamCollector::new(None);
) -> Vec<MarkdownLogicalLine> {
let mut collector = MarkdownStreamCollector::new();
let mut out = Vec::new();
for d in deltas {
collector.push_delta(d);
@@ -120,9 +145,18 @@ mod tests {
use super::*;
use ratatui::style::Color;
fn logical_line_text(line: &MarkdownLogicalLine) -> String {
line.initial_indent
.spans
.iter()
.chain(line.content.spans.iter())
.map(|s| s.content.as_ref())
.collect()
}
#[tokio::test]
async fn no_commit_until_newline() {
let mut c = super::MarkdownStreamCollector::new(None);
let mut c = super::MarkdownStreamCollector::new();
c.push_delta("Hello, world");
let out = c.commit_complete_lines();
assert!(out.is_empty(), "should not commit without newline");
@@ -133,7 +167,7 @@ mod tests {
#[tokio::test]
async fn finalize_commits_partial_line() {
let mut c = super::MarkdownStreamCollector::new(None);
let mut c = super::MarkdownStreamCollector::new();
c.push_delta("Line without newline");
let out = c.finalize_and_drain();
assert_eq!(out.len(), 1);
@@ -145,10 +179,10 @@ mod tests {
assert_eq!(out.len(), 1);
let l = &out[0];
assert_eq!(
l.style.fg,
l.line_style.fg,
Some(Color::Green),
"expected blockquote line fg green, got {:?}",
l.style.fg
l.line_style.fg
);
}
@@ -159,28 +193,23 @@ mod tests {
let non_blank: Vec<_> = out
.into_iter()
.filter(|l| {
let s = l
.spans
.iter()
.map(|sp| sp.content.clone())
.collect::<Vec<_>>()
.join("");
let t = s.trim();
let t = logical_line_text(l);
let t = t.trim();
// Ignore quote-only blank lines like ">" inserted at paragraph boundaries.
!(t.is_empty() || t == ">")
})
.collect();
assert_eq!(non_blank.len(), 2);
assert_eq!(non_blank[0].style.fg, Some(Color::Green));
assert_eq!(non_blank[1].style.fg, Some(Color::Green));
assert_eq!(non_blank[0].line_style.fg, Some(Color::Green));
assert_eq!(non_blank[1].line_style.fg, Some(Color::Green));
}
#[tokio::test]
async fn e2e_stream_blockquote_with_list_items_is_green() {
let out = super::simulate_stream_markdown_for_tests(&["> - item 1\n> - item 2\n"], true);
assert_eq!(out.len(), 2);
assert_eq!(out[0].style.fg, Some(Color::Green));
assert_eq!(out[1].style.fg, Some(Color::Green));
assert_eq!(out[0].line_style.fg, Some(Color::Green));
assert_eq!(out[1].line_style.fg, Some(Color::Green));
}
#[tokio::test]
@@ -194,19 +223,17 @@ mod tests {
];
let out = super::simulate_stream_markdown_for_tests(&md, true);
// Find the line that contains the third-level ordered text
let find_idx = out.iter().position(|l| {
l.spans
.iter()
.map(|s| s.content.clone())
.collect::<String>()
.contains("Third level (ordered)")
});
let find_idx = out
.iter()
.position(|l| logical_line_text(l).contains("Third level (ordered)"));
let idx = find_idx.expect("expected third-level ordered line");
let line = &out[idx];
// Expect at least one span on this line to be styled light blue
let has_light_blue = line
.initial_indent
.spans
.iter()
.chain(line.content.spans.iter())
.any(|s| s.style.fg == Some(ratatui::style::Color::LightBlue));
assert!(
has_light_blue,
@@ -218,54 +245,18 @@ mod tests {
async fn e2e_stream_blockquote_wrap_preserves_green_style() {
let long = "> This is a very long quoted line that should wrap across multiple columns to verify style preservation.";
let out = super::simulate_stream_markdown_for_tests(&[long, "\n"], true);
// Wrap to a narrow width to force multiple output lines.
let wrapped =
crate::wrapping::word_wrap_lines(out.iter(), crate::wrapping::RtOptions::new(24));
// Filter out purely blank lines
let non_blank: Vec<_> = wrapped
.into_iter()
.filter(|l| {
let s = l
.spans
.iter()
.map(|sp| sp.content.clone())
.collect::<Vec<_>>()
.join("");
!s.trim().is_empty()
})
.collect();
assert!(
non_blank.len() >= 2,
"expected wrapped blockquote to span multiple lines"
);
for (i, l) in non_blank.iter().enumerate() {
assert_eq!(
l.spans[0].style.fg,
Some(Color::Green),
"wrapped line {} should preserve green style, got {:?}",
i,
l.spans[0].style.fg
);
}
assert_eq!(out.len(), 1);
assert_eq!(out[0].line_style.fg, Some(Color::Green));
}
#[tokio::test]
async fn heading_starts_on_new_line_when_following_paragraph() {
// Stream a paragraph line, then a heading on the next line.
// Expect two distinct rendered lines: "Hello." and "Heading".
let mut c = super::MarkdownStreamCollector::new(None);
let mut c = super::MarkdownStreamCollector::new();
c.push_delta("Hello.\n");
let out1 = c.commit_complete_lines();
let s1: Vec<String> = out1
.iter()
.map(|l| {
l.spans
.iter()
.map(|s| s.content.clone())
.collect::<Vec<_>>()
.join("")
})
.collect();
let s1: Vec<String> = out1.iter().map(logical_line_text).collect();
assert_eq!(
out1.len(),
1,
@@ -276,32 +267,14 @@ mod tests {
c.push_delta("## Heading\n");
let out2 = c.commit_complete_lines();
let s2: Vec<String> = out2
.iter()
.map(|l| {
l.spans
.iter()
.map(|s| s.content.clone())
.collect::<Vec<_>>()
.join("")
})
.collect();
let s2: Vec<String> = out2.iter().map(logical_line_text).collect();
assert_eq!(
s2,
vec!["", "## Heading"],
"expected a blank separator then the heading line"
);
let line_to_string = |l: &ratatui::text::Line<'_>| -> String {
l.spans
.iter()
.map(|s| s.content.clone())
.collect::<Vec<_>>()
.join("")
};
assert_eq!(line_to_string(&out1[0]), "Hello.");
assert_eq!(line_to_string(&out2[1]), "## Heading");
assert_eq!(logical_line_text(&out1[0]), "Hello.");
assert_eq!(logical_line_text(&out2[1]), "## Heading");
}
#[tokio::test]
@@ -309,7 +282,7 @@ mod tests {
// Paragraph without trailing newline, then a chunk that starts with the newline
// and the heading text, then a final newline. The collector should first commit
// only the paragraph line, and later commit the heading as its own line.
let mut c = super::MarkdownStreamCollector::new(None);
let mut c = super::MarkdownStreamCollector::new();
c.push_delta("Sounds good!");
// No commit yet
assert!(c.commit_complete_lines().is_empty());
@@ -317,16 +290,7 @@ mod tests {
// Introduce the newline that completes the paragraph and the start of the heading.
c.push_delta("\n## Adding Bird subcommand");
let out1 = c.commit_complete_lines();
let s1: Vec<String> = out1
.iter()
.map(|l| {
l.spans
.iter()
.map(|s| s.content.clone())
.collect::<Vec<_>>()
.join("")
})
.collect();
let s1: Vec<String> = out1.iter().map(logical_line_text).collect();
assert_eq!(
s1,
vec!["Sounds good!"],
@@ -336,16 +300,7 @@ mod tests {
// Now finish the heading line with the trailing newline.
c.push_delta("\n");
let out2 = c.commit_complete_lines();
let s2: Vec<String> = out2
.iter()
.map(|l| {
l.spans
.iter()
.map(|s| s.content.clone())
.collect::<Vec<_>>()
.join("")
})
.collect();
let s2: Vec<String> = out2.iter().map(logical_line_text).collect();
assert_eq!(
s2,
vec!["", "## Adding Bird subcommand"],
@@ -353,18 +308,8 @@ mod tests {
);
// Sanity check raw markdown rendering for a simple line does not produce spurious extras.
let mut rendered: Vec<ratatui::text::Line<'static>> = Vec::new();
crate::markdown::append_markdown("Hello.\n", None, &mut rendered);
let rendered_strings: Vec<String> = rendered
.iter()
.map(|l| {
l.spans
.iter()
.map(|s| s.content.clone())
.collect::<Vec<_>>()
.join("")
})
.collect();
let rendered = crate::markdown_render::render_markdown_logical_lines("Hello.\n");
let rendered_strings: Vec<String> = rendered.iter().map(logical_line_text).collect();
assert_eq!(
rendered_strings,
vec!["Hello."],
@@ -372,17 +317,8 @@ mod tests {
);
}
fn lines_to_plain_strings(lines: &[ratatui::text::Line<'_>]) -> Vec<String> {
lines
.iter()
.map(|l| {
l.spans
.iter()
.map(|s| s.content.clone())
.collect::<Vec<_>>()
.join("")
})
.collect()
fn lines_to_plain_strings(lines: &[MarkdownLogicalLine]) -> Vec<String> {
lines.iter().map(logical_line_text).collect()
}
#[tokio::test]
@@ -413,8 +349,7 @@ mod tests {
let streamed = simulate_stream_markdown_for_tests(&deltas, true);
let streamed_str = lines_to_plain_strings(&streamed);
let mut rendered_all: Vec<ratatui::text::Line<'static>> = Vec::new();
crate::markdown::append_markdown(input, None, &mut rendered_all);
let rendered_all = crate::markdown_render::render_markdown_logical_lines(input);
let rendered_all_str = lines_to_plain_strings(&rendered_all);
assert_eq!(
@@ -433,9 +368,8 @@ mod tests {
let target_suffix = "1. Third level (ordered)";
let mut found = None;
for line in &streamed {
let s: String = line.spans.iter().map(|sp| sp.content.clone()).collect();
if s.contains(target_suffix) {
found = Some(line.clone());
if logical_line_text(line).contains(target_suffix) {
found = Some(line);
break;
}
}
@@ -443,22 +377,22 @@ mod tests {
panic!("expected to find the third-level ordered list line; got: {streamed_strs:?}")
});
// The marker (including indent and "1.") is expected to be in the first span
// and colored LightBlue; following content should be default color.
// The marker (including indent and "1.") should include LightBlue styling.
let has_light_blue = line
.initial_indent
.spans
.iter()
.chain(line.content.spans.iter())
.any(|sp| sp.style.fg == Some(Color::LightBlue));
assert!(
!line.spans.is_empty(),
"expected non-empty spans for the third-level line"
);
let marker_span = &line.spans[0];
assert_eq!(
marker_span.style.fg,
Some(Color::LightBlue),
"expected LightBlue 3rd-level ordered marker, got {:?}",
marker_span.style.fg
has_light_blue,
"expected LightBlue marker styling on: {:?}",
logical_line_text(line)
);
// Find the first non-empty non-space content span and verify it is default color.
let mut content_fg = None;
for sp in &line.spans[1..] {
for sp in line.content.spans.iter() {
let t = sp.content.trim();
if !t.is_empty() {
content_fg = Some(sp.style.fg);
@@ -519,8 +453,7 @@ mod tests {
let streamed_strs = lines_to_plain_strings(&streamed);
let full: String = deltas.iter().copied().collect();
let mut rendered_all: Vec<ratatui::text::Line<'static>> = Vec::new();
crate::markdown::append_markdown(&full, None, &mut rendered_all);
let rendered_all = crate::markdown_render::render_markdown_logical_lines(&full);
let rendered_all_strs = lines_to_plain_strings(&rendered_all);
assert_eq!(
@@ -605,10 +538,11 @@ mod tests {
let streamed = simulate_stream_markdown_for_tests(&deltas, true);
let streamed_strs = lines_to_plain_strings(&streamed);
// Compute a full render for diagnostics only.
// Also assert streamed output matches a full render.
let full: String = deltas.iter().copied().collect();
let mut rendered_all: Vec<ratatui::text::Line<'static>> = Vec::new();
crate::markdown::append_markdown(&full, None, &mut rendered_all);
let rendered_all = crate::markdown_render::render_markdown_logical_lines(&full);
let rendered_all_strs = lines_to_plain_strings(&rendered_all);
assert_eq!(streamed_strs, rendered_all_strs);
// Also assert exact expected plain strings for clarity.
let expected = vec![
@@ -634,8 +568,7 @@ mod tests {
let streamed = simulate_stream_markdown_for_tests(deltas, true);
let streamed_strs = lines_to_plain_strings(&streamed);
let full: String = deltas.iter().copied().collect();
let mut rendered: Vec<ratatui::text::Line<'static>> = Vec::new();
crate::markdown::append_markdown(&full, None, &mut rendered);
let rendered = crate::markdown_render::render_markdown_logical_lines(&full);
let rendered_strs = lines_to_plain_strings(&rendered);
assert_eq!(streamed_strs, rendered_strs, "full:\n---\n{full}\n---");
}
codex-rs/tui2/src/snapshots/codex_tui2__history_cell__tests__agent_message_cell_reflow_on_resize.snap
+25
View File
@@ -0,0 +1,25 @@
---
source: tui2/src/history_cell.rs
expression: snapshot
---
narrow:
• - This is a long list item
that should reflow when
the viewport width
changes. The old
streaming implementation
used to bake soft wraps
into hard line breaks.
> A blockquote line that
> is also long enough to
> wrap and should reflow
> cleanly.
wide:
• - This is a long list item that should reflow when the viewport width changes.
The old streaming implementation used to bake soft wraps into hard line
breaks.
> A blockquote line that is also long enough to wrap and should reflow
> cleanly.
codex-rs/tui2/src/snapshots/codex_tui2__history_cell__tests__agent_message_cell_reflow_on_resize_vt100.snap
+20
View File
@@ -0,0 +1,20 @@
---
source: tui2/src/history_cell.rs
expression: "format!(\"narrow:\\n{narrow}\\n\\nwide:\\n{wide}\")"
---
narrow:
• - This is a long list item
that should reflow when
the viewport width
changes.
> A blockquote that also
> reflows across widths.
wide:
• - This is a long list item that should reflow when the viewport
width changes.
> A blockquote that also reflows across widths.
codex-rs/tui2/src/streaming/controller.rs
+46 -13
View File
@@ -1,6 +1,25 @@
//! Orchestrates streaming assistant output into immutable transcript cells.
//!
//! The UI receives assistant output as a sequence of deltas. TUI2 wants to:
//!
//! - render incrementally during streaming,
//! - keep the transcript model append-only (emit immutable history cells),
//! - avoid duplicating content or showing partial final lines, and
//! - preserve resize reflow by not baking width-derived wraps into stored cells.
//!
//! [`StreamController`] glues together:
//!
//! - newline-gated delta accumulation (`MarkdownStreamCollector`),
//! - commit-tick animation (`StreamState` queue), and
//! - history cell emission (`AgentMessageCell::new_logical`).
//!
//! Each emitted cell contains **logical markdown lines** plus wrap metadata. The cell wraps those
//! lines at render time using the current viewport width and returns soft-wrap joiners for
//! copy/paste fidelity.
use crate::history_cell::HistoryCell;
use crate::history_cell::{self};
use ratatui::text::Line;
use crate::markdown_render::MarkdownLogicalLine;
use super::StreamState;
@@ -13,15 +32,19 @@ pub(crate) struct StreamController {
}
impl StreamController {
pub(crate) fn new(width: Option<usize>) -> Self {
/// Create a new controller for one assistant message stream.
pub(crate) fn new() -> Self {
Self {
state: StreamState::new(width),
state: StreamState::new(),
finishing_after_drain: false,
header_emitted: false,
}
}
/// Push a delta; if it contains a newline, commit completed lines and start animation.
/// Push a streaming delta and enqueue newly completed logical lines.
///
/// Returns `true` when at least one logical line was committed and should trigger commit-tick
/// animation.
pub(crate) fn push(&mut self, delta: &str) -> bool {
let state = &mut self.state;
if !delta.is_empty() {
@@ -38,7 +61,10 @@ impl StreamController {
false
}
/// Finalize the active stream. Drain and emit now.
/// Finalize the active stream and emit any remaining logical lines.
///
/// This forces the final "partial" line to be committed (if present) and resets the controller
/// so it is ready for the next stream.
pub(crate) fn finalize(&mut self) -> Option<Box<dyn HistoryCell>> {
// Finalize collector first.
let remaining = {
@@ -62,21 +88,28 @@ impl StreamController {
self.emit(out_lines)
}
/// Step animation: commit at most one queued line and handle end-of-drain cleanup.
/// Advance the commit-tick animation by at most one logical line.
///
/// Returns `(cell, idle)` where:
/// - `cell` is a new immutable history cell to append to the transcript (if any output is ready)
/// - `idle` is `true` once the queue is fully drained.
pub(crate) fn on_commit_tick(&mut self) -> (Option<Box<dyn HistoryCell>>, bool) {
let step = self.state.step();
(self.emit(step), self.state.is_idle())
}
fn emit(&mut self, lines: Vec<Line<'static>>) -> Option<Box<dyn HistoryCell>> {
fn emit(&mut self, lines: Vec<MarkdownLogicalLine>) -> Option<Box<dyn HistoryCell>> {
if lines.is_empty() {
return None;
}
Some(Box::new(history_cell::AgentMessageCell::new(lines, {
let header_emitted = self.header_emitted;
self.header_emitted = true;
!header_emitted
})))
Some(Box::new(history_cell::AgentMessageCell::new_logical(
lines,
{
let header_emitted = self.header_emitted;
self.header_emitted = true;
!header_emitted
},
)))
}
}
@@ -99,7 +132,7 @@ mod tests {
#[tokio::test]
async fn controller_loose_vs_tight_with_commit_ticks_matches_full() {
let mut ctrl = StreamController::new(None);
let mut ctrl = StreamController::new();
let mut lines = Vec::new();
// Exact deltas from the session log (section: Loose vs. tight list items)
codex-rs/tui2/src/streaming/mod.rs
+28 -8
View File
@@ -1,39 +1,59 @@
//! Streaming state for newline-gated assistant output.
//!
//! The streaming pipeline in `tui2` is split into:
//!
//! - [`crate::markdown_stream::MarkdownStreamCollector`]: accumulates raw deltas and commits
//! completed *logical* markdown lines (width-agnostic).
//! - [`StreamState`]: a small queue that supports "commit tick" animation by releasing at most one
//! logical line per tick.
//! - [`controller::StreamController`]: orchestration (header emission, finalize/drain semantics,
//! and converting queued logical lines into `HistoryCell`s).
//!
//! Keeping the queued units as logical lines (not wrapped visual lines) is essential for resize
//! reflow: visual wrapping depends on the current viewport width and must be performed at render
//! time inside the relevant history cell.
use std::collections::VecDeque;
use ratatui::text::Line;
use crate::markdown_render::MarkdownLogicalLine;
use crate::markdown_stream::MarkdownStreamCollector;
pub(crate) mod controller;
pub(crate) struct StreamState {
pub(crate) collector: MarkdownStreamCollector,
queued_lines: VecDeque<Line<'static>>,
queued_lines: VecDeque<MarkdownLogicalLine>,
pub(crate) has_seen_delta: bool,
}
impl StreamState {
pub(crate) fn new(width: Option<usize>) -> Self {
/// Create a fresh streaming state for one assistant message.
pub(crate) fn new() -> Self {
Self {
collector: MarkdownStreamCollector::new(width),
collector: MarkdownStreamCollector::new(),
queued_lines: VecDeque::new(),
has_seen_delta: false,
}
}
/// Reset state for the next stream.
pub(crate) fn clear(&mut self) {
self.collector.clear();
self.queued_lines.clear();
self.has_seen_delta = false;
}
pub(crate) fn step(&mut self) -> Vec<Line<'static>> {
/// Pop at most one queued logical line (for commit-tick animation).
pub(crate) fn step(&mut self) -> Vec<MarkdownLogicalLine> {
self.queued_lines.pop_front().into_iter().collect()
}
pub(crate) fn drain_all(&mut self) -> Vec<Line<'static>> {
/// Drain all queued logical lines (used on finalize).
pub(crate) fn drain_all(&mut self) -> Vec<MarkdownLogicalLine> {
self.queued_lines.drain(..).collect()
}
/// True when there is no queued output waiting to be emitted by commit ticks.
pub(crate) fn is_idle(&self) -> bool {
self.queued_lines.is_empty()
}
pub(crate) fn enqueue(&mut self, lines: Vec<Line<'static>>) {
/// Enqueue newly committed logical lines.
pub(crate) fn enqueue(&mut self, lines: Vec<MarkdownLogicalLine>) {
self.queued_lines.extend(lines);
}
}
codex-rs/tui2/src/tui.rs
+1 -1
View File
@@ -95,7 +95,7 @@ pub fn restore() -> Result<()> {
Ok(())
}
/// Initialize the terminal (inline viewport; history stays in normal scrollback)
/// Initialize the terminal (inline viewport; no always-on scrollback printing).
pub fn init() -> Result<Terminal> {
if !stdin().is_terminal() {
return Err(std::io::Error::other("stdin is not a terminal"));