fix: don't truncate at new lines (#6907)

This commit is contained in:
Ahmed Ibrahim
2025-11-19 09:05:48 -08:00
committed by GitHub
Unverified
parent d5dfba2509
commit d62cab9a06
5 changed files with 8 additions and 18 deletions
+2 -13
View File
@@ -282,22 +282,11 @@ fn truncate_on_boundary(input: &str, max_len: usize) -> &str {
}
fn pick_prefix_end(s: &str, left_budget: usize) -> usize {
if let Some(head) = s.get(..left_budget)
&& let Some(i) = head.rfind('\n')
{
return i + 1;
}
truncate_on_boundary(s, left_budget).len()
}
fn pick_suffix_start(s: &str, right_budget: usize) -> usize {
let start_tail = s.len().saturating_sub(right_budget);
if let Some(tail) = s.get(start_tail..)
&& let Some(i) = tail.find('\n')
{
return start_tail + i + 1;
}
let mut idx = start_tail.min(s.len());
while idx < s.len() && !s.is_char_boundary(idx) {
idx += 1;
@@ -420,7 +409,7 @@ mod tests {
fn truncate_middle_tokens_handles_utf8_content() {
let s = "πŸ˜€πŸ˜€πŸ˜€πŸ˜€πŸ˜€πŸ˜€πŸ˜€πŸ˜€πŸ˜€πŸ˜€\nsecond line with text\n";
let (out, tokens) = truncate_with_token_budget(s, TruncationPolicy::Tokens(8));
assert_eq!(out, "πŸ˜€πŸ˜€πŸ˜€πŸ˜€β€¦8 tokens truncated…");
assert_eq!(out, "πŸ˜€πŸ˜€πŸ˜€πŸ˜€β€¦8 tokens truncated… line with text\n");
assert_eq!(tokens, Some(16));
}
@@ -428,7 +417,7 @@ mod tests {
fn truncate_middle_bytes_handles_utf8_content() {
let s = "πŸ˜€πŸ˜€πŸ˜€πŸ˜€πŸ˜€πŸ˜€πŸ˜€πŸ˜€πŸ˜€πŸ˜€\nsecond line with text\n";
let out = truncate_text(s, TruncationPolicy::Bytes(20));
assert_eq!(out, "πŸ˜€πŸ˜€β€¦31 chars truncated…");
assert_eq!(out, "πŸ˜€πŸ˜€β€¦21 chars truncated…with text\n");
}
#[test]