mirror of
https://github.com/pchuan98/codex.git
synced 2026-07-01 00:31:56 +08:00
fix: don't truncate at new lines (#6907)
This commit is contained in:
committed by
GitHub
Unverified
parent
d5dfba2509
commit
d62cab9a06
@@ -282,22 +282,11 @@ fn truncate_on_boundary(input: &str, max_len: usize) -> &str {
|
||||
}
|
||||
|
||||
fn pick_prefix_end(s: &str, left_budget: usize) -> usize {
|
||||
if let Some(head) = s.get(..left_budget)
|
||||
&& let Some(i) = head.rfind('\n')
|
||||
{
|
||||
return i + 1;
|
||||
}
|
||||
truncate_on_boundary(s, left_budget).len()
|
||||
}
|
||||
|
||||
fn pick_suffix_start(s: &str, right_budget: usize) -> usize {
|
||||
let start_tail = s.len().saturating_sub(right_budget);
|
||||
if let Some(tail) = s.get(start_tail..)
|
||||
&& let Some(i) = tail.find('\n')
|
||||
{
|
||||
return start_tail + i + 1;
|
||||
}
|
||||
|
||||
let mut idx = start_tail.min(s.len());
|
||||
while idx < s.len() && !s.is_char_boundary(idx) {
|
||||
idx += 1;
|
||||
@@ -420,7 +409,7 @@ mod tests {
|
||||
fn truncate_middle_tokens_handles_utf8_content() {
|
||||
let s = "ππππππππππ\nsecond line with text\n";
|
||||
let (out, tokens) = truncate_with_token_budget(s, TruncationPolicy::Tokens(8));
|
||||
assert_eq!(out, "ππππβ¦8 tokens truncatedβ¦");
|
||||
assert_eq!(out, "ππππβ¦8 tokens truncatedβ¦ line with text\n");
|
||||
assert_eq!(tokens, Some(16));
|
||||
}
|
||||
|
||||
@@ -428,7 +417,7 @@ mod tests {
|
||||
fn truncate_middle_bytes_handles_utf8_content() {
|
||||
let s = "ππππππππππ\nsecond line with text\n";
|
||||
let out = truncate_text(s, TruncationPolicy::Bytes(20));
|
||||
assert_eq!(out, "ππβ¦31 chars truncatedβ¦");
|
||||
assert_eq!(out, "ππβ¦21 chars truncatedβ¦with text\n");
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
Reference in New Issue
Block a user