This commit is contained in:
Andras Schmelczer 2025-07-06 13:04:56 +01:00
parent 6d56177ca8
commit 78fe3fd6fd
No known key found for this signature in database
GPG key ID: FC8F2C3D3D1A718C
5 changed files with 7 additions and 7 deletions

View file

@ -11,7 +11,7 @@ use super::token::Token;
pub fn line_tokenizer(text: &str) -> Vec<Token<String>> {
let mut result = Vec::new();
let mut line_start = 0;
let mut chars = text.char_indices().peekable();
while let Some((i, c)) = chars.next() {
if c == '\n' {
@ -32,12 +32,12 @@ pub fn line_tokenizer(text: &str) -> Vec<Token<String>> {
line_start = i + 2;
}
}
// Add final line if any
if line_start < text.len() {
result.push(text[line_start..].into());
}
result
}