26 lines
583 B
Rust
26 lines
583 B
Rust
use super::token::Token;
|
|
|
|
/// Splits text into UTF-8 characters.
|
|
///
|
|
/// ```not_rust
|
|
/// "Hey!" -> ["H", "e", "y", "!"]
|
|
/// ```
|
|
pub fn character_tokenizer(text: &str) -> Vec<Token<String>> {
|
|
text.chars()
|
|
.map(|char| Token::new(char.to_string(), char.to_string(), true, true))
|
|
.collect()
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use insta::assert_debug_snapshot;
|
|
|
|
use super::*;
|
|
|
|
#[test]
|
|
fn test_with_snapshots() {
|
|
assert_debug_snapshot!(character_tokenizer(""));
|
|
|
|
assert_debug_snapshot!(character_tokenizer(" hello, \nwhere are you?"));
|
|
}
|
|
}
|