7 lines
208 B
Rust
7 lines
208 B
Rust
use super::token::Token;
|
|
|
|
pub fn word_tokenizer(text: &str) -> Vec<Token<String>> {
|
|
text.split_inclusive(char::is_whitespace)
|
|
.map(|s| Token::new(s.to_owned(), s.to_owned()))
|
|
.collect()
|
|
}
|