Apply more lints

This commit is contained in:
Andras Schmelczer 2024-12-18 22:14:09 +00:00
parent 0c92bf959b
commit 1e1dd7b877
No known key found for this signature in database
GPG key ID: FC8F2C3D3D1A718C
24 changed files with 61 additions and 63 deletions

View file

@ -1,6 +0,0 @@
use token::Token;
pub mod token;
pub mod word_tokenizer;
pub type Tokenizer<T> = dyn Fn(&str) -> Vec<Token<T>>;

View file

@ -17,8 +17,8 @@ where
impl From<&str> for Token<String> {
fn from(s: &str) -> Self {
Token {
normalised: s.to_string(),
original: s.to_string(),
normalised: s.to_owned(),
original: s.to_owned(),
}
}
}

View file

@ -2,6 +2,6 @@ use super::token::Token;
pub fn word_tokenizer(text: &str) -> Vec<Token<String>> {
text.split_inclusive(char::is_whitespace)
.map(|s| Token::new(s.to_string(), s.to_string()))
.map(|s| Token::new(s.to_owned(), s.to_owned()))
.collect()
}