Enable more lints
This commit is contained in:
parent
48f301d8ab
commit
aa78e258e7
2 changed files with 14 additions and 11 deletions
|
|
@ -5,7 +5,9 @@ use super::token::Token;
|
|||
///
|
||||
/// ## Example
|
||||
///
|
||||
/// "Hi there!" -> ["Hi", " " ", "there!"]
|
||||
/// ```not_rust
|
||||
/// "Hi there!" -> ["Hi", " ", "there!"]
|
||||
/// ```
|
||||
pub fn word_tokenizer(text: &str) -> Vec<Token<String>> {
|
||||
let mut result: Vec<Token<String>> = Vec::new();
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue