Enable more lints

This commit is contained in:
Andras Schmelczer 2025-04-06 11:43:08 +01:00
parent 48f301d8ab
commit aa78e258e7
No known key found for this signature in database
GPG key ID: FC8F2C3D3D1A718C
2 changed files with 14 additions and 11 deletions

View file

@ -5,7 +5,9 @@ use super::token::Token;
///
/// ## Example
///
/// "Hi there!" -> ["Hi", " " ", "there!"]
/// ```not_rust
/// "Hi there!" -> ["Hi", " ", "there!"]
/// ```
pub fn word_tokenizer(text: &str) -> Vec<Token<String>> {
let mut result: Vec<Token<String>> = Vec::new();