andras/vault-link
1
0
Fork 0
Code Issues Pull requests 18 Projects Releases 55 Packages Wiki Activity Actions 1

Add tokenizer

Browse source
This commit is contained in:
Andras Schmelczer 2024-11-24 22:32:06 +00:00
parent e910d9c5f4
commit 331e264399
No known key found for this signature in database
GPG key ID: FC8F2C3D3D1A718C
7 changed files with 119 additions and 33 deletions
Download patch file Download diff file Expand all files Collapse all files

7
backend/reconcile/src/tokenizer/word_tokenizer.rs Normal file
View file

@ -0,0 +1,7 @@
use super::token::Token;
pub fn word_tokenizer(text: &str) -> Vec<Token<String>> {
text.split_inclusive(char::is_whitespace)
.map(|s| Token::new(s.to_string(), s.to_string()))
.collect()
}