Store tokens instead of text in insert

2024-11-28 20:51:05 +00:00 · 2024-11-28 20:51:05 +00:00 · 44f9a44f63
commit 44f9a44f63
parent 3391d2c0ec
10 changed files with 144 additions and 74 deletions
--- a/backend/reconcile/src/tokenizer/mod.rs
+++ b/backend/reconcile/src/tokenizer/mod.rs
@ -1,2 +1,6 @@
+use token::Token;
+
 pub mod token;
 pub mod word_tokenizer;
+
+pub type Tokenizer<T> = dyn Fn(&str) -> Vec<Token<T>>;