andras/reconcile
1
0
Fork 0
Code Issues Pull requests 6 Projects Releases Packages Wiki Activity Actions

Merge crates

Browse source
This commit is contained in:
Andras Schmelczer 2025-06-15 11:30:07 +01:00
parent 82e77eec89
commit bcbac03228
No known key found for this signature in database
GPG key ID: FC8F2C3D3D1A718C
60 changed files with 73 additions and 248 deletions
Download patch file Download diff file Expand all files Collapse all files

7
src/tokenizer.rs Normal file
View file

@ -0,0 +1,7 @@
use token::Token;
pub mod token;
pub mod word_tokenizer;
/// A trait for tokenizers that take a string and return a list of tokens.
pub type Tokenizer<T> = dyn Fn(&str) -> Vec<Token<T>>;