6 lines
114 B
Rust
6 lines
114 B
Rust
use token::Token;
|
|
|
|
pub mod token;
|
|
pub mod word_tokenizer;
|
|
|
|
pub type Tokenizer<T> = dyn Fn(&str) -> Vec<Token<T>>;
|