andras/reconcile
1
0
Fork 0
Code Issues Pull requests 6 Projects Releases Packages Wiki Activity Actions

Rename TextWithHistory to SpanWithHistory

Browse source
This commit is contained in:
Andras Schmelczer 2025-07-04 03:14:18 +01:00
parent ae5940718e
commit 8bd803c9b2
7 changed files with 100 additions and 30 deletions
Download patch file Download diff file Expand all files Collapse all files

4
src/tokenizer.rs
View file

@ -1,5 +1,5 @@
mod word_tokenizer;
mod character_tokenizer;
mod word_tokenizer;
use std::ops::Deref;
@ -36,7 +36,7 @@ impl Deref for BuiltinTokenizer {
fn deref(&self) -> &Self::Target {
match self {
BuiltinTokenizer::Character =>&character_tokenizer::character_tokenizer,
BuiltinTokenizer::Character => &character_tokenizer::character_tokenizer,
BuiltinTokenizer::Word => &word_tokenizer::word_tokenizer,
#[cfg(feature = "wasm")]
BuiltinTokenizer::__Invalid => panic!("Unexpected tokenizer type"),