Rename package
This commit is contained in:
parent
ce4af278ac
commit
7e9ea69a08
41 changed files with 58 additions and 209 deletions
|
|
@ -1,39 +0,0 @@
|
|||
---
|
||||
source: reconcile/src/tokenizer/word_tokenizer.rs
|
||||
expression: "word_tokenizer(\" hello, \\nwhere are you?\")"
|
||||
snapshot_kind: text
|
||||
---
|
||||
[
|
||||
Token {
|
||||
normalised: " ",
|
||||
original: " ",
|
||||
},
|
||||
Token {
|
||||
normalised: "hello,",
|
||||
original: "hello,",
|
||||
},
|
||||
Token {
|
||||
normalised: " \n",
|
||||
original: " \n",
|
||||
},
|
||||
Token {
|
||||
normalised: "where",
|
||||
original: "where",
|
||||
},
|
||||
Token {
|
||||
normalised: " ",
|
||||
original: " ",
|
||||
},
|
||||
Token {
|
||||
normalised: "are",
|
||||
original: "are",
|
||||
},
|
||||
Token {
|
||||
normalised: " ",
|
||||
original: " ",
|
||||
},
|
||||
Token {
|
||||
normalised: "you?",
|
||||
original: "you?",
|
||||
},
|
||||
]
|
||||
|
|
@ -1,5 +1,6 @@
|
|||
---
|
||||
source: src/tokenizer/character_tokenizer.rs
|
||||
expression: "character_tokenizer(\"\")"
|
||||
snapshot_kind: text
|
||||
---
|
||||
[]
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
---
|
||||
source: reconcile/src/tokenizer/word_tokenizer.rs
|
||||
source: src/tokenizer/word_tokenizer.rs
|
||||
expression: "word_tokenizer(\"\")"
|
||||
snapshot_kind: text
|
||||
---
|
||||
Loading…
Add table
Add a link
Reference in a new issue