andras/reconcile
andras/reconcile
1
0
Fork 0
Code Issues Pull requests 6 Projects Releases Packages Wiki Activity Actions

Add Markdown tokeniser #48

Merged
schmelczer merged 5 commits from asch/md-tokeniser into main 2026-03-11 20:47:44 +00:00
Conversation 0 Commits 5 Files changed 29 +62
2 changed files with 62 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files
Showing only changes of commit 09b5c606ea - Show all commits

Update tests

Prev Next
Andras Schmelczer 2026-03-11 20:43:41 +00:00
commit 09b5c606ea

37
src/tokenizer/snapshots/reconcile_text__tokenizer__line_tokenizer__tests__with_snapshots-11.snap.new Normal file
View file

@ -0,0 +1,37 @@
---
source: src/tokenizer/line_tokenizer.rs
assertion_line: 78
expression: "line_tokenizer(\"Mixed\\r\\nand\\rbare\")"
---
[
Token {
normalized: "Mixed",
original: "Mixed",
is_left_joinable: true,
is_right_joinable: true,
},
Token {
normalized: "\r\n",
original: "\r\n",
is_left_joinable: true,
is_right_joinable: true,
},
Token {
normalized: "and",
original: "and",
is_left_joinable: true,
is_right_joinable: true,
},
Token {
normalized: "\r",
original: "\r",
is_left_joinable: true,
is_right_joinable: true,
},
Token {
normalized: "bare",
original: "bare",
is_left_joinable: true,
is_right_joinable: true,
},
]

25
src/tokenizer/snapshots/reconcile_text__tokenizer__markdown_tokenizer__tests__headings-2.snap.new Normal file
View file

@ -0,0 +1,25 @@
---
source: src/tokenizer/markdown_tokenizer.rs
assertion_line: 199
expression: "markdown_tokenizer(\"## Sub heading\")"
---
[
Token {
normalized: "## Sub",
original: "## Sub",
is_left_joinable: false,
is_right_joinable: true,
},
Token {
normalized: " heading",
original: " ",
is_left_joinable: true,
is_right_joinable: true,
},
Token {
normalized: "heading",
original: "heading",
is_left_joinable: true,
is_right_joinable: true,
},
]