diff --git a/src/tokenizer/snapshots/reconcile_text__tokenizer__line_tokenizer__tests__with_snapshots-10.snap b/src/tokenizer/snapshots/reconcile_text__tokenizer__line_tokenizer__tests__with_snapshots-10.snap new file mode 100644 index 0000000..22daf9f --- /dev/null +++ b/src/tokenizer/snapshots/reconcile_text__tokenizer__line_tokenizer__tests__with_snapshots-10.snap @@ -0,0 +1,36 @@ +--- +source: src/tokenizer/line_tokenizer.rs +expression: "line_tokenizer(\"Old\\rMac\\rStyle\")" +--- +[ + Token { + normalized: "Old", + original: "Old", + is_left_joinable: true, + is_right_joinable: true, + }, + Token { + normalized: "\r", + original: "\r", + is_left_joinable: true, + is_right_joinable: true, + }, + Token { + normalized: "Mac", + original: "Mac", + is_left_joinable: true, + is_right_joinable: true, + }, + Token { + normalized: "\r", + original: "\r", + is_left_joinable: true, + is_right_joinable: true, + }, + Token { + normalized: "Style", + original: "Style", + is_left_joinable: true, + is_right_joinable: true, + }, +] diff --git a/src/tokenizer/snapshots/reconcile_text__tokenizer__markdown_tokenizer__tests__blockquote.snap b/src/tokenizer/snapshots/reconcile_text__tokenizer__markdown_tokenizer__tests__blockquote.snap new file mode 100644 index 0000000..6bbf317 --- /dev/null +++ b/src/tokenizer/snapshots/reconcile_text__tokenizer__markdown_tokenizer__tests__blockquote.snap @@ -0,0 +1,48 @@ +--- +source: src/tokenizer/markdown_tokenizer.rs +expression: "markdown_tokenizer(\"> quoted text\\n> more quoted\")" +--- +[ + Token { + normalized: "> quoted", + original: "> quoted", + is_left_joinable: false, + is_right_joinable: true, + }, + Token { + normalized: " text", + original: " ", + is_left_joinable: true, + is_right_joinable: true, + }, + Token { + normalized: "text", + original: "text", + is_left_joinable: true, + is_right_joinable: true, + }, + Token { + normalized: "\n", + original: "\n", + is_left_joinable: false, + is_right_joinable: false, + }, + Token { + normalized: "> more", + original: "> more", + is_left_joinable: false, + is_right_joinable: true, + }, + Token { + normalized: " quoted", + original: " ", + is_left_joinable: true, + is_right_joinable: true, + }, + Token { + normalized: "quoted", + original: "quoted", + is_left_joinable: true, + is_right_joinable: true, + }, +] diff --git a/src/tokenizer/snapshots/reconcile_text__tokenizer__markdown_tokenizer__tests__bold_not_confused_with_list.snap b/src/tokenizer/snapshots/reconcile_text__tokenizer__markdown_tokenizer__tests__bold_not_confused_with_list.snap new file mode 100644 index 0000000..92a0ffa --- /dev/null +++ b/src/tokenizer/snapshots/reconcile_text__tokenizer__markdown_tokenizer__tests__bold_not_confused_with_list.snap @@ -0,0 +1,24 @@ +--- +source: src/tokenizer/markdown_tokenizer.rs +expression: "markdown_tokenizer(\"**bold text**\")" +--- +[ + Token { + normalized: "**bold", + original: "**bold", + is_left_joinable: true, + is_right_joinable: true, + }, + Token { + normalized: " text**", + original: " ", + is_left_joinable: true, + is_right_joinable: true, + }, + Token { + normalized: "text**", + original: "text**", + is_left_joinable: true, + is_right_joinable: true, + }, +] diff --git a/src/tokenizer/snapshots/reconcile_text__tokenizer__markdown_tokenizer__tests__code_fence.snap b/src/tokenizer/snapshots/reconcile_text__tokenizer__markdown_tokenizer__tests__code_fence.snap new file mode 100644 index 0000000..a951f58 --- /dev/null +++ b/src/tokenizer/snapshots/reconcile_text__tokenizer__markdown_tokenizer__tests__code_fence.snap @@ -0,0 +1,72 @@ +--- +source: src/tokenizer/markdown_tokenizer.rs +expression: "markdown_tokenizer(\"```rust\\nlet x = 1;\\n```\")" +--- +[ + Token { + normalized: "```rust", + original: "```rust", + is_left_joinable: true, + is_right_joinable: true, + }, + Token { + normalized: "\n", + original: "\n", + is_left_joinable: false, + is_right_joinable: false, + }, + Token { + normalized: "let", + original: "let", + is_left_joinable: true, + is_right_joinable: true, + }, + Token { + normalized: " x", + original: " ", + is_left_joinable: true, + is_right_joinable: true, + }, + Token { + normalized: "x", + original: "x", + is_left_joinable: true, + is_right_joinable: true, + }, + Token { + normalized: " =", + original: " ", + is_left_joinable: true, + is_right_joinable: true, + }, + Token { + normalized: "=", + original: "=", + is_left_joinable: true, + is_right_joinable: true, + }, + Token { + normalized: " 1;", + original: " ", + is_left_joinable: true, + is_right_joinable: true, + }, + Token { + normalized: "1;", + original: "1;", + is_left_joinable: true, + is_right_joinable: true, + }, + Token { + normalized: "\n", + original: "\n", + is_left_joinable: false, + is_right_joinable: false, + }, + Token { + normalized: "```", + original: "```", + is_left_joinable: true, + is_right_joinable: true, + }, +] diff --git a/src/tokenizer/snapshots/reconcile_text__tokenizer__markdown_tokenizer__tests__crlf.snap b/src/tokenizer/snapshots/reconcile_text__tokenizer__markdown_tokenizer__tests__crlf.snap new file mode 100644 index 0000000..d1817c2 --- /dev/null +++ b/src/tokenizer/snapshots/reconcile_text__tokenizer__markdown_tokenizer__tests__crlf.snap @@ -0,0 +1,48 @@ +--- +source: src/tokenizer/markdown_tokenizer.rs +expression: "markdown_tokenizer(\"Line 1\\r\\nLine 2\")" +--- +[ + Token { + normalized: "Line", + original: "Line", + is_left_joinable: true, + is_right_joinable: true, + }, + Token { + normalized: " 1", + original: " ", + is_left_joinable: true, + is_right_joinable: true, + }, + Token { + normalized: "1", + original: "1", + is_left_joinable: true, + is_right_joinable: true, + }, + Token { + normalized: "\r\n", + original: "\r\n", + is_left_joinable: false, + is_right_joinable: false, + }, + Token { + normalized: "Line", + original: "Line", + is_left_joinable: true, + is_right_joinable: true, + }, + Token { + normalized: " 2", + original: " ", + is_left_joinable: true, + is_right_joinable: true, + }, + Token { + normalized: "2", + original: "2", + is_left_joinable: true, + is_right_joinable: true, + }, +] diff --git a/src/tokenizer/snapshots/reconcile_text__tokenizer__markdown_tokenizer__tests__empty.snap b/src/tokenizer/snapshots/reconcile_text__tokenizer__markdown_tokenizer__tests__empty.snap new file mode 100644 index 0000000..aa65c19 --- /dev/null +++ b/src/tokenizer/snapshots/reconcile_text__tokenizer__markdown_tokenizer__tests__empty.snap @@ -0,0 +1,5 @@ +--- +source: src/tokenizer/markdown_tokenizer.rs +expression: "markdown_tokenizer(\"\")" +--- +[] diff --git a/src/tokenizer/snapshots/reconcile_text__tokenizer__markdown_tokenizer__tests__heading_only.snap b/src/tokenizer/snapshots/reconcile_text__tokenizer__markdown_tokenizer__tests__heading_only.snap new file mode 100644 index 0000000..0d10e0f --- /dev/null +++ b/src/tokenizer/snapshots/reconcile_text__tokenizer__markdown_tokenizer__tests__heading_only.snap @@ -0,0 +1,12 @@ +--- +source: src/tokenizer/markdown_tokenizer.rs +expression: "markdown_tokenizer(\"# \")" +--- +[ + Token { + normalized: "# ", + original: "# ", + is_left_joinable: false, + is_right_joinable: false, + }, +] diff --git a/src/tokenizer/snapshots/reconcile_text__tokenizer__markdown_tokenizer__tests__headings.snap b/src/tokenizer/snapshots/reconcile_text__tokenizer__markdown_tokenizer__tests__headings.snap new file mode 100644 index 0000000..0c3e98e --- /dev/null +++ b/src/tokenizer/snapshots/reconcile_text__tokenizer__markdown_tokenizer__tests__headings.snap @@ -0,0 +1,24 @@ +--- +source: src/tokenizer/markdown_tokenizer.rs +expression: "markdown_tokenizer(\"# Hello world\")" +--- +[ + Token { + normalized: "# Hello", + original: "# Hello", + is_left_joinable: false, + is_right_joinable: true, + }, + Token { + normalized: " world", + original: " ", + is_left_joinable: true, + is_right_joinable: true, + }, + Token { + normalized: "world", + original: "world", + is_left_joinable: true, + is_right_joinable: true, + }, +] diff --git a/src/tokenizer/snapshots/reconcile_text__tokenizer__markdown_tokenizer__tests__indented_list.snap b/src/tokenizer/snapshots/reconcile_text__tokenizer__markdown_tokenizer__tests__indented_list.snap new file mode 100644 index 0000000..5ea77f7 --- /dev/null +++ b/src/tokenizer/snapshots/reconcile_text__tokenizer__markdown_tokenizer__tests__indented_list.snap @@ -0,0 +1,36 @@ +--- +source: src/tokenizer/markdown_tokenizer.rs +expression: "markdown_tokenizer(\" - nested item\\n - deeper\")" +--- +[ + Token { + normalized: " - nested", + original: " - nested", + is_left_joinable: false, + is_right_joinable: true, + }, + Token { + normalized: " item", + original: " ", + is_left_joinable: true, + is_right_joinable: true, + }, + Token { + normalized: "item", + original: "item", + is_left_joinable: true, + is_right_joinable: true, + }, + Token { + normalized: "\n", + original: "\n", + is_left_joinable: false, + is_right_joinable: false, + }, + Token { + normalized: " - deeper", + original: " - deeper", + is_left_joinable: false, + is_right_joinable: true, + }, +] diff --git a/src/tokenizer/snapshots/reconcile_text__tokenizer__markdown_tokenizer__tests__inline_formatting.snap b/src/tokenizer/snapshots/reconcile_text__tokenizer__markdown_tokenizer__tests__inline_formatting.snap new file mode 100644 index 0000000..6744a2b --- /dev/null +++ b/src/tokenizer/snapshots/reconcile_text__tokenizer__markdown_tokenizer__tests__inline_formatting.snap @@ -0,0 +1,60 @@ +--- +source: src/tokenizer/markdown_tokenizer.rs +expression: "markdown_tokenizer(\"Some **bold** and *italic* text\")" +--- +[ + Token { + normalized: "Some", + original: "Some", + is_left_joinable: true, + is_right_joinable: true, + }, + Token { + normalized: " **bold**", + original: " ", + is_left_joinable: true, + is_right_joinable: true, + }, + Token { + normalized: "**bold**", + original: "**bold**", + is_left_joinable: true, + is_right_joinable: true, + }, + Token { + normalized: " and", + original: " ", + is_left_joinable: true, + is_right_joinable: true, + }, + Token { + normalized: "and", + original: "and", + is_left_joinable: true, + is_right_joinable: true, + }, + Token { + normalized: " *italic*", + original: " ", + is_left_joinable: true, + is_right_joinable: true, + }, + Token { + normalized: "*italic*", + original: "*italic*", + is_left_joinable: true, + is_right_joinable: true, + }, + Token { + normalized: " text", + original: " ", + is_left_joinable: true, + is_right_joinable: true, + }, + Token { + normalized: "text", + original: "text", + is_left_joinable: true, + is_right_joinable: true, + }, +] diff --git a/src/tokenizer/snapshots/reconcile_text__tokenizer__markdown_tokenizer__tests__link.snap b/src/tokenizer/snapshots/reconcile_text__tokenizer__markdown_tokenizer__tests__link.snap new file mode 100644 index 0000000..79e3afb --- /dev/null +++ b/src/tokenizer/snapshots/reconcile_text__tokenizer__markdown_tokenizer__tests__link.snap @@ -0,0 +1,36 @@ +--- +source: src/tokenizer/markdown_tokenizer.rs +expression: "markdown_tokenizer(\"Click [here](https://example.com) now\")" +--- +[ + Token { + normalized: "Click", + original: "Click", + is_left_joinable: true, + is_right_joinable: true, + }, + Token { + normalized: " [here](https://example.com)", + original: " ", + is_left_joinable: true, + is_right_joinable: true, + }, + Token { + normalized: "[here](https://example.com)", + original: "[here](https://example.com)", + is_left_joinable: true, + is_right_joinable: true, + }, + Token { + normalized: " now", + original: " ", + is_left_joinable: true, + is_right_joinable: true, + }, + Token { + normalized: "now", + original: "now", + is_left_joinable: true, + is_right_joinable: true, + }, +] diff --git a/src/tokenizer/snapshots/reconcile_text__tokenizer__markdown_tokenizer__tests__list_with_star_marker.snap b/src/tokenizer/snapshots/reconcile_text__tokenizer__markdown_tokenizer__tests__list_with_star_marker.snap new file mode 100644 index 0000000..3273a24 --- /dev/null +++ b/src/tokenizer/snapshots/reconcile_text__tokenizer__markdown_tokenizer__tests__list_with_star_marker.snap @@ -0,0 +1,48 @@ +--- +source: src/tokenizer/markdown_tokenizer.rs +expression: "markdown_tokenizer(\"* item one\\n* item two\")" +--- +[ + Token { + normalized: "* item", + original: "* item", + is_left_joinable: false, + is_right_joinable: true, + }, + Token { + normalized: " one", + original: " ", + is_left_joinable: true, + is_right_joinable: true, + }, + Token { + normalized: "one", + original: "one", + is_left_joinable: true, + is_right_joinable: true, + }, + Token { + normalized: "\n", + original: "\n", + is_left_joinable: false, + is_right_joinable: false, + }, + Token { + normalized: "* item", + original: "* item", + is_left_joinable: false, + is_right_joinable: true, + }, + Token { + normalized: " two", + original: " ", + is_left_joinable: true, + is_right_joinable: true, + }, + Token { + normalized: "two", + original: "two", + is_left_joinable: true, + is_right_joinable: true, + }, +] diff --git a/src/tokenizer/snapshots/reconcile_text__tokenizer__markdown_tokenizer__tests__mixed_content.snap b/src/tokenizer/snapshots/reconcile_text__tokenizer__markdown_tokenizer__tests__mixed_content.snap new file mode 100644 index 0000000..78458af --- /dev/null +++ b/src/tokenizer/snapshots/reconcile_text__tokenizer__markdown_tokenizer__tests__mixed_content.snap @@ -0,0 +1,120 @@ +--- +source: src/tokenizer/markdown_tokenizer.rs +expression: "markdown_tokenizer(\"# Title\\n\\nSome text with **bold**.\\n\\n- list item\\n- another item\")" +--- +[ + Token { + normalized: "# Title", + original: "# Title", + is_left_joinable: false, + is_right_joinable: true, + }, + Token { + normalized: "\n", + original: "\n", + is_left_joinable: false, + is_right_joinable: false, + }, + Token { + normalized: "\n", + original: "\n", + is_left_joinable: false, + is_right_joinable: false, + }, + Token { + normalized: "Some", + original: "Some", + is_left_joinable: true, + is_right_joinable: true, + }, + Token { + normalized: " text", + original: " ", + is_left_joinable: true, + is_right_joinable: true, + }, + Token { + normalized: "text", + original: "text", + is_left_joinable: true, + is_right_joinable: true, + }, + Token { + normalized: " with", + original: " ", + is_left_joinable: true, + is_right_joinable: true, + }, + Token { + normalized: "with", + original: "with", + is_left_joinable: true, + is_right_joinable: true, + }, + Token { + normalized: " **bold**.", + original: " ", + is_left_joinable: true, + is_right_joinable: true, + }, + Token { + normalized: "**bold**.", + original: "**bold**.", + is_left_joinable: true, + is_right_joinable: true, + }, + Token { + normalized: "\n", + original: "\n", + is_left_joinable: false, + is_right_joinable: false, + }, + Token { + normalized: "\n", + original: "\n", + is_left_joinable: false, + is_right_joinable: false, + }, + Token { + normalized: "- list", + original: "- list", + is_left_joinable: false, + is_right_joinable: true, + }, + Token { + normalized: " item", + original: " ", + is_left_joinable: true, + is_right_joinable: true, + }, + Token { + normalized: "item", + original: "item", + is_left_joinable: true, + is_right_joinable: true, + }, + Token { + normalized: "\n", + original: "\n", + is_left_joinable: false, + is_right_joinable: false, + }, + Token { + normalized: "- another", + original: "- another", + is_left_joinable: false, + is_right_joinable: true, + }, + Token { + normalized: " item", + original: " ", + is_left_joinable: true, + is_right_joinable: true, + }, + Token { + normalized: "item", + original: "item", + is_left_joinable: true, + is_right_joinable: true, + }, +] diff --git a/src/tokenizer/snapshots/reconcile_text__tokenizer__markdown_tokenizer__tests__multiline_paragraph.snap b/src/tokenizer/snapshots/reconcile_text__tokenizer__markdown_tokenizer__tests__multiline_paragraph.snap new file mode 100644 index 0000000..244f515 --- /dev/null +++ b/src/tokenizer/snapshots/reconcile_text__tokenizer__markdown_tokenizer__tests__multiline_paragraph.snap @@ -0,0 +1,78 @@ +--- +source: src/tokenizer/markdown_tokenizer.rs +expression: "markdown_tokenizer(\"First line\\nSecond line\\n\\nNew paragraph\")" +--- +[ + Token { + normalized: "First", + original: "First", + is_left_joinable: true, + is_right_joinable: true, + }, + Token { + normalized: " line", + original: " ", + is_left_joinable: true, + is_right_joinable: true, + }, + Token { + normalized: "line", + original: "line", + is_left_joinable: true, + is_right_joinable: true, + }, + Token { + normalized: "\n", + original: "\n", + is_left_joinable: false, + is_right_joinable: false, + }, + Token { + normalized: "Second", + original: "Second", + is_left_joinable: true, + is_right_joinable: true, + }, + Token { + normalized: " line", + original: " ", + is_left_joinable: true, + is_right_joinable: true, + }, + Token { + normalized: "line", + original: "line", + is_left_joinable: true, + is_right_joinable: true, + }, + Token { + normalized: "\n", + original: "\n", + is_left_joinable: false, + is_right_joinable: false, + }, + Token { + normalized: "\n", + original: "\n", + is_left_joinable: false, + is_right_joinable: false, + }, + Token { + normalized: "New", + original: "New", + is_left_joinable: true, + is_right_joinable: true, + }, + Token { + normalized: " paragraph", + original: " ", + is_left_joinable: true, + is_right_joinable: true, + }, + Token { + normalized: "paragraph", + original: "paragraph", + is_left_joinable: true, + is_right_joinable: true, + }, +] diff --git a/src/tokenizer/snapshots/reconcile_text__tokenizer__markdown_tokenizer__tests__ordered_list.snap b/src/tokenizer/snapshots/reconcile_text__tokenizer__markdown_tokenizer__tests__ordered_list.snap new file mode 100644 index 0000000..c465053 --- /dev/null +++ b/src/tokenizer/snapshots/reconcile_text__tokenizer__markdown_tokenizer__tests__ordered_list.snap @@ -0,0 +1,36 @@ +--- +source: src/tokenizer/markdown_tokenizer.rs +expression: "markdown_tokenizer(\"1. first\\n2. second\\n3. third\")" +--- +[ + Token { + normalized: "1. first", + original: "1. first", + is_left_joinable: false, + is_right_joinable: true, + }, + Token { + normalized: "\n", + original: "\n", + is_left_joinable: false, + is_right_joinable: false, + }, + Token { + normalized: "2. second", + original: "2. second", + is_left_joinable: false, + is_right_joinable: true, + }, + Token { + normalized: "\n", + original: "\n", + is_left_joinable: false, + is_right_joinable: false, + }, + Token { + normalized: "3. third", + original: "3. third", + is_left_joinable: false, + is_right_joinable: true, + }, +] diff --git a/src/tokenizer/snapshots/reconcile_text__tokenizer__markdown_tokenizer__tests__ordered_task_list.snap b/src/tokenizer/snapshots/reconcile_text__tokenizer__markdown_tokenizer__tests__ordered_task_list.snap new file mode 100644 index 0000000..55a8001 --- /dev/null +++ b/src/tokenizer/snapshots/reconcile_text__tokenizer__markdown_tokenizer__tests__ordered_task_list.snap @@ -0,0 +1,48 @@ +--- +source: src/tokenizer/markdown_tokenizer.rs +expression: "markdown_tokenizer(\"1. [ ] first task\\n2. [x] second task\")" +--- +[ + Token { + normalized: "1. [ ] first", + original: "1. [ ] first", + is_left_joinable: false, + is_right_joinable: true, + }, + Token { + normalized: " task", + original: " ", + is_left_joinable: true, + is_right_joinable: true, + }, + Token { + normalized: "task", + original: "task", + is_left_joinable: true, + is_right_joinable: true, + }, + Token { + normalized: "\n", + original: "\n", + is_left_joinable: false, + is_right_joinable: false, + }, + Token { + normalized: "2. [x] second", + original: "2. [x] second", + is_left_joinable: false, + is_right_joinable: true, + }, + Token { + normalized: " task", + original: " ", + is_left_joinable: true, + is_right_joinable: true, + }, + Token { + normalized: "task", + original: "task", + is_left_joinable: true, + is_right_joinable: true, + }, +] diff --git a/src/tokenizer/snapshots/reconcile_text__tokenizer__markdown_tokenizer__tests__plain_text.snap b/src/tokenizer/snapshots/reconcile_text__tokenizer__markdown_tokenizer__tests__plain_text.snap new file mode 100644 index 0000000..0785a2f --- /dev/null +++ b/src/tokenizer/snapshots/reconcile_text__tokenizer__markdown_tokenizer__tests__plain_text.snap @@ -0,0 +1,24 @@ +--- +source: src/tokenizer/markdown_tokenizer.rs +expression: "markdown_tokenizer(\"Hello world\")" +--- +[ + Token { + normalized: "Hello", + original: "Hello", + is_left_joinable: true, + is_right_joinable: true, + }, + Token { + normalized: " world", + original: " ", + is_left_joinable: true, + is_right_joinable: true, + }, + Token { + normalized: "world", + original: "world", + is_left_joinable: true, + is_right_joinable: true, + }, +] diff --git a/src/tokenizer/snapshots/reconcile_text__tokenizer__markdown_tokenizer__tests__task_list.snap b/src/tokenizer/snapshots/reconcile_text__tokenizer__markdown_tokenizer__tests__task_list.snap new file mode 100644 index 0000000..9789b13 --- /dev/null +++ b/src/tokenizer/snapshots/reconcile_text__tokenizer__markdown_tokenizer__tests__task_list.snap @@ -0,0 +1,48 @@ +--- +source: src/tokenizer/markdown_tokenizer.rs +expression: "markdown_tokenizer(\"- [ ] todo\\n- [x] done\\n- [X] also done\")" +--- +[ + Token { + normalized: "- [ ] todo", + original: "- [ ] todo", + is_left_joinable: false, + is_right_joinable: true, + }, + Token { + normalized: "\n", + original: "\n", + is_left_joinable: false, + is_right_joinable: false, + }, + Token { + normalized: "- [x] done", + original: "- [x] done", + is_left_joinable: false, + is_right_joinable: true, + }, + Token { + normalized: "\n", + original: "\n", + is_left_joinable: false, + is_right_joinable: false, + }, + Token { + normalized: "- [X] also", + original: "- [X] also", + is_left_joinable: false, + is_right_joinable: true, + }, + Token { + normalized: " done", + original: " ", + is_left_joinable: true, + is_right_joinable: true, + }, + Token { + normalized: "done", + original: "done", + is_left_joinable: true, + is_right_joinable: true, + }, +] diff --git a/src/tokenizer/snapshots/reconcile_text__tokenizer__markdown_tokenizer__tests__unicode.snap b/src/tokenizer/snapshots/reconcile_text__tokenizer__markdown_tokenizer__tests__unicode.snap new file mode 100644 index 0000000..e5d1beb --- /dev/null +++ b/src/tokenizer/snapshots/reconcile_text__tokenizer__markdown_tokenizer__tests__unicode.snap @@ -0,0 +1,60 @@ +--- +source: src/tokenizer/markdown_tokenizer.rs +expression: "markdown_tokenizer(\"# \\u{1F600} Héllo\\n- \\u{00E9}lément\\n> \\u{4F60}\\u{597D} world\")" +--- +[ + Token { + normalized: "# 😀", + original: "# 😀", + is_left_joinable: false, + is_right_joinable: true, + }, + Token { + normalized: " Héllo", + original: " ", + is_left_joinable: true, + is_right_joinable: true, + }, + Token { + normalized: "Héllo", + original: "Héllo", + is_left_joinable: true, + is_right_joinable: true, + }, + Token { + normalized: "\n", + original: "\n", + is_left_joinable: false, + is_right_joinable: false, + }, + Token { + normalized: "- élément", + original: "- élément", + is_left_joinable: false, + is_right_joinable: true, + }, + Token { + normalized: "\n", + original: "\n", + is_left_joinable: false, + is_right_joinable: false, + }, + Token { + normalized: "> 你好", + original: "> 你好", + is_left_joinable: false, + is_right_joinable: true, + }, + Token { + normalized: " world", + original: " ", + is_left_joinable: true, + is_right_joinable: true, + }, + Token { + normalized: "world", + original: "world", + is_left_joinable: true, + is_right_joinable: true, + }, +] diff --git a/src/tokenizer/snapshots/reconcile_text__tokenizer__markdown_tokenizer__tests__unordered_list.snap b/src/tokenizer/snapshots/reconcile_text__tokenizer__markdown_tokenizer__tests__unordered_list.snap new file mode 100644 index 0000000..b50f920 --- /dev/null +++ b/src/tokenizer/snapshots/reconcile_text__tokenizer__markdown_tokenizer__tests__unordered_list.snap @@ -0,0 +1,72 @@ +--- +source: src/tokenizer/markdown_tokenizer.rs +expression: "markdown_tokenizer(\"- item one\\n- item two\\n- item three\")" +--- +[ + Token { + normalized: "- item", + original: "- item", + is_left_joinable: false, + is_right_joinable: true, + }, + Token { + normalized: " one", + original: " ", + is_left_joinable: true, + is_right_joinable: true, + }, + Token { + normalized: "one", + original: "one", + is_left_joinable: true, + is_right_joinable: true, + }, + Token { + normalized: "\n", + original: "\n", + is_left_joinable: false, + is_right_joinable: false, + }, + Token { + normalized: "- item", + original: "- item", + is_left_joinable: false, + is_right_joinable: true, + }, + Token { + normalized: " two", + original: " ", + is_left_joinable: true, + is_right_joinable: true, + }, + Token { + normalized: "two", + original: "two", + is_left_joinable: true, + is_right_joinable: true, + }, + Token { + normalized: "\n", + original: "\n", + is_left_joinable: false, + is_right_joinable: false, + }, + Token { + normalized: "- item", + original: "- item", + is_left_joinable: false, + is_right_joinable: true, + }, + Token { + normalized: " three", + original: " ", + is_left_joinable: true, + is_right_joinable: true, + }, + Token { + normalized: "three", + original: "three", + is_left_joinable: true, + is_right_joinable: true, + }, +]