From 7b81034625a13f9d36c9a25e6bc361966bc71164 Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Wed, 11 Mar 2026 20:43:41 +0000 Subject: [PATCH] Update tests --- ...kenizer__tests__with_snapshots-11.snap.new | 37 +++++++++++++++++++ ...down_tokenizer__tests__headings-2.snap.new | 25 +++++++++++++ 2 files changed, 62 insertions(+) create mode 100644 src/tokenizer/snapshots/reconcile_text__tokenizer__line_tokenizer__tests__with_snapshots-11.snap.new create mode 100644 src/tokenizer/snapshots/reconcile_text__tokenizer__markdown_tokenizer__tests__headings-2.snap.new diff --git a/src/tokenizer/snapshots/reconcile_text__tokenizer__line_tokenizer__tests__with_snapshots-11.snap.new b/src/tokenizer/snapshots/reconcile_text__tokenizer__line_tokenizer__tests__with_snapshots-11.snap.new new file mode 100644 index 0000000..36f12f9 --- /dev/null +++ b/src/tokenizer/snapshots/reconcile_text__tokenizer__line_tokenizer__tests__with_snapshots-11.snap.new @@ -0,0 +1,37 @@ +--- +source: src/tokenizer/line_tokenizer.rs +assertion_line: 78 +expression: "line_tokenizer(\"Mixed\\r\\nand\\rbare\")" +--- +[ + Token { + normalized: "Mixed", + original: "Mixed", + is_left_joinable: true, + is_right_joinable: true, + }, + Token { + normalized: "\r\n", + original: "\r\n", + is_left_joinable: true, + is_right_joinable: true, + }, + Token { + normalized: "and", + original: "and", + is_left_joinable: true, + is_right_joinable: true, + }, + Token { + normalized: "\r", + original: "\r", + is_left_joinable: true, + is_right_joinable: true, + }, + Token { + normalized: "bare", + original: "bare", + is_left_joinable: true, + is_right_joinable: true, + }, +] diff --git a/src/tokenizer/snapshots/reconcile_text__tokenizer__markdown_tokenizer__tests__headings-2.snap.new b/src/tokenizer/snapshots/reconcile_text__tokenizer__markdown_tokenizer__tests__headings-2.snap.new new file mode 100644 index 0000000..0f35315 --- /dev/null +++ b/src/tokenizer/snapshots/reconcile_text__tokenizer__markdown_tokenizer__tests__headings-2.snap.new @@ -0,0 +1,25 @@ +--- +source: src/tokenizer/markdown_tokenizer.rs +assertion_line: 199 +expression: "markdown_tokenizer(\"## Sub heading\")" +--- +[ + Token { + normalized: "## Sub", + original: "## Sub", + is_left_joinable: false, + is_right_joinable: true, + }, + Token { + normalized: " heading", + original: " ", + is_left_joinable: true, + is_right_joinable: true, + }, + Token { + normalized: "heading", + original: "heading", + is_left_joinable: true, + is_right_joinable: true, + }, +]