Use stable rust

This commit is contained in:
Andras Schmelczer 2026-03-11 21:07:23 +00:00
parent 31993762de
commit c2144a2634
18 changed files with 118 additions and 44 deletions

View file

@ -1,6 +1,5 @@
---
source: src/tokenizer/line_tokenizer.rs
assertion_line: 78
expression: "line_tokenizer(\"Mixed\\r\\nand\\rbare\")"
---
[

View file

@ -1,6 +1,5 @@
---
source: src/tokenizer/markdown_tokenizer.rs
assertion_line: 199
expression: "markdown_tokenizer(\"## Sub heading\")"
---
[

View file

@ -0,0 +1,24 @@
---
source: src/tokenizer/markdown_tokenizer.rs
expression: "markdown_tokenizer(\"###### Deep heading\")"
---
[
Token {
normalized: "###### Deep",
original: "###### Deep",
is_left_joinable: false,
is_right_joinable: true,
},
Token {
normalized: " heading",
original: " ",
is_left_joinable: true,
is_right_joinable: true,
},
Token {
normalized: "heading",
original: "heading",
is_left_joinable: true,
is_right_joinable: true,
},
]

View file

@ -30,7 +30,9 @@ where
/// Trivial implementation of Token when the normalized form is the same as the
/// original string
impl From<&str> for Token<String> {
fn from(text: &str) -> Self { Token::new(text.to_owned(), text.to_owned(), true, true) }
fn from(text: &str) -> Self {
Token::new(text.to_owned(), text.to_owned(), true, true)
}
}
impl<T> Token<T>
@ -51,18 +53,28 @@ where
}
}
pub fn original(&self) -> &str { &self.original }
pub fn original(&self) -> &str {
&self.original
}
pub fn set_normalized(&mut self, normalized: T) { self.normalized = normalized; }
pub fn set_normalized(&mut self, normalized: T) {
self.normalized = normalized;
}
pub fn normalized(&self) -> &T { &self.normalized }
pub fn normalized(&self) -> &T {
&self.normalized
}
pub fn get_original_length(&self) -> usize { self.original.chars().count() }
pub fn get_original_length(&self) -> usize {
self.original.chars().count()
}
}
impl<T> PartialEq for Token<T>
where
T: PartialEq + Clone + Debug,
{
fn eq(&self, other: &Self) -> bool { self.normalized == other.normalized }
fn eq(&self, other: &Self) -> bool {
self.normalized == other.normalized
}
}