Fix whitespaces

backend/reconcile/src/tokenizer/token.rs

@@ -15,12 +15,7 @@ where
 }
 
 impl From<&str> for Token<String> {
-    fn from(s: &str) -> Self {
-        Token {
-            normalised: s.to_owned(),
-            original: s.to_owned(),
-        }
-    }
+    fn from(s: &str) -> Self { Token::new(s.trim().to_owned(), s.to_owned()) }
 }
 
 impl<T> Token<T>

backend/reconcile/src/tokenizer/word_tokenizer.rs

@@ -1,7 +1,48 @@
 use super::token::Token;
 
+/// Splits on whitespace keeping the leading whitespace.
+///
+///     
+/// ## Example
+///
+/// "Hi there!" -> ["Hi", " there!"]
 pub fn word_tokenizer(text: &str) -> Vec<Token<String>> {
-    text.split_inclusive(char::is_whitespace)
-        .map(|s| Token::new(s.to_owned(), s.to_owned()))
-        .collect()
+    let mut result: Vec<Token<String>> = Vec::new();
+
+    let mut last_whitespace = 0;
+    let mut previous_char_is_whitespace = true;
+
+    for (i, c) in text.char_indices() {
+        let is_current_char_whitespace = c.is_whitespace();
+        if !previous_char_is_whitespace && is_current_char_whitespace {
+            result.push(text[last_whitespace..i].into());
+            last_whitespace = i;
+        }
+
+        previous_char_is_whitespace = is_current_char_whitespace;
+    }
+
+    if last_whitespace < text.len() {
+        result.push(text[last_whitespace..].into());
+    }
+
+    result
+}
+
+#[cfg(test)]
+mod tests {
+    use insta::assert_debug_snapshot;
+
+    use super::*;
+
+    #[test]
+    fn test_with_snapshots() {
+        assert_debug_snapshot!(word_tokenizer("Hi there!"));
+
+        assert_debug_snapshot!(word_tokenizer(""));
+
+        assert_debug_snapshot!(word_tokenizer(" what? "));
+
+        assert_debug_snapshot!(word_tokenizer(" hello, \nwhere are you?"));
+    }
 }