Add tokenizer

backend/reconcile/src/operation_transformation/mod.rs

@@ -4,8 +4,9 @@ mod operation;
 
 pub use edited_text::EditedText;
 pub use operation::Operation;
+use std::hash::Hash;
 
-use crate::errors::SyncLibError;
+use crate::{errors::SyncLibError, tokenizer::token::Token};
 
 pub fn reconcile(original: &str, left: &str, right: &str) -> Result<String, SyncLibError> {
     let left_operations = EditedText::from_strings(original, left);
@@ -15,6 +16,23 @@ pub fn reconcile(original: &str, left: &str, right: &str) -> Result<String, Sync
     merged_operations.apply()
 }
 
+pub fn reconcile_with_tokenizer<F, T>(
+    original: &str,
+    left: &str,
+    right: &str,
+    tokenizer: &F,
+) -> Result<String, SyncLibError>
+where
+    F: Fn(&str) -> Vec<Token<T>>,
+    T: PartialEq + Hash + Clone,
+{
+    let left_operations = EditedText::from_strings_with_tokenizer(original, left, tokenizer);
+    let right_operations = EditedText::from_strings_with_tokenizer(original, right, tokenizer);
+
+    let merged_operations = left_operations.merge(right_operations);
+    merged_operations.apply()
+}
+
 #[cfg(test)]
 mod test {
     use std::{fs, ops::Range, path::Path};