wip

backend/reconcile/src/diffs/lcs.rs

@@ -0,0 +1,165 @@
+//! LCS diff algorithm.
+//!
+//! * time: `O((NM)D log (M)D)`
+//! * space `O(MN)`
+use std::collections::BTreeMap;
+use std::ops::{Index, Range};
+
+use crate::tokenizer::token::Token;
+
+use super::raw_operation::RawOperation;
+use super::utils::{common_prefix_len, common_suffix_len};
+
+/// LCS diff algorithm.
+/// Copied from https://github.com/mitsuhiko/similar/blob/7e15c44de11a1cd61e1149189929e189ef977fd8/src/algorithms/lcs.rs
+pub fn diff(old: &[Token], new: &[Token]) -> Vec<RawOperation> {
+    let common_prefix_len = common_prefix_len(old, 0..old.len(), new, 0..new.len());
+    let common_suffix_len = common_suffix_len(
+        old,
+        common_prefix_len..old.len(),
+        new,
+        common_prefix_len..new.len(),
+    );
+
+    let maybe_table = make_table(
+        old,
+        common_prefix_len..(old.len() - common_suffix_len),
+        new,
+        common_prefix_len..(new.len() - common_suffix_len),
+    );
+    let mut old_idx = 0;
+    let mut new_idx = 0;
+    let new_len = new.len() - common_prefix_len - common_suffix_len;
+    let old_len = old.len() - common_prefix_len - common_suffix_len;
+
+    let mut result: Vec<RawOperation> = Vec::new();
+    if common_prefix_len > 0 {
+        result.push(RawOperation::Equal(old[0..common_prefix_len].to_vec()));
+    }
+
+    if let Some(table) = maybe_table {
+        while new_idx < new_len && old_idx < old_len {
+            let old_orig_idx = common_prefix_len + old_idx;
+            let new_orig_idx = common_prefix_len + new_idx;
+
+            if new[new_orig_idx] == old[old_orig_idx] {
+                result.push(RawOperation::Equal(vec![old[old_orig_idx].clone()]));
+                old_idx += 1;
+                new_idx += 1;
+            } else if table.get(&(new_idx, old_idx + 1)).unwrap_or(&0)
+                >= table.get(&(new_idx + 1, old_idx)).unwrap_or(&0)
+            {
+                result.push(RawOperation::Delete(vec![old[old_orig_idx].clone()]));
+                old_idx += 1;
+            } else {
+                result.push(RawOperation::Insert(vec![new[new_orig_idx].clone()]));
+                new_idx += 1;
+            }
+        }
+    } else {
+        let old_orig_idx = common_prefix_len + old_idx;
+        let new_orig_idx = common_prefix_len + new_idx;
+
+        result.push(RawOperation::Delete(
+            old[old_orig_idx..old_orig_idx + old_len].to_vec(),
+        ));
+        result.push(RawOperation::Insert(
+            new[new_orig_idx..new_orig_idx + new_len].to_vec(),
+        ));
+    }
+
+    if old_idx < old_len {
+        result.push(RawOperation::Delete(
+            old[common_prefix_len + old_idx..common_prefix_len + old_len].to_vec(),
+        ));
+        old_idx += old_len - old_idx;
+    }
+
+    if new_idx < new_len {
+        result.push(RawOperation::Insert(
+            new[common_prefix_len + new_idx..common_prefix_len + new_len].to_vec(),
+        ));
+    }
+
+    if common_suffix_len > 0 {
+        result.push(RawOperation::Equal(
+            old[old_len + common_prefix_len..old_len + common_prefix_len + common_suffix_len]
+                .to_vec(),
+        ));
+    }
+
+    result
+}
+
+fn make_table<Old, New>(
+    old: &Old,
+    old_range: Range<usize>,
+    new: &New,
+    new_range: Range<usize>,
+) -> Option<BTreeMap<(usize, usize), u32>>
+where
+    Old: Index<usize> + ?Sized,
+    New: Index<usize> + ?Sized,
+    New::Output: PartialEq<Old::Output>,
+{
+    let old_len = old_range.len();
+    let new_len = new_range.len();
+    let mut table = BTreeMap::new();
+
+    for i in (0..new_len).rev() {
+        for j in (0..old_len).rev() {
+            let val = if new[i] == old[j] {
+                table.get(&(i + 1, j + 1)).unwrap_or(&0) + 1
+            } else {
+                *table
+                    .get(&(i + 1, j))
+                    .unwrap_or(&0)
+                    .max(table.get(&(i, j + 1)).unwrap_or(&0))
+            };
+            if val > 0 {
+                table.insert((i, j), val);
+            }
+        }
+    }
+
+    Some(table)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use pretty_assertions::assert_eq;
+    use std::collections::BTreeMap;
+
+    #[test]
+    fn test_table() {
+        let table = make_table(&vec![2, 3], 0..2, &vec![0, 1, 2], 0..3).unwrap();
+        let expected = {
+            let mut m = BTreeMap::new();
+            m.insert((1, 0), 1);
+            m.insert((0, 0), 1);
+            m.insert((2, 0), 1);
+            m
+        };
+        assert_eq!(table, expected);
+    }
+
+    #[test]
+    fn test_empty_examples() {
+        assert_eq!(diff(&[], &[]), vec![]);
+        assert_eq!(
+            diff(&[Token::new("a".to_string(), "a".to_string())], &[]),
+            vec![RawOperation::Delete(vec![Token::new(
+                "a".to_string(),
+                "a".to_string()
+            )])]
+        );
+        assert_eq!(
+            diff(&[], &[Token::new("a".to_string(), "a".to_string())]),
+            vec![RawOperation::Insert(vec![Token::new(
+                "a".to_string(),
+                "a".to_string()
+            )])]
+        );
+    }
+}

backend/reconcile/src/diffs/mod.rs

@@ -0,0 +1,4 @@
+pub mod lcs;
+pub mod myers;
+pub mod raw_operation;
+mod utils;

backend/reconcile/src/diffs/myers.rs

@@ -0,0 +1,310 @@
+//! Taken from https://github.com/mitsuhiko/similar/blob/7e15c44de11a1cd61e1149189929e189ef977fd8/src/algorithms/myers.rs
+//! Myers' diff algorithm.
+//!
+//! * time: `O((N+M)D)`
+//! * space `O(N+M)`
+//!
+//! See [the original article by Eugene W. Myers](http://www.xmailserver.org/diff2.pdf)
+//! describing it.
+//!
+//! The implementation of this algorithm is based on the implementation by
+//! Brandon Williams.
+//!
+//! # Heuristics
+//!
+//! At present this implementation of Myers' does not implement any more advanced
+//! heuristics that would solve some pathological cases.  For instance passing two
+//! large and completely distinct sequences to the algorithm will make it spin
+//! without making reasonable progress.  Currently the only protection in the
+//! library against this is to pass a deadline to the diffing algorithm.
+//!
+//! For potential improvements here see [similar#15](https://github.com/mitsuhiko/similar/issues/15).
+
+use std::ops::{Index, IndexMut, Range};
+use std::time::Instant;
+use std::vec;
+
+use crate::tokenizer::token::Token;
+
+use super::raw_operation::RawOperation;
+use super::utils::{common_prefix_len, common_suffix_len};
+
+/// Myers' diff algorithm.
+///
+/// Diff `old`, between indices `old_range` and `new` between indices `new_range`.
+pub fn diff(old: &[Token], new: &[Token]) -> Vec<RawOperation> {
+    diff_deadline(old, new, None)
+}
+
+/// Myers' diff algorithm with deadline.
+///
+/// Diff `old`, between indices `old_range` and `new` between indices `new_range`.
+///
+/// This diff is done with an optional deadline that defines the maximal
+/// execution time permitted before it bails and falls back to an approximation.
+pub fn diff_deadline(old: &[Token], new: &[Token], deadline: Option<Instant>) -> Vec<RawOperation> {
+    let max_d = max_d(old.len(), new.len());
+    let mut vb = V::new(max_d);
+    let mut vf = V::new(max_d);
+    let mut result: Vec<RawOperation> = vec![];
+    conquer(
+        old,
+        0..old.len(),
+        new,
+        0..new.len(),
+        &mut vf,
+        &mut vb,
+        &mut result,
+        deadline,
+    );
+    result
+}
+
+// A D-path is a path which starts at (0,0) that has exactly D non-diagonal
+// edges. All D-paths consist of a (D - 1)-path followed by a non-diagonal edge
+// and then a possibly empty sequence of diagonal edges called a snake.
+
+/// `V` contains the endpoints of the furthest reaching `D-paths`. For each
+/// recorded endpoint `(x,y)` in diagonal `k`, we only need to retain `x` because
+/// `y` can be computed from `x - k`. In other words, `V` is an array of integers
+/// where `V[k]` contains the row index of the endpoint of the furthest reaching
+/// path in diagonal `k`.
+///
+/// We can't use a traditional Vec to represent `V` since we use `k` as an index
+/// and it can take on negative values. So instead `V` is represented as a
+/// light-weight wrapper around a Vec plus an `offset` which is the maximum value
+/// `k` can take on in order to map negative `k`'s back to a value >= 0.
+#[derive(Debug)]
+struct V {
+    offset: isize,
+    v: Vec<usize>, // Look into initializing this to -1 and storing isize
+}
+
+impl V {
+    fn new(max_d: usize) -> Self {
+        Self {
+            offset: max_d as isize,
+            v: vec![0; 2 * max_d],
+        }
+    }
+
+    fn len(&self) -> usize {
+        self.v.len()
+    }
+}
+
+impl Index<isize> for V {
+    type Output = usize;
+
+    fn index(&self, index: isize) -> &Self::Output {
+        &self.v[(index + self.offset) as usize]
+    }
+}
+
+impl IndexMut<isize> for V {
+    fn index_mut(&mut self, index: isize) -> &mut Self::Output {
+        &mut self.v[(index + self.offset) as usize]
+    }
+}
+
+fn max_d(len1: usize, len2: usize) -> usize {
+    // XXX look into reducing the need to have the additional '+ 1'
+    (len1 + len2 + 1) / 2 + 1
+}
+
+#[inline(always)]
+fn split_at(range: Range<usize>, at: usize) -> (Range<usize>, Range<usize>) {
+    (range.start..at, at..range.end)
+}
+
+/// A `Snake` is a sequence of diagonal edges in the edit graph.  Normally
+/// a snake has a start end end point (and it is possible for a snake to have
+/// a length of zero, meaning the start and end points are the same) however
+/// we do not need the end point which is why it's not implemented here.
+///
+/// The divide part of a divide-and-conquer strategy. A D-path has D+1 snakes
+/// some of which may be empty. The divide step requires finding the ceil(D/2) +
+/// 1 or middle snake of an optimal D-path. The idea for doing so is to
+/// simultaneously run the basic algorithm in both the forward and reverse
+/// directions until furthest reaching forward and reverse paths starting at
+/// opposing corners 'overlap'.
+fn find_middle_snake(
+    old: &[Token],
+    old_range: Range<usize>,
+    new: &[Token],
+    new_range: Range<usize>,
+    vf: &mut V,
+    vb: &mut V,
+    deadline: Option<Instant>,
+) -> Option<(usize, usize)> {
+    let n = old_range.len();
+    let m = new_range.len();
+
+    // By Lemma 1 in the paper, the optimal edit script length is odd or even as
+    // `delta` is odd or even.
+    let delta = n as isize - m as isize;
+    let odd = delta & 1 == 1;
+
+    // The initial point at (0, -1)
+    vf[1] = 0;
+    // The initial point at (N, M+1)
+    vb[1] = 0;
+
+    // We only need to explore ceil(D/2) + 1
+    let d_max = max_d(n, m);
+    assert!(vf.len() >= d_max);
+    assert!(vb.len() >= d_max);
+
+    for d in 0..d_max as isize {
+        // are we running for too long?
+        if let Some(deadline) = deadline {
+            if Instant::now() > deadline {
+                break;
+            }
+        }
+
+        // Forward path
+        for k in (-d..=d).rev().step_by(2) {
+            let mut x = if k == -d || (k != d && vf[k - 1] < vf[k + 1]) {
+                vf[k + 1]
+            } else {
+                vf[k - 1] + 1
+            };
+            let y = (x as isize - k) as usize;
+
+            // The coordinate of the start of a snake
+            let (x0, y0) = (x, y);
+            //  While these sequences are identical, keep moving through the
+            //  graph with no cost
+            if x < old_range.len() && y < new_range.len() {
+                let advance = common_prefix_len(
+                    old,
+                    old_range.start + x..old_range.end,
+                    new,
+                    new_range.start + y..new_range.end,
+                );
+                x += advance;
+            }
+
+            // This is the new best x value
+            vf[k] = x;
+
+            // Only check for connections from the forward search when N - M is
+            // odd and when there is a reciprocal k line coming from the other
+            // direction.
+            if odd && (k - delta).abs() <= (d - 1) {
+                // TODO optimize this so we don't have to compare against n
+                if vf[k] + vb[-(k - delta)] >= n {
+                    // Return the snake
+                    return Some((x0 + old_range.start, y0 + new_range.start));
+                }
+            }
+        }
+
+        // Backward path
+        for k in (-d..=d).rev().step_by(2) {
+            let mut x = if k == -d || (k != d && vb[k - 1] < vb[k + 1]) {
+                vb[k + 1]
+            } else {
+                vb[k - 1] + 1
+            };
+            let mut y = (x as isize - k) as usize;
+
+            // The coordinate of the start of a snake
+            if x < n && y < m {
+                let advance = common_suffix_len(
+                    old,
+                    old_range.start..old_range.start + n - x,
+                    new,
+                    new_range.start..new_range.start + m - y,
+                );
+                x += advance;
+                y += advance;
+            }
+
+            // This is the new best x value
+            vb[k] = x;
+
+            if !odd && (k - delta).abs() <= d {
+                // TODO optimize this so we don't have to compare against n
+                if vb[k] + vf[-(k - delta)] >= n {
+                    // Return the snake
+                    return Some((n - x + old_range.start, m - y + new_range.start));
+                }
+            }
+        }
+
+        // TODO: Maybe there's an opportunity to optimize and bail early?
+    }
+
+    // deadline reached
+    None
+}
+
+fn conquer(
+    old: &[Token],
+    mut old_range: Range<usize>,
+    new: &[Token],
+    mut new_range: Range<usize>,
+    vf: &mut V,
+    vb: &mut V,
+    result: &mut Vec<RawOperation>,
+    deadline: Option<Instant>,
+) {
+    // Check for common prefix
+    let common_prefix_len = common_prefix_len(old, old_range.clone(), new, new_range.clone());
+    if common_prefix_len > 0 {
+        result.push(RawOperation::Equal(
+            old[old_range.start..old_range.start + common_prefix_len].to_vec(),
+        ));
+    }
+    old_range.start += common_prefix_len;
+    new_range.start += common_prefix_len;
+
+    // Check for common suffix
+    let common_suffix_len = common_suffix_len(old, old_range.clone(), new, new_range.clone());
+    let common_suffix = (
+        old_range.end - common_suffix_len,
+        new_range.end - common_suffix_len,
+    );
+    old_range.end -= common_suffix_len;
+    new_range.end -= common_suffix_len;
+
+    if old_range.is_empty() && new_range.is_empty() {
+        // Do nothing
+    } else if new_range.is_empty() {
+        result.push(RawOperation::Delete(
+            old[old_range.start..old_range.start + old_range.len()].to_vec(),
+        ));
+    } else if old_range.is_empty() {
+        result.push(RawOperation::Insert(
+            new[new_range.start..new_range.start + new_range.len()].to_vec(),
+        ));
+    } else if let Some((x_start, y_start)) = find_middle_snake(
+        old,
+        old_range.clone(),
+        new,
+        new_range.clone(),
+        vf,
+        vb,
+        deadline,
+    ) {
+        let (old_a, old_b) = split_at(old_range, x_start);
+        let (new_a, new_b) = split_at(new_range, y_start);
+        conquer(old, old_a, new, new_a, vf, vb, result, deadline);
+        conquer(old, old_b, new, new_b, vf, vb, result, deadline);
+    } else {
+        result.push(RawOperation::Delete(
+            old[old_range.start..old_range.end].to_vec(),
+        ));
+        result.push(RawOperation::Insert(
+            new[new_range.start..new_range.end].to_vec(),
+        ));
+    }
+
+    if common_suffix_len > 0 {
+        result.push(RawOperation::Equal(
+            old[common_suffix.0..common_suffix.0 + common_suffix_len].to_vec(),
+        ));
+    }
+}

backend/reconcile/src/diffs/raw_operation.rs

@@ -0,0 +1,47 @@
+use crate::tokenizer::token::Token;
+
+#[derive(Debug, Clone, PartialEq)]
+pub enum RawOperation {
+    Insert(Vec<Token>),
+    Delete(Vec<Token>),
+    Equal(Vec<Token>),
+}
+
+impl RawOperation {
+    pub fn tokens(&self) -> &Vec<Token> {
+        match self {
+            RawOperation::Insert(tokens) => tokens,
+            RawOperation::Delete(tokens) => tokens,
+            RawOperation::Equal(tokens) => tokens,
+        }
+    }
+
+    pub fn original_text_length(&self) -> usize {
+        self.tokens()
+            .iter()
+            .map(|t| t.original.chars().count())
+            .sum()
+    }
+
+    pub fn get_original_text(self) -> String {
+        self.tokens().iter().map(|t| t.original.clone()).collect()
+    }
+
+    /// Extends the operation with another operation if returning the new operation.
+    /// Only operations of the same type can be used to extend. If the operations are of different
+    /// types, returns None.
+    pub fn extend(&self, other: &RawOperation) -> Option<RawOperation> {
+        match (self, other) {
+            (RawOperation::Insert(tokens1), RawOperation::Insert(tokens2)) => Some(
+                RawOperation::Insert(tokens1.iter().chain(tokens2.iter()).cloned().collect()),
+            ),
+            (RawOperation::Delete(tokens1), RawOperation::Delete(tokens2)) => Some(
+                RawOperation::Delete(tokens1.iter().chain(tokens2.iter()).cloned().collect()),
+            ),
+            (RawOperation::Equal(tokens1), RawOperation::Equal(tokens2)) => Some(
+                RawOperation::Equal(tokens1.iter().chain(tokens2.iter()).cloned().collect()),
+            ),
+            _ => None,
+        }
+    }
+}

backend/reconcile/src/diffs/utils.rs

@@ -0,0 +1,86 @@
+use std::ops::{Index, Range};
+
+/// Given two lookups and ranges calculates the length of the common prefix.
+/// Copied from https://github.com/mitsuhiko/similar/blob/7e15c44de11a1cd61e1149189929e189ef977fd8/src/algorithms/utils.rs
+pub fn common_prefix_len<Old, New>(
+    old: &Old,
+    old_range: Range<usize>,
+    new: &New,
+    new_range: Range<usize>,
+) -> usize
+where
+    Old: Index<usize> + ?Sized,
+    New: Index<usize> + ?Sized,
+    New::Output: PartialEq<Old::Output>,
+{
+    new_range
+        .zip(old_range)
+        .take_while(|x| new[x.0] == old[x.1])
+        .count()
+}
+
+/// Given two lookups and ranges calculates the length of common suffix.
+/// Copied from https://github.com/mitsuhiko/similar/blob/7e15c44de11a1cd61e1149189929e189ef977fd8/src/algorithms/utils.rs
+pub fn common_suffix_len<Old, New>(
+    old: &Old,
+    old_range: Range<usize>,
+    new: &New,
+    new_range: Range<usize>,
+) -> usize
+where
+    Old: Index<usize> + ?Sized,
+    New: Index<usize> + ?Sized,
+    New::Output: PartialEq<Old::Output>,
+{
+    new_range
+        .rev()
+        .zip(old_range.rev())
+        .take_while(|x| new[x.0] == old[x.1])
+        .count()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use pretty_assertions::assert_eq;
+
+    #[test]
+    fn test_common_prefix_len() {
+        assert_eq!(
+            common_prefix_len("".as_bytes(), 0..0, "".as_bytes(), 0..0),
+            0
+        );
+        assert_eq!(
+            common_prefix_len("foobarbaz".as_bytes(), 0..9, "foobarblah".as_bytes(), 0..10),
+            7
+        );
+        assert_eq!(
+            common_prefix_len("foobarbaz".as_bytes(), 0..9, "blablabla".as_bytes(), 0..9),
+            0
+        );
+        assert_eq!(
+            common_prefix_len("foobarbaz".as_bytes(), 3..9, "foobarblah".as_bytes(), 3..10),
+            4
+        );
+    }
+
+    #[test]
+    fn test_common_suffix_len() {
+        assert_eq!(
+            common_suffix_len("".as_bytes(), 0..0, "".as_bytes(), 0..0),
+            0
+        );
+        assert_eq!(
+            common_suffix_len("1234".as_bytes(), 0..4, "X0001234".as_bytes(), 0..8),
+            4
+        );
+        assert_eq!(
+            common_suffix_len("1234".as_bytes(), 0..4, "Xxxx".as_bytes(), 0..4),
+            0
+        );
+        assert_eq!(
+            common_suffix_len("1234".as_bytes(), 2..4, "01234".as_bytes(), 2..5),
+            2
+        );
+    }
+}