Compare commits

...
Sign in to create a new pull request.

23 commits

Author SHA1 Message Date
ea76c5ecc6
Add editorconfig 2025-04-02 22:06:29 +01:00
b51cf5c04e
Merge branch 'main' into asch/cursors 2025-04-02 21:32:36 +01:00
afee6eb680
Merge branch 'main' of https://github.com/schmelczer/obsidian-shared-sync 2025-04-02 21:32:20 +01:00
31a81921a1
Move cursor after file updates 2025-04-02 21:32:08 +01:00
5deb10ab8b
Add cursor position conversions 2025-04-02 21:29:48 +01:00
a2c51a9d5d
Fix tests 2025-04-02 21:27:36 +01:00
40323c33ee
Make JS API usable 2025-04-02 21:07:33 +01:00
565372b526
Enable tests 2025-04-02 21:07:14 +01:00
426e9f07b7
Fix compile error 2025-04-02 20:37:44 +01:00
5ed7e0ad3a
Move tests 2025-04-02 20:37:36 +01:00
38a8e6b774
Fix web tests 2025-04-01 22:53:50 +01:00
89b87efa59
Fix tests 2025-04-01 22:50:55 +01:00
680486c4b5
Implement cursor merging 2025-04-01 22:49:02 +01:00
941100d715
Lint 2025-04-01 22:46:03 +01:00
688fd2f1b1
Update API and add cursor position tests 2025-04-01 22:45:26 +01:00
168fb44b07
Fix double document creation on first sync 2025-04-01 22:42:14 +01:00
3744b0a633
Add apply_merge_context to cursor 2025-04-01 22:41:52 +01:00
998ee387d2
Add integration tests 2025-04-01 22:40:05 +01:00
23c288b1eb
Clean up 2025-04-01 20:53:38 +01:00
e9e2328f03
Expose merge_text_with_cursors on API 2025-04-01 20:10:03 +01:00
71ccd7b61d
Make Side printable 2025-03-30 21:27:04 +01:00
c9c0ffecf1
Add cursor types 2025-03-30 21:26:04 +01:00
23ba0d2c82
Add comments 2025-03-30 12:24:08 +01:00
49 changed files with 1105 additions and 141 deletions

11
.editorconfig Normal file
View file

@ -0,0 +1,11 @@
# https://editorconfig.org
root = true
[*]
end_of_line = lf
insert_final_newline = true
trim_trailing_whitespace = true
charset = utf-8
indent_style = space
indent_size = 4

1
backend/Cargo.lock generated
View file

@ -1897,6 +1897,7 @@ dependencies = [
"insta",
"pretty_assertions",
"serde",
"serde_yaml",
"test-case",
]

View file

@ -7,7 +7,7 @@ license.workspace = true
repository.workspace = true
[dependencies]
serde = { version = "1.0.219", optional = true }
serde = { version = "1.0.219", optional = true, features = ["derive"] }
[features]
serde = [ "dep:serde" ]
@ -15,6 +15,8 @@ serde = [ "dep:serde" ]
[dev-dependencies]
insta = "1.42.2"
pretty_assertions = "1.4.1"
serde = { version = "1.0.219", features = ["derive"] }
serde_yaml ="0.9.34"
test-case = "3.3.1"
[lints]

View file

@ -3,5 +3,8 @@ mod operation_transformation;
mod tokenizer;
mod utils;
pub use operation_transformation::{EditedText, reconcile, reconcile_with_tokenizer};
pub use tokenizer::token::Token;
pub use operation_transformation::{
CursorPosition, EditedText, TextWithCursors, reconcile, reconcile_with_cursors,
reconcile_with_tokenizer,
};
pub use tokenizer::{Tokenizer, token::Token};

View file

@ -1,41 +1,42 @@
mod cursor;
mod edited_text;
mod merge_context;
mod operation;
pub use cursor::{CursorPosition, TextWithCursors};
pub use edited_text::EditedText;
pub use operation::Operation;
use crate::tokenizer::Tokenizer;
use crate::Tokenizer;
#[must_use]
pub fn reconcile(original: &str, left: &str, right: &str) -> String {
// Common trivial cases
if left == right {
return left.to_owned();
}
reconcile_with_cursors(original, left.into(), right.into())
.text
.to_string()
}
if original == left {
return right.to_owned();
}
if original == right {
return left.to_owned();
}
// 3-way merge
#[must_use]
pub fn reconcile_with_cursors<'a>(
original: &'a str,
left: TextWithCursors<'a>,
right: TextWithCursors<'a>,
) -> TextWithCursors<'static> {
let left_operations = EditedText::from_strings(original, left);
let right_operations = EditedText::from_strings(original, right);
let merged_operations = left_operations.merge(right_operations);
merged_operations.apply()
TextWithCursors::new_owned(merged_operations.apply(), merged_operations.cursors)
}
pub fn reconcile_with_tokenizer<F, T>(
#[must_use]
pub fn reconcile_with_tokenizer<'a, F, T>(
original: &str,
left: &str,
right: &str,
left: TextWithCursors<'a>,
right: TextWithCursors<'a>,
tokenizer: &Tokenizer<T>,
) -> String
) -> TextWithCursors<'static>
where
T: PartialEq + Clone + std::fmt::Debug,
{
@ -43,7 +44,8 @@ where
let right_operations = EditedText::from_strings_with_tokenizer(original, right, tokenizer);
let merged_operations = left_operations.merge(right_operations);
merged_operations.apply()
TextWithCursors::new_owned(merged_operations.apply(), merged_operations.cursors)
}
#[cfg(test)]
@ -54,6 +56,7 @@ mod test {
use test_case::test_matrix;
use super::*;
use crate::CursorPosition;
#[test]
fn test_merges() {
@ -172,6 +175,188 @@ mod test {
" |7ca2b36d-6ee7-49eb-8eb1-d77e4cc1a001| |cd9195cc-103a-4f13-90c8-4fba0ba421ee| |d39156cc-cfd6-42a8-b70a-75020896069d| |fbad794c-9c47-41f2-a343-490284ecb5a0| |dup| |dup| ");
}
#[test]
fn test_cursor_position_no_updates() {
let original = "hello world";
let left = TextWithCursors::new(
"hello world",
vec![CursorPosition {
id: 0,
char_index: 0,
}],
);
let right = TextWithCursors::new(
"hello world",
vec![CursorPosition {
id: 1,
char_index: 5,
}],
);
let merged = reconcile_with_cursors(original, left, right);
assert_eq!(
merged,
TextWithCursors::new(
"hello world",
vec![
CursorPosition {
id: 0,
char_index: 0
},
CursorPosition {
id: 1,
char_index: 5
}
]
)
);
}
#[test]
fn test_cursor_position_updates_with_inserts() {
let original = "hi";
let left = TextWithCursors::new(
"hi there",
vec![CursorPosition {
id: 0,
char_index: 7,
}],
);
let right = TextWithCursors::new(
"hi world!",
vec![
CursorPosition {
id: 1,
char_index: 9,
},
CursorPosition {
id: 2,
char_index: 1,
},
],
);
let merged = reconcile_with_cursors(original, left, right);
assert_eq!(
merged,
TextWithCursors::new(
"hi there world!",
vec![
CursorPosition {
id: 2,
char_index: 1,
},
CursorPosition {
id: 0,
char_index: 7
},
CursorPosition {
id: 1,
char_index: 15
},
]
)
);
}
#[test]
fn test_cursor_position_updates_with_deleted() {
let original = "a b c d";
let left = TextWithCursors::new(
"a b d",
vec![CursorPosition {
id: 0,
char_index: 1, // after a
}],
);
let right = TextWithCursors::new(
"c d",
vec![CursorPosition {
id: 1,
char_index: 1, // after c
}],
);
let merged = reconcile_with_cursors(original, left, right);
assert_eq!(
merged,
TextWithCursors::new(
" d",
vec![
CursorPosition {
id: 0,
char_index: 0
},
CursorPosition {
id: 1,
char_index: 1
}
]
)
);
}
#[test]
fn test_cursor_complex() {
let original = "this is some complex text to test cursor positions";
let left = TextWithCursors::new(
"this is really complex text for testing cursor positions",
vec![
CursorPosition {
id: 0,
char_index: 8,
}, // after "this is "
CursorPosition {
id: 1,
char_index: 22,
}, // after "this is really complex text"
],
);
let right = TextWithCursors::new(
"that was some complex sample to test cursor movements",
vec![
CursorPosition {
id: 2,
char_index: 5,
}, // after "that "
CursorPosition {
id: 3,
char_index: 29,
}, // after "some complex sample "
],
);
let merged = reconcile_with_cursors(original, left, right);
assert_eq!(
merged,
TextWithCursors::new(
"that was really complex sample for testing cursor movements",
vec![
CursorPosition {
id: 2,
char_index: 5
}, // unchanged
CursorPosition {
id: 0,
char_index: 9
}, // before "really"
CursorPosition {
id: 1,
char_index: 23
}, // inside of "s|ample" because "text" got replaced by "sample"
CursorPosition {
id: 3,
char_index: 31
}, // before "for"
]
)
);
}
#[test_matrix( [
"pride_and_prejudice.txt",
"romeo_and_juliet.txt",
@ -200,7 +385,7 @@ mod test {
let files = [file_name_1, file_name_2, file_name_3];
let permutations = [range_1, range_2, range_3];
let root = Path::new("test/resources/");
let root = Path::new("tests/resources/");
let contents = files
.iter()

View file

@ -0,0 +1,68 @@
use std::borrow::Cow;
#[cfg(feature = "serde")]
use serde::{Deserialize, Serialize};
use super::merge_context::MergeContext;
use crate::operation_transformation::Operation;
// CursorPosition represents the position of an identifiable cursor in a text
// document based on its (UTF-8) character index.
#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
#[derive(Debug, Clone, PartialEq, Default)]
pub struct CursorPosition {
pub id: usize,
pub char_index: usize,
}
impl CursorPosition {
#[must_use]
pub fn apply_merge_context<T>(&self, context: &MergeContext<T>) -> Self
where
T: PartialEq + Clone + std::fmt::Debug,
{
let char_index = match context.last_operation() {
Some(Operation::Delete { index, .. }) => (*index) as i64,
_ => self.char_index as i64 + context.shift,
};
CursorPosition {
id: self.id,
char_index: char_index.max(0) as usize,
}
}
}
#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
#[derive(Debug, Clone, PartialEq, Default)]
pub struct TextWithCursors<'a> {
pub text: Cow<'a, str>,
pub cursors: Vec<CursorPosition>,
}
impl<'a> TextWithCursors<'a> {
#[must_use]
pub fn new(text: &'a str, cursors: Vec<CursorPosition>) -> Self {
Self {
text: text.into(),
cursors,
}
}
#[must_use]
pub fn new_owned(text: String, cursors: Vec<CursorPosition>) -> Self {
Self {
text: text.into(),
cursors,
}
}
}
impl<'a> From<&'a str> for TextWithCursors<'a> {
fn from(text: &'a str) -> Self {
Self {
text: text.into(),
cursors: Vec::new(),
}
}
}

View file

@ -3,7 +3,7 @@ use core::iter;
#[cfg(feature = "serde")]
use serde::{Deserialize, Serialize};
use super::Operation;
use super::{CursorPosition, Operation, TextWithCursors};
use crate::{
diffs::{myers::diff, raw_operation::RawOperation},
operation_transformation::merge_context::MergeContext,
@ -29,6 +29,7 @@ where
{
text: &'a str,
operations: Vec<OrderedOperation<T>>,
pub(crate) cursors: Vec<CursorPosition>,
}
impl<'a> EditedText<'a, String> {
@ -39,7 +40,7 @@ impl<'a> EditedText<'a, String> {
/// word tokenizer is used to tokenize the text which splits the text on
/// whitespaces.
#[must_use]
pub fn from_strings(original: &'a str, updated: &str) -> Self {
pub fn from_strings(original: &'a str, updated: TextWithCursors<'a>) -> Self {
Self::from_strings_with_tokenizer(original, updated, &word_tokenizer)
}
}
@ -55,17 +56,18 @@ where
/// function is used to tokenize the text.
pub fn from_strings_with_tokenizer(
original: &'a str,
updated: &str,
updated: TextWithCursors<'a>,
tokenizer: &Tokenizer<T>,
) -> Self {
let original_tokens = (tokenizer)(original);
let updated_tokens = (tokenizer)(updated);
let updated_tokens = (tokenizer)(&updated.text);
let diff: Vec<RawOperation<T>> = diff(&original_tokens, &updated_tokens);
Self::new(
original,
Self::cook_operations(Self::elongate_operations(diff)).collect(),
updated.cursors,
)
}
@ -170,7 +172,11 @@ where
/// Create a new `EditedText` with the given operations.
/// The operations must be in the order in which they are meant to be
/// applied. The operations must not overlap.
fn new(text: &'a str, operations: Vec<OrderedOperation<T>>) -> Self {
fn new(
text: &'a str,
operations: Vec<OrderedOperation<T>>,
mut cursors: Vec<CursorPosition>,
) -> Self {
operations
.iter()
.zip(operations.iter().skip(1))
@ -183,7 +189,13 @@ where
);
});
Self { text, operations }
cursors.sort_by_key(|cursor| cursor.char_index);
Self {
text,
operations,
cursors,
}
}
#[must_use]
@ -196,50 +208,110 @@ where
let mut left_merge_context = MergeContext::default();
let mut right_merge_context = MergeContext::default();
Self::new(
self.text,
self.operations
.into_iter()
.map(|op| (op, Side::Left))
.merge_sorted_by_key(
other.operations.into_iter().map(|op| (op, Side::Right)),
|(operation, _)| {
(
operation.order,
// Operations on the left and right must come in the same order so that
// inserts can be merged with other inserts and deletes with deletes.
usize::from(matches!(operation.operation, Operation::Delete { .. })),
// Make sure that the ordering is deterministic regardless which text
// is left or right.
match &operation.operation {
Operation::Insert { text, .. } => text
.iter()
.map(super::super::tokenizer::token::Token::original)
.collect::<String>(),
Operation::Delete {
deleted_character_count,
..
} => deleted_character_count.to_string(),
},
let mut merged_cursors = Vec::with_capacity(self.cursors.len() + other.cursors.len());
let mut left_cursors = self.cursors.iter().peekable();
let mut right_cursors = other.cursors.iter().peekable();
let merged_operations = self
.operations
.into_iter()
// The current text is always the left; the other operation is the right side.
.map(|op| (op, Side::Left))
.merge_sorted_by_key(
other.operations.into_iter().map(|op| (op, Side::Right)),
|(operation, _)| {
(
operation.order,
// Operations on the left and right must come in the same order so that
// inserts can be merged with other inserts and deletes with deletes.
usize::from(matches!(operation.operation, Operation::Delete { .. })),
// Make sure that the ordering is deterministic regardless which text
// is left or right.
match &operation.operation {
Operation::Insert { text, .. } => text
.iter()
.map(super::super::tokenizer::token::Token::original)
.collect::<String>(),
Operation::Delete {
deleted_character_count,
..
} => deleted_character_count.to_string(),
},
)
},
)
.flat_map(|(OrderedOperation { order, operation }, side)| {
match side {
Side::Left => {
while let Some(cursor) = left_cursors
.next_if(|cursor| cursor.char_index <= operation.start_index())
{
right_merge_context.consume_last_operation_if_it_is_too_behind(
cursor.char_index as i64,
);
merged_cursors.push(cursor.apply_merge_context(&right_merge_context));
}
while let Some(cursor) = right_cursors.next_if(|cursor| {
cursor.char_index as i64
<= operation.start_index() as i64 + right_merge_context.shift
- left_merge_context.shift
}) {
left_merge_context.consume_last_operation_if_it_is_too_behind(
cursor.char_index as i64,
);
merged_cursors.push(cursor.apply_merge_context(&left_merge_context));
}
operation.merge_operations_with_context(
&mut right_merge_context,
&mut left_merge_context,
)
},
)
.flat_map(|(OrderedOperation { order, operation }, side)| {
match side {
Side::Left => operation.merge_operations_with_context(
&mut right_merge_context,
&mut left_merge_context,
),
Side::Right => operation.merge_operations_with_context(
&mut left_merge_context,
&mut right_merge_context,
),
}
.map(|operation| OrderedOperation { order, operation })
.into_iter()
})
.collect(),
)
Side::Right => {
while let Some(cursor) = right_cursors
.next_if(|cursor| cursor.char_index <= operation.start_index())
{
left_merge_context.consume_last_operation_if_it_is_too_behind(
cursor.char_index as i64,
);
merged_cursors.push(cursor.apply_merge_context(&left_merge_context));
}
while let Some(cursor) = left_cursors.next_if(|cursor| {
cursor.char_index as i64
<= operation.start_index() as i64 + left_merge_context.shift
- right_merge_context.shift
}) {
right_merge_context.consume_last_operation_if_it_is_too_behind(
cursor.char_index as i64,
);
merged_cursors.push(cursor.apply_merge_context(&right_merge_context));
}
operation.merge_operations_with_context(
&mut left_merge_context,
&mut right_merge_context,
)
}
}
.map(|operation| OrderedOperation { order, operation })
.into_iter()
})
.collect();
for cursor in left_cursors {
right_merge_context
.consume_last_operation_if_it_is_too_behind(cursor.char_index as i64);
merged_cursors.push(cursor.apply_merge_context(&right_merge_context));
}
for cursor in right_cursors {
left_merge_context.consume_last_operation_if_it_is_too_behind(cursor.char_index as i64);
merged_cursors.push(cursor.apply_merge_context(&left_merge_context));
}
Self::new(self.text, merged_operations, merged_cursors)
}
/// Apply the operations to the text and return the resulting text.
@ -268,7 +340,7 @@ mod tests {
let left = "hello world! How are you? Adam";
let right = "Hello, my friend! How are you doing? Albert";
let operations = EditedText::from_strings(left, right);
let operations = EditedText::from_strings(left, right.into());
insta::assert_debug_snapshot!(operations);
@ -280,7 +352,7 @@ mod tests {
fn test_calculate_operations_with_no_diff() {
let text = "hello world!";
let operations = EditedText::from_strings(text, text);
let operations = EditedText::from_strings(text, text.into());
assert_eq!(operations.operations.len(), 0);
@ -296,8 +368,8 @@ mod tests {
let right = "Hello world! How are you?";
let expected = "Hello world! How are you? I'm Andras.";
let operations_1 = EditedText::from_strings(original, left);
let operations_2 = EditedText::from_strings(original, right);
let operations_1 = EditedText::from_strings(original, left.into());
let operations_2 = EditedText::from_strings(original, right.into());
let operations = operations_1.merge(operations_2);
assert_eq!(operations.apply(), expected);

View file

@ -2,7 +2,7 @@ use core::fmt::Debug;
use crate::operation_transformation::Operation;
#[derive(Clone)]
#[derive(Clone, Debug)]
pub struct MergeContext<T>
where
T: PartialEq + Clone + std::fmt::Debug,
@ -23,26 +23,19 @@ where
}
}
impl<T> Debug for MergeContext<T>
where
T: PartialEq + Clone + std::fmt::Debug,
{
fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
f.debug_struct("MergeContext")
.field("last_operation", &self.last_operation)
.field("shift", &self.shift)
.finish()
}
}
impl<T> MergeContext<T>
where
T: PartialEq + Clone + std::fmt::Debug,
{
pub fn last_operation(&self) -> Option<&Operation<T>> { self.last_operation.as_ref() }
pub fn replace_last_operation(&mut self, operation: Option<Operation<T>>) {
self.last_operation = operation;
}
/// Replace the last delete operation (if there was one) with a new one
/// while applying it to the shift.
/// while applying it to the `shift` in case the last operation
/// was a delete.
pub fn consume_and_replace_last_operation(&mut self, operation: Option<Operation<T>>) {
if let Some(Operation::Delete {
deleted_character_count,
@ -55,32 +48,22 @@ where
self.last_operation = operation;
}
pub fn replace_last_operation(&mut self, operation: Option<Operation<T>>) {
self.last_operation = operation;
}
/// Remove the last operation (if there was one) in case it is behind the
/// threshold operation. This changes the shift in case the last operation
/// threshold operation. This updates the `shift` in case the last operation
/// was a delete.
pub fn consume_last_operation_if_it_is_too_behind(
&mut self,
threshold_operation: &Operation<T>,
) {
pub fn consume_last_operation_if_it_is_too_behind(&mut self, threshold_index: i64) {
if let Some(last_operation) = self.last_operation.as_ref() {
if let Operation::Delete {
deleted_character_count,
..
} = last_operation
{
if threshold_operation.start_index() as i64 + self.shift
> last_operation.end_index() as i64
{
if threshold_index + self.shift > last_operation.end_index() as i64 {
self.shift -= *deleted_character_count as i64;
self.last_operation = None;
}
} else if let Operation::Insert { .. } = last_operation {
if threshold_operation.start_index() as i64 + self.shift
- last_operation.len() as i64
if threshold_index + self.shift - last_operation.len() as i64
> last_operation.end_index() as i64
{
self.last_operation = None;

View file

@ -189,7 +189,7 @@ where
affecting_context: &mut MergeContext<T>,
produced_context: &mut MergeContext<T>,
) -> Option<Operation<T>> {
affecting_context.consume_last_operation_if_it_is_too_behind(&self);
affecting_context.consume_last_operation_if_it_is_too_behind(self.start_index() as i64);
let operation = self.with_shifted_index(affecting_context.shift);
match (operation, affecting_context.last_operation()) {

View file

@ -23,4 +23,5 @@ EditedText {
operation: <delete ' you? Adam' from index 43>,
},
],
cursors: [],
}

View file

@ -1,4 +1,16 @@
use std::fmt::Display;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Side {
Left,
Right,
}
impl Display for Side {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Side::Left => write!(f, "Left"),
Side::Right => write!(f, "Right"),
}
}
}

View file

@ -0,0 +1,98 @@
use std::{fs, path::Path};
use pretty_assertions::assert_eq;
use reconcile::{CursorPosition, TextWithCursors};
use serde::Deserialize;
/// `ExampleDocument` represents a test case for the reconciliation process.
/// It contains a parent string, left and right strings with cursor positions,
/// and the expected result after reconciliation.
///
/// '|' characters in the left, right, and expected strings are treated as
/// cursor positions and are converted into `CursorPosition` objects.
#[derive(Debug, Deserialize, Clone, PartialEq, Eq)]
pub struct ExampleDocument {
parent: String,
left: String,
right: String,
expected: String,
}
impl ExampleDocument {
/// Creates a new `ExampleDocument` instance from a YAML file.
///
/// # Panics
///
/// If the file cannot be opened or parsed, the program will panic.
#[must_use]
pub fn from_yaml(path: &Path) -> Self {
let file = fs::File::open(path).expect("Failed to open example file");
serde_yaml::from_reader(file).expect("Failed to parse example file")
}
#[must_use]
pub fn parent(&self) -> String { self.parent.clone() }
#[must_use]
pub fn left(&self) -> TextWithCursors<'static> {
ExampleDocument::string_to_text_with_cursors(&self.left)
}
#[must_use]
pub fn right(&self) -> TextWithCursors<'static> {
ExampleDocument::string_to_text_with_cursors(&self.right)
}
/// Asserts that the result string matches the expected string,
/// including cursor positions.
///
/// # Panics
///
/// If the result string does not match the expected string, the program
/// will panic.
pub fn assert_eq(&self, result: &TextWithCursors<'static>) {
let result_str = ExampleDocument::text_with_cursors_to_string(result);
assert_eq!(result_str, self.expected);
}
/// Asserts that the result string matches the expected string,
/// ignoring cursor positions.
///
/// # Panics
///
/// If the result string does not match the expected string, the program
/// will panic.
pub fn assert_eq_without_cursors(&self, result: &str) {
assert_eq!(
result,
ExampleDocument::string_to_text_with_cursors(&self.expected).text,
);
}
fn text_with_cursors_to_string(text: &TextWithCursors<'_>) -> String {
let mut result = text.text.clone().into_owned();
for (i, cursor) in text.cursors.iter().enumerate() {
result.insert(cursor.char_index + i, '|');
}
result
}
fn string_to_text_with_cursors(text: &str) -> TextWithCursors<'static> {
let cursors = Self::parse_cursors(text);
let text = text.replace('|', "");
TextWithCursors::new_owned(text, cursors)
}
fn parse_cursors(text: &str) -> Vec<CursorPosition> {
let mut cursors = Vec::new();
for (i, c) in text.chars().enumerate() {
if c == '|' {
cursors.push(CursorPosition {
id: 0,
char_index: i - cursors.len(),
});
}
}
cursors
}
}

View file

@ -0,0 +1,6 @@
# The `|` characters denote cursor positions which are stripped before the actual reconcile logic is run
---
parent: You're Annual Savings Statement is available in our online portal
left: Your| annual record is available in our online portal|
right: You're Annual Savings information| is available online
expected: Your| annual record information| is available online|

View file

@ -0,0 +1,4 @@
parent: marketplace
left: market| place
right: market|space
expected: market| placemarket|space

View file

@ -0,0 +1,4 @@
parent: Please remember to bring your laptop and charger
left: Please remember to bring your laptop|
right: Please remember to bring your |new |laptop and charger
expected: Please remember to bring your |new |laptop|

View file

@ -0,0 +1,4 @@
parent: Party A shall pay Party B
left: Party C shall pay Party B
right: Party A shall receive from Party B
expected: Party C shall receive from Party B

View file

@ -0,0 +1,4 @@
parent: Please submit your assignment by Friday
left: Please submit your |completed |assignment by Friday
right: Please submit your assignment |online |by Friday
expected: Please submit your |completed |assignment |online |by Friday

View file

@ -0,0 +1,4 @@
parent:
left: hi my friend|
right: hi there|
expected: hi my friend| there|

View file

@ -0,0 +1,4 @@
parent: Buy milk and eggs
left: Buy organic milk| and eggs|
right: Buy milk and eggs| and bread
expected: Buy organic milk| and eggs|| and bread

View file

@ -0,0 +1,4 @@
parent: Meeting at 2pm in 会议室
left: Meeting at |3pm in the 会议室
right: Team meeting at 2pm in conference room|
expected: Team meeting at |3pm in conference room| the

View file

@ -0,0 +1,4 @@
parent: Send the report to the team
left: Send the |detailed |report to the |entire |team
right: Send the |quarterly |detailed |report to the team
expected: Send the |detailed |quarterly |detailed ||report to the |entire |team

View file

@ -0,0 +1,4 @@
parent: Ready, Set go
left: Ready! Set go|
right: Ready, Set, go!|
expected: Ready! Set, go!||

View file

@ -0,0 +1,4 @@
parent: "Total: $100"
left: "Total: |$150"
right: "Total: |€100"
expected: "Total: |$150 |€100"

View file

@ -0,0 +1,4 @@
parent: Start middle end
left: Start [important] middle end|
right: Start middle [critical] end|
expected: Start [important] middle [critical] end||

View file

@ -0,0 +1,4 @@
parent: A B C D
left: A X B D|
right: A B Y|
expected: A X B Y||

View file

@ -0,0 +1,46 @@
mod example_document;
use std::{fs, path::Path};
use example_document::ExampleDocument;
use reconcile::{reconcile, reconcile_with_cursors};
#[test]
fn test_with_examples() {
let examples_dir = Path::new("tests/examples");
let mut entries = fs::read_dir(examples_dir)
.expect("Failed to read examples directory")
.collect::<Vec<_>>();
entries.sort_by_key(|entry| {
let path = entry
.as_ref()
.expect("Failed to read directory entry")
.path();
path.file_name()
.and_then(|name| name.to_str())
.and_then(|name| name.split('.').next().unwrap().parse::<i32>().ok())
.unwrap_or_default()
});
for entry in entries {
let entry = entry.expect("Failed to read directory entry");
let path = entry.path();
if path.is_file() && path.extension().and_then(|ext| ext.to_str()) == Some("yml") {
let doc = ExampleDocument::from_yaml(&path);
println!("Testing with example from {}", path.display());
doc.assert_eq_without_cursors(&reconcile(
&doc.parent(),
&doc.left().text,
&doc.right().text,
));
doc.assert_eq(&reconcile_with_cursors(
&doc.parent(),
doc.left(),
doc.right(),
));
}
}
}

View file

@ -0,0 +1,88 @@
use wasm_bindgen::prelude::*;
/// Wrapper type to expose `TextWithCursors` to JS.
#[wasm_bindgen]
#[derive(Debug, Clone, PartialEq)]
pub struct TextWithCursors {
text: String,
cursors: Vec<CursorPosition>,
}
#[wasm_bindgen]
impl TextWithCursors {
#[wasm_bindgen(constructor)]
#[must_use]
pub fn new(text: String, cursors: Vec<CursorPosition>) -> Self { Self { text, cursors } }
#[must_use]
pub fn text(&self) -> String { self.text.clone() }
#[must_use]
pub fn cursors(&self) -> Vec<CursorPosition> { self.cursors.clone() }
}
impl From<TextWithCursors> for reconcile::TextWithCursors<'_> {
fn from(owned: TextWithCursors) -> Self {
reconcile::TextWithCursors::new_owned(
owned.text.to_string(),
owned
.cursors
.into_iter()
.map(std::convert::Into::into)
.collect(),
)
}
}
impl From<reconcile::TextWithCursors<'_>> for TextWithCursors {
fn from(text_with_cursors: reconcile::TextWithCursors<'_>) -> Self {
TextWithCursors {
text: text_with_cursors.text.into_owned(),
cursors: text_with_cursors
.cursors
.into_iter()
.map(std::convert::Into::into)
.collect(),
}
}
}
/// Wrapper type to expose `CursorPosition` to JS.
#[wasm_bindgen]
#[derive(Debug, Clone, PartialEq)]
pub struct CursorPosition {
id: usize,
char_index: usize,
}
#[wasm_bindgen]
impl CursorPosition {
#[wasm_bindgen(constructor)]
#[must_use]
pub fn new(id: usize, char_index: usize) -> Self { Self { id, char_index } }
#[must_use]
pub fn id(&self) -> usize { self.id }
#[wasm_bindgen(js_name = characterPosition)]
#[must_use]
pub fn char_index(&self) -> usize { self.char_index }
}
impl From<CursorPosition> for reconcile::CursorPosition {
fn from(owned: CursorPosition) -> Self {
reconcile::CursorPosition {
id: owned.id,
char_index: owned.char_index,
}
}
}
impl From<reconcile::CursorPosition> for CursorPosition {
fn from(cursor: reconcile::CursorPosition) -> Self {
CursorPosition {
id: cursor.id,
char_index: cursor.char_index,
}
}
}

View file

@ -8,12 +8,15 @@
//! # Modules
//!
//! - `errors`: Contains error types used in this crate.
use core::str;
use base64::{Engine as _, engine::general_purpose::STANDARD};
use cursor::TextWithCursors;
use errors::SyncLibError;
use wasm_bindgen::prelude::*;
pub mod cursor;
pub mod errors;
/// Encode binary data for easy transport over HTTP. Inverse of
@ -93,7 +96,7 @@ pub fn merge(parent: &[u8], left: &[u8], right: &[u8]) -> Vec<u8> {
}
}
/// WASM wrapper around `reconcile::reconcile` for text merging.
/// WASM wrapper around `reconcile::reconcile` for merging text.
#[wasm_bindgen(js_name = mergeText)]
#[must_use]
pub fn merge_text(parent: &str, left: &str, right: &str) -> String {
@ -102,6 +105,19 @@ pub fn merge_text(parent: &str, left: &str, right: &str) -> String {
reconcile::reconcile(parent, left, right)
}
/// WASM wrapper around `reconcile::reconcile_with_cursors` for merging text.
#[wasm_bindgen(js_name = mergeTextWithCursors)]
#[must_use]
pub fn merge_text_with_cursors(
parent: &str,
left: TextWithCursors,
right: TextWithCursors,
) -> TextWithCursors {
set_panic_hook();
reconcile::reconcile_with_cursors(parent, left.into(), right.into()).into()
}
/// Heuristically determine if the given data is a binary or a text file's
/// content.
#[wasm_bindgen(js_name = isBinary)]

View file

@ -1,5 +1,8 @@
use insta::assert_debug_snapshot;
use sync_lib::*;
use sync_lib::{
cursor::{CursorPosition, TextWithCursors},
*,
};
use wasm_bindgen_test::*;
#[wasm_bindgen_test(unsupported = test)]
@ -23,11 +26,44 @@ fn test_base64_to_bytes_error() {
}
#[wasm_bindgen_test(unsupported = test)]
fn merge_text() {
fn test_merge() {
let left = b"hello ";
let right = b"world";
let result = merge(b"", left, right);
assert_eq!(result, b"hello world");
let left = b"\0binary";
let right = b"other";
let result = merge(b"", left, right);
assert_eq!(result, right);
}
#[wasm_bindgen_test(unsupported = test)]
fn test_merge_text() {
let left = "hello ";
let right = "world";
let result = merge_text("", left, right);
assert_eq!(result, "hello world");
}
#[wasm_bindgen_test(unsupported = test)]
fn test_merge_text_with_cursors() {
let result = merge_text_with_cursors(
"hi",
TextWithCursors::new("hi world".to_owned(), vec![]),
TextWithCursors::new(
"hi".to_owned(),
vec![CursorPosition::new(0, 1), CursorPosition::new(1, 2)],
),
);
assert_eq!(
result,
TextWithCursors::new(
"hi world".to_owned(),
vec![CursorPosition::new(0, 1), CursorPosition::new(1, 2)]
),
);
}
#[wasm_bindgen_test(unsupported = test)]

View file

@ -1,3 +1,5 @@
use log::info;
use crate::{
app_state::{AppState, database::models::VaultId},
config::user_config::{AllowListedVaults, User, VaultAccess},
@ -13,10 +15,16 @@ pub fn auth(app_state: &AppState, token: &str, vault: &VaultId) -> Result<User,
.cloned()
.ok_or_else(|| unauthenticated_error(anyhow::anyhow!("Invalid token")))?;
info!("User `{}` authenticated", user.name);
if match user.vault_access {
VaultAccess::AllowAccessToAll => true,
VaultAccess::AllowList(AllowListedVaults { ref allowed }) => allowed.contains(vault),
} {
info!(
"User `{}` is authorised to access to vault `{}`",
user.name, vault
);
Ok(user)
} else {
Err(permission_denied_error(anyhow::anyhow!(

View file

@ -0,0 +1,3 @@
module.exports = {
preset: "ts-jest/presets/js-with-babel-esm"
};

View file

@ -6,7 +6,7 @@
"scripts": {
"dev": "webpack watch --mode development",
"build": "webpack --mode production",
"test": "jest --passWithNoTests",
"test": "jest",
"version": "node version-bump.mjs"
},
"keywords": [],
@ -36,4 +36,4 @@
"webpack": "^5.98.0",
"webpack-cli": "^6.0.1"
}
}
}

View file

@ -1,6 +1,12 @@
import type { Stat, Vault, Workspace } from "obsidian";
import { MarkdownView, normalizePath } from "obsidian";
import type { FileSystemOperations, RelativePath } from "sync-client";
import type {
FileSystemOperations,
RelativePath,
TextWithCursors
} from "sync-client";
import { lineAndColumnToPosition } from "./utils/line-and-column-to-position";
import { positionToLineAndColumn } from "./utils/position-to-line-and-column";
export class ObsidianFileSystemOperations implements FileSystemOperations {
public constructor(
@ -42,20 +48,50 @@ export class ObsidianFileSystemOperations implements FileSystemOperations {
public async atomicUpdateText(
path: RelativePath,
updater: (currentContent: string) => string
updater: (current: TextWithCursors) => TextWithCursors
): Promise<string> {
path = normalizePath(path);
const view = this.workspace.getActiveViewOfType(MarkdownView);
if (view?.file?.path === path) {
const result = updater(view.editor.getValue());
const position = view.editor.getCursor();
view.editor.setValue(result);
view.editor.setCursor(position);
return result;
const cursor = view.editor.getCursor();
const text = view.editor.getValue();
const result = updater({
text,
cursors: [
{
id: 0,
characterPosition: lineAndColumnToPosition(
text,
cursor.line,
cursor.ch
)
}
]
});
view.editor.setValue(result.text);
result.cursors.forEach((movedCursor) => {
const { line, column } = positionToLineAndColumn(
result.text,
movedCursor.characterPosition
);
view.editor.setCursor(line, column);
});
return result.text;
}
return this.vault.adapter.process(path, updater);
return this.vault.adapter.process(
path,
(text) =>
updater({
text,
cursors: []
}).text
);
}
public async getFileSize(path: RelativePath): Promise<number> {

View file

@ -0,0 +1,43 @@
import { lineAndColumnToPosition } from "./line-and-column-to-position";
describe("lineAndColumnToPosition", () => {
it("should return the correct position for the first line", () => {
const text = "Hello\nWorld";
const position = lineAndColumnToPosition(text, 0, 3);
expect(position).toBe(3);
});
it("should return the correct position for the second line", () => {
const text = "Hello\nWorld";
const position = lineAndColumnToPosition(text, 1, 2);
expect(position).toBe(8);
});
it("should return the correct position for an empty string", () => {
const text = "";
const position = lineAndColumnToPosition(text, 0, 0);
expect(position).toBe(0);
});
it("should handle a single-line string correctly", () => {
const text = "SingleLine";
const position = lineAndColumnToPosition(text, 0, 5);
expect(position).toBe(5);
});
it("should handle multi-line strings with varying lengths", () => {
const text = "Line1\nLongerLine2\nShort3";
const position = lineAndColumnToPosition(text, 2, 4);
expect(position).toBe(22);
});
it("should throw an error if the line number is out of range", () => {
const text = "Line1\nLine2";
expect(() => lineAndColumnToPosition(text, 3, 0)).toThrow();
});
it("should throw an error if the column number is out of range", () => {
const text = "Line1\nLine2";
expect(() => lineAndColumnToPosition(text, 1, 10)).toThrow();
});
});

View file

@ -0,0 +1,34 @@
/**
* Converts line and column coordinates to an absolute character position in a text string.
*
* @param line - The zero-based line number
* @param column - The zero-based column number
* @param text - The text string to calculate position in
* @returns The absolute character position (zero-based index) in the text string
* @throws Error if line number is out of range
* @throws Error if column number is out of range
*/
export function lineAndColumnToPosition(
text: string,
line: number,
column: number
): number {
const lines = text.split("\n");
if (line >= lines.length) {
throw new Error(`Line number ${line} is out of range.`);
}
if (column > lines[line].length) {
throw new Error(`Column number ${column} is out of range.`);
}
let position = 0;
for (let i = 0; i < line; i++) {
position += lines[i].length + 1;
}
position += column;
return position;
}

View file

@ -0,0 +1,69 @@
import { positionToLineAndColumn } from "./position-to-line-and-column";
describe("positionToLineAndColumn", () => {
test("converts position to line and column in a single line text", () => {
const text = "Hello, world!";
expect(positionToLineAndColumn(text, 0)).toEqual({
line: 0,
column: 1
});
expect(positionToLineAndColumn(text, 7)).toEqual({
line: 0,
column: 8
});
expect(positionToLineAndColumn(text, 12)).toEqual({
line: 0,
column: 13
});
});
test("converts position to line and column in multi-line text", () => {
const text = "First line\nSecond line\nThird line";
expect(positionToLineAndColumn(text, 0)).toEqual({
line: 0,
column: 1
});
expect(positionToLineAndColumn(text, 10)).toEqual({
line: 0,
column: 11
});
expect(positionToLineAndColumn(text, 15)).toEqual({
line: 1,
column: 5
});
expect(positionToLineAndColumn(text, 26)).toEqual({
line: 2,
column: 4
});
});
test("handles positions at line breaks", () => {
const text = "Line\nBreak";
expect(positionToLineAndColumn(text, 4)).toEqual({
line: 0,
column: 5
});
expect(positionToLineAndColumn(text, 5)).toEqual({
line: 1,
column: 1
});
});
test("handles empty input", () => {
expect(positionToLineAndColumn("", 0)).toEqual({ line: 0, column: 1 });
});
test("handles positions at the end of text", () => {
const text = "End";
expect(positionToLineAndColumn(text, 3)).toEqual({
line: 0,
column: 4
});
});
test("throws error for position out of range", () => {
const text = "Short text";
expect(() => positionToLineAndColumn(text, 15)).toThrow();
expect(() => positionToLineAndColumn(text, -1)).toThrow();
});
});

View file

@ -0,0 +1,30 @@
/**
* Converts a character position in text to line and column numbers.
*
* @param text The text content to analyze
* @param position The character position to convert
* @returns An object containing line and column numbers (0-based index for line, 1-based index for column)
* @throws Will throw an error if the position is negative or exceeds the text length
*/
export function positionToLineAndColumn(
text: string,
position: number
): { line: number; column: number } {
if (position < 0) {
throw new Error("Position cannot be negative");
}
if (position > text.length) {
throw new Error(
`Position ${position} exceeds text length ${text.length}`
);
}
const textUpToPosition = text.substring(0, position);
const lines = textUpToPosition.split("\n");
const line = lines.length - 1; // 0-based index
const column = lines[lines.length - 1].length + 1; // 1-based index
return { line, column };
}

View file

@ -6,7 +6,10 @@ import type {
import { FileOperations } from "./file-operations";
import { Logger } from "../tracing/logger";
import { assertSetContainsExactly } from "../utils/assert-set-contains-exactly";
import type { FileSystemOperations } from "./filesystem-operations";
import type {
FileSystemOperations,
TextWithCursors
} from "./filesystem-operations";
import init, { base64ToBytes } from "sync_lib";
import fs from "fs";
@ -43,7 +46,7 @@ class FakeFileSystemOperations implements FileSystemOperations {
}
public async atomicUpdateText(
_path: RelativePath,
_updater: (currentContent: string) => string
_updater: (current: TextWithCursors) => TextWithCursors
): Promise<string> {
throw new Error("Method not implemented.");
}

View file

@ -1,7 +1,16 @@
import type { Logger } from "../tracing/logger";
import type { FileSystemOperations } from "./filesystem-operations";
import type {
FileSystemOperations,
TextWithCursors
} from "./filesystem-operations";
import type { Database, RelativePath } from "../persistence/database";
import { isBinary, isFileTypeMergable, mergeText } from "sync_lib";
import {
CursorPosition,
isBinary,
isFileTypeMergable,
mergeTextWithCursors,
TextWithCursors as RustTextWithCursors
} from "sync_lib";
import { SafeFileSystemOperations } from "./safe-filesystem-operations";
export class FileOperations {
@ -90,18 +99,45 @@ export class FileOperations {
const expectedText = new TextDecoder().decode(expectedContent); // this comes from a previous read which must only have \n line endings
const newText = new TextDecoder().decode(newContent); // this comes from the server which stores text with \n line endings
await this.fs.atomicUpdateText(path, (currentText) => {
currentText = currentText.replace(this.nativeLineEndings, "\n");
await this.fs.atomicUpdateText(
path,
({ text, cursors }: TextWithCursors): TextWithCursors => {
text = text.replace(this.nativeLineEndings, "\n");
this.logger.debug(
`Performing a 3-way merge for ${path} with the expected content`
);
this.logger.debug(
`Performing a 3-way merge for ${path} with the expected content`
);
return mergeText(expectedText, currentText, newText).replace(
"\n",
this.nativeLineEndings
);
});
const left = new RustTextWithCursors(
text,
cursors.map(
(cursor) =>
new CursorPosition(
cursor.id,
cursor.characterPosition
)
)
);
const right = new RustTextWithCursors(newText, []);
const merged = mergeTextWithCursors(expectedText, left, right);
const resultText = merged
.text()
.replace("\n", this.nativeLineEndings);
const resultCursors = merged.cursors().map((cursor) => ({
id: cursor.id(),
characterPosition: cursor.characterPosition()
}));
merged.free();
return {
text: resultText,
cursors: resultCursors
};
}
);
}
public async delete(path: RelativePath): Promise<void> {

View file

@ -1,5 +1,15 @@
import type { RelativePath } from "../persistence/database";
export interface Cursor {
id: number;
characterPosition: number;
}
export interface TextWithCursors {
text: string;
cursors: Cursor[];
}
export interface FileSystemOperations {
// List all files that should be synced.
listAllFiles: () => Promise<RelativePath[]>;
@ -13,7 +23,7 @@ export interface FileSystemOperations {
// Atomically update the content of a text file.
atomicUpdateText: (
path: RelativePath,
updater: (currentContent: string) => string
updater: (current: TextWithCursors) => TextWithCursors
) => Promise<string>;
// Get the size of a file in bytes.

View file

@ -1,5 +1,8 @@
import type { RelativePath } from "../persistence/database";
import type { FileSystemOperations } from "./filesystem-operations";
import type {
FileSystemOperations,
TextWithCursors
} from "./filesystem-operations";
import type { Logger } from "../tracing/logger";
import { Locks } from "../utils/locks";
import { FileNotFoundError } from "./file-not-found-error";
@ -44,7 +47,7 @@ export class SafeFileSystemOperations implements FileSystemOperations {
public async atomicUpdateText(
path: RelativePath,
updater: (currentContent: string) => string
updater: (current: TextWithCursors) => TextWithCursors
): Promise<string> {
this.logger.debug(`Atomically updating file '${path}'`);
return this.safeOperation(

View file

@ -8,7 +8,11 @@ export { Logger, LogLevel, LogLine } from "./tracing/logger";
export { type SyncSettings } from "./persistence/settings";
export { rateLimit } from "./utils/rate-limit";
export type { RelativePath, StoredDatabase } from "./persistence/database";
export type { FileSystemOperations } from "./file-operations/filesystem-operations";
export type {
FileSystemOperations,
TextWithCursors,
Cursor
} from "./file-operations/filesystem-operations";
export type { PersistenceProvider } from "./persistence/persistence";
export type { NetworkConnectionStatus } from "./sync-client";

View file

@ -266,6 +266,8 @@ export class Syncer {
wsUri.protocol = wsUri.protocol === "https" ? "wss" : "ws";
wsUri.pathname = `/vaults/${settings.vaultName}/ws`;
this.logger.info(`Connecting to WebSocket at ${wsUri.toString()}`);
if (
typeof globalThis !== "undefined" &&
typeof globalThis.WebSocket === "undefined"
@ -288,6 +290,7 @@ export class Syncer {
// The JS WebSocket API doesn't support setting headers, so we have to send the token as a message
this.applyRemoteChangesWebSocket.onopen = (): void => {
this.logger.info("WebSocket connection opened");
this.applyRemoteChangesWebSocket?.send(settings.token);
this.webSocketStatusChangeListeners.forEach((listener) => {
listener();
@ -476,7 +479,10 @@ export class Syncer {
.filter(
(remoteDocument) =>
allLocalFiles.includes(remoteDocument.relativePath) &&
!remoteDocument.isDeleted
!remoteDocument.isDeleted &&
this.database.getDocumentByDocumentId(
remoteDocument.documentId
) === undefined
)
.forEach((remoteDocument) => {
this.database.createNewEmptyDocument(

View file

@ -313,7 +313,10 @@ export class MockAgent extends MockClient {
`Decided to update file ${file} with ${content}`
);
this.doNotTouchWhileOffline.push(file);
await this.atomicUpdateText(file, (old) => old + ` ${content} `);
await this.atomicUpdateText(file, (old) => ({
text: old.text + ` ${content} `,
cursors: []
}));
}
private async deleteFileAction(files: RelativePath[]): Promise<void> {

View file

@ -1,4 +1,4 @@
import type { StoredDatabase } from "sync-client";
import type { StoredDatabase, TextWithCursors } from "sync-client";
import { assert } from "../utils/assert";
import {
type RelativePath,
@ -87,14 +87,14 @@ export class MockClient implements FileSystemOperations {
public async atomicUpdateText(
path: RelativePath,
updater: (currentContent: string) => string
updater: (currentContent: TextWithCursors) => TextWithCursors
): Promise<string> {
const file = this.localFiles.get(path);
if (!file) {
throw new Error(`File ${path} does not exist`);
}
const currentContent = new TextDecoder().decode(file);
const newContent = updater(currentContent);
const newContent = updater({ text: currentContent, cursors: [] }).text;
const newContentUint8Array = new TextEncoder().encode(newContent);
this.localFiles.set(path, newContentUint8Array);