From 23ba0d2c8231db8fa548337fc6c8aa2a3195208c Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sun, 30 Mar 2025 12:24:08 +0100 Subject: [PATCH 01/21] Add comments --- backend/sync_server/src/server/auth.rs | 8 ++++++++ frontend/sync-client/src/sync-operations/syncer.ts | 3 +++ 2 files changed, 11 insertions(+) diff --git a/backend/sync_server/src/server/auth.rs b/backend/sync_server/src/server/auth.rs index 06bfe5db..7a7d2197 100644 --- a/backend/sync_server/src/server/auth.rs +++ b/backend/sync_server/src/server/auth.rs @@ -1,3 +1,5 @@ +use log::info; + use crate::{ app_state::{AppState, database::models::VaultId}, config::user_config::{AllowListedVaults, User, VaultAccess}, @@ -13,10 +15,16 @@ pub fn auth(app_state: &AppState, token: &str, vault: &VaultId) -> Result true, VaultAccess::AllowList(AllowListedVaults { ref allowed }) => allowed.contains(vault), } { + info!( + "User `{}` is authorised to access to vault `{}`", + user.name, vault + ); Ok(user) } else { Err(permission_denied_error(anyhow::anyhow!( diff --git a/frontend/sync-client/src/sync-operations/syncer.ts b/frontend/sync-client/src/sync-operations/syncer.ts index 7bad88e4..8ec83694 100644 --- a/frontend/sync-client/src/sync-operations/syncer.ts +++ b/frontend/sync-client/src/sync-operations/syncer.ts @@ -266,6 +266,8 @@ export class Syncer { wsUri.protocol = wsUri.protocol === "https" ? "wss" : "ws"; wsUri.pathname = `/vaults/${settings.vaultName}/ws`; + this.logger.info(`Connecting to WebSocket at ${wsUri.toString()}`); + if ( typeof globalThis !== "undefined" && typeof globalThis.WebSocket === "undefined" @@ -288,6 +290,7 @@ export class Syncer { // The JS WebSocket API doesn't support setting headers, so we have to send the token as a message this.applyRemoteChangesWebSocket.onopen = (): void => { + this.logger.info("WebSocket connection opened"); this.applyRemoteChangesWebSocket?.send(settings.token); this.webSocketStatusChangeListeners.forEach((listener) => { listener(); -- 2.47.2 From c9c0ffecf19b44f4e492e5c7467b0bd39452a7d7 Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sun, 30 Mar 2025 21:26:04 +0100 Subject: [PATCH 02/21] Add cursor types --- backend/reconcile/src/cursor.rs | 45 +++++++++++++++++++++ backend/sync_lib/src/cursor.rs | 71 +++++++++++++++++++++++++++++++++ 2 files changed, 116 insertions(+) create mode 100644 backend/reconcile/src/cursor.rs create mode 100644 backend/sync_lib/src/cursor.rs diff --git a/backend/reconcile/src/cursor.rs b/backend/reconcile/src/cursor.rs new file mode 100644 index 00000000..7ec9402c --- /dev/null +++ b/backend/reconcile/src/cursor.rs @@ -0,0 +1,45 @@ +use std::borrow::Cow; + +#[cfg(feature = "serde")] +use serde::{Deserialize, Serialize}; + +// CursorPosition is a wrapper around usize to represent the position of an +// identifiable cursor in a text document based on the character index. +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +#[derive(Debug, Clone, PartialEq, Default)] +pub struct CursorPosition { + pub id: usize, + pub char_index: usize, +} + +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +#[derive(Debug, Clone, PartialEq, Default)] +pub struct TextWithCursors<'a> { + pub text: Cow<'a, str>, + pub cursors: Vec, +} + +impl<'a> TextWithCursors<'a> { + pub fn new(text: &'a str, cursors: Vec) -> Self { + Self { + text: text.into(), + cursors, + } + } + + pub fn new_owned(text: String, cursors: Vec) -> Self { + Self { + text: text.into(), + cursors, + } + } +} + +impl<'a> From<&'a str> for TextWithCursors<'a> { + fn from(text: &'a str) -> Self { + Self { + text: text.into(), + cursors: Vec::new(), + } + } +} diff --git a/backend/sync_lib/src/cursor.rs b/backend/sync_lib/src/cursor.rs new file mode 100644 index 00000000..4b5b9a6d --- /dev/null +++ b/backend/sync_lib/src/cursor.rs @@ -0,0 +1,71 @@ +use reconcile::{CursorPosition, TextWithCursors}; +use wasm_bindgen::prelude::*; + +/// Wrapper type to expose `TextWithCursors` to JS. +#[wasm_bindgen] +#[derive(Debug, Clone, PartialEq)] +pub struct OwnedTextWithCursors { + text: String, + cursors: Vec, +} + +impl OwnedTextWithCursors { + pub fn new(text: impl Into, cursors: Vec) -> Self { + Self { + text: text.into(), + cursors, + } + } +} + +impl From for TextWithCursors<'_> { + fn from(owned: OwnedTextWithCursors) -> Self { + TextWithCursors::new_owned( + owned.text.to_string(), + owned + .cursors + .into_iter() + .map(|cursor| cursor.into()) + .collect(), + ) + } +} + +impl From> for OwnedTextWithCursors { + fn from(text_with_cursors: TextWithCursors<'_>) -> Self { + OwnedTextWithCursors { + text: text_with_cursors.text.into_owned(), + cursors: text_with_cursors + .cursors + .into_iter() + .map(|cursor| cursor.into()) + .collect(), + } + } +} + +/// Wrapper type to expose `CursorPosition` to JS. +#[wasm_bindgen] +#[derive(Debug, Clone, PartialEq)] +pub struct OwnedCursorPosition { + pub id: usize, + pub char_index: usize, +} + +impl From for CursorPosition { + fn from(owned: OwnedCursorPosition) -> Self { + CursorPosition { + id: owned.id, + char_index: owned.char_index, + } + } +} + +impl From for OwnedCursorPosition { + fn from(cursor: CursorPosition) -> Self { + OwnedCursorPosition { + id: cursor.id, + char_index: cursor.char_index, + } + } +} -- 2.47.2 From 71ccd7b61dd322127c96492675cd414e8dc6f33e Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sun, 30 Mar 2025 21:27:04 +0100 Subject: [PATCH 03/21] Make Side printable --- backend/reconcile/src/utils/side.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/backend/reconcile/src/utils/side.rs b/backend/reconcile/src/utils/side.rs index bfeee2c4..825fa9e2 100644 --- a/backend/reconcile/src/utils/side.rs +++ b/backend/reconcile/src/utils/side.rs @@ -1,4 +1,16 @@ +use std::fmt::Display; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum Side { Left, Right, } + +impl Display for Side { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Side::Left => write!(f, "Left"), + Side::Right => write!(f, "Right"), + } + } +} -- 2.47.2 From e9e2328f03b845c9df1d61e67e91bbe0e551d904 Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Tue, 1 Apr 2025 20:10:03 +0100 Subject: [PATCH 04/21] Expose merge_text_with_cursors on API --- backend/reconcile/src/lib.rs | 8 +++-- backend/sync_lib/src/lib.rs | 18 ++++++++++- backend/sync_lib/tests/web.rs | 58 +++++++++++++++++++++++++++++++++-- 3 files changed, 79 insertions(+), 5 deletions(-) diff --git a/backend/reconcile/src/lib.rs b/backend/reconcile/src/lib.rs index 9c5bb764..8023cc15 100644 --- a/backend/reconcile/src/lib.rs +++ b/backend/reconcile/src/lib.rs @@ -1,7 +1,11 @@ +mod cursor; mod diffs; mod operation_transformation; mod tokenizer; mod utils; -pub use operation_transformation::{EditedText, reconcile, reconcile_with_tokenizer}; -pub use tokenizer::token::Token; +pub use cursor::{CursorPosition, TextWithCursors}; +pub use operation_transformation::{ + EditedText, reconcile, reconcile_with_cursors, reconcile_with_tokenizer, +}; +pub use tokenizer::{Tokenizer, token::Token}; diff --git a/backend/sync_lib/src/lib.rs b/backend/sync_lib/src/lib.rs index 6f27e055..dc35e27f 100644 --- a/backend/sync_lib/src/lib.rs +++ b/backend/sync_lib/src/lib.rs @@ -8,12 +8,15 @@ //! # Modules //! //! - `errors`: Contains error types used in this crate. + use core::str; use base64::{Engine as _, engine::general_purpose::STANDARD}; +use cursor::OwnedTextWithCursors; use errors::SyncLibError; use wasm_bindgen::prelude::*; +pub mod cursor; pub mod errors; /// Encode binary data for easy transport over HTTP. Inverse of @@ -93,7 +96,7 @@ pub fn merge(parent: &[u8], left: &[u8], right: &[u8]) -> Vec { } } -/// WASM wrapper around `reconcile::reconcile` for text merging. +/// WASM wrapper around `reconcile::reconcile` for merging text. #[wasm_bindgen(js_name = mergeText)] #[must_use] pub fn merge_text(parent: &str, left: &str, right: &str) -> String { @@ -102,6 +105,19 @@ pub fn merge_text(parent: &str, left: &str, right: &str) -> String { reconcile::reconcile(parent, left, right) } +/// WASM wrapper around `reconcile::reconcile_with_cursors` for merging text. +#[wasm_bindgen(js_name = mergeText)] +#[must_use] +pub fn merge_text_with_cursors( + parent: &str, + left: OwnedTextWithCursors, + right: OwnedTextWithCursors, +) -> OwnedTextWithCursors { + set_panic_hook(); + + reconcile::reconcile_with_cursors(parent, left.into(), right.into()).into() +} + /// Heuristically determine if the given data is a binary or a text file's /// content. #[wasm_bindgen(js_name = isBinary)] diff --git a/backend/sync_lib/tests/web.rs b/backend/sync_lib/tests/web.rs index e45cbea6..017e2dc2 100644 --- a/backend/sync_lib/tests/web.rs +++ b/backend/sync_lib/tests/web.rs @@ -1,5 +1,8 @@ use insta::assert_debug_snapshot; -use sync_lib::*; +use sync_lib::{ + cursor::{OwnedCursorPosition, OwnedTextWithCursors}, + *, +}; use wasm_bindgen_test::*; #[wasm_bindgen_test(unsupported = test)] @@ -23,11 +26,62 @@ fn test_base64_to_bytes_error() { } #[wasm_bindgen_test(unsupported = test)] -fn merge_text() { +fn test_merge() { let left = b"hello "; let right = b"world"; let result = merge(b"", left, right); assert_eq!(result, b"hello world"); + + let left = b"\0binary"; + let right = b"other"; + let result = merge(b"", left, right); + assert_eq!(result, right); +} + +#[wasm_bindgen_test(unsupported = test)] +fn test_merge_text() { + let left = "hello "; + let right = "world"; + let result = merge_text("", left, right); + assert_eq!(result, "hello world"); +} + +#[wasm_bindgen_test(unsupported = test)] +fn test_merge_text_with_cursors() { + let result = merge_text_with_cursors( + "hi", + OwnedTextWithCursors::new("hi world", vec![]), + OwnedTextWithCursors::new( + "hi", + vec![ + OwnedCursorPosition { + id: 0, + char_index: 1, + }, + OwnedCursorPosition { + id: 1, + char_index: 2, + }, + ], + ), + ); + + assert_eq!( + result, + OwnedTextWithCursors::new( + "hi world", + vec![ + OwnedCursorPosition { + id: 0, + char_index: 1, + }, + OwnedCursorPosition { + id: 1, + char_index: 8, + } + ] + ), + ); } #[wasm_bindgen_test(unsupported = test)] -- 2.47.2 From 23c288b1eb776fadf72f122a44d45aa03df17110 Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Tue, 1 Apr 2025 20:53:38 +0100 Subject: [PATCH 05/21] Clean up --- .../operation_transformation/merge_context.rs | 39 ++++++------------- 1 file changed, 11 insertions(+), 28 deletions(-) diff --git a/backend/reconcile/src/operation_transformation/merge_context.rs b/backend/reconcile/src/operation_transformation/merge_context.rs index 980389df..d45f08ad 100644 --- a/backend/reconcile/src/operation_transformation/merge_context.rs +++ b/backend/reconcile/src/operation_transformation/merge_context.rs @@ -2,7 +2,7 @@ use core::fmt::Debug; use crate::operation_transformation::Operation; -#[derive(Clone)] +#[derive(Clone, Debug)] pub struct MergeContext where T: PartialEq + Clone + std::fmt::Debug, @@ -23,26 +23,19 @@ where } } -impl Debug for MergeContext -where - T: PartialEq + Clone + std::fmt::Debug, -{ - fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - f.debug_struct("MergeContext") - .field("last_operation", &self.last_operation) - .field("shift", &self.shift) - .finish() - } -} - impl MergeContext where T: PartialEq + Clone + std::fmt::Debug, { pub fn last_operation(&self) -> Option<&Operation> { self.last_operation.as_ref() } + pub fn replace_last_operation(&mut self, operation: Option>) { + self.last_operation = operation; + } + /// Replace the last delete operation (if there was one) with a new one - /// while applying it to the shift. + /// while applying it to the `shift` in case the last operation + /// was a delete. pub fn consume_and_replace_last_operation(&mut self, operation: Option>) { if let Some(Operation::Delete { deleted_character_count, @@ -55,32 +48,22 @@ where self.last_operation = operation; } - pub fn replace_last_operation(&mut self, operation: Option>) { - self.last_operation = operation; - } - /// Remove the last operation (if there was one) in case it is behind the - /// threshold operation. This changes the shift in case the last operation + /// threshold operation. This updates the `shift` in case the last operation /// was a delete. - pub fn consume_last_operation_if_it_is_too_behind( - &mut self, - threshold_operation: &Operation, - ) { + pub fn consume_last_operation_if_it_is_too_behind(&mut self, threshold_index: i64) { if let Some(last_operation) = self.last_operation.as_ref() { if let Operation::Delete { deleted_character_count, .. } = last_operation { - if threshold_operation.start_index() as i64 + self.shift - > last_operation.end_index() as i64 - { + if threshold_index + self.shift > last_operation.end_index() as i64 { self.shift -= *deleted_character_count as i64; self.last_operation = None; } } else if let Operation::Insert { .. } = last_operation { - if threshold_operation.start_index() as i64 + self.shift - - last_operation.len() as i64 + if threshold_index + self.shift - last_operation.len() as i64 > last_operation.end_index() as i64 { self.last_operation = None; -- 2.47.2 From 998ee387d20ba06e4c0aa6c0f7ea4f09d16ec7e5 Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Tue, 1 Apr 2025 22:40:05 +0100 Subject: [PATCH 06/21] Add integration tests --- backend/Cargo.lock | 1 + backend/reconcile/Cargo.toml | 4 +- backend/reconcile/test/examples/1.yml | 6 ++ backend/reconcile/test/examples/10.yml | 5 ++ backend/reconcile/test/examples/11.yml | 4 ++ backend/reconcile/test/examples/12.yml | 4 ++ backend/reconcile/test/examples/13.yml | 4 ++ backend/reconcile/test/examples/2.yml | 4 ++ backend/reconcile/test/examples/3.yml | 4 ++ backend/reconcile/test/examples/4.yml | 4 ++ backend/reconcile/test/examples/5.yml | 4 ++ backend/reconcile/test/examples/6.yml | 4 ++ backend/reconcile/test/examples/7.yml | 4 ++ backend/reconcile/test/examples/8.yml | 4 ++ backend/reconcile/test/examples/9.yml | 4 ++ backend/reconcile/tests/example_document.rs | 75 +++++++++++++++++++++ backend/reconcile/tests/test.rs | 47 +++++++++++++ 17 files changed, 181 insertions(+), 1 deletion(-) create mode 100644 backend/reconcile/test/examples/1.yml create mode 100644 backend/reconcile/test/examples/10.yml create mode 100644 backend/reconcile/test/examples/11.yml create mode 100644 backend/reconcile/test/examples/12.yml create mode 100644 backend/reconcile/test/examples/13.yml create mode 100644 backend/reconcile/test/examples/2.yml create mode 100644 backend/reconcile/test/examples/3.yml create mode 100644 backend/reconcile/test/examples/4.yml create mode 100644 backend/reconcile/test/examples/5.yml create mode 100644 backend/reconcile/test/examples/6.yml create mode 100644 backend/reconcile/test/examples/7.yml create mode 100644 backend/reconcile/test/examples/8.yml create mode 100644 backend/reconcile/test/examples/9.yml create mode 100644 backend/reconcile/tests/example_document.rs create mode 100644 backend/reconcile/tests/test.rs diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 5a5628f7..5ab0fa12 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -1881,6 +1881,7 @@ dependencies = [ "insta", "pretty_assertions", "serde", + "serde_yaml", "test-case", ] diff --git a/backend/reconcile/Cargo.toml b/backend/reconcile/Cargo.toml index 99e7d6e6..9ef29647 100644 --- a/backend/reconcile/Cargo.toml +++ b/backend/reconcile/Cargo.toml @@ -7,7 +7,7 @@ license.workspace = true repository.workspace = true [dependencies] -serde = { version = "1.0.219", optional = true } +serde = { version = "1.0.219", optional = true, features = ["derive"] } [features] serde = [ "dep:serde" ] @@ -15,6 +15,8 @@ serde = [ "dep:serde" ] [dev-dependencies] insta = "1.41.1" pretty_assertions = "1.4.1" +serde = { version = "1.0.219", features = ["derive"] } +serde_yaml ="0.9.34" test-case = "3.3.1" [lints] diff --git a/backend/reconcile/test/examples/1.yml b/backend/reconcile/test/examples/1.yml new file mode 100644 index 00000000..ce90f51f --- /dev/null +++ b/backend/reconcile/test/examples/1.yml @@ -0,0 +1,6 @@ +# The `|` characters denote cursor positions which are stripped before the actual reconcile logic is run +--- +parent: You're Annual Savings Statement is available in our online portal +left: Your| annual record is available in our online portal| +right: You're Annual Savings information| is available online +expected: Your| annual record information| is available online| diff --git a/backend/reconcile/test/examples/10.yml b/backend/reconcile/test/examples/10.yml new file mode 100644 index 00000000..5b2183dc --- /dev/null +++ b/backend/reconcile/test/examples/10.yml @@ -0,0 +1,5 @@ +parent: marketplace +left: market| place +right: market|space +expected: market| placemarket|space + \ No newline at end of file diff --git a/backend/reconcile/test/examples/11.yml b/backend/reconcile/test/examples/11.yml new file mode 100644 index 00000000..549717de --- /dev/null +++ b/backend/reconcile/test/examples/11.yml @@ -0,0 +1,4 @@ +parent: Please remember to bring your laptop and charger +left: Please remember to bring your laptop| +right: Please remember to bring your |new |laptop and charger +expected: Please remember to bring your |new |laptop| \ No newline at end of file diff --git a/backend/reconcile/test/examples/12.yml b/backend/reconcile/test/examples/12.yml new file mode 100644 index 00000000..879b398f --- /dev/null +++ b/backend/reconcile/test/examples/12.yml @@ -0,0 +1,4 @@ +parent: Party A shall pay Party B +left: Party C shall pay Party B +right: Party A shall receive from Party B +expected: Party C shall receive from Party B diff --git a/backend/reconcile/test/examples/13.yml b/backend/reconcile/test/examples/13.yml new file mode 100644 index 00000000..e12c635b --- /dev/null +++ b/backend/reconcile/test/examples/13.yml @@ -0,0 +1,4 @@ +parent: Please submit your assignment by Friday +left: Please submit your |completed |assignment by Friday +right: Please submit your assignment |online |by Friday +expected: Please submit your |completed |assignment |online |by Friday diff --git a/backend/reconcile/test/examples/2.yml b/backend/reconcile/test/examples/2.yml new file mode 100644 index 00000000..77a03755 --- /dev/null +++ b/backend/reconcile/test/examples/2.yml @@ -0,0 +1,4 @@ +parent: +left: hi my friend| +right: hi there| +expected: hi my friend| there| diff --git a/backend/reconcile/test/examples/3.yml b/backend/reconcile/test/examples/3.yml new file mode 100644 index 00000000..8e2dd222 --- /dev/null +++ b/backend/reconcile/test/examples/3.yml @@ -0,0 +1,4 @@ +parent: Buy milk and eggs +left: Buy organic milk| and eggs| +right: Buy milk and eggs| and bread +expected: Buy organic milk| and eggs|| and bread diff --git a/backend/reconcile/test/examples/4.yml b/backend/reconcile/test/examples/4.yml new file mode 100644 index 00000000..f06d3287 --- /dev/null +++ b/backend/reconcile/test/examples/4.yml @@ -0,0 +1,4 @@ +parent: Meeting at 2pm in 会议室 +left: Meeting at |3pm in the 会议室 +right: Team meeting at 2pm in conference room| +expected: Team meeting at |3pm in conference room| the diff --git a/backend/reconcile/test/examples/5.yml b/backend/reconcile/test/examples/5.yml new file mode 100644 index 00000000..b1f3f45d --- /dev/null +++ b/backend/reconcile/test/examples/5.yml @@ -0,0 +1,4 @@ +parent: Send the report to the team +left: Send the |detailed |report to the |entire |team +right: Send the |quarterly |detailed |report to the team +expected: Send the |detailed |quarterly |detailed ||report to the |entire |team \ No newline at end of file diff --git a/backend/reconcile/test/examples/6.yml b/backend/reconcile/test/examples/6.yml new file mode 100644 index 00000000..16d25fb2 --- /dev/null +++ b/backend/reconcile/test/examples/6.yml @@ -0,0 +1,4 @@ +parent: Ready, Set go +left: Ready! Set go| +right: Ready, Set, go!| +expected: Ready! Set, go!|| diff --git a/backend/reconcile/test/examples/7.yml b/backend/reconcile/test/examples/7.yml new file mode 100644 index 00000000..579e9271 --- /dev/null +++ b/backend/reconcile/test/examples/7.yml @@ -0,0 +1,4 @@ +parent: "Total: $100" +left: "Total: |$150" +right: "Total: |€100" +expected: "Total: |$150 |€100" diff --git a/backend/reconcile/test/examples/8.yml b/backend/reconcile/test/examples/8.yml new file mode 100644 index 00000000..6c316ef6 --- /dev/null +++ b/backend/reconcile/test/examples/8.yml @@ -0,0 +1,4 @@ +parent: Start middle end +left: Start [important] middle end| +right: Start middle [critical] end| +expected: Start [important] middle [critical] end|| diff --git a/backend/reconcile/test/examples/9.yml b/backend/reconcile/test/examples/9.yml new file mode 100644 index 00000000..6f534b76 --- /dev/null +++ b/backend/reconcile/test/examples/9.yml @@ -0,0 +1,4 @@ +parent: A B C D +left: A X B D| +right: A B Y| +expected: A X B Y|| diff --git a/backend/reconcile/tests/example_document.rs b/backend/reconcile/tests/example_document.rs new file mode 100644 index 00000000..6bf845be --- /dev/null +++ b/backend/reconcile/tests/example_document.rs @@ -0,0 +1,75 @@ +use std::{fs, path::Path}; + +use pretty_assertions::assert_eq; +use reconcile::{CursorPosition, TextWithCursors, reconcile_with_cursors}; +use serde::Deserialize; + +/// ExampleDocument represents a test case for the reconciliation process. +/// It contains a parent string, left and right strings with cursor positions, +/// and the expected result after reconciliation. +/// +/// '|' characters in the left, right, and expected strings are treated as +/// cursor positions and are converted into `CursorPosition` objects. +#[derive(Debug, Deserialize, Clone, PartialEq, Eq)] +pub struct ExampleDocument { + parent: String, + left: String, + right: String, + expected: String, +} + +impl ExampleDocument { + pub fn from_yaml(path: &Path) -> Self { + let file = fs::File::open(path).expect("Failed to open example file"); + serde_yaml::from_reader(file).expect("Failed to parse example file") + } + + pub fn parent(&self) -> String { self.parent.clone() } + + pub fn left(&self) -> TextWithCursors<'static> { + ExampleDocument::string_to_text_with_cursors(&self.left) + } + + pub fn right(&self) -> TextWithCursors<'static> { + ExampleDocument::string_to_text_with_cursors(&self.right) + } + + pub fn assert_eq(&self, result: TextWithCursors<'static>) { + let result_str = ExampleDocument::text_with_cursors_to_string(&result); + assert_eq!(result_str, self.expected); + } + + pub fn assert_eq_without_cursors(&self, result: &str) { + assert_eq!( + result, + ExampleDocument::string_to_text_with_cursors(&self.expected).text, + ); + } + + fn text_with_cursors_to_string<'a>(text: &TextWithCursors<'a>) -> String { + let mut result = text.text.clone().into_owned(); + for (i, cursor) in text.cursors.iter().enumerate() { + result.insert(cursor.char_index + i, '|'); + } + result + } + + fn string_to_text_with_cursors(text: &str) -> TextWithCursors<'static> { + let cursors = Self::parse_cursors(&text); + let text = text.replace("|", ""); + TextWithCursors::new_owned(text, cursors) + } + + fn parse_cursors(text: &str) -> Vec { + let mut cursors = Vec::new(); + for (i, c) in text.chars().enumerate() { + if c == '|' { + cursors.push(CursorPosition { + id: 0, + char_index: i - cursors.len(), + }); + } + } + cursors + } +} diff --git a/backend/reconcile/tests/test.rs b/backend/reconcile/tests/test.rs new file mode 100644 index 00000000..4b8382d0 --- /dev/null +++ b/backend/reconcile/tests/test.rs @@ -0,0 +1,47 @@ +mod example_document; +use std::{fs, path::Path}; + +use example_document::ExampleDocument; +use reconcile::{CursorPosition, TextWithCursors, reconcile, reconcile_with_cursors}; +use serde::Deserialize; + +#[test] +fn test_with_examples() { + let examples_dir = Path::new("test/examples"); + let mut entries = fs::read_dir(examples_dir) + .expect("Failed to read examples directory") + .collect::>(); + + entries.sort_by_key(|entry| { + let path = entry + .as_ref() + .expect("Failed to read directory entry") + .path(); + path.file_name() + .and_then(|name| name.to_str()) + .and_then(|name| name.split(".").next().unwrap().parse::().ok()) + .unwrap_or_default() + }); + + for entry in entries { + let entry = entry.expect("Failed to read directory entry"); + let path = entry.path(); + + if path.is_file() && path.extension().and_then(|ext| ext.to_str()) == Some("yml") { + let doc = ExampleDocument::from_yaml(&path); + println!("Testing with example from {}", path.display()); + + doc.assert_eq_without_cursors(&reconcile( + &doc.parent(), + &doc.left().text, + &doc.right().text, + )); + + doc.assert_eq(reconcile_with_cursors( + &doc.parent(), + doc.left(), + doc.right(), + )); + } + } +} -- 2.47.2 From 3744b0a633c9b53c20de9710de68d3bf38ad5b8e Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Tue, 1 Apr 2025 22:41:04 +0100 Subject: [PATCH 07/21] Add apply_merge_context to cursor --- backend/reconcile/src/lib.rs | 5 ++--- .../{ => operation_transformation}/cursor.rs | 20 +++++++++++++++++++ 2 files changed, 22 insertions(+), 3 deletions(-) rename backend/reconcile/src/{ => operation_transformation}/cursor.rs (67%) diff --git a/backend/reconcile/src/lib.rs b/backend/reconcile/src/lib.rs index 8023cc15..a04ae853 100644 --- a/backend/reconcile/src/lib.rs +++ b/backend/reconcile/src/lib.rs @@ -1,11 +1,10 @@ -mod cursor; mod diffs; mod operation_transformation; mod tokenizer; mod utils; -pub use cursor::{CursorPosition, TextWithCursors}; pub use operation_transformation::{ - EditedText, reconcile, reconcile_with_cursors, reconcile_with_tokenizer, + CursorPosition, EditedText, TextWithCursors, reconcile, reconcile_with_cursors, + reconcile_with_tokenizer, }; pub use tokenizer::{Tokenizer, token::Token}; diff --git a/backend/reconcile/src/cursor.rs b/backend/reconcile/src/operation_transformation/cursor.rs similarity index 67% rename from backend/reconcile/src/cursor.rs rename to backend/reconcile/src/operation_transformation/cursor.rs index 7ec9402c..60abff22 100644 --- a/backend/reconcile/src/cursor.rs +++ b/backend/reconcile/src/operation_transformation/cursor.rs @@ -3,6 +3,9 @@ use std::borrow::Cow; #[cfg(feature = "serde")] use serde::{Deserialize, Serialize}; +use super::merge_context::MergeContext; +use crate::operation_transformation::Operation; + // CursorPosition is a wrapper around usize to represent the position of an // identifiable cursor in a text document based on the character index. #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] @@ -12,6 +15,23 @@ pub struct CursorPosition { pub char_index: usize, } +impl CursorPosition { + pub fn apply_merge_context(&self, context: &MergeContext) -> Self + where + T: PartialEq + Clone + std::fmt::Debug, + { + let char_index = match context.last_operation() { + Some(Operation::Delete { index, .. }) => (*index) as i64, + _ => self.char_index as i64 + context.shift, + }; + + CursorPosition { + id: self.id, + char_index: char_index.max(0) as usize, + } + } +} + #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[derive(Debug, Clone, PartialEq, Default)] pub struct TextWithCursors<'a> { -- 2.47.2 From 168fb44b0744f1f0fc696e669d06b5dac2a1f98e Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Tue, 1 Apr 2025 22:42:14 +0100 Subject: [PATCH 08/21] Fix double document creation on first sync --- frontend/sync-client/src/sync-operations/syncer.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/frontend/sync-client/src/sync-operations/syncer.ts b/frontend/sync-client/src/sync-operations/syncer.ts index 8ec83694..ec6b2288 100644 --- a/frontend/sync-client/src/sync-operations/syncer.ts +++ b/frontend/sync-client/src/sync-operations/syncer.ts @@ -479,7 +479,10 @@ export class Syncer { .filter( (remoteDocument) => allLocalFiles.includes(remoteDocument.relativePath) && - !remoteDocument.isDeleted + !remoteDocument.isDeleted && + this.database.getDocumentByDocumentId( + remoteDocument.documentId + ) === undefined ) .forEach((remoteDocument) => { this.database.createNewEmptyDocument( -- 2.47.2 From 688fd2f1b13ef3d459ec798f3146d9720397dbf7 Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Tue, 1 Apr 2025 22:45:26 +0100 Subject: [PATCH 09/21] Update API and add cursor position tests --- .../reconcile/src/operation_transformation.rs | 225 ++++++++++++++++-- 1 file changed, 205 insertions(+), 20 deletions(-) diff --git a/backend/reconcile/src/operation_transformation.rs b/backend/reconcile/src/operation_transformation.rs index a71bc65a..0674a254 100644 --- a/backend/reconcile/src/operation_transformation.rs +++ b/backend/reconcile/src/operation_transformation.rs @@ -1,41 +1,42 @@ +mod cursor; mod edited_text; mod merge_context; mod operation; +pub use cursor::{CursorPosition, TextWithCursors}; pub use edited_text::EditedText; pub use operation::Operation; -use crate::tokenizer::Tokenizer; +use crate::Tokenizer; #[must_use] pub fn reconcile(original: &str, left: &str, right: &str) -> String { - // Common trivial cases - if left == right { - return left.to_owned(); - } + reconcile_with_cursors(original, left.into(), right.into()) + .text + .to_string() +} - if original == left { - return right.to_owned(); - } - - if original == right { - return left.to_owned(); - } - - // 3-way merge +#[must_use] +pub fn reconcile_with_cursors<'a>( + original: &'a str, + left: TextWithCursors<'a>, + right: TextWithCursors<'a>, +) -> TextWithCursors<'static> { let left_operations = EditedText::from_strings(original, left); let right_operations = EditedText::from_strings(original, right); let merged_operations = left_operations.merge(right_operations); - merged_operations.apply() + + TextWithCursors::new_owned(merged_operations.apply(), merged_operations.cursors) } -pub fn reconcile_with_tokenizer( +#[must_use] +pub fn reconcile_with_tokenizer<'a, F, T>( original: &str, - left: &str, - right: &str, + left: TextWithCursors<'a>, + right: TextWithCursors<'a>, tokenizer: &Tokenizer, -) -> String +) -> TextWithCursors<'static> where T: PartialEq + Clone + std::fmt::Debug, { @@ -43,7 +44,8 @@ where let right_operations = EditedText::from_strings_with_tokenizer(original, right, tokenizer); let merged_operations = left_operations.merge(right_operations); - merged_operations.apply() + + TextWithCursors::new_owned(merged_operations.apply(), merged_operations.cursors) } #[cfg(test)] @@ -54,6 +56,7 @@ mod test { use test_case::test_matrix; use super::*; + use crate::CursorPosition; #[test] fn test_merges() { @@ -172,6 +175,188 @@ mod test { " |7ca2b36d-6ee7-49eb-8eb1-d77e4cc1a001| |cd9195cc-103a-4f13-90c8-4fba0ba421ee| |d39156cc-cfd6-42a8-b70a-75020896069d| |fbad794c-9c47-41f2-a343-490284ecb5a0| |dup| |dup| "); } + #[test] + fn test_cursor_position_no_updates() { + let original = "hello world"; + let left = TextWithCursors::new( + "hello world", + vec![CursorPosition { + id: 0, + char_index: 0, + }], + ); + let right = TextWithCursors::new( + "hello world", + vec![CursorPosition { + id: 1, + char_index: 5, + }], + ); + + let merged = reconcile_with_cursors(original, left, right); + + assert_eq!( + merged, + TextWithCursors::new( + "hello world", + vec![ + CursorPosition { + id: 0, + char_index: 0 + }, + CursorPosition { + id: 1, + char_index: 5 + } + ] + ) + ); + } + + #[test] + fn test_cursor_position_updates_with_inserts() { + let original = "hi"; + let left = TextWithCursors::new( + "hi there", + vec![CursorPosition { + id: 0, + char_index: 7, + }], + ); + let right = TextWithCursors::new( + "hi world!", + vec![ + CursorPosition { + id: 1, + char_index: 9, + }, + CursorPosition { + id: 2, + char_index: 1, + }, + ], + ); + + let merged = reconcile_with_cursors(original, left, right); + + assert_eq!( + merged, + TextWithCursors::new( + "hi there world!", + vec![ + CursorPosition { + id: 2, + char_index: 1, + }, + CursorPosition { + id: 0, + char_index: 7 + }, + CursorPosition { + id: 1, + char_index: 15 + }, + ] + ) + ); + } + + #[test] + fn test_cursor_position_updates_with_deleted() { + let original = "a b c d"; + let left = TextWithCursors::new( + "a b d", + vec![CursorPosition { + id: 0, + char_index: 1, // after a + }], + ); + let right = TextWithCursors::new( + "c d", + vec![CursorPosition { + id: 1, + char_index: 1, // after c + }], + ); + + let merged = reconcile_with_cursors(original, left, right); + + assert_eq!( + merged, + TextWithCursors::new( + " d", + vec![ + CursorPosition { + id: 0, + char_index: 0 + }, + CursorPosition { + id: 1, + char_index: 0 + } + ] + ) + ); + } + + #[test] + fn test_cursor_complex() { + let original = "this is some complex text to test cursor positions"; + let left = TextWithCursors::new( + "this is really complex text for testing cursor positions", + vec![ + CursorPosition { + id: 0, + char_index: 8, + }, // after "this is " + CursorPosition { + id: 1, + char_index: 22, + }, // after "this is really complex text" + ], + ); + let right = TextWithCursors::new( + "that was some complex sample to test cursor movements", + vec![ + CursorPosition { + id: 2, + char_index: 5, + }, // after "that " + CursorPosition { + id: 3, + char_index: 29, + }, // after "some complex sample " + ], + ); + + let merged = reconcile_with_cursors(original, left, right); + + assert_eq!( + merged, + TextWithCursors::new( + "that was really complex sample for testing cursor movements", + vec![ + CursorPosition { + id: 0, + char_index: 9 + }, // before "really" + CursorPosition { + id: 1, + char_index: 25 + }, // inside of "s|ample" because "text" got replaced by "sample" + CursorPosition { + id: 2, + char_index: 5 + }, // unchanged + CursorPosition { + id: 3, + char_index: 31 + }, // before "for" + ] + ) + ); + } + #[test_matrix( [ "pride_and_prejudice.txt", "romeo_and_juliet.txt", -- 2.47.2 From 941100d7151ec9caaf0a79fcf6657f51e518600f Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Tue, 1 Apr 2025 22:46:03 +0100 Subject: [PATCH 10/21] Lint --- .../src/operation_transformation/cursor.rs | 3 ++ backend/reconcile/tests/example_document.rs | 37 +++++++++++++++---- backend/reconcile/tests/test.rs | 5 +-- backend/sync_lib/src/cursor.rs | 4 +- 4 files changed, 37 insertions(+), 12 deletions(-) diff --git a/backend/reconcile/src/operation_transformation/cursor.rs b/backend/reconcile/src/operation_transformation/cursor.rs index 60abff22..0f2ee509 100644 --- a/backend/reconcile/src/operation_transformation/cursor.rs +++ b/backend/reconcile/src/operation_transformation/cursor.rs @@ -16,6 +16,7 @@ pub struct CursorPosition { } impl CursorPosition { + #[must_use] pub fn apply_merge_context(&self, context: &MergeContext) -> Self where T: PartialEq + Clone + std::fmt::Debug, @@ -40,6 +41,7 @@ pub struct TextWithCursors<'a> { } impl<'a> TextWithCursors<'a> { + #[must_use] pub fn new(text: &'a str, cursors: Vec) -> Self { Self { text: text.into(), @@ -47,6 +49,7 @@ impl<'a> TextWithCursors<'a> { } } + #[must_use] pub fn new_owned(text: String, cursors: Vec) -> Self { Self { text: text.into(), diff --git a/backend/reconcile/tests/example_document.rs b/backend/reconcile/tests/example_document.rs index 6bf845be..75f5bdab 100644 --- a/backend/reconcile/tests/example_document.rs +++ b/backend/reconcile/tests/example_document.rs @@ -1,10 +1,10 @@ use std::{fs, path::Path}; use pretty_assertions::assert_eq; -use reconcile::{CursorPosition, TextWithCursors, reconcile_with_cursors}; +use reconcile::{CursorPosition, TextWithCursors}; use serde::Deserialize; -/// ExampleDocument represents a test case for the reconciliation process. +/// `ExampleDocument` represents a test case for the reconciliation process. /// It contains a parent string, left and right strings with cursor positions, /// and the expected result after reconciliation. /// @@ -19,26 +19,49 @@ pub struct ExampleDocument { } impl ExampleDocument { + /// Creates a new `ExampleDocument` instance from a YAML file. + /// + /// # Panics + /// + /// If the file cannot be opened or parsed, the program will panic. + #[must_use] pub fn from_yaml(path: &Path) -> Self { let file = fs::File::open(path).expect("Failed to open example file"); serde_yaml::from_reader(file).expect("Failed to parse example file") } + #[must_use] pub fn parent(&self) -> String { self.parent.clone() } + #[must_use] pub fn left(&self) -> TextWithCursors<'static> { ExampleDocument::string_to_text_with_cursors(&self.left) } + #[must_use] pub fn right(&self) -> TextWithCursors<'static> { ExampleDocument::string_to_text_with_cursors(&self.right) } - pub fn assert_eq(&self, result: TextWithCursors<'static>) { - let result_str = ExampleDocument::text_with_cursors_to_string(&result); + /// Asserts that the result string matches the expected string, + /// including cursor positions. + /// + /// # Panics + /// + /// If the result string does not match the expected string, the program + /// will panic. + pub fn assert_eq(&self, result: &TextWithCursors<'static>) { + let result_str = ExampleDocument::text_with_cursors_to_string(result); assert_eq!(result_str, self.expected); } + /// Asserts that the result string matches the expected string, + /// ignoring cursor positions. + /// + /// # Panics + /// + /// If the result string does not match the expected string, the program + /// will panic. pub fn assert_eq_without_cursors(&self, result: &str) { assert_eq!( result, @@ -46,7 +69,7 @@ impl ExampleDocument { ); } - fn text_with_cursors_to_string<'a>(text: &TextWithCursors<'a>) -> String { + fn text_with_cursors_to_string(text: &TextWithCursors<'_>) -> String { let mut result = text.text.clone().into_owned(); for (i, cursor) in text.cursors.iter().enumerate() { result.insert(cursor.char_index + i, '|'); @@ -55,8 +78,8 @@ impl ExampleDocument { } fn string_to_text_with_cursors(text: &str) -> TextWithCursors<'static> { - let cursors = Self::parse_cursors(&text); - let text = text.replace("|", ""); + let cursors = Self::parse_cursors(text); + let text = text.replace('|', ""); TextWithCursors::new_owned(text, cursors) } diff --git a/backend/reconcile/tests/test.rs b/backend/reconcile/tests/test.rs index 4b8382d0..6c7b3ec7 100644 --- a/backend/reconcile/tests/test.rs +++ b/backend/reconcile/tests/test.rs @@ -2,8 +2,7 @@ mod example_document; use std::{fs, path::Path}; use example_document::ExampleDocument; -use reconcile::{CursorPosition, TextWithCursors, reconcile, reconcile_with_cursors}; -use serde::Deserialize; +use reconcile::{reconcile, reconcile_with_cursors}; #[test] fn test_with_examples() { @@ -19,7 +18,7 @@ fn test_with_examples() { .path(); path.file_name() .and_then(|name| name.to_str()) - .and_then(|name| name.split(".").next().unwrap().parse::().ok()) + .and_then(|name| name.split('.').next().unwrap().parse::().ok()) .unwrap_or_default() }); diff --git a/backend/sync_lib/src/cursor.rs b/backend/sync_lib/src/cursor.rs index 4b5b9a6d..6b61a078 100644 --- a/backend/sync_lib/src/cursor.rs +++ b/backend/sync_lib/src/cursor.rs @@ -25,7 +25,7 @@ impl From for TextWithCursors<'_> { owned .cursors .into_iter() - .map(|cursor| cursor.into()) + .map(std::convert::Into::into) .collect(), ) } @@ -38,7 +38,7 @@ impl From> for OwnedTextWithCursors { cursors: text_with_cursors .cursors .into_iter() - .map(|cursor| cursor.into()) + .map(std::convert::Into::into) .collect(), } } -- 2.47.2 From 680486c4b51d427fddd57a67d1addb5e7bae4c60 Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Tue, 1 Apr 2025 22:49:02 +0100 Subject: [PATCH 11/21] Implement cursor merging --- .../operation_transformation/edited_text.rs | 176 ++++++++++++------ .../src/operation_transformation/operation.rs | 2 +- backend/reconcile/tests/test.rs | 2 +- 3 files changed, 126 insertions(+), 54 deletions(-) diff --git a/backend/reconcile/src/operation_transformation/edited_text.rs b/backend/reconcile/src/operation_transformation/edited_text.rs index 8a7013e4..8fc2ed96 100644 --- a/backend/reconcile/src/operation_transformation/edited_text.rs +++ b/backend/reconcile/src/operation_transformation/edited_text.rs @@ -3,7 +3,7 @@ use core::iter; #[cfg(feature = "serde")] use serde::{Deserialize, Serialize}; -use super::Operation; +use super::{CursorPosition, Operation, TextWithCursors}; use crate::{ diffs::{myers::diff, raw_operation::RawOperation}, operation_transformation::merge_context::MergeContext, @@ -29,6 +29,7 @@ where { text: &'a str, operations: Vec>, + pub(crate) cursors: Vec, } impl<'a> EditedText<'a, String> { @@ -39,7 +40,7 @@ impl<'a> EditedText<'a, String> { /// word tokenizer is used to tokenize the text which splits the text on /// whitespaces. #[must_use] - pub fn from_strings(original: &'a str, updated: &str) -> Self { + pub fn from_strings(original: &'a str, updated: TextWithCursors<'a>) -> Self { Self::from_strings_with_tokenizer(original, updated, &word_tokenizer) } } @@ -55,17 +56,18 @@ where /// function is used to tokenize the text. pub fn from_strings_with_tokenizer( original: &'a str, - updated: &str, + updated: TextWithCursors<'a>, tokenizer: &Tokenizer, ) -> Self { let original_tokens = (tokenizer)(original); - let updated_tokens = (tokenizer)(updated); + let updated_tokens = (tokenizer)(&updated.text); let diff: Vec> = diff(&original_tokens, &updated_tokens); Self::new( original, Self::cook_operations(Self::elongate_operations(diff)).collect(), + updated.cursors, ) } @@ -170,7 +172,11 @@ where /// Create a new `EditedText` with the given operations. /// The operations must be in the order in which they are meant to be /// applied. The operations must not overlap. - fn new(text: &'a str, operations: Vec>) -> Self { + fn new( + text: &'a str, + operations: Vec>, + mut cursors: Vec, + ) -> Self { operations .iter() .zip(operations.iter().skip(1)) @@ -183,7 +189,13 @@ where ); }); - Self { text, operations } + cursors.sort_by_key(|cursor| cursor.char_index); + + Self { + text, + operations, + cursors, + } } #[must_use] @@ -196,50 +208,110 @@ where let mut left_merge_context = MergeContext::default(); let mut right_merge_context = MergeContext::default(); - Self::new( - self.text, - self.operations - .into_iter() - .map(|op| (op, Side::Left)) - .merge_sorted_by_key( - other.operations.into_iter().map(|op| (op, Side::Right)), - |(operation, _)| { - ( - operation.order, - // Operations on the left and right must come in the same order so that - // inserts can be merged with other inserts and deletes with deletes. - usize::from(matches!(operation.operation, Operation::Delete { .. })), - // Make sure that the ordering is deterministic regardless which text - // is left or right. - match &operation.operation { - Operation::Insert { text, .. } => text - .iter() - .map(super::super::tokenizer::token::Token::original) - .collect::(), - Operation::Delete { - deleted_character_count, - .. - } => deleted_character_count.to_string(), - }, + let mut merged_cursors = Vec::with_capacity(self.cursors.len() + other.cursors.len()); + let mut left_cursors = self.cursors.iter().peekable(); + let mut right_cursors = other.cursors.iter().peekable(); + + let merged_operations = self + .operations + .into_iter() + // The current text is always the left; the other operation is the right side. + .map(|op| (op, Side::Left)) + .merge_sorted_by_key( + other.operations.into_iter().map(|op| (op, Side::Right)), + |(operation, _)| { + ( + operation.order, + // Operations on the left and right must come in the same order so that + // inserts can be merged with other inserts and deletes with deletes. + usize::from(matches!(operation.operation, Operation::Delete { .. })), + // Make sure that the ordering is deterministic regardless which text + // is left or right. + match &operation.operation { + Operation::Insert { text, .. } => text + .iter() + .map(super::super::tokenizer::token::Token::original) + .collect::(), + Operation::Delete { + deleted_character_count, + .. + } => deleted_character_count.to_string(), + }, + ) + }, + ) + .flat_map(|(OrderedOperation { order, operation }, side)| { + match side { + Side::Left => { + while let Some(cursor) = left_cursors + .next_if(|cursor| cursor.char_index <= operation.start_index()) + { + right_merge_context.consume_last_operation_if_it_is_too_behind( + cursor.char_index as i64, + ); + merged_cursors.push(cursor.apply_merge_context(&right_merge_context)); + } + + while let Some(cursor) = right_cursors.next_if(|cursor| { + cursor.char_index as i64 + <= operation.start_index() as i64 + right_merge_context.shift + - left_merge_context.shift + }) { + left_merge_context.consume_last_operation_if_it_is_too_behind( + cursor.char_index as i64, + ); + merged_cursors.push(cursor.apply_merge_context(&left_merge_context)); + } + + operation.merge_operations_with_context( + &mut right_merge_context, + &mut left_merge_context, ) - }, - ) - .flat_map(|(OrderedOperation { order, operation }, side)| { - match side { - Side::Left => operation.merge_operations_with_context( - &mut right_merge_context, - &mut left_merge_context, - ), - Side::Right => operation.merge_operations_with_context( - &mut left_merge_context, - &mut right_merge_context, - ), } - .map(|operation| OrderedOperation { order, operation }) - .into_iter() - }) - .collect(), - ) + Side::Right => { + while let Some(cursor) = right_cursors + .next_if(|cursor| cursor.char_index <= operation.start_index()) + { + left_merge_context.consume_last_operation_if_it_is_too_behind( + cursor.char_index as i64, + ); + merged_cursors.push(cursor.apply_merge_context(&left_merge_context)); + } + + while let Some(cursor) = left_cursors.next_if(|cursor| { + cursor.char_index as i64 + <= operation.start_index() as i64 + left_merge_context.shift + - right_merge_context.shift + }) { + right_merge_context.consume_last_operation_if_it_is_too_behind( + cursor.char_index as i64, + ); + merged_cursors.push(cursor.apply_merge_context(&right_merge_context)); + } + + operation.merge_operations_with_context( + &mut left_merge_context, + &mut right_merge_context, + ) + } + } + .map(|operation| OrderedOperation { order, operation }) + .into_iter() + }) + .collect(); + + for cursor in left_cursors { + right_merge_context + .consume_last_operation_if_it_is_too_behind(cursor.char_index as i64); + merged_cursors.push(cursor.apply_merge_context(&right_merge_context)); + } + + for cursor in right_cursors { + left_merge_context.consume_last_operation_if_it_is_too_behind(cursor.char_index as i64); + merged_cursors.push(cursor.apply_merge_context(&left_merge_context)); + } + + Self::new(self.text, merged_operations, merged_cursors) } /// Apply the operations to the text and return the resulting text. @@ -268,7 +340,7 @@ mod tests { let left = "hello world! How are you? Adam"; let right = "Hello, my friend! How are you doing? Albert"; - let operations = EditedText::from_strings(left, right); + let operations = EditedText::from_strings(left, right.into()); insta::assert_debug_snapshot!(operations); @@ -280,7 +352,7 @@ mod tests { fn test_calculate_operations_with_no_diff() { let text = "hello world!"; - let operations = EditedText::from_strings(text, text); + let operations = EditedText::from_strings(text, text.into()); assert_eq!(operations.operations.len(), 0); @@ -296,8 +368,8 @@ mod tests { let right = "Hello world! How are you?"; let expected = "Hello world! How are you? I'm Andras."; - let operations_1 = EditedText::from_strings(original, left); - let operations_2 = EditedText::from_strings(original, right); + let operations_1 = EditedText::from_strings(original, left.into()); + let operations_2 = EditedText::from_strings(original, right.into()); let operations = operations_1.merge(operations_2); assert_eq!(operations.apply(), expected); diff --git a/backend/reconcile/src/operation_transformation/operation.rs b/backend/reconcile/src/operation_transformation/operation.rs index d0d285b0..68eab6ae 100644 --- a/backend/reconcile/src/operation_transformation/operation.rs +++ b/backend/reconcile/src/operation_transformation/operation.rs @@ -189,7 +189,7 @@ where affecting_context: &mut MergeContext, produced_context: &mut MergeContext, ) -> Option> { - affecting_context.consume_last_operation_if_it_is_too_behind(&self); + affecting_context.consume_last_operation_if_it_is_too_behind(self.start_index() as i64); let operation = self.with_shifted_index(affecting_context.shift); match (operation, affecting_context.last_operation()) { diff --git a/backend/reconcile/tests/test.rs b/backend/reconcile/tests/test.rs index 6c7b3ec7..3c677ea2 100644 --- a/backend/reconcile/tests/test.rs +++ b/backend/reconcile/tests/test.rs @@ -36,7 +36,7 @@ fn test_with_examples() { &doc.right().text, )); - doc.assert_eq(reconcile_with_cursors( + doc.assert_eq(&reconcile_with_cursors( &doc.parent(), doc.left(), doc.right(), -- 2.47.2 From 89b87efa599acb62a9e91bd49ffd56a7ea9dad83 Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Tue, 1 Apr 2025 22:50:55 +0100 Subject: [PATCH 12/21] Fix tests --- backend/reconcile/src/operation_transformation.rs | 12 ++++++------ ...on__edited_text__tests__calculate_operations.snap | 1 + 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/backend/reconcile/src/operation_transformation.rs b/backend/reconcile/src/operation_transformation.rs index 0674a254..4338ca6e 100644 --- a/backend/reconcile/src/operation_transformation.rs +++ b/backend/reconcile/src/operation_transformation.rs @@ -292,7 +292,7 @@ mod test { }, CursorPosition { id: 1, - char_index: 0 + char_index: 1 } ] ) @@ -336,18 +336,18 @@ mod test { TextWithCursors::new( "that was really complex sample for testing cursor movements", vec![ + CursorPosition { + id: 2, + char_index: 5 + }, // unchanged CursorPosition { id: 0, char_index: 9 }, // before "really" CursorPosition { id: 1, - char_index: 25 + char_index: 23 }, // inside of "s|ample" because "text" got replaced by "sample" - CursorPosition { - id: 2, - char_index: 5 - }, // unchanged CursorPosition { id: 3, char_index: 31 diff --git a/backend/reconcile/src/operation_transformation/snapshots/reconcile__operation_transformation__edited_text__tests__calculate_operations.snap b/backend/reconcile/src/operation_transformation/snapshots/reconcile__operation_transformation__edited_text__tests__calculate_operations.snap index 0630f986..f08083f4 100644 --- a/backend/reconcile/src/operation_transformation/snapshots/reconcile__operation_transformation__edited_text__tests__calculate_operations.snap +++ b/backend/reconcile/src/operation_transformation/snapshots/reconcile__operation_transformation__edited_text__tests__calculate_operations.snap @@ -23,4 +23,5 @@ EditedText { operation: , }, ], + cursors: [], } -- 2.47.2 From 38a8e6b77452f6d781d140d7cfd7259a061398dd Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Tue, 1 Apr 2025 22:53:50 +0100 Subject: [PATCH 13/21] Fix web tests --- backend/sync_lib/tests/web.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/sync_lib/tests/web.rs b/backend/sync_lib/tests/web.rs index 017e2dc2..0ce76b39 100644 --- a/backend/sync_lib/tests/web.rs +++ b/backend/sync_lib/tests/web.rs @@ -77,7 +77,7 @@ fn test_merge_text_with_cursors() { }, OwnedCursorPosition { id: 1, - char_index: 8, + char_index: 2, } ] ), -- 2.47.2 From 5ed7e0ad3a60f80fa7e1ea8fa2f1bfc2bdc5e347 Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Wed, 2 Apr 2025 20:37:36 +0100 Subject: [PATCH 14/21] Move tests --- backend/reconcile/src/operation_transformation.rs | 2 +- backend/reconcile/{test => tests}/examples/1.yml | 0 backend/reconcile/{test => tests}/examples/10.yml | 0 backend/reconcile/{test => tests}/examples/11.yml | 0 backend/reconcile/{test => tests}/examples/12.yml | 0 backend/reconcile/{test => tests}/examples/13.yml | 0 backend/reconcile/{test => tests}/examples/2.yml | 0 backend/reconcile/{test => tests}/examples/3.yml | 0 backend/reconcile/{test => tests}/examples/4.yml | 0 backend/reconcile/{test => tests}/examples/5.yml | 0 backend/reconcile/{test => tests}/examples/6.yml | 0 backend/reconcile/{test => tests}/examples/7.yml | 0 backend/reconcile/{test => tests}/examples/8.yml | 0 backend/reconcile/{test => tests}/examples/9.yml | 0 backend/reconcile/{test => tests}/resources/kun_lu.txt | 0 .../reconcile/{test => tests}/resources/pride_and_prejudice.txt | 0 .../reconcile/{test => tests}/resources/romeo_and_juliet.txt | 0 .../reconcile/{test => tests}/resources/room_with_a_view.txt | 0 backend/reconcile/tests/test.rs | 2 +- 19 files changed, 2 insertions(+), 2 deletions(-) rename backend/reconcile/{test => tests}/examples/1.yml (100%) rename backend/reconcile/{test => tests}/examples/10.yml (100%) rename backend/reconcile/{test => tests}/examples/11.yml (100%) rename backend/reconcile/{test => tests}/examples/12.yml (100%) rename backend/reconcile/{test => tests}/examples/13.yml (100%) rename backend/reconcile/{test => tests}/examples/2.yml (100%) rename backend/reconcile/{test => tests}/examples/3.yml (100%) rename backend/reconcile/{test => tests}/examples/4.yml (100%) rename backend/reconcile/{test => tests}/examples/5.yml (100%) rename backend/reconcile/{test => tests}/examples/6.yml (100%) rename backend/reconcile/{test => tests}/examples/7.yml (100%) rename backend/reconcile/{test => tests}/examples/8.yml (100%) rename backend/reconcile/{test => tests}/examples/9.yml (100%) rename backend/reconcile/{test => tests}/resources/kun_lu.txt (100%) rename backend/reconcile/{test => tests}/resources/pride_and_prejudice.txt (100%) rename backend/reconcile/{test => tests}/resources/romeo_and_juliet.txt (100%) rename backend/reconcile/{test => tests}/resources/room_with_a_view.txt (100%) diff --git a/backend/reconcile/src/operation_transformation.rs b/backend/reconcile/src/operation_transformation.rs index 4338ca6e..8c95d397 100644 --- a/backend/reconcile/src/operation_transformation.rs +++ b/backend/reconcile/src/operation_transformation.rs @@ -385,7 +385,7 @@ mod test { let files = [file_name_1, file_name_2, file_name_3]; let permutations = [range_1, range_2, range_3]; - let root = Path::new("test/resources/"); + let root = Path::new("tests/resources/"); let contents = files .iter() diff --git a/backend/reconcile/test/examples/1.yml b/backend/reconcile/tests/examples/1.yml similarity index 100% rename from backend/reconcile/test/examples/1.yml rename to backend/reconcile/tests/examples/1.yml diff --git a/backend/reconcile/test/examples/10.yml b/backend/reconcile/tests/examples/10.yml similarity index 100% rename from backend/reconcile/test/examples/10.yml rename to backend/reconcile/tests/examples/10.yml diff --git a/backend/reconcile/test/examples/11.yml b/backend/reconcile/tests/examples/11.yml similarity index 100% rename from backend/reconcile/test/examples/11.yml rename to backend/reconcile/tests/examples/11.yml diff --git a/backend/reconcile/test/examples/12.yml b/backend/reconcile/tests/examples/12.yml similarity index 100% rename from backend/reconcile/test/examples/12.yml rename to backend/reconcile/tests/examples/12.yml diff --git a/backend/reconcile/test/examples/13.yml b/backend/reconcile/tests/examples/13.yml similarity index 100% rename from backend/reconcile/test/examples/13.yml rename to backend/reconcile/tests/examples/13.yml diff --git a/backend/reconcile/test/examples/2.yml b/backend/reconcile/tests/examples/2.yml similarity index 100% rename from backend/reconcile/test/examples/2.yml rename to backend/reconcile/tests/examples/2.yml diff --git a/backend/reconcile/test/examples/3.yml b/backend/reconcile/tests/examples/3.yml similarity index 100% rename from backend/reconcile/test/examples/3.yml rename to backend/reconcile/tests/examples/3.yml diff --git a/backend/reconcile/test/examples/4.yml b/backend/reconcile/tests/examples/4.yml similarity index 100% rename from backend/reconcile/test/examples/4.yml rename to backend/reconcile/tests/examples/4.yml diff --git a/backend/reconcile/test/examples/5.yml b/backend/reconcile/tests/examples/5.yml similarity index 100% rename from backend/reconcile/test/examples/5.yml rename to backend/reconcile/tests/examples/5.yml diff --git a/backend/reconcile/test/examples/6.yml b/backend/reconcile/tests/examples/6.yml similarity index 100% rename from backend/reconcile/test/examples/6.yml rename to backend/reconcile/tests/examples/6.yml diff --git a/backend/reconcile/test/examples/7.yml b/backend/reconcile/tests/examples/7.yml similarity index 100% rename from backend/reconcile/test/examples/7.yml rename to backend/reconcile/tests/examples/7.yml diff --git a/backend/reconcile/test/examples/8.yml b/backend/reconcile/tests/examples/8.yml similarity index 100% rename from backend/reconcile/test/examples/8.yml rename to backend/reconcile/tests/examples/8.yml diff --git a/backend/reconcile/test/examples/9.yml b/backend/reconcile/tests/examples/9.yml similarity index 100% rename from backend/reconcile/test/examples/9.yml rename to backend/reconcile/tests/examples/9.yml diff --git a/backend/reconcile/test/resources/kun_lu.txt b/backend/reconcile/tests/resources/kun_lu.txt similarity index 100% rename from backend/reconcile/test/resources/kun_lu.txt rename to backend/reconcile/tests/resources/kun_lu.txt diff --git a/backend/reconcile/test/resources/pride_and_prejudice.txt b/backend/reconcile/tests/resources/pride_and_prejudice.txt similarity index 100% rename from backend/reconcile/test/resources/pride_and_prejudice.txt rename to backend/reconcile/tests/resources/pride_and_prejudice.txt diff --git a/backend/reconcile/test/resources/romeo_and_juliet.txt b/backend/reconcile/tests/resources/romeo_and_juliet.txt similarity index 100% rename from backend/reconcile/test/resources/romeo_and_juliet.txt rename to backend/reconcile/tests/resources/romeo_and_juliet.txt diff --git a/backend/reconcile/test/resources/room_with_a_view.txt b/backend/reconcile/tests/resources/room_with_a_view.txt similarity index 100% rename from backend/reconcile/test/resources/room_with_a_view.txt rename to backend/reconcile/tests/resources/room_with_a_view.txt diff --git a/backend/reconcile/tests/test.rs b/backend/reconcile/tests/test.rs index 3c677ea2..1139dc16 100644 --- a/backend/reconcile/tests/test.rs +++ b/backend/reconcile/tests/test.rs @@ -6,7 +6,7 @@ use reconcile::{reconcile, reconcile_with_cursors}; #[test] fn test_with_examples() { - let examples_dir = Path::new("test/examples"); + let examples_dir = Path::new("tests/examples"); let mut entries = fs::read_dir(examples_dir) .expect("Failed to read examples directory") .collect::>(); -- 2.47.2 From 426e9f07b7efec3dca38c80bac6588d1907debbf Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Wed, 2 Apr 2025 20:37:44 +0100 Subject: [PATCH 15/21] Fix compile error --- backend/sync_lib/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/sync_lib/src/lib.rs b/backend/sync_lib/src/lib.rs index dc35e27f..6383983e 100644 --- a/backend/sync_lib/src/lib.rs +++ b/backend/sync_lib/src/lib.rs @@ -106,7 +106,7 @@ pub fn merge_text(parent: &str, left: &str, right: &str) -> String { } /// WASM wrapper around `reconcile::reconcile_with_cursors` for merging text. -#[wasm_bindgen(js_name = mergeText)] +#[wasm_bindgen(js_name = mergeTextWithCursors)] #[must_use] pub fn merge_text_with_cursors( parent: &str, -- 2.47.2 From 565372b526e0befda566aa15e9d0cde45868f9d7 Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Wed, 2 Apr 2025 21:07:14 +0100 Subject: [PATCH 16/21] Enable tests --- frontend/obsidian-plugin/jest.config.js | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 frontend/obsidian-plugin/jest.config.js diff --git a/frontend/obsidian-plugin/jest.config.js b/frontend/obsidian-plugin/jest.config.js new file mode 100644 index 00000000..8c1027ee --- /dev/null +++ b/frontend/obsidian-plugin/jest.config.js @@ -0,0 +1,3 @@ +module.exports = { + preset: "ts-jest/presets/js-with-babel-esm" +}; -- 2.47.2 From 40323c33ee06bbe5b45b438ccd102abe533d210a Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Wed, 2 Apr 2025 21:07:33 +0100 Subject: [PATCH 17/21] Make JS API usable --- backend/sync_lib/src/cursor.rs | 67 +++++++++++++++++++++------------- backend/sync_lib/src/lib.rs | 8 ++-- backend/sync_lib/tests/web.rs | 34 ++++------------- 3 files changed, 54 insertions(+), 55 deletions(-) diff --git a/backend/sync_lib/src/cursor.rs b/backend/sync_lib/src/cursor.rs index 6b61a078..2f7135eb 100644 --- a/backend/sync_lib/src/cursor.rs +++ b/backend/sync_lib/src/cursor.rs @@ -1,26 +1,29 @@ -use reconcile::{CursorPosition, TextWithCursors}; use wasm_bindgen::prelude::*; /// Wrapper type to expose `TextWithCursors` to JS. #[wasm_bindgen] #[derive(Debug, Clone, PartialEq)] -pub struct OwnedTextWithCursors { +pub struct TextWithCursors { text: String, - cursors: Vec, + cursors: Vec, } -impl OwnedTextWithCursors { - pub fn new(text: impl Into, cursors: Vec) -> Self { - Self { - text: text.into(), - cursors, - } - } +#[wasm_bindgen] +impl TextWithCursors { + #[wasm_bindgen(constructor)] + #[must_use] + pub fn new(text: String, cursors: Vec) -> Self { Self { text, cursors } } + + #[must_use] + pub fn text(&self) -> String { self.text.clone() } + + #[must_use] + pub fn cursors(&self) -> Vec { self.cursors.clone() } } -impl From for TextWithCursors<'_> { - fn from(owned: OwnedTextWithCursors) -> Self { - TextWithCursors::new_owned( +impl From for reconcile::TextWithCursors<'_> { + fn from(owned: TextWithCursors) -> Self { + reconcile::TextWithCursors::new_owned( owned.text.to_string(), owned .cursors @@ -31,9 +34,9 @@ impl From for TextWithCursors<'_> { } } -impl From> for OwnedTextWithCursors { - fn from(text_with_cursors: TextWithCursors<'_>) -> Self { - OwnedTextWithCursors { +impl From> for TextWithCursors { + fn from(text_with_cursors: reconcile::TextWithCursors<'_>) -> Self { + TextWithCursors { text: text_with_cursors.text.into_owned(), cursors: text_with_cursors .cursors @@ -47,23 +50,37 @@ impl From> for OwnedTextWithCursors { /// Wrapper type to expose `CursorPosition` to JS. #[wasm_bindgen] #[derive(Debug, Clone, PartialEq)] -pub struct OwnedCursorPosition { - pub id: usize, - pub char_index: usize, +pub struct CursorPosition { + id: usize, + char_index: usize, } -impl From for CursorPosition { - fn from(owned: OwnedCursorPosition) -> Self { - CursorPosition { +#[wasm_bindgen] +impl CursorPosition { + #[wasm_bindgen(constructor)] + #[must_use] + pub fn new(id: usize, char_index: usize) -> Self { Self { id, char_index } } + + #[must_use] + pub fn id(&self) -> usize { self.id } + + #[wasm_bindgen(js_name = characterPosition)] + #[must_use] + pub fn char_index(&self) -> usize { self.char_index } +} + +impl From for reconcile::CursorPosition { + fn from(owned: CursorPosition) -> Self { + reconcile::CursorPosition { id: owned.id, char_index: owned.char_index, } } } -impl From for OwnedCursorPosition { - fn from(cursor: CursorPosition) -> Self { - OwnedCursorPosition { +impl From for CursorPosition { + fn from(cursor: reconcile::CursorPosition) -> Self { + CursorPosition { id: cursor.id, char_index: cursor.char_index, } diff --git a/backend/sync_lib/src/lib.rs b/backend/sync_lib/src/lib.rs index 6383983e..d2a54cf9 100644 --- a/backend/sync_lib/src/lib.rs +++ b/backend/sync_lib/src/lib.rs @@ -12,7 +12,7 @@ use core::str; use base64::{Engine as _, engine::general_purpose::STANDARD}; -use cursor::OwnedTextWithCursors; +use cursor::TextWithCursors; use errors::SyncLibError; use wasm_bindgen::prelude::*; @@ -110,9 +110,9 @@ pub fn merge_text(parent: &str, left: &str, right: &str) -> String { #[must_use] pub fn merge_text_with_cursors( parent: &str, - left: OwnedTextWithCursors, - right: OwnedTextWithCursors, -) -> OwnedTextWithCursors { + left: TextWithCursors, + right: TextWithCursors, +) -> TextWithCursors { set_panic_hook(); reconcile::reconcile_with_cursors(parent, left.into(), right.into()).into() diff --git a/backend/sync_lib/tests/web.rs b/backend/sync_lib/tests/web.rs index 0ce76b39..cf82aa7e 100644 --- a/backend/sync_lib/tests/web.rs +++ b/backend/sync_lib/tests/web.rs @@ -1,6 +1,6 @@ use insta::assert_debug_snapshot; use sync_lib::{ - cursor::{OwnedCursorPosition, OwnedTextWithCursors}, + cursor::{CursorPosition, TextWithCursors}, *, }; use wasm_bindgen_test::*; @@ -50,36 +50,18 @@ fn test_merge_text() { fn test_merge_text_with_cursors() { let result = merge_text_with_cursors( "hi", - OwnedTextWithCursors::new("hi world", vec![]), - OwnedTextWithCursors::new( - "hi", - vec![ - OwnedCursorPosition { - id: 0, - char_index: 1, - }, - OwnedCursorPosition { - id: 1, - char_index: 2, - }, - ], + TextWithCursors::new("hi world".to_owned(), vec![]), + TextWithCursors::new( + "hi".to_owned(), + vec![CursorPosition::new(0, 1), CursorPosition::new(1, 2)], ), ); assert_eq!( result, - OwnedTextWithCursors::new( - "hi world", - vec![ - OwnedCursorPosition { - id: 0, - char_index: 1, - }, - OwnedCursorPosition { - id: 1, - char_index: 2, - } - ] + TextWithCursors::new( + "hi world".to_owned(), + vec![CursorPosition::new(0, 1), CursorPosition::new(1, 2)] ), ); } -- 2.47.2 From a2c51a9d5d9fe9fa320f588898506b51258f836d Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Wed, 2 Apr 2025 21:27:36 +0100 Subject: [PATCH 18/21] Fix tests --- frontend/obsidian-plugin/package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/obsidian-plugin/package.json b/frontend/obsidian-plugin/package.json index 749941aa..d54690e4 100644 --- a/frontend/obsidian-plugin/package.json +++ b/frontend/obsidian-plugin/package.json @@ -6,7 +6,7 @@ "scripts": { "dev": "webpack watch --mode development", "build": "webpack --mode production", - "test": "jest --passWithNoTests", + "test": "jest", "version": "node version-bump.mjs" }, "keywords": [], @@ -36,4 +36,4 @@ "webpack": "^5.98.0", "webpack-cli": "^6.0.1" } -} +} \ No newline at end of file -- 2.47.2 From 5deb10ab8b89807ba4d845d4b81a55fdac483e57 Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Wed, 2 Apr 2025 21:29:48 +0100 Subject: [PATCH 19/21] Add cursor position conversions --- .../utils/line-and-column-to-position.test.ts | 43 ++++++++++++ .../src/utils/line-and-column-to-position.ts | 34 +++++++++ .../utils/position-to-line-and-column.test.ts | 69 +++++++++++++++++++ .../src/utils/position-to-line-and-column.ts | 30 ++++++++ 4 files changed, 176 insertions(+) create mode 100644 frontend/obsidian-plugin/src/utils/line-and-column-to-position.test.ts create mode 100644 frontend/obsidian-plugin/src/utils/line-and-column-to-position.ts create mode 100644 frontend/obsidian-plugin/src/utils/position-to-line-and-column.test.ts create mode 100644 frontend/obsidian-plugin/src/utils/position-to-line-and-column.ts diff --git a/frontend/obsidian-plugin/src/utils/line-and-column-to-position.test.ts b/frontend/obsidian-plugin/src/utils/line-and-column-to-position.test.ts new file mode 100644 index 00000000..b98f66e5 --- /dev/null +++ b/frontend/obsidian-plugin/src/utils/line-and-column-to-position.test.ts @@ -0,0 +1,43 @@ +import { lineAndColumnToPosition } from "./line-and-column-to-position"; + +describe("lineAndColumnToPosition", () => { + it("should return the correct position for the first line", () => { + const text = "Hello\nWorld"; + const position = lineAndColumnToPosition(text, 0, 3); + expect(position).toBe(3); + }); + + it("should return the correct position for the second line", () => { + const text = "Hello\nWorld"; + const position = lineAndColumnToPosition(text, 1, 2); + expect(position).toBe(8); + }); + + it("should return the correct position for an empty string", () => { + const text = ""; + const position = lineAndColumnToPosition(text, 0, 0); + expect(position).toBe(0); + }); + + it("should handle a single-line string correctly", () => { + const text = "SingleLine"; + const position = lineAndColumnToPosition(text, 0, 5); + expect(position).toBe(5); + }); + + it("should handle multi-line strings with varying lengths", () => { + const text = "Line1\nLongerLine2\nShort3"; + const position = lineAndColumnToPosition(text, 2, 4); + expect(position).toBe(22); + }); + + it("should throw an error if the line number is out of range", () => { + const text = "Line1\nLine2"; + expect(() => lineAndColumnToPosition(text, 3, 0)).toThrow(); + }); + + it("should throw an error if the column number is out of range", () => { + const text = "Line1\nLine2"; + expect(() => lineAndColumnToPosition(text, 1, 10)).toThrow(); + }); +}); diff --git a/frontend/obsidian-plugin/src/utils/line-and-column-to-position.ts b/frontend/obsidian-plugin/src/utils/line-and-column-to-position.ts new file mode 100644 index 00000000..0bc114c7 --- /dev/null +++ b/frontend/obsidian-plugin/src/utils/line-and-column-to-position.ts @@ -0,0 +1,34 @@ +/** + * Converts line and column coordinates to an absolute character position in a text string. + * + * @param line - The zero-based line number + * @param column - The zero-based column number + * @param text - The text string to calculate position in + * @returns The absolute character position (zero-based index) in the text string + * @throws Error if line number is out of range + * @throws Error if column number is out of range + */ +export function lineAndColumnToPosition( + text: string, + line: number, + column: number +): number { + const lines = text.split("\n"); + + if (line >= lines.length) { + throw new Error(`Line number ${line} is out of range.`); + } + + if (column > lines[line].length) { + throw new Error(`Column number ${column} is out of range.`); + } + + let position = 0; + for (let i = 0; i < line; i++) { + position += lines[i].length + 1; + } + + position += column; + + return position; +} diff --git a/frontend/obsidian-plugin/src/utils/position-to-line-and-column.test.ts b/frontend/obsidian-plugin/src/utils/position-to-line-and-column.test.ts new file mode 100644 index 00000000..e5d3bac5 --- /dev/null +++ b/frontend/obsidian-plugin/src/utils/position-to-line-and-column.test.ts @@ -0,0 +1,69 @@ +import { positionToLineAndColumn } from "./position-to-line-and-column"; + +describe("positionToLineAndColumn", () => { + test("converts position to line and column in a single line text", () => { + const text = "Hello, world!"; + expect(positionToLineAndColumn(text, 0)).toEqual({ + line: 0, + column: 1 + }); + expect(positionToLineAndColumn(text, 7)).toEqual({ + line: 0, + column: 8 + }); + expect(positionToLineAndColumn(text, 12)).toEqual({ + line: 0, + column: 13 + }); + }); + + test("converts position to line and column in multi-line text", () => { + const text = "First line\nSecond line\nThird line"; + expect(positionToLineAndColumn(text, 0)).toEqual({ + line: 0, + column: 1 + }); + expect(positionToLineAndColumn(text, 10)).toEqual({ + line: 0, + column: 11 + }); + expect(positionToLineAndColumn(text, 15)).toEqual({ + line: 1, + column: 5 + }); + expect(positionToLineAndColumn(text, 26)).toEqual({ + line: 2, + column: 4 + }); + }); + + test("handles positions at line breaks", () => { + const text = "Line\nBreak"; + expect(positionToLineAndColumn(text, 4)).toEqual({ + line: 0, + column: 5 + }); + expect(positionToLineAndColumn(text, 5)).toEqual({ + line: 1, + column: 1 + }); + }); + + test("handles empty input", () => { + expect(positionToLineAndColumn("", 0)).toEqual({ line: 0, column: 1 }); + }); + + test("handles positions at the end of text", () => { + const text = "End"; + expect(positionToLineAndColumn(text, 3)).toEqual({ + line: 0, + column: 4 + }); + }); + + test("throws error for position out of range", () => { + const text = "Short text"; + expect(() => positionToLineAndColumn(text, 15)).toThrow(); + expect(() => positionToLineAndColumn(text, -1)).toThrow(); + }); +}); diff --git a/frontend/obsidian-plugin/src/utils/position-to-line-and-column.ts b/frontend/obsidian-plugin/src/utils/position-to-line-and-column.ts new file mode 100644 index 00000000..a9c81881 --- /dev/null +++ b/frontend/obsidian-plugin/src/utils/position-to-line-and-column.ts @@ -0,0 +1,30 @@ +/** + * Converts a character position in text to line and column numbers. + * + * @param text The text content to analyze + * @param position The character position to convert + * @returns An object containing line and column numbers (0-based index for line, 1-based index for column) + * @throws Will throw an error if the position is negative or exceeds the text length + */ +export function positionToLineAndColumn( + text: string, + position: number +): { line: number; column: number } { + if (position < 0) { + throw new Error("Position cannot be negative"); + } + + if (position > text.length) { + throw new Error( + `Position ${position} exceeds text length ${text.length}` + ); + } + + const textUpToPosition = text.substring(0, position); + const lines = textUpToPosition.split("\n"); + + const line = lines.length - 1; // 0-based index + const column = lines[lines.length - 1].length + 1; // 1-based index + + return { line, column }; +} -- 2.47.2 From 31a81921a182f2c33d876be57c8662cfa14a48b3 Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Wed, 2 Apr 2025 21:32:08 +0100 Subject: [PATCH 20/21] Move cursor after file updates --- .../src/obsidian-file-system.ts | 52 +++++++++++++--- .../file-operations/file-operations.test.ts | 7 ++- .../src/file-operations/file-operations.ts | 60 +++++++++++++++---- .../file-operations/filesystem-operations.ts | 12 +++- .../safe-filesystem-operations.ts | 7 ++- frontend/sync-client/src/index.ts | 6 +- frontend/test-client/src/agent/mock-agent.ts | 5 +- frontend/test-client/src/agent/mock-client.ts | 6 +- 8 files changed, 125 insertions(+), 30 deletions(-) diff --git a/frontend/obsidian-plugin/src/obsidian-file-system.ts b/frontend/obsidian-plugin/src/obsidian-file-system.ts index 24e06af9..60c49328 100644 --- a/frontend/obsidian-plugin/src/obsidian-file-system.ts +++ b/frontend/obsidian-plugin/src/obsidian-file-system.ts @@ -1,6 +1,12 @@ import type { Stat, Vault, Workspace } from "obsidian"; import { MarkdownView, normalizePath } from "obsidian"; -import type { FileSystemOperations, RelativePath } from "sync-client"; +import type { + FileSystemOperations, + RelativePath, + TextWithCursors +} from "sync-client"; +import { lineAndColumnToPosition } from "./utils/line-and-column-to-position"; +import { positionToLineAndColumn } from "./utils/position-to-line-and-column"; export class ObsidianFileSystemOperations implements FileSystemOperations { public constructor( @@ -42,20 +48,50 @@ export class ObsidianFileSystemOperations implements FileSystemOperations { public async atomicUpdateText( path: RelativePath, - updater: (currentContent: string) => string + updater: (current: TextWithCursors) => TextWithCursors ): Promise { path = normalizePath(path); const view = this.workspace.getActiveViewOfType(MarkdownView); + if (view?.file?.path === path) { - const result = updater(view.editor.getValue()); - const position = view.editor.getCursor(); - view.editor.setValue(result); - view.editor.setCursor(position); - return result; + const cursor = view.editor.getCursor(); + const text = view.editor.getValue(); + const result = updater({ + text, + cursors: [ + { + id: 0, + characterPosition: lineAndColumnToPosition( + text, + cursor.line, + cursor.ch + ) + } + ] + }); + + view.editor.setValue(result.text); + + result.cursors.forEach((movedCursor) => { + const { line, column } = positionToLineAndColumn( + result.text, + movedCursor.characterPosition + ); + view.editor.setCursor(line, column); + }); + + return result.text; } - return this.vault.adapter.process(path, updater); + return this.vault.adapter.process( + path, + (text) => + updater({ + text, + cursors: [] + }).text + ); } public async getFileSize(path: RelativePath): Promise { diff --git a/frontend/sync-client/src/file-operations/file-operations.test.ts b/frontend/sync-client/src/file-operations/file-operations.test.ts index 4f7dd491..2529bab2 100644 --- a/frontend/sync-client/src/file-operations/file-operations.test.ts +++ b/frontend/sync-client/src/file-operations/file-operations.test.ts @@ -6,7 +6,10 @@ import type { import { FileOperations } from "./file-operations"; import { Logger } from "../tracing/logger"; import { assertSetContainsExactly } from "../utils/assert-set-contains-exactly"; -import type { FileSystemOperations } from "./filesystem-operations"; +import type { + FileSystemOperations, + TextWithCursors +} from "./filesystem-operations"; import init, { base64ToBytes } from "sync_lib"; import fs from "fs"; @@ -43,7 +46,7 @@ class FakeFileSystemOperations implements FileSystemOperations { } public async atomicUpdateText( _path: RelativePath, - _updater: (currentContent: string) => string + _updater: (current: TextWithCursors) => TextWithCursors ): Promise { throw new Error("Method not implemented."); } diff --git a/frontend/sync-client/src/file-operations/file-operations.ts b/frontend/sync-client/src/file-operations/file-operations.ts index 6cac74f3..e6e42c9d 100644 --- a/frontend/sync-client/src/file-operations/file-operations.ts +++ b/frontend/sync-client/src/file-operations/file-operations.ts @@ -1,7 +1,16 @@ import type { Logger } from "../tracing/logger"; -import type { FileSystemOperations } from "./filesystem-operations"; +import type { + FileSystemOperations, + TextWithCursors +} from "./filesystem-operations"; import type { Database, RelativePath } from "../persistence/database"; -import { isBinary, isFileTypeMergable, mergeText } from "sync_lib"; +import { + CursorPosition, + isBinary, + isFileTypeMergable, + mergeTextWithCursors, + TextWithCursors as RustTextWithCursors +} from "sync_lib"; import { SafeFileSystemOperations } from "./safe-filesystem-operations"; export class FileOperations { @@ -90,18 +99,45 @@ export class FileOperations { const expectedText = new TextDecoder().decode(expectedContent); // this comes from a previous read which must only have \n line endings const newText = new TextDecoder().decode(newContent); // this comes from the server which stores text with \n line endings - await this.fs.atomicUpdateText(path, (currentText) => { - currentText = currentText.replace(this.nativeLineEndings, "\n"); + await this.fs.atomicUpdateText( + path, + ({ text, cursors }: TextWithCursors): TextWithCursors => { + text = text.replace(this.nativeLineEndings, "\n"); - this.logger.debug( - `Performing a 3-way merge for ${path} with the expected content` - ); + this.logger.debug( + `Performing a 3-way merge for ${path} with the expected content` + ); - return mergeText(expectedText, currentText, newText).replace( - "\n", - this.nativeLineEndings - ); - }); + const left = new RustTextWithCursors( + text, + cursors.map( + (cursor) => + new CursorPosition( + cursor.id, + cursor.characterPosition + ) + ) + ); + const right = new RustTextWithCursors(newText, []); + const merged = mergeTextWithCursors(expectedText, left, right); + + const resultText = merged + .text() + .replace("\n", this.nativeLineEndings); + + const resultCursors = merged.cursors().map((cursor) => ({ + id: cursor.id(), + characterPosition: cursor.characterPosition() + })); + + merged.free(); + + return { + text: resultText, + cursors: resultCursors + }; + } + ); } public async delete(path: RelativePath): Promise { diff --git a/frontend/sync-client/src/file-operations/filesystem-operations.ts b/frontend/sync-client/src/file-operations/filesystem-operations.ts index 19d319ba..175490d4 100644 --- a/frontend/sync-client/src/file-operations/filesystem-operations.ts +++ b/frontend/sync-client/src/file-operations/filesystem-operations.ts @@ -1,5 +1,15 @@ import type { RelativePath } from "../persistence/database"; +export interface Cursor { + id: number; + characterPosition: number; +} + +export interface TextWithCursors { + text: string; + cursors: Cursor[]; +} + export interface FileSystemOperations { // List all files that should be synced. listAllFiles: () => Promise; @@ -13,7 +23,7 @@ export interface FileSystemOperations { // Atomically update the content of a text file. atomicUpdateText: ( path: RelativePath, - updater: (currentContent: string) => string + updater: (current: TextWithCursors) => TextWithCursors ) => Promise; // Get the size of a file in bytes. diff --git a/frontend/sync-client/src/file-operations/safe-filesystem-operations.ts b/frontend/sync-client/src/file-operations/safe-filesystem-operations.ts index 8b2a547a..433f1d75 100644 --- a/frontend/sync-client/src/file-operations/safe-filesystem-operations.ts +++ b/frontend/sync-client/src/file-operations/safe-filesystem-operations.ts @@ -1,5 +1,8 @@ import type { RelativePath } from "../persistence/database"; -import type { FileSystemOperations } from "./filesystem-operations"; +import type { + FileSystemOperations, + TextWithCursors +} from "./filesystem-operations"; import type { Logger } from "../tracing/logger"; import { Locks } from "../utils/locks"; import { FileNotFoundError } from "./file-not-found-error"; @@ -44,7 +47,7 @@ export class SafeFileSystemOperations implements FileSystemOperations { public async atomicUpdateText( path: RelativePath, - updater: (currentContent: string) => string + updater: (current: TextWithCursors) => TextWithCursors ): Promise { this.logger.debug(`Atomically updating file '${path}'`); return this.safeOperation( diff --git a/frontend/sync-client/src/index.ts b/frontend/sync-client/src/index.ts index 0a03d0ae..e5760ead 100644 --- a/frontend/sync-client/src/index.ts +++ b/frontend/sync-client/src/index.ts @@ -8,7 +8,11 @@ export { Logger, LogLevel, LogLine } from "./tracing/logger"; export { type SyncSettings } from "./persistence/settings"; export { rateLimit } from "./utils/rate-limit"; export type { RelativePath, StoredDatabase } from "./persistence/database"; -export type { FileSystemOperations } from "./file-operations/filesystem-operations"; +export type { + FileSystemOperations, + TextWithCursors, + Cursor +} from "./file-operations/filesystem-operations"; export type { PersistenceProvider } from "./persistence/persistence"; export type { NetworkConnectionStatus } from "./sync-client"; diff --git a/frontend/test-client/src/agent/mock-agent.ts b/frontend/test-client/src/agent/mock-agent.ts index 945fd7dd..9939d53c 100644 --- a/frontend/test-client/src/agent/mock-agent.ts +++ b/frontend/test-client/src/agent/mock-agent.ts @@ -313,7 +313,10 @@ export class MockAgent extends MockClient { `Decided to update file ${file} with ${content}` ); this.doNotTouchWhileOffline.push(file); - await this.atomicUpdateText(file, (old) => old + ` ${content} `); + await this.atomicUpdateText(file, (old) => ({ + text: old.text + ` ${content} `, + cursors: [] + })); } private async deleteFileAction(files: RelativePath[]): Promise { diff --git a/frontend/test-client/src/agent/mock-client.ts b/frontend/test-client/src/agent/mock-client.ts index 5aa3dd6c..29d808f8 100644 --- a/frontend/test-client/src/agent/mock-client.ts +++ b/frontend/test-client/src/agent/mock-client.ts @@ -1,4 +1,4 @@ -import type { StoredDatabase } from "sync-client"; +import type { StoredDatabase, TextWithCursors } from "sync-client"; import { assert } from "../utils/assert"; import { type RelativePath, @@ -87,14 +87,14 @@ export class MockClient implements FileSystemOperations { public async atomicUpdateText( path: RelativePath, - updater: (currentContent: string) => string + updater: (currentContent: TextWithCursors) => TextWithCursors ): Promise { const file = this.localFiles.get(path); if (!file) { throw new Error(`File ${path} does not exist`); } const currentContent = new TextDecoder().decode(file); - const newContent = updater(currentContent); + const newContent = updater({ text: currentContent, cursors: [] }).text; const newContentUint8Array = new TextEncoder().encode(newContent); this.localFiles.set(path, newContentUint8Array); -- 2.47.2 From ea76c5ecc69081b23a44393d79debe417d629bc6 Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Wed, 2 Apr 2025 22:06:29 +0100 Subject: [PATCH 21/21] Add editorconfig --- .editorconfig | 11 +++++++++++ .../reconcile/src/operation_transformation/cursor.rs | 4 ++-- backend/reconcile/tests/examples/10.yml | 1 - backend/reconcile/tests/examples/11.yml | 2 +- backend/reconcile/tests/examples/5.yml | 2 +- 5 files changed, 15 insertions(+), 5 deletions(-) create mode 100644 .editorconfig diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 00000000..5773d4e4 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,11 @@ +# https://editorconfig.org + +root = true + +[*] +end_of_line = lf +insert_final_newline = true +trim_trailing_whitespace = true +charset = utf-8 +indent_style = space +indent_size = 4 diff --git a/backend/reconcile/src/operation_transformation/cursor.rs b/backend/reconcile/src/operation_transformation/cursor.rs index 0f2ee509..c17f560c 100644 --- a/backend/reconcile/src/operation_transformation/cursor.rs +++ b/backend/reconcile/src/operation_transformation/cursor.rs @@ -6,8 +6,8 @@ use serde::{Deserialize, Serialize}; use super::merge_context::MergeContext; use crate::operation_transformation::Operation; -// CursorPosition is a wrapper around usize to represent the position of an -// identifiable cursor in a text document based on the character index. +// CursorPosition represents the position of an identifiable cursor in a text +// document based on its (UTF-8) character index. #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[derive(Debug, Clone, PartialEq, Default)] pub struct CursorPosition { diff --git a/backend/reconcile/tests/examples/10.yml b/backend/reconcile/tests/examples/10.yml index 5b2183dc..0ee73838 100644 --- a/backend/reconcile/tests/examples/10.yml +++ b/backend/reconcile/tests/examples/10.yml @@ -2,4 +2,3 @@ parent: marketplace left: market| place right: market|space expected: market| placemarket|space - \ No newline at end of file diff --git a/backend/reconcile/tests/examples/11.yml b/backend/reconcile/tests/examples/11.yml index 549717de..d576c04d 100644 --- a/backend/reconcile/tests/examples/11.yml +++ b/backend/reconcile/tests/examples/11.yml @@ -1,4 +1,4 @@ parent: Please remember to bring your laptop and charger left: Please remember to bring your laptop| right: Please remember to bring your |new |laptop and charger -expected: Please remember to bring your |new |laptop| \ No newline at end of file +expected: Please remember to bring your |new |laptop| diff --git a/backend/reconcile/tests/examples/5.yml b/backend/reconcile/tests/examples/5.yml index b1f3f45d..aac8a98c 100644 --- a/backend/reconcile/tests/examples/5.yml +++ b/backend/reconcile/tests/examples/5.yml @@ -1,4 +1,4 @@ parent: Send the report to the team left: Send the |detailed |report to the |entire |team right: Send the |quarterly |detailed |report to the team -expected: Send the |detailed |quarterly |detailed ||report to the |entire |team \ No newline at end of file +expected: Send the |detailed |quarterly |detailed ||report to the |entire |team -- 2.47.2