Fix syncing when network latency is present (#4)

* WIP

* Add debug

* Dedupe inserts

* Add deterministic ordering

* Fix whitespaces

* Update insta

* Add integration test script

* Rename

* Add test

* Working for non-deletes

* omg it mostly works for deletes

* Isdeleted fix

* remove created dates

* update api

* Take document id

* No max attempt

* works

* Use string uuids

* .

* working!!!! (hopefully)

* Improve bundling

* Add module

* lint

* .

* lint

* Fix CI

* use toolchain

* clean up

* Add useSlowFileEvents

* Delete fuzz

* Fix CI

* use docker

* fix script

* clean up

* Clean up

* change node version

* Build docker image on every commit

* fix ci

* 1 db per vault

* Add scritps folder

* Bump versions

* Lint

* .

* Fix tests for real

* Style

* .

* try

* Consistent ordering

* Fix tests

* hmm

* .

* Clean up diff

* Fixes

* .

* Fix version bump

* .

* .

* .

.github/workflows/check.yml

@@ -17,11 +17,14 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - name: Setup
+      - name: Setup Node.js environment
+        uses: actions/setup-node@v4.2.0
+        with:
+          node-version: "22.x"
+          check-latest: true
+
+      - name: Setup rust
         run: |
-          rustup install nightly
-          rustup default nightly
-          rustup component add clippy rustfmt
           cargo install sqlx-cli
           cd backend
           sqlx database create --database-url sqlite://db.sqlite3
@@ -44,7 +47,7 @@ jobs:
           cd backend
           cargo test --verbose
           cd sync_lib
-          # wasm-pack test --node # todo: fix this in CI
+          wasm-pack test --node
 
       - name: Lint frontend
         run: |

.github/workflows/e2e.yml

@@ -0,0 +1,45 @@
+name: E2E tests
+
+on:
+  push:
+    branches: ["master"]
+  pull_request:
+    branches: ["master"]
+
+env:
+  CARGO_TERM_COLOR: always
+  RUSTFLAGS: "-Dwarnings"
+
+jobs:
+  build:
+    runs-on: self-hosted
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup Node.js environment
+        uses: actions/setup-node@v4.2.0
+        with:
+          node-version: "22.x"
+          check-latest: true
+
+      - name: Setup rust
+        run: |
+          cargo install sqlx-cli wasm-pack
+          cd backend
+          sqlx database create --database-url sqlite://db.sqlite3
+          sqlx migrate run --source sync_server/src/database/migrations --database-url sqlite://db.sqlite3
+
+      - name: Build wasm
+        run: |
+          cd backend
+          wasm-pack build --target web sync_lib
+
+      - name: E2E tests
+        run: |
+          cd backend
+          RUST_BACKTRACE=1 cargo run -p sync_server &
+          cd ../frontend
+          npm ci
+          cd ..
+          scripts/e2e.sh 32

.github/workflows/publish-docker.yml

@@ -7,8 +7,9 @@ name: Publish server Docker image
 
 on:
   push:
-    tags:
+    branches: ["master"]
-      - "*"
+  pull_request:
+    branches: ["master"]
 
 env:
   # Use docker.io for Docker Hub if empty
@@ -17,8 +18,9 @@ env:
   IMAGE_NAME: ${{ github.repository }}
 
 jobs:
-  build-docker:
+  publish-docker:
-    runs-on: ubuntu-latest
+    runs-on: self-hosted
+
     permissions:
       contents: read
       packages: write
@@ -33,7 +35,7 @@ jobs:
       # Install the cosign tool except on PR
       # https://github.com/sigstore/cosign-installer
       - name: Install cosign
-        if: github.event_name != 'pull_request'
+        if: ${{ github.ref_type == 'tag' }}
         uses: sigstore/cosign-installer@59acb6260d9c0ba8f4a2f9d9b48431a222b68e20 #v3.5.0
         with:
           cosign-release: "v2.2.4"
@@ -47,7 +49,7 @@ jobs:
       # Login against a Docker registry except on PR
       # https://github.com/docker/login-action
       - name: Log into registry ${{ env.REGISTRY }}
-        if: github.event_name != 'pull_request'
+        if: ${{ github.ref_type == 'tag' }}
         uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0
         with:
           registry: ${{ env.REGISTRY }}
@@ -69,7 +71,7 @@ jobs:
         uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v5.0.0
         with:
           context: backend
-          push: ${{ github.event_name != 'pull_request' }}
+          push: ${{ github.ref_type == 'tag' }}
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
           cache-from: type=gha
@@ -81,7 +83,7 @@ jobs:
       # transparency data even for private images, pass --force to cosign below.
       # https://github.com/sigstore/cosign
       - name: Sign the published Docker image
-        if: ${{ github.event_name != 'pull_request' }}
+        if: ${{ github.ref_type == 'tag' }}
         env:
           # https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions#using-an-intermediate-environment-variable
           TAGS: ${{ steps.meta.outputs.tags }}

.github/workflows/publish-plugin.yml

@@ -2,29 +2,27 @@ name: Publish Obsidian plugin
 
 on:
   push:
-    tags:
+    tags: ["*"]
-      - "*"
 
 env:
   CARGO_TERM_COLOR: always
 
 jobs:
-  build-plugin:
+  publish-plugin:
     runs-on: self-hosted
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
-      - name: Use Node.js
+      - name: Setup Node.js environment
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4.2.0
         with:
-          node-version: "18.x"
+          node-version: "22.x"
+          check-latest: true
 
       - name: Build wasm
         run: |
           cd backend
-          rustup install nightly
-          rustup default nightly
           cargo install wasm-pack
           wasm-pack build --target web sync_lib
 

.gitignore

@@ -10,6 +10,7 @@ backend/target
 frontend/*/dist
 
 backend/db.sqlite3*
+backend/databases
 backend/config.yml
 
 *.log

README.md

@@ -1,65 +1,56 @@
-## VaultLink self-hosted Obsidian sync plugin
+# VaultLink self-hosted Obsidian plugin for file syncing
 
 [![Check](https://github.com/schmelczer/vault-link/actions/workflows/check.yml/badge.svg)](https://github.com/schmelczer/vault-link/actions/workflows/check.yml)
+[![E2E tests](https://github.com/schmelczer/vault-link/actions/workflows/e2e.yml/badge.svg)](https://github.com/schmelczer/vault-link/actions/workflows/e2e.yml)
 [![Publish server Docker image](https://github.com/schmelczer/vault-link/actions/workflows/publish-docker.yml/badge.svg)](https://github.com/schmelczer/vault-link/actions/workflows/publish-docker.yml)
 [![Publish Obsidian plugin](https://github.com/schmelczer/vault-link/actions/workflows/publish-plugin.yml/badge.svg)](https://github.com/schmelczer/vault-link/actions/workflows/publish-plugin.yml)
 
 
-## Install [nvm](https://github.com/nvm-sh/nvm)
+## Develop
+
+### Install [nvm](https://github.com/nvm-sh/nvm)
 
 - `curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh | bash`
-- `nvm install 20`
+- `nvm install 22`
-- `nvm use 20`
+- `nvm use 22`
-- Optionally set the system-wide default: `nvm alias default 20`
+- Optionally set the system-wide default: `nvm alias default 22`
 
 
-## Set up Rust
+### Set up Rust
 
 - Install [`rustup`](https://rustup.rs): `curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh`
-- `sudo apt install llvm -y`
-- `rustup self update`
-- `rustup update`
-- `rustup install nightly`
-- `rustup default nightly`
-- `rustup component add llvm-tools-preview`
-- `cargo install cargo-generate cargo-fuzz cargo-insta rustfilt cargo-binutils`
 - Install [`wasm-pack`](https://rustwasm.github.io/wasm-pack/installer): `curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh`
 - `cargo install cargo-insta sqlx-cli cargo-edit`
 
-
+### Install Obsidian on Linux
-## Publish new version 
 
 ```sh
-./bump-version.sh patch
-```
-
-
-## Update HTTP API TS bindings
-
-```sh 
-npm install -g openapi-typescript
-openapi-typescript http://localhost:3030/api.json --output frontend/sync-client/src/services/types.ts
-```
-
-```
-
-todo: enable
-[workspace.lints.clippy]
-single_call_fn = { level = "allow", priority = 1 }
-absolute_paths = { level = "allow", priority = 1 }
-arithmetic_side_effects = { level = "allow", priority = 1 }
-similar_names = { level = "allow", priority = 1 }
-self_named_module_files = { level = "allow", priority = 1 }
-single_char_lifetime_names = { level = "allow", priority = 1 }
-missing_docs_in_private_items = { level = "allow", priority = 1 }
-question_mark_used =  { level = "allow", priority = 1 }
-implicit_return = { level = "allow", priority = 1 }
-pedantic = { level = "warn", priority = 0 }
-cargo = { level = "warn", priority = 0 }
-
-```
-
 apt install flatpak
 flatpak remote-add --if-not-exists flathub https://dl.flathub.org/repo/flathub.flatpakrepo
 flatpak install flathub md.obsidian.Obsidian
 flatpak run md.obsidian.Obsidian
+```
+
+### Scripts
+
+#### Update HTTP API TS bindings
+
+```sh 
+scripts/update-api-types.sh
+```
+
+#### Publish new version 
+
+```sh
+scripts/bump-version.sh patch
+```
+
+
+#### Run E2E tests
+
+```sh 
+scripts/e2e.sh
+```
+
+And to clean up the logs & database files, run `scripts/clean-up.sh`
+```

backend/Cargo.lock

@@ -105,12 +105,6 @@ dependencies = [
  "backtrace",
 ]
 
-[[package]]
-name = "arbitrary"
-version = "1.4.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dde20b3d026af13f561bdd0f15edf01fc734f0dafcedbaf42bba506a9517f223"
-
 [[package]]
 name = "async-trait"
 version = "0.1.85"
@@ -381,8 +375,6 @@ version = "1.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f34d93e62b03caf570cccc334cbc6c2fceca82f39211051345108adcba3eebdc"
 dependencies = [
- "jobserver",
- "libc",
  "shlex",
 ]
 
@@ -1228,15 +1220,6 @@ version = "1.0.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674"
 
-[[package]]
-name = "jobserver"
-version = "0.1.32"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0"
-dependencies = [
- "libc",
-]
-
 [[package]]
 name = "js-sys"
 version = "0.3.76"
@@ -1290,16 +1273,6 @@ version = "0.2.167"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "09d6582e104315a817dff97f75133544b2e094ee22447d2acf4a74e189ba06fc"
 
-[[package]]
-name = "libfuzzer-sys"
-version = "0.4.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9b9569d2f74e257076d8c6bfa73fb505b46b851e51ddaecc825944aa3bed17fa"
-dependencies = [
- "arbitrary",
- "cc",
-]
-
 [[package]]
 name = "libm"
 version = "0.2.11"
@@ -1781,14 +1754,6 @@ dependencies = [
  "test-case",
 ]
 
-[[package]]
-name = "reconcile-fuzz"
-version = "0.0.30"
-dependencies = [
- "libfuzzer-sys",
- "reconcile",
-]
-
 [[package]]
 name = "redox_syscall"
 version = "0.5.7"

backend/Cargo.toml

@@ -2,7 +2,6 @@
 resolver = "2"
 members = [
     "reconcile",
-    "fuzz",
     "sync_server", 
     "sync_lib"
 ]
@@ -57,3 +56,19 @@ uninlined_format_args = "warn"
 unnested_or_patterns = "warn"
 unused_self = "warn"
 verbose_file_reads = "warn"
+
+cast_possible_truncation = { level = "allow", priority = 1 }
+doc_link_with_quotes = { level = "allow", priority = 1 }
+cast_sign_loss = { level = "allow", priority = 1 }
+cast_possible_wrap = { level = "allow", priority = 1 }
+struct_field_names = { level = "allow", priority = 1 }
+single_call_fn = { level = "allow", priority = 1 }
+absolute_paths = { level = "allow", priority = 1 }
+arithmetic_side_effects = { level = "allow", priority = 1 }
+similar_names = { level = "allow", priority = 1 }
+self_named_module_files = { level = "allow", priority = 1 }
+single_char_lifetime_names = { level = "allow", priority = 1 }
+missing_docs_in_private_items = { level = "allow", priority = 1 }
+question_mark_used =  { level = "allow", priority = 1 }
+implicit_return = { level = "allow", priority = 1 }
+pedantic = { level = "warn", priority = 0 }

backend/Dockerfile

@@ -3,8 +3,6 @@ FROM rust:1.83 AS builder
 WORKDIR /usr/src/backend
 
 RUN apt update && apt install -y musl-tools
-RUN rustup install nightly && rustup default nightly
-RUN rustup target add x86_64-unknown-linux-musl
 RUN cargo install sqlx-cli
 
 COPY . .
@@ -23,7 +21,7 @@ RUN apk add --no-cache curl
 
 COPY --from=builder /usr/src/backend/target/x86_64-unknown-linux-musl/release/sync_server /app/sync_server
 
-VOLUME /data
+VOLUME /data/databases
 EXPOSE 3000/tcp
 
 WORKDIR /data

backend/fuzz/.gitignore

@@ -1,4 +0,0 @@
-target
-corpus
-artifacts
-coverage

backend/fuzz/Cargo.toml

@@ -1,25 +0,0 @@
-[package]
-name = "reconcile-fuzz"
-version.workspace = true
-edition.workspace = true
-authors.workspace = true
-license.workspace = true
-repository.workspace = true
-publish = false
-
-[package.metadata]
-cargo-fuzz = true
-
-[dependencies]
-libfuzzer-sys = "0.4"
-reconcile = { path = "../reconcile" }
-
-[[bin]]
-name = "reconcile"
-path = "fuzz_targets/reconcile.rs"
-test = false
-doc = false
-bench = false
-
-[lints]
-workspace = true

backend/fuzz/fuzz_targets/reconcile.rs

@@ -1,8 +0,0 @@
-#![no_main]
-
-use libfuzzer_sys::fuzz_target;
-
-fuzz_target!(|texts: (String, String, String)| {
-    let (original, left, right) = texts;
-    let _ = reconcile::reconcile(&original, &left, &right);
-});

backend/reconcile/src/diffs/myers.rs

@@ -38,7 +38,7 @@ use crate::{
 /// execution time permitted before it bails and falls back to an approximation.
 pub fn diff<T>(old: &[Token<T>], new: &[Token<T>]) -> Vec<RawOperation<T>>
 where
-    T: PartialEq + Clone,
+    T: PartialEq + Clone + std::fmt::Debug,
 {
     let max_d = (old.len() + new.len()).div_ceil(2) + 1;
     let mut vb = V::new(max_d);
@@ -99,7 +99,6 @@ impl IndexMut<isize> for V {
     }
 }
 
-#[inline(always)]
 fn split_at(range: Range<usize>, at: usize) -> (Range<usize>, Range<usize>) {
     (range.start..at, at..range.end)
 }
@@ -124,7 +123,7 @@ fn find_middle_snake<T>(
     vb: &mut V,
 ) -> Option<(usize, usize)>
 where
-    T: PartialEq + Clone,
+    T: PartialEq + Clone + std::fmt::Debug,
 {
     let n = old_range.len();
     let m = new_range.len();
@@ -230,7 +229,7 @@ fn conquer<T>(
     vb: &mut V,
     result: &mut Vec<RawOperation<T>>,
 ) where
-    T: PartialEq + Clone,
+    T: PartialEq + Clone + std::fmt::Debug,
 {
     // Check for common prefix
     let common_prefix_len = common_prefix_len(old, old_range.clone(), new, new_range.clone());

backend/reconcile/src/diffs/raw_operation.rs

@@ -3,7 +3,7 @@ use crate::tokenizer::token::Token;
 #[derive(Debug, Clone, PartialEq)]
 pub enum RawOperation<T>
 where
-    T: PartialEq + Clone,
+    T: PartialEq + Clone + std::fmt::Debug,
 {
     Insert(Vec<Token<T>>),
     Delete(Vec<Token<T>>),
@@ -12,13 +12,13 @@ where
 
 impl<T> RawOperation<T>
 where
-    T: PartialEq + Clone,
+    T: PartialEq + Clone + std::fmt::Debug,
 {
     pub fn tokens(&self) -> &Vec<Token<T>> {
         match self {
-            RawOperation::Insert(tokens) => tokens,
+            RawOperation::Insert(tokens)
-            RawOperation::Delete(tokens) => tokens,
+            | RawOperation::Delete(tokens)
-            RawOperation::Equal(tokens) => tokens,
+            | RawOperation::Equal(tokens) => tokens,
         }
     }
 

backend/reconcile/src/operation_transformation.rs

@@ -37,7 +37,7 @@ pub fn reconcile_with_tokenizer<F, T>(
     tokenizer: &Tokenizer<T>,
 ) -> String
 where
-    T: PartialEq + Clone,
+    T: PartialEq + Clone + std::fmt::Debug,
 {
     let left_operations = EditedText::from_strings_with_tokenizer(original, left, tokenizer);
     let right_operations = EditedText::from_strings_with_tokenizer(original, right, tokenizer);
@@ -73,7 +73,8 @@ mod test {
             "original_1 edit_1 original_3",
         );
 
-        // One deleted a large range, the other deleted subranges and inserted as well
+        // One deleted a large range, the other deleted subranges and inserted as
+        // well
         test_merge_both_ways(
             "original_1 original_2 original_3 original_4 original_5",
             "original_1 original_5",
@@ -120,9 +121,6 @@ mod test {
             "hi, my friend!",
         );
 
-        // test_merge_both_ways("hello world", "world !", "hi hello world", "hi world
-        // !");
-
         test_merge_both_ways(
             "both delete the same word",
             "both the same word",
@@ -147,7 +145,33 @@ mod test {
         );
     }
 
-    #[ignore = "it's too slow"]
+    #[test]
+    fn test_reconcile_idempotent_inserts() {
+        // Both inserted the same prefix; this should get deduped
+        test_merge_both_ways(
+            "hi ",
+            "hi there ",
+            "hi there my friend ",
+            "hi there my friend ",
+        );
+
+        // The prefix of the 2nd appears on the 1st so it shouldn't get duplicated
+        test_merge_both_ways(
+            "hi ",
+            "hi there you ",
+            "hi there my friend ",
+            "hi there my friend you ",
+        );
+
+        test_merge_both_ways("a", "a b c", "a b c d", "a b c d");
+
+        test_merge_both_ways(
+            "      |7ca2b36d-6ee7-49eb-8eb1-d77e4cc1a001|   ",
+              "      |7ca2b36d-6ee7-49eb-8eb1-d77e4cc1a001|      |cd9195cc-103a-4f13-90c8-4fba0ba421ee|      |d39156cc-cfd6-42a8-b70a-75020896069d|      |fbad794c-9c47-41f2-a343-490284ecb5a0|      |dup|   ",
+             "       |7ca2b36d-6ee7-49eb-8eb1-d77e4cc1a001|      |cd9195cc-103a-4f13-90c8-4fba0ba421ee|      |dup|   ",
+            "      |7ca2b36d-6ee7-49eb-8eb1-d77e4cc1a001|      |cd9195cc-103a-4f13-90c8-4fba0ba421ee|      |d39156cc-cfd6-42a8-b70a-75020896069d|      |fbad794c-9c47-41f2-a343-490284ecb5a0|      |dup|      |dup|   ");
+    }
+
     #[test_matrix( [
         "pride_and_prejudice.txt",
         "romeo_and_juliet.txt",

backend/reconcile/src/operation_transformation/edited_text.rs

@@ -25,7 +25,7 @@ use crate::{
 #[derive(Debug, Clone, PartialEq, Default)]
 pub struct EditedText<'a, T>
 where
-    T: PartialEq + Clone,
+    T: PartialEq + Clone + std::fmt::Debug,
 {
     text: &'a str,
     operations: Vec<OrderedOperation<T>>,
@@ -46,7 +46,7 @@ impl<'a> EditedText<'a, String> {
 
 impl<'a, T> EditedText<'a, T>
 where
-    T: PartialEq + Clone,
+    T: PartialEq + Clone + std::fmt::Debug,
 {
     /// Create an `EditedText` from the given original (old) and updated (new)
     /// strings. The returned `EditedText` represents the changes from the
@@ -65,7 +65,6 @@ where
 
         Self::new(
             original,
-            // Self::cook_operations(diff),
             Self::cook_operations(Self::elongate_operations(diff)).collect(),
         )
     }
@@ -191,7 +190,7 @@ where
     pub fn merge(self, other: Self) -> Self {
         debug_assert_eq!(
             self.text, other.text,
-            "EditedText-s must be derived from the same text to be mergable"
+            "`EditedText`-s must be derived from the same text to be mergable"
         );
 
         let mut left_merge_context = MergeContext::default();
@@ -207,9 +206,21 @@ where
                     |(operation, _)| {
                         (
                             operation.order,
-                            // Operations on left and right must come in the same order so that
+                            // Operations on the left and right must come in the same order so that
                             // inserts can be merged with other inserts and deletes with deletes.
                             usize::from(matches!(operation.operation, Operation::Delete { .. })),
+                            // Make sure that the ordering is deterministic regardless which text
+                            // is left or right.
+                            match &operation.operation {
+                                Operation::Insert { text, .. } => text
+                                    .iter()
+                                    .map(super::super::tokenizer::token::Token::original)
+                                    .collect::<String>(),
+                                Operation::Delete {
+                                    deleted_character_count,
+                                    ..
+                                } => deleted_character_count.to_string(),
+                            },
                         )
                     },
                 )
@@ -232,6 +243,7 @@ where
     }
 
     /// Apply the operations to the text and return the resulting text.
+    #[must_use]
     pub fn apply(&self) -> String {
         let mut builder: StringBuilder<'_> = StringBuilder::new(self.text);
 
@@ -282,7 +294,7 @@ mod tests {
         let original = "hello world! ...";
         let left = "Hello world! I'm Andras.";
         let right = "Hello world! How are you?";
-        let expected = "Hello world! I'm Andras.How are you?";
+        let expected = "Hello world! How are you? I'm Andras.";
 
         let operations_1 = EditedText::from_strings(original, left);
         let operations_2 = EditedText::from_strings(original, right);

backend/reconcile/src/operation_transformation/merge_context.rs

@@ -5,7 +5,7 @@ use crate::operation_transformation::Operation;
 #[derive(Clone)]
 pub struct MergeContext<T>
 where
-    T: PartialEq + Clone,
+    T: PartialEq + Clone + std::fmt::Debug,
 {
     last_operation: Option<Operation<T>>,
     pub shift: i64,
@@ -13,7 +13,7 @@ where
 
 impl<T> Default for MergeContext<T>
 where
-    T: PartialEq + Clone,
+    T: PartialEq + Clone + std::fmt::Debug,
 {
     fn default() -> Self {
         MergeContext {
@@ -25,7 +25,7 @@ where
 
 impl<T> Debug for MergeContext<T>
 where
-    T: PartialEq + Clone,
+    T: PartialEq + Clone + std::fmt::Debug,
 {
     fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
         f.debug_struct("MergeContext")
@@ -37,7 +37,7 @@ where
 
 impl<T> MergeContext<T>
 where
-    T: PartialEq + Clone,
+    T: PartialEq + Clone + std::fmt::Debug,
 {
     pub fn last_operation(&self) -> Option<&Operation<T>> { self.last_operation.as_ref() }
 

backend/reconcile/src/operation_transformation/operation.rs

@@ -1,7 +1,5 @@
-use core::{
+use core::fmt::{Debug, Display};
-    fmt::{Debug, Display},
+use std::ops::Range;
-    ops::Range,
-};
 
 #[cfg(feature = "serde")]
 use serde::{Deserialize, Serialize};
@@ -9,7 +7,10 @@ use serde::{Deserialize, Serialize};
 use super::merge_context::MergeContext;
 use crate::{
     Token,
-    utils::{find_common_overlap::find_common_overlap, string_builder::StringBuilder},
+    utils::{
+        find_longest_prefix_contained_within::find_longest_prefix_contained_within,
+        string_builder::StringBuilder,
+    },
 };
 
 /// Represents a change that can be applied to a text document.
@@ -19,7 +20,7 @@ use crate::{
 #[derive(Clone, PartialEq)]
 pub enum Operation<T>
 where
-    T: PartialEq + Clone,
+    T: PartialEq + Clone + std::fmt::Debug,
 {
     Insert {
         index: usize,
@@ -37,7 +38,7 @@ where
 
 impl<T> Operation<T>
 where
-    T: PartialEq + Clone,
+    T: PartialEq + Clone + std::fmt::Debug,
 {
     /// Creates an insert operation with the given index and text.
     /// If the text is empty (meaning that the operation would be a no-op),
@@ -81,15 +82,8 @@ where
         })
     }
 
-    /// Tries to apply the operation to the given `ropey::Rope` text, returning
+    /// Applies the operation to the given `StringBuilder`, returning the
-    /// the modified text.
+    /// modified `StringBuilder`.
-    ///
-    /// # Errors
-    ///
-    /// Returns a `SyncLibError::OperationApplicationError` if the operation
-    /// cannot be applied.
-    ///
-    /// # Panics
     ///
     /// When compiled in debug mode, panics if a delete operation is attempted
     /// on a range of text that does not match the text to be deleted.
@@ -114,7 +108,7 @@ where
 
                 builder.delete(self.range());
             }
-        };
+        }
 
         builder
     }
@@ -122,8 +116,7 @@ where
     /// Returns the index of the first character that the operation affects.
     pub fn start_index(&self) -> usize {
         match self {
-            Operation::Insert { index, .. } => *index,
+            Operation::Insert { index, .. } | Operation::Delete { index, .. } => *index,
-            Operation::Delete { index, .. } => *index,
         }
     }
 
@@ -137,6 +130,7 @@ where
     }
 
     /// Returns the range of indices of characters that the operation affects.
+    #[allow(clippy::range_plus_one)]
     pub fn range(&self) -> Range<usize> { self.start_index()..self.end_index() + 1 }
 
     /// Returns the number of affected characters. It is always greater than 0
@@ -212,17 +206,20 @@ where
                     ..
                 }),
             ) => {
-                let offset_in_tokens = find_common_overlap(previous_inserted_text, &text);
+                // In case the current insert's prefix appears in the previously inserted text,
-                let trimmed_length_in_tokens = previous_inserted_text.len() - offset_in_tokens;
+                // we can trim the current insert to only include the non-overlapping part.
-                let trimmed_length = previous_inserted_text
+                // This way, we don't end up duplicating text.
+                let offset_in_tokens =
+                    find_longest_prefix_contained_within(previous_inserted_text, &text);
+                let offset_in_length = text
                     .iter()
-                    .skip(offset_in_tokens)
+                    .take(offset_in_tokens)
                     .map(Token::get_original_length)
                     .sum::<usize>();
                 let trimmed_operation =
-                    Operation::create_insert(index, text[trimmed_length_in_tokens..].to_vec());
+                    Operation::create_insert(index, text[offset_in_tokens..].to_vec());
 
-                affecting_context.shift -= trimmed_length as i64;
+                affecting_context.shift -= offset_in_length as i64;
                 produced_context.shift += trimmed_operation
                     .as_ref()
                     .map(Operation::len)
@@ -297,7 +294,7 @@ where
 
 impl<T> Display for Operation<T>
 where
-    T: PartialEq + Clone,
+    T: PartialEq + Clone + std::fmt::Debug,
 {
     fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
         match self {
@@ -341,7 +338,7 @@ where
 
 impl<T> Debug for Operation<T>
 where
-    T: PartialEq + Clone,
+    T: PartialEq + Clone + std::fmt::Debug,
 {
     fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { write!(f, "{self}") }
 }
@@ -353,7 +350,7 @@ mod tests {
     use super::*;
 
     #[test]
-    #[should_panic]
+    #[should_panic(expected = "Shifted index must be non-negative")]
     fn test_shifting_error() {
         insta::assert_debug_snapshot!(
             Operation::create_insert(1, vec!["hi".into()])

backend/reconcile/src/operation_transformation/snapshots/reconcile__operation_transformation__edited_text__tests__calculate_operations.snap

@@ -8,19 +8,19 @@ EditedText {
     operations: [
         OrderedOperation {
             order: 0,
-            operation: <insert 'Hello, my friend! ' from index 0>,
+            operation: <insert 'Hello, my friend!' from index 0>,
         },
         OrderedOperation {
             order: 0,
-            operation: <delete 'hello world! ' from index 18>,
+            operation: <delete 'hello world!' from index 17>,
         },
         OrderedOperation {
-            order: 21,
+            order: 20,
-            operation: <insert 'you doing? Albert' from index 26>,
+            operation: <insert ' you doing? Albert' from index 25>,
         },
         OrderedOperation {
-            order: 21,
+            order: 20,
-            operation: <delete 'you?  Adam' from index 43>,
+            operation: <delete ' you?  Adam' from index 43>,
         },
     ],
 }

backend/reconcile/src/tokenizer/snapshots/reconcile__tokenizer__word_tokenizer__tests__with_snapshots-2.snap

@@ -0,0 +1,6 @@
+---
+source: reconcile/src/tokenizer/word_tokenizer.rs
+expression: "word_tokenizer(\"\")"
+snapshot_kind: text
+---
+[]

backend/reconcile/src/tokenizer/snapshots/reconcile__tokenizer__word_tokenizer__tests__with_snapshots-3.snap

@@ -0,0 +1,15 @@
+---
+source: reconcile/src/tokenizer/word_tokenizer.rs
+expression: "word_tokenizer(\" what? \")"
+snapshot_kind: text
+---
+[
+    Token {
+        normalised: "what?",
+        original: " what?",
+    },
+    Token {
+        normalised: "",
+        original: " ",
+    },
+]

backend/reconcile/src/tokenizer/snapshots/reconcile__tokenizer__word_tokenizer__tests__with_snapshots-4.snap

@@ -0,0 +1,23 @@
+---
+source: reconcile/src/tokenizer/word_tokenizer.rs
+expression: "word_tokenizer(\" hello, \\nwhere are you?\")"
+snapshot_kind: text
+---
+[
+    Token {
+        normalised: "hello,",
+        original: " hello,",
+    },
+    Token {
+        normalised: "where",
+        original: " \nwhere",
+    },
+    Token {
+        normalised: "are",
+        original: " are",
+    },
+    Token {
+        normalised: "you?",
+        original: " you?",
+    },
+]

backend/reconcile/src/tokenizer/snapshots/reconcile__tokenizer__word_tokenizer__tests__with_snapshots.snap

@@ -0,0 +1,15 @@
+---
+source: reconcile/src/tokenizer/word_tokenizer.rs
+expression: "word_tokenizer(\"Hi there!\")"
+snapshot_kind: text
+---
+[
+    Token {
+        normalised: "Hi",
+        original: "Hi",
+    },
+    Token {
+        normalised: "there!",
+        original: " there!",
+    },
+]

backend/reconcile/src/tokenizer/token.rs

@@ -8,24 +8,19 @@ use serde::{Deserialize, Serialize};
 #[derive(Debug, Clone)]
 pub struct Token<T>
 where
-    T: PartialEq + Clone,
+    T: PartialEq + Clone + std::fmt::Debug,
 {
     normalised: T,
     original: String,
 }
 
 impl From<&str> for Token<String> {
-    fn from(s: &str) -> Self {
+    fn from(s: &str) -> Self { Token::new(s.trim().to_owned(), s.to_owned()) }
-        Token {
-            normalised: s.to_owned(),
-            original: s.to_owned(),
-        }
-    }
 }
 
 impl<T> Token<T>
 where
-    T: PartialEq + Clone,
+    T: PartialEq + Clone + std::fmt::Debug,
 {
     pub fn new(normalised: T, original: String) -> Self {
         Token {
@@ -43,7 +38,7 @@ where
 
 impl<T> PartialEq for Token<T>
 where
-    T: PartialEq + Clone,
+    T: PartialEq + Clone + std::fmt::Debug,
 {
     fn eq(&self, other: &Self) -> bool { self.normalised == other.normalised }
 }

backend/reconcile/src/tokenizer/word_tokenizer.rs

@@ -1,7 +1,48 @@
 use super::token::Token;
 
+/// Splits on whitespace keeping the leading whitespace.
+///
+///     
+/// ## Example
+///
+/// "Hi there!" -> ["Hi", " there!"]
 pub fn word_tokenizer(text: &str) -> Vec<Token<String>> {
-    text.split_inclusive(char::is_whitespace)
+    let mut result: Vec<Token<String>> = Vec::new();
-        .map(|s| Token::new(s.to_owned(), s.to_owned()))
+
-        .collect()
+    let mut last_whitespace = 0;
+    let mut previous_char_is_whitespace = true;
+
+    for (i, c) in text.char_indices() {
+        let is_current_char_whitespace = c.is_whitespace();
+        if !previous_char_is_whitespace && is_current_char_whitespace {
+            result.push(text[last_whitespace..i].into());
+            last_whitespace = i;
+        }
+
+        previous_char_is_whitespace = is_current_char_whitespace;
+    }
+
+    if last_whitespace < text.len() {
+        result.push(text[last_whitespace..].into());
+    }
+
+    result
+}
+
+#[cfg(test)]
+mod tests {
+    use insta::assert_debug_snapshot;
+
+    use super::*;
+
+    #[test]
+    fn test_with_snapshots() {
+        assert_debug_snapshot!(word_tokenizer("Hi there!"));
+
+        assert_debug_snapshot!(word_tokenizer(""));
+
+        assert_debug_snapshot!(word_tokenizer(" what? "));
+
+        assert_debug_snapshot!(word_tokenizer(" hello, \nwhere are you?"));
+    }
 }

backend/reconcile/src/utils.rs

@@ -1,6 +1,6 @@
 pub mod common_prefix_len;
 pub mod common_suffix_len;
-pub mod find_common_overlap;
+pub mod find_longest_prefix_contained_within;
 pub mod merge_iters;
 pub mod ordered_operation;
 pub mod side;

backend/reconcile/src/utils/find_common_overlap.rs

@@ -1,71 +0,0 @@
-use crate::Token;
-
-/// Given two lists of tokens, returns the offset in the first (old) list from
-/// which the two lists have the same tokens until the end of the first list.
-/// Thus, the suffix of the old list from the offset to the end is equal to a
-/// prefix of the new list.
-///
-/// If there is no overlap, the function returns the maxmium offset, the length
-/// of the old list.
-///
-/// ## Example
-///
-/// ```not_rust
-/// old: [0, 1, 9, 0, 2, 5]
-/// new:       [9, 0, 2, 5, 1]
-/// ```
-/// > results in an offset of 2
-pub fn find_common_overlap<T>(old: &[Token<T>], new: &[Token<T>]) -> usize
-where
-    T: PartialEq + Clone,
-{
-    let minimum_offset = old.len().saturating_sub(new.len());
-    for offset in minimum_offset..old.len() {
-        if old.iter().skip(offset).zip(new.iter()).all(|(a, b)| a == b) {
-            return offset;
-        }
-    }
-
-    old.len()
-}
-
-#[cfg(test)]
-mod tests {
-    use pretty_assertions::assert_eq;
-
-    use super::*;
-
-    #[test]
-    fn test_common_overlap() {
-        assert_eq!(find_common_overlap(&["".into()], &["".into()]), 0);
-
-        assert_eq!(
-            find_common_overlap(
-                &["a".into(), "b".into(), "c".into()],
-                &["b".into(), "c".into(), "a".into()]
-            ),
-            1
-        );
-
-        assert_eq!(
-            find_common_overlap(
-                &["a".into(), "a".into(), "a".into()],
-                &["a".into(), "b".into(), "c".into()]
-            ),
-            2
-        );
-
-        assert_eq!(
-            find_common_overlap(
-                &["a".into(), "b".into(), "c".into()],
-                &["d".into(), "e".into(), "a".into()]
-            ),
-            3
-        );
-
-        assert_eq!(
-            find_common_overlap(&["a".into(), "a".into()], &["a".into()]),
-            1
-        );
-    }
-}

backend/reconcile/src/utils/find_longest_prefix_contained_within.rs

@@ -0,0 +1,103 @@
+use crate::Token;
+
+/// Given two lists of tokens, returns `length` where `old` list somewhere
+/// within contains the `length` prefix of the `new` list.
+///
+/// ## Example
+///
+/// ```not_rust
+/// old: [0, 1, 9, 0, 2, 5]
+/// new:       [9, 0, 2, 5, 1]
+/// ```
+/// > results in an length of 4
+///
+///
+/// ```not_rust
+/// old: [0, 1, 9, 0, 2, 5]
+/// new:          [0, 2]
+/// ```
+/// > results in an length of 2
+///
+/// ```not_rust
+/// old: [0, 1, 9, 0, 2, 5]
+/// new:          [0, 4]
+/// ```
+/// > results in an length of 1
+pub fn find_longest_prefix_contained_within<T>(old: &[Token<T>], new: &[Token<T>]) -> usize
+where
+    T: PartialEq + Clone + std::fmt::Debug,
+{
+    let max_possible = new.len().min(old.len());
+
+    for len in (1..=max_possible).rev() {
+        let prefix = &new[..len];
+        if old.windows(len).any(|window| window == prefix) {
+            return len;
+        }
+    }
+
+    0
+}
+
+#[cfg(test)]
+mod tests {
+    use pretty_assertions::assert_eq;
+
+    use super::*;
+
+    #[test]
+    fn test_common_overlap() {
+        assert_eq!(
+            find_longest_prefix_contained_within(&["".into()], &["".into()]),
+            1
+        );
+
+        assert_eq!(
+            find_longest_prefix_contained_within(
+                &["a".into(), "b".into(), "c".into()],
+                &["b".into(), "c".into(), "a".into()]
+            ),
+            2
+        );
+
+        assert_eq!(
+            find_longest_prefix_contained_within(
+                &["a".into(), "b".into(), "c".into()],
+                &["b".into(), "c".into()]
+            ),
+            2
+        );
+
+        assert_eq!(
+            find_longest_prefix_contained_within(
+                &["a".into(), "b".into(), "c".into()],
+                &["b".into()]
+            ),
+            1
+        );
+
+        assert_eq!(
+            find_longest_prefix_contained_within(
+                &["a".into(), "b".into(), "c".into(), "b".into(), "a".into()],
+                &["b".into(), "a".into()]
+            ),
+            2
+        );
+
+        assert_eq!(
+            find_longest_prefix_contained_within(
+                &["a".into(), "a".into(), "a".into()],
+                &["a".into(), "b".into(), "c".into()]
+            ),
+            1
+        );
+
+        assert_eq!(
+            find_longest_prefix_contained_within(
+                &["a".into(), "b".into(), "c".into()],
+                &["d".into(), "e".into(), "a".into()]
+            ),
+            0
+        );
+    }
+}

backend/reconcile/src/utils/merge_iters.rs

@@ -46,8 +46,7 @@ where
         };
 
         match order {
-            Some(Ordering::Less) | None => self.left.next(),
+            Some(Ordering::Less | Ordering::Equal) | None => self.left.next(),
-            Some(Ordering::Equal) => self.left.next(),
             Some(Ordering::Greater) => self.right.next(),
         }
     }

backend/reconcile/src/utils/ordered_operation.rs

@@ -7,7 +7,7 @@ use crate::operation_transformation::Operation;
 #[derive(Debug, Clone, PartialEq)]
 pub struct OrderedOperation<T>
 where
-    T: PartialEq + Clone,
+    T: PartialEq + Clone + std::fmt::Debug,
 {
     pub order: usize,
     pub operation: Operation<T>,

backend/rust-toolchain.toml

@@ -0,0 +1,4 @@
+[toolchain]
+channel = "nightly-2025-03-14"
+targets = [ "x86_64-unknown-linux-gnu", "x86_64-unknown-linux-musl" ]
+profile = "default"

backend/sync_lib/src/lib.rs

@@ -18,7 +18,20 @@ pub mod errors;
 
 /// Encode binary data for easy transport over HTTP. Inverse of
 /// `base64_to_bytes`.
+///
+/// # Arguments
+///
+/// - `input`: The binary data to encode.
+///
+/// # Returns
+///
+/// The base64-encoded string.
+///
+/// # Panics
+///
+/// If the input is not valid UTF-8.
 #[wasm_bindgen(js_name = bytesToBase64)]
+#[must_use]
 pub fn bytes_to_base64(input: &[u8]) -> String {
     set_panic_hook();
 
@@ -26,6 +39,19 @@ pub fn bytes_to_base64(input: &[u8]) -> String {
 }
 
 /// Inverse of `bytes_to_base64`.
+/// Decode base64-encoded data into binary data.
+///
+/// # Arguments
+///
+/// - `input`: The base64-encoded string.
+///
+/// # Returns
+///
+/// The decoded binary data.
+///
+/// # Errors
+///
+/// If the input is not valid base64.
 #[wasm_bindgen(js_name = base64ToBytes)]
 pub fn base64_to_bytes(input: &str) -> Result<Vec<u8>, SyncLibError> {
     set_panic_hook();
@@ -36,7 +62,22 @@ pub fn base64_to_bytes(input: &str) -> Result<Vec<u8>, SyncLibError> {
 /// Merge two documents with a common parent. Relies on `reconcile::reconcile`
 /// for texts and returns the right document as-is if either of the updated
 /// documents is binary.
+///
+/// # Arguments
+///
+/// - `parent`: The common parent document.
+/// - `left`: The left document updated by one user.
+/// - `right`: The right document updated by another user.
+///
+/// # Returns
+///
+/// The merged document.
+///
+/// # Panics
+///
+/// If any of the input documents are not valid UTF-8 strings.
 #[wasm_bindgen]
+#[must_use]
 pub fn merge(parent: &[u8], left: &[u8], right: &[u8]) -> Vec<u8> {
     set_panic_hook();
 
@@ -54,6 +95,7 @@ pub fn merge(parent: &[u8], left: &[u8], right: &[u8]) -> Vec<u8> {
 
 /// WASM wrapper around `reconcile::reconcile` for text merging.
 #[wasm_bindgen(js_name = mergeText)]
+#[must_use]
 pub fn merge_text(parent: &str, left: &str, right: &str) -> String {
     set_panic_hook();
 
@@ -63,10 +105,11 @@ pub fn merge_text(parent: &str, left: &str, right: &str) -> String {
 /// Heuristically determine if the given data is a binary or a text file's
 /// content.
 #[wasm_bindgen(js_name = isBinary)]
+#[must_use]
 pub fn is_binary(data: &[u8]) -> bool {
     set_panic_hook();
 
-    if data.iter().any(|&b| b == 0) {
+    if data.contains(&0) {
         // Even though the NUL character is valid in UTF-8, it's highly suspicious in
         // human-readable text.
         return true;
@@ -77,6 +120,7 @@ pub fn is_binary(data: &[u8]) -> bool {
 
 /// We don't want to support merging structured data like JSON, YAML, etc.
 #[wasm_bindgen(js_name = isFileTypeMergable)]
+#[must_use]
 pub fn is_file_type_mergable(path_or_file_name: &str) -> bool {
     set_panic_hook();
 

backend/sync_lib/tests/web.rs

@@ -26,7 +26,8 @@ fn test_base64_to_bytes_error() {
 fn merge_text() {
     let left = b"hello ";
     let right = b"world";
-    assert_eq!(merge(b"", left, right), b"hello world".to_vec());
+    let result = merge(b"", left, right);
+    assert_eq!(result, b"hello world");
 }
 
 #[wasm_bindgen_test(unsupported = test)]

backend/sync_server/src/config.rs

@@ -42,9 +42,12 @@ impl Config {
     }
 
     pub async fn load_from_file(path: &Path) -> Result<Self> {
-        let contents = fs::read_to_string(path)
+        let contents = fs::read_to_string(path).await.with_context(|| {
-            .await
+            format!(
-            .with_context(|| format!("Cannot load configuration from disk from ({path:?})"))?;
+                "Cannot load configuration from disk from {}",
+                path.display()
+            )
+        })?;
 
         let config = serde_yaml::from_str(&contents).context("Failed to parse configuration")?;
 

backend/sync_server/src/config/database_config.rs

@@ -1,31 +1,33 @@
+use std::path::PathBuf;
+
 use log::debug;
 use serde::{Deserialize, Serialize};
 
-use crate::consts::{DEFAULT_MAX_CONNECTIONS, DEFAULT_SQLITE_URL};
+use crate::consts::{DEFAULT_DATABASES_DIRECTORY_PATH, DEFAULT_MAX_CONNECTIONS};
 
 #[derive(Debug, Deserialize, Serialize, Clone)]
 pub struct DatabaseConfig {
-    #[serde(default = "default_sqlite_url")]
+    #[serde(default = "default_databases_directory_path")]
-    pub sqlite_url: String,
+    pub databases_directory_path: PathBuf,
 
     #[serde(default = "default_max_connections")]
     pub max_connections: u32,
 }
 
-fn default_sqlite_url() -> String {
+fn default_databases_directory_path() -> PathBuf {
-    debug!("Using default sqlite url: {}", DEFAULT_SQLITE_URL);
+    debug!("Using default databases directory path: {DEFAULT_DATABASES_DIRECTORY_PATH:?}");
-    DEFAULT_SQLITE_URL.to_owned()
+    PathBuf::from(DEFAULT_DATABASES_DIRECTORY_PATH)
 }
 
 fn default_max_connections() -> u32 {
-    debug!("Using default max connections: {}", DEFAULT_MAX_CONNECTIONS);
+    debug!("Using default max connections: {DEFAULT_MAX_CONNECTIONS}");
     DEFAULT_MAX_CONNECTIONS
 }
 
 impl Default for DatabaseConfig {
     fn default() -> Self {
         Self {
-            sqlite_url: default_sqlite_url(),
+            databases_directory_path: default_databases_directory_path(),
             max_connections: default_max_connections(),
         }
     }

backend/sync_server/src/config/server_config.rs

@@ -15,20 +15,17 @@ pub struct ServerConfig {
 }
 
 fn default_host() -> String {
-    debug!("Using default server host: {}", DEFAULT_HOST);
+    debug!("Using default server host: {DEFAULT_HOST}");
     DEFAULT_HOST.to_owned()
 }
 
 fn default_port() -> u16 {
-    debug!("Using default server port: {}", DEFAULT_PORT);
+    debug!("Using default server port: {DEFAULT_PORT}");
     DEFAULT_PORT
 }
 
 fn default_max_body_size_mb() -> usize {
-    debug!(
+    debug!("Using default max body size (MB): {DEFAULT_MAX_BODY_SIZE_MB}");
-        "Using default max body size (MB): {}",
-        DEFAULT_MAX_BODY_SIZE_MB
-    );
     DEFAULT_MAX_BODY_SIZE_MB
 }
 

backend/sync_server/src/consts.rs

@@ -1,5 +1,5 @@
 pub const CONFIG_PATH: &str = "config.yml";
-pub const DEFAULT_SQLITE_URL: &str = "db.sqlite3";
+pub const DEFAULT_DATABASES_DIRECTORY_PATH: &str = "databases";
 pub const DEFAULT_HOST: &str = "127.0.0.1";
 pub const DEFAULT_PORT: u16 = 3000;
 pub const DEFAULT_MAX_CONNECTIONS: u32 = 12;

backend/sync_server/src/database.rs

@@ -1,4 +1,5 @@
-use core::{str::FromStr as _, time::Duration};
+use core::time::Duration;
+use std::{collections::HashMap, sync::Arc};
 
 use anyhow::{Context as _, Result};
 use models::{
@@ -7,19 +8,66 @@ use models::{
 use sqlx::{sqlite::SqliteConnectOptions, types::chrono::Utc};
 pub mod models;
 use sqlx::{Pool, Sqlite, sqlite::SqlitePoolOptions};
+use tokio::sync::Mutex;
+use uuid::fmt::Hyphenated;
 
 use crate::config::database_config::DatabaseConfig;
 
 #[derive(Clone, Debug)]
 pub struct Database {
-    connection_pool: Pool<Sqlite>,
+    config: DatabaseConfig,
+    connection_pools: Arc<Mutex<HashMap<VaultId, Pool<Sqlite>>>>,
 }
 
 pub type Transaction<'a> = sqlx::Transaction<'a, Sqlite>;
 
 impl Database {
     pub async fn try_new(config: &DatabaseConfig) -> Result<Self> {
-        let connection_options = SqliteConnectOptions::from_str(&config.sqlite_url)?
+        tokio::fs::create_dir_all(&config.databases_directory_path)
+            .await
+            .with_context(|| {
+                format!(
+                    "Failed to create databases directory: {}",
+                    config.databases_directory_path.to_string_lossy()
+                )
+            })?;
+
+        let mut connection_pools = std::collections::HashMap::new();
+
+        let mut entries = tokio::fs::read_dir(&config.databases_directory_path).await?;
+        while let Some(entry) = entries.next_entry().await? {
+            if !entry.file_name().to_string_lossy().ends_with(".sqlite") {
+                continue;
+            }
+
+            let vault: VaultId = entry
+                .file_name()
+                .to_string_lossy()
+                .trim_end_matches(".sqlite")
+                .to_owned();
+
+            connection_pools.insert(
+                vault.clone(),
+                Self::create_vault_database(config, &vault).await?,
+            );
+        }
+
+        Ok(Self {
+            config: config.clone(),
+            connection_pools: Arc::new(Mutex::new(connection_pools)),
+        })
+    }
+
+    async fn create_vault_database(
+        config: &DatabaseConfig,
+        vault: &VaultId,
+    ) -> Result<Pool<Sqlite>> {
+        let file_name = config
+            .databases_directory_path
+            .join(format!("{vault}.sqlite"));
+
+        let connection_options = SqliteConnectOptions::new()
+            .filename(file_name.clone())
             .create_if_missing(true)
             .busy_timeout(Duration::from_secs(3600))
             .journal_mode(sqlx::sqlite::SqliteJournalMode::Wal);
@@ -29,18 +77,11 @@ impl Database {
             .test_before_acquire(true)
             .connect_with(connection_options)
             .await
-            .with_context(|| {
+            .with_context(|| format!("Cannot open database at {}", file_name.display()))?;
-                format!(
-                    "Cannot connect to database with url: {}",
-                    &config.sqlite_url
-                )
-            })?;
 
         Self::run_migrations(&pool).await?;
 
-        Ok(Self {
+        Ok(pool)
-            connection_pool: pool,
-        })
     }
 
     async fn run_migrations(pool: &Pool<Sqlite>) -> Result<()> {
@@ -50,17 +91,38 @@ impl Database {
             .context("Cannot check for pending migrations")
     }
 
+    async fn get_connection_pool(&mut self, vault: &VaultId) -> Result<Pool<Sqlite>> {
+        let mut pools = self.connection_pools.lock().await;
+        if !pools.contains_key(vault) {
+            let pool = Self::create_vault_database(&self.config, vault).await?;
+            pools.insert(vault.clone(), pool);
+        }
+
+        let pool = pools
+            .get(vault)
+            .expect("Pool was just inserted or already exists");
+
+        Ok(pool.clone())
+    }
+
     /// Attempting to write from this transaction might result in a
     /// database locked error. Use this transaction for read-only operations.
-    pub async fn create_readonly_transaction(&self) -> Result<Transaction<'_>> {
+    pub async fn create_readonly_transaction(
-        self.connection_pool
+        &mut self,
+        vault: &VaultId,
+    ) -> Result<Transaction<'static>> {
+        self.get_connection_pool(vault)
+            .await?
             .begin()
             .await
             .context("Cannot create transaction")
     }
 
-    pub async fn create_write_transaction(&self) -> Result<Transaction<'_>> {
+    pub async fn create_write_transaction(
-        let mut transaction = self.create_readonly_transaction().await?;
+        &mut self,
+        vault: &VaultId,
+    ) -> Result<Transaction<'static>> {
+        let mut transaction = self.create_readonly_transaction(vault).await?;
 
         // sqlx doesn't support immediate transactions for sqlite: https://github.com/launchbadge/sqlx/issues/481
         sqlx::query!("END; BEGIN IMMEDIATE;")
@@ -72,7 +134,7 @@ impl Database {
 
     /// Return the latest state of all documents in the vault
     pub async fn get_latest_documents(
-        &self,
+        &mut self,
         vault: &VaultId,
         transaction: Option<&mut Transaction<'_>>,
     ) -> Result<Vec<DocumentVersionWithoutContent>> {
@@ -80,24 +142,22 @@ impl Database {
             DocumentVersionWithoutContent,
             r#"
             select 
-                vault_id,
                 vault_update_id,
-                document_id as "document_id: uuid::Uuid", 
+                document_id as "document_id: Hyphenated", 
                 relative_path,
-                created_date as "created_date: chrono::DateTime<Utc>",
                 updated_date as "updated_date: chrono::DateTime<Utc>",
                 is_deleted
             from latest_document_versions
-            where vault_id = ?
             order by vault_update_id desc
             "#,
-            vault,
         );
 
         if let Some(transaction) = transaction {
             query.fetch_all(&mut **transaction).await
         } else {
-            query.fetch_all(&self.connection_pool).await
+            query
+                .fetch_all(&self.get_connection_pool(vault).await?)
+                .await
         }
         .context("Cannot fetch latest documents")
     }
@@ -105,7 +165,7 @@ impl Database {
     /// Return the latest state of all documents (including deleted) in the
     /// vault which have changed since the given update id
     pub async fn get_latest_documents_since(
-        &self,
+        &mut self,
         vault: &VaultId,
         vault_update_id: VaultUpdateId,
         transaction: Option<&mut Transaction<'_>>,
@@ -114,25 +174,24 @@ impl Database {
             DocumentVersionWithoutContent,
             r#"
             select
-                vault_id,
                 vault_update_id,
-                document_id as "document_id: uuid::Uuid",
+                document_id as "document_id: Hyphenated",
                 relative_path,
-                created_date as "created_date: chrono::DateTime<Utc>",
                 updated_date as "updated_date: chrono::DateTime<Utc>",
                 is_deleted
             from latest_document_versions
-            where vault_id = ? and vault_update_id > ?
+            where vault_update_id > ?
             order by vault_update_id desc
             "#,
-            vault,
             vault_update_id
         );
 
         if let Some(transaction) = transaction {
             query.fetch_all(&mut **transaction).await
         } else {
-            query.fetch_all(&self.connection_pool).await
+            query
+                .fetch_all(&self.get_connection_pool(vault).await?)
+                .await
         }
         .with_context(|| {
             format!("Cannot fetch latest documents since vault_update_id {vault_update_id}")
@@ -140,7 +199,7 @@ impl Database {
     }
 
     pub async fn get_max_update_id_in_vault(
-        &self,
+        &mut self,
         vault: &VaultId,
         transaction: Option<&mut Transaction<'_>>,
     ) -> Result<i64> {
@@ -148,22 +207,22 @@ impl Database {
             r#"
             select coalesce(max(vault_update_id), 0) as max_vault_update_id
             from documents
-            where vault_id = ?
             "#,
-            vault
         );
 
         if let Some(transaction) = transaction {
             query.fetch_one(&mut **transaction).await
         } else {
-            query.fetch_one(&self.connection_pool).await
+            query
+                .fetch_one(&self.get_connection_pool(vault).await?)
+                .await
         }
         .map(|row| row.max_vault_update_id)
         .context("Cannot fetch max update id in vault")
     }
 
     pub async fn get_latest_document_by_path(
-        &self,
+        &mut self,
         vault: &VaultId,
         relative_path: &str,
         transaction: Option<&mut Transaction<'_>>,
@@ -172,68 +231,67 @@ impl Database {
             StoredDocumentVersion,
             r#"
             select 
-                vault_id,
                 vault_update_id,
-                document_id as "document_id: uuid::Uuid", 
+                document_id as "document_id: Hyphenated", 
                 relative_path,
-                created_date as "created_date: chrono::DateTime<Utc>",
                 updated_date as "updated_date: chrono::DateTime<Utc>",
                 content,
                 is_deleted
             from latest_document_versions
-            where vault_id = ? and relative_path = ?
+            where relative_path = ?
             order by vault_update_id desc  -- `latest_document_versions` only contains a single latest version of each document, however,
                                            -- multiple documents can have the same `relative_path`, if they have been deleted. That's
                                            -- why we only care about the latest version of the document with the given relative path.
             limit 1
             "#,
-            vault,
             relative_path
         );
 
         if let Some(transaction) = transaction {
             query.fetch_optional(&mut **transaction).await
         } else {
-            query.fetch_optional(&self.connection_pool).await
+            query
+                .fetch_optional(&self.get_connection_pool(vault).await?)
+                .await
         }
         .context("Cannot fetch latest document version")
     }
 
     pub async fn get_latest_document(
-        &self,
+        &mut self,
         vault: &VaultId,
         document_id: &DocumentId,
         transaction: Option<&mut Transaction<'_>>,
     ) -> Result<Option<StoredDocumentVersion>> {
+        let document_id = document_id.as_hyphenated();
         let query = sqlx::query_as!(
             StoredDocumentVersion,
             r#"
             select 
-                vault_id,
                 vault_update_id,
-                document_id as "document_id: uuid::Uuid", 
+                document_id as "document_id: Hyphenated", 
                 relative_path,
-                created_date as "created_date: chrono::DateTime<Utc>",
                 updated_date as "updated_date: chrono::DateTime<Utc>",
                 content,
                 is_deleted
             from latest_document_versions
-            where vault_id = ? and document_id = ?
+            where document_id = ?
             "#,
-            vault,
             document_id
         );
 
         if let Some(transaction) = transaction {
             query.fetch_optional(&mut **transaction).await
         } else {
-            query.fetch_optional(&self.connection_pool).await
+            query
+                .fetch_optional(&self.get_connection_pool(vault).await?)
+                .await
         }
         .context("Cannot fetch latest document version")
     }
 
     pub async fn get_document_version(
-        &self,
+        &mut self,
         vault: &VaultId,
         vault_update_id: VaultUpdateId,
         transaction: Option<&mut Transaction<'_>>,
@@ -242,52 +300,49 @@ impl Database {
             StoredDocumentVersion,
             r#"
             select 
-                vault_id,
                 vault_update_id,
-                document_id as "document_id: uuid::Uuid", 
+                document_id as "document_id: Hyphenated", 
                 relative_path,
-                created_date as "created_date: chrono::DateTime<Utc>",
                 updated_date as "updated_date: chrono::DateTime<Utc>",
                 content,
                 is_deleted
             from documents
-            where vault_id = ? and vault_update_id = ?"#,
+            where vault_update_id = ?"#,
-            vault,
             vault_update_id
         );
 
         if let Some(transaction) = transaction {
             query.fetch_optional(&mut **transaction).await
         } else {
-            query.fetch_optional(&self.connection_pool).await
+            query
+                .fetch_optional(&self.get_connection_pool(vault).await?)
+                .await
         }
         .context("Cannot fetch document version")
     }
 
     pub async fn insert_document_version(
-        &self,
+        &mut self,
+        vault: &VaultId,
         version: &StoredDocumentVersion,
         transaction: Option<&mut Transaction<'_>>,
     ) -> Result<()> {
+        let document_id = version.document_id.as_hyphenated();
         let query = sqlx::query!(
             r#"
             insert into documents (
-                vault_id,
                 vault_update_id,
                 document_id, 
                 relative_path,
-                created_date,
                 updated_date,
                 content,
                 is_deleted
             )
-            values (?, ?, ?, ?, ?, ?, ?, ?)
+            values (?, ?, ?, ?, ?, ?)
             "#,
-            version.vault_id,
             version.vault_update_id,
-            version.document_id,
+            document_id,
             version.relative_path,
-            version.created_date,
             version.updated_date,
             version.content,
             version.is_deleted
@@ -296,7 +351,7 @@ impl Database {
         if let Some(transaction) = transaction {
             query.execute(&mut **transaction).await
         } else {
-            query.execute(&self.connection_pool).await
+            query.execute(&self.get_connection_pool(vault).await?).await
         }
         .context("Cannot insert document version")?;
 

backend/sync_server/src/database/migrations/20241207143519_bootstrap.sql

@@ -1,25 +1,21 @@
 CREATE TABLE IF NOT EXISTS documents (
-    vault_id TEXT NOT NULL,
+    vault_update_id INTEGER NOT NULL PRIMARY KEY,
-    vault_update_id INTEGER NOT NULL,
     document_id TEXT NOT NULL,
     relative_path TEXT NOT NULL,
-    created_date TIMESTAMP NOT NULL,
     updated_date TIMESTAMP NOT NULL,
     content BLOB NOT NULL,
-    is_deleted BOOLEAN NOT NULL,
+    is_deleted BOOLEAN NOT NULL
-    PRIMARY KEY (vault_id, vault_update_id)
 );
 
 CREATE VIEW IF NOT EXISTS latest_document_versions AS
 SELECT d.*
 FROM documents d
 INNER JOIN (
-    SELECT vault_id, MAX(vault_update_id) AS max_version_id
+    SELECT MAX(vault_update_id) AS max_version_id
     FROM documents
-    GROUP BY vault_id, document_id
+    GROUP BY document_id
 ) max_versions
-ON d.vault_id = max_versions.vault_id
+ON d.vault_update_id = max_versions.max_version_id;
-AND d.vault_update_id = max_versions.max_version_id;
 
 CREATE INDEX IF NOT EXISTS idx_documents_vault_id_relative_path
-ON documents (vault_id, relative_path);
+ON documents (relative_path);

backend/sync_server/src/database/models.rs

@@ -9,30 +9,24 @@ pub type DocumentId = uuid::Uuid;
 
 #[derive(Debug, Clone)]
 pub struct StoredDocumentVersion {
-    pub vault_id: VaultId,
     pub vault_update_id: VaultUpdateId,
     pub document_id: DocumentId,
     pub relative_path: String,
-    pub created_date: DateTime<Utc>,
     pub updated_date: DateTime<Utc>,
     pub content: Vec<u8>,
     pub is_deleted: bool,
 }
 
 impl PartialEq<Self> for StoredDocumentVersion {
-    fn eq(&self, other: &Self) -> bool {
+    fn eq(&self, other: &Self) -> bool { self.vault_update_id == other.vault_update_id }
-        self.vault_id == other.vault_id && self.vault_update_id == other.vault_update_id
-    }
 }
 
 #[derive(Debug, Clone, Serialize, JsonSchema)]
 #[serde(rename_all = "camelCase")]
 pub struct DocumentVersionWithoutContent {
-    pub vault_id: VaultId,
     pub vault_update_id: VaultUpdateId,
     pub document_id: DocumentId,
     pub relative_path: String,
-    pub created_date: DateTime<Utc>,
     pub updated_date: DateTime<Utc>,
     pub is_deleted: bool,
 }
@@ -40,11 +34,9 @@ pub struct DocumentVersionWithoutContent {
 impl From<StoredDocumentVersion> for DocumentVersionWithoutContent {
     fn from(value: StoredDocumentVersion) -> Self {
         Self {
-            vault_id: value.vault_id,
             vault_update_id: value.vault_update_id,
             document_id: value.document_id,
             relative_path: value.relative_path,
-            created_date: value.created_date,
             updated_date: value.updated_date,
             is_deleted: value.is_deleted,
         }
@@ -54,11 +46,9 @@ impl From<StoredDocumentVersion> for DocumentVersionWithoutContent {
 #[derive(Debug, Clone, Serialize, JsonSchema)]
 #[serde(rename_all = "camelCase")]
 pub struct DocumentVersion {
-    pub vault_id: VaultId,
     pub vault_update_id: VaultUpdateId,
     pub document_id: DocumentId,
     pub relative_path: String,
-    pub created_date: DateTime<Utc>,
     pub updated_date: DateTime<Utc>,
     pub content_base64: String,
     pub is_deleted: bool,
@@ -67,11 +57,9 @@ pub struct DocumentVersion {
 impl From<StoredDocumentVersion> for DocumentVersion {
     fn from(value: StoredDocumentVersion) -> Self {
         Self {
-            vault_id: value.vault_id,
             vault_update_id: value.vault_update_id,
             document_id: value.document_id,
             relative_path: value.relative_path,
-            created_date: value.created_date,
             updated_date: value.updated_date,
             content_base64: bytes_to_base64(&value.content),
             is_deleted: value.is_deleted,

backend/sync_server/src/errors.rs

@@ -33,12 +33,12 @@ pub enum SyncServerError {
 impl SyncServerError {
     pub fn serialize(&self) -> SerializedError {
         match self {
-            Self::InitError(error) => error.into(),
+            Self::InitError(error)
-            Self::ClientError(error) => error.into(),
+            | Self::ClientError(error)
-            Self::ServerError(error) => error.into(),
+            | Self::ServerError(error)
-            Self::NotFound(error) => error.into(),
+            | Self::NotFound(error)
-            Self::Unauthorized(error) => error.into(),
+            | Self::Unauthorized(error)
-            Self::PermissionDeniedError(error) => error.into(),
+            | Self::PermissionDeniedError(error) => error.into(),
         }
     }
 }
@@ -48,9 +48,10 @@ impl IntoResponse for SyncServerError {
         let body = Json(self.serialize());
 
         match self {
-            Self::InitError(_) => (StatusCode::INTERNAL_SERVER_ERROR, body).into_response(),
+            Self::InitError(_) | Self::ServerError(_) => {
+                (StatusCode::INTERNAL_SERVER_ERROR, body).into_response()
+            }
             Self::ClientError(_) => (StatusCode::BAD_REQUEST, body).into_response(),
-            Self::ServerError(_) => (StatusCode::INTERNAL_SERVER_ERROR, body).into_response(),
             Self::NotFound(_) => (StatusCode::NOT_FOUND, body).into_response(),
             Self::Unauthorized(_) => (StatusCode::UNAUTHORIZED, body).into_response(),
             Self::PermissionDeniedError(_) => (StatusCode::FORBIDDEN, body).into_response(),

backend/sync_server/src/server.rs

@@ -16,6 +16,7 @@ use axum::{
     extract::{DefaultBodyLimit, Request},
     http::{self, HeaderValue, Method},
     response::IntoResponse,
+    routing::IntoMakeService,
 };
 use log::{error, info};
 use tokio::signal;
@@ -30,7 +31,10 @@ use tower_http::{
 };
 use tracing::{Level, info_span};
 
-use crate::errors::{SerializedError, not_found_error};
+use crate::{
+    config::server_config::ServerConfig,
+    errors::{SerializedError, not_found_error},
+};
 mod app_state;
 mod auth;
 mod create_document;
@@ -52,24 +56,9 @@ pub async fn create_server() -> Result<()> {
         .await
         .context("Failed to initialise app state")?;
 
-    let address = format!(
+    let server_config = app_state.config.server.clone();
-        "{}:{}",
-        &app_state.config.server.host, &app_state.config.server.port
-    );
-
-    let mut api = OpenApi {
-        info: Info {
-            title: "VaultLink sync server".to_owned(),
-            summary: Some(
-                "Simple API for syncing documents between concurrent clients.".to_owned(),
-            ),
-            description: Some(include_str!("../README.md").to_owned()),
-            version: env!("CARGO_PKG_VERSION").to_owned(),
-            ..Info::default()
-        },
-        ..OpenApi::default()
-    };
 
+    let mut api = create_open_api();
     let app = ApiRouter::new()
         .api_route("/ping", get(ping::ping))
         .api_route(
@@ -140,11 +129,42 @@ pub async fn create_server() -> Result<()> {
                 .allow_methods([Method::GET, Method::POST, Method::PUT, Method::DELETE]),
         )
         .with_state(app_state)
-        .finish_api_with(&mut api, api_docs)
+        .finish_api_with(&mut api, add_api_docs_error_example)
         .layer(Extension(Arc::new(api))) // https://github.com/tamasfe/aide/blob/507f4a8822bc0c13cbda0f589da1e0f4cbcdb812/examples/example-axum/src/main.rs#L39
         .fallback(handler_404)
         .into_make_service();
 
+    start_server(app, &server_config).await
+}
+
+async fn serve_api(Extension(api): Extension<Arc<OpenApi>>) -> impl IntoResponse { Json(api) }
+
+fn create_open_api() -> OpenApi {
+    OpenApi {
+        info: Info {
+            title: "VaultLink sync server".to_owned(),
+            summary: Some(
+                "Simple API for syncing documents between concurrent clients.".to_owned(),
+            ),
+            description: Some(include_str!("../README.md").to_owned()),
+            version: env!("CARGO_PKG_VERSION").to_owned(),
+            ..Info::default()
+        },
+        ..OpenApi::default()
+    }
+}
+
+fn add_api_docs_error_example(api: TransformOpenApi<'_>) -> TransformOpenApi<'_> {
+    api.default_response_with::<Json<SerializedError>, _>(|res| {
+        res.example(SerializedError {
+            message: "An error has occurred".to_owned(),
+            causes: vec![],
+        })
+    })
+}
+
+async fn start_server(app: IntoMakeService<axum::Router>, config: &ServerConfig) -> Result<()> {
+    let address = format!("{}:{}", config.host, config.port);
     let listener = tokio::net::TcpListener::bind(address.clone())
         .await
         .with_context(|| format!("Failed to bind to address: {address}"))?;
@@ -163,17 +183,6 @@ pub async fn create_server() -> Result<()> {
         .context("Failed to start server")
 }
 
-async fn serve_api(Extension(api): Extension<Arc<OpenApi>>) -> impl IntoResponse { Json(api) }
-
-fn api_docs(api: TransformOpenApi<'_>) -> TransformOpenApi<'_> {
-    api.default_response_with::<Json<SerializedError>, _>(|res| {
-        res.example(SerializedError {
-            message: "An error has occurred".to_owned(),
-            causes: vec![],
-        })
-    })
-}
-
 async fn shutdown_signal() {
     let ctrl_c = async {
         signal::ctrl_c()
@@ -193,8 +202,8 @@ async fn shutdown_signal() {
     let terminate = std::future::pending::<()>();
 
     tokio::select! {
-        _ = ctrl_c => {},
+        () = ctrl_c => {},
-        _ = terminate => {},
+        () = terminate => {},
     }
 }
 

backend/sync_server/src/server/create_document.rs

@@ -6,7 +6,6 @@ use axum_extra::{
     headers::{Authorization, authorization::Bearer},
 };
 use axum_jsonschema::Json;
-use chrono::{DateTime, Utc};
 use schemars::JsonSchema;
 use serde::Deserialize;
 use sync_lib::base64_to_bytes;
@@ -17,7 +16,7 @@ use super::{
     requests::{CreateDocumentVersion, CreateDocumentVersionMultipart},
 };
 use crate::{
-    database::models::{DocumentVersionWithoutContent, StoredDocumentVersion, VaultId},
+    database::models::{DocumentId, DocumentVersionWithoutContent, StoredDocumentVersion, VaultId},
     errors::{SyncServerError, client_error, server_error},
     utils::sanitize_path,
 };
@@ -44,8 +43,8 @@ pub async fn create_document_multipart(
         auth_header,
         state,
         vault_id,
+        request.document_id,
         request.relative_path,
-        request.created_date,
         request.content.contents.to_vec(),
     )
     .await
@@ -69,8 +68,8 @@ pub async fn create_document_json(
         auth_header,
         state,
         vault_id,
+        request.document_id,
         request.relative_path,
-        request.created_date,
         content_bytes,
     )
     .await
@@ -78,20 +77,39 @@ pub async fn create_document_json(
 
 async fn internal_create_document(
     auth_header: Authorization<Bearer>,
-    state: AppState,
+    mut state: AppState,
     vault_id: VaultId,
+    document_id: Option<DocumentId>,
     relative_path: String,
-    created_date: DateTime<Utc>,
     content: Vec<u8>,
 ) -> Result<Json<DocumentVersionWithoutContent>, SyncServerError> {
     auth(&state, auth_header.token())?;
 
     let mut transaction = state
         .database
-        .create_write_transaction()
+        .create_write_transaction(&vault_id)
         .await
         .map_err(server_error)?;
 
+    let document_id = match document_id {
+        Some(document_id) => {
+            let existing_version = state
+                .database
+                .get_latest_document(&vault_id, &document_id, Some(&mut transaction))
+                .await
+                .map_err(server_error)?;
+
+            if existing_version.is_some() {
+                return Err(client_error(anyhow::anyhow!(
+                    "Document with the same ID already exists"
+                )));
+            }
+
+            document_id
+        }
+        None => uuid::Uuid::new_v4(),
+    };
+
     let last_update_id = state
         .database
         .get_max_update_id_in_vault(&vault_id, Some(&mut transaction))
@@ -101,19 +119,17 @@ async fn internal_create_document(
     let sanitized_relative_path = sanitize_path(&relative_path);
 
     let new_version = StoredDocumentVersion {
-        vault_id,
         vault_update_id: last_update_id + 1,
-        document_id: uuid::Uuid::new_v4(),
+        document_id,
         relative_path: sanitized_relative_path,
         content,
-        created_date,
         updated_date: chrono::Utc::now(),
         is_deleted: false,
     };
 
     state
         .database
-        .insert_document_version(&new_version, Some(&mut transaction))
+        .insert_document_version(&vault_id, &new_version, Some(&mut transaction))
         .await
         .map_err(server_error)?;
 

backend/sync_server/src/server/delete_document.rs

@@ -10,7 +10,7 @@ use serde::Deserialize;
 
 use super::{app_state::AppState, auth::auth, requests::DeleteDocumentVersion};
 use crate::{
-    database::models::{DocumentId, StoredDocumentVersion, VaultId},
+    database::models::{DocumentId, DocumentVersionWithoutContent, StoredDocumentVersion, VaultId},
     errors::{SyncServerError, server_error},
     utils::sanitize_path,
 };
@@ -29,14 +29,14 @@ pub async fn delete_document(
         vault_id,
         document_id,
     }): Path<PathParams>,
-    State(state): State<AppState>,
+    State(mut state): State<AppState>,
     Json(request): Json<DeleteDocumentVersion>,
-) -> Result<(), SyncServerError> {
+) -> Result<Json<DocumentVersionWithoutContent>, SyncServerError> {
     auth(&state, auth_header.token())?;
 
     let mut transaction = state
         .database
-        .create_write_transaction()
+        .create_write_transaction(&vault_id)
         .await
         .map_err(server_error)?;
 
@@ -47,19 +47,17 @@ pub async fn delete_document(
         .map_err(server_error)?;
 
     let new_version = StoredDocumentVersion {
-        vault_id,
         vault_update_id: last_update_id + 1,
         document_id,
         relative_path: sanitize_path(&request.relative_path),
         content: vec![],
-        created_date: request.created_date,
         updated_date: chrono::Utc::now(),
         is_deleted: true,
     };
 
     state
         .database
-        .insert_document_version(&new_version, Some(&mut transaction))
+        .insert_document_version(&vault_id, &new_version, Some(&mut transaction))
         .await
         .map_err(server_error)?;
 
@@ -69,5 +67,5 @@ pub async fn delete_document(
         .context("Failed to commit successful transaction")
         .map_err(server_error)?;
 
-    Ok(())
+    Ok(Json(new_version.into()))
 }

backend/sync_server/src/server/fetch_document_version.rs

@@ -30,7 +30,7 @@ pub async fn fetch_document_version(
         document_id,
         vault_update_id,
     }): Path<PathParams>,
-    State(state): State<AppState>,
+    State(mut state): State<AppState>,
 ) -> Result<Json<DocumentVersion>, SyncServerError> {
     auth(&state, auth_header.token())?;
 
@@ -39,12 +39,14 @@ pub async fn fetch_document_version(
         .get_document_version(&vault_id, vault_update_id, None)
         .await
         .map_err(server_error)?
-        .map(Ok)
+        .map_or_else(
-        .unwrap_or_else(|| {
+            || {
-            Err(not_found_error(anyhow!(
+                Err(not_found_error(anyhow!(
-                "Document with vault update id `{vault_update_id}` not found",
+                    "Document with vault update id `{vault_update_id}` not found",
-            )))
+                )))
-        })?;
+            },
+            Ok,
+        )?;
 
     if result.document_id != document_id {
         return Err(not_found_error(anyhow!(

backend/sync_server/src/server/fetch_document_version_content.rs

@@ -32,7 +32,7 @@ pub async fn fetch_document_version_content(
         document_id,
         vault_update_id,
     }): Path<PathParams>,
-    State(state): State<AppState>,
+    State(mut state): State<AppState>,
 ) -> Result<Bytes, SyncServerError> {
     auth(&state, auth_header.token())?;
 
@@ -41,12 +41,14 @@ pub async fn fetch_document_version_content(
         .get_document_version(&vault_id, vault_update_id, None)
         .await
         .map_err(server_error)?
-        .map(Ok)
+        .map_or_else(
-        .unwrap_or_else(|| {
+            || {
-            Err(not_found_error(anyhow!(
+                Err(not_found_error(anyhow!(
-                "Document with vault update id `{vault_update_id}` not found",
+                    "Document with vault update id `{vault_update_id}` not found",
-            )))
+                )))
-        })?;
+            },
+            Ok,
+        )?;
 
     if result.document_id != document_id {
         return Err(not_found_error(anyhow!(

backend/sync_server/src/server/fetch_latest_document_version.rs

@@ -28,7 +28,7 @@ pub async fn fetch_latest_document_version(
         vault_id,
         document_id,
     }): Path<PathParams>,
-    State(state): State<AppState>,
+    State(mut state): State<AppState>,
 ) -> Result<Json<DocumentVersion>, SyncServerError> {
     auth(&state, auth_header.token())?;
 
@@ -37,12 +37,14 @@ pub async fn fetch_latest_document_version(
         .get_latest_document(&vault_id, &document_id, None)
         .await
         .map_err(server_error)?
-        .map(Ok)
+        .map_or_else(
-        .unwrap_or_else(|| {
+            || {
-            Err(not_found_error(anyhow!(
+                Err(not_found_error(anyhow!(
-                "Document with id `{document_id}` not found",
+                    "Document with id `{document_id}` not found",
-            )))
+                )))
-        })?;
+            },
+            Ok,
+        )?;
 
     Ok(Json(latest_version.into()))
 }

backend/sync_server/src/server/fetch_latest_documents.rs

@@ -30,7 +30,7 @@ pub async fn fetch_latest_documents(
     TypedHeader(auth_header): TypedHeader<Authorization<Bearer>>,
     Path(PathParams { vault_id }): Path<PathParams>,
     Query(QueryParams { since_update_id }): Query<QueryParams>,
-    State(state): State<AppState>,
+    State(mut state): State<AppState>,
 ) -> Result<Json<FetchLatestDocumentsResponse>, SyncServerError> {
     auth(&state, auth_header.token())?;
 

backend/sync_server/src/server/requests.rs

@@ -1,24 +1,27 @@
 use aide_axum_typed_multipart::FieldData;
 use axum::body::Bytes;
 use axum_typed_multipart::TryFromMultipart;
-use chrono::{DateTime, Utc};
 use schemars::JsonSchema;
 use serde::{self, Deserialize};
 
-use crate::database::models::VaultUpdateId;
+use crate::database::models::{DocumentId, VaultUpdateId};
 
 #[derive(Debug, Deserialize, JsonSchema)]
 #[serde(rename_all = "camelCase")]
 pub struct CreateDocumentVersion {
+    /// The client can decide the document id (if it wishes to) in order
+    /// to help with syncing. If the client does not provide a document id,
+    /// the server will generate one. If the client provides a document id
+    /// it must not already exist in the database.
+    pub document_id: Option<DocumentId>,
     pub relative_path: String,
-    pub created_date: DateTime<Utc>,
     pub content_base64: String,
 }
 
 #[derive(Debug, TryFromMultipart, JsonSchema)]
 pub struct CreateDocumentVersionMultipart {
+    pub document_id: Option<DocumentId>,
     pub relative_path: String,
-    pub created_date: DateTime<Utc>,
     #[form_data(limit = "unlimited")]
     pub content: FieldData<Bytes>,
 }
@@ -28,7 +31,6 @@ pub struct CreateDocumentVersionMultipart {
 pub struct UpdateDocumentVersion {
     pub parent_version_id: VaultUpdateId,
     pub relative_path: String,
-    pub created_date: DateTime<Utc>,
     pub content_base64: String,
 }
 
@@ -37,7 +39,6 @@ pub struct UpdateDocumentVersion {
 pub struct UpdateDocumentVersionMultipart {
     pub parent_version_id: VaultUpdateId,
     pub relative_path: String,
-    pub created_date: DateTime<Utc>,
     #[form_data(limit = "unlimited")]
     pub content: FieldData<Bytes>,
 }
@@ -46,5 +47,4 @@ pub struct UpdateDocumentVersionMultipart {
 #[serde(rename_all = "camelCase")]
 pub struct DeleteDocumentVersion {
     pub relative_path: String,
-    pub created_date: DateTime<Utc>,
 }

backend/sync_server/src/server/update_document.rs

@@ -6,7 +6,6 @@ use axum_extra::{
     headers::{Authorization, authorization::Bearer},
 };
 use axum_jsonschema::Json;
-use chrono::{DateTime, Utc};
 use log::info;
 use schemars::JsonSchema;
 use serde::Deserialize;
@@ -50,7 +49,6 @@ pub async fn update_document_multipart(
         document_id,
         request.parent_version_id,
         request.relative_path,
-        request.created_date,
         request.content.contents.to_vec(),
     )
     .await
@@ -77,21 +75,19 @@ pub async fn update_document_json(
         document_id,
         request.parent_version_id,
         request.relative_path,
-        request.created_date,
         content_bytes,
     )
     .await
 }
 
-#[allow(clippy::too_many_arguments)]
+#[allow(clippy::too_many_arguments, clippy::too_many_lines)]
 async fn internal_update_document(
     auth_header: Authorization<Bearer>,
-    state: AppState,
+    mut state: AppState,
     vault_id: VaultId,
     document_id: DocumentId,
     parent_version_id: VaultUpdateId,
     relative_path: String,
-    created_date: DateTime<Utc>,
     content: Vec<u8>,
 ) -> Result<Json<DocumentUpdateResponse>, SyncServerError> {
     auth(&state, auth_header.token())?;
@@ -114,7 +110,7 @@ async fn internal_update_document(
 
     let mut transaction = state
         .database
-        .create_write_transaction()
+        .create_write_transaction(&vault_id)
         .await
         .map_err(server_error)?;
 
@@ -138,6 +134,18 @@ async fn internal_update_document(
             Ok,
         )?;
 
+    if latest_version.is_deleted {
+        transaction
+            .rollback()
+            .await
+            .context("Failed to roll back transaction")
+            .map_err(server_error)?;
+
+        return Ok(Json(DocumentUpdateResponse::FastForwardUpdate(
+            latest_version.into(),
+        )));
+    }
+
     let sanitized_relative_path = sanitize_path(&relative_path);
 
     // Return the latest version if the content and path are the same as the latest
@@ -168,7 +176,7 @@ async fn internal_update_document(
     let new_relative_path = if parent_document.relative_path == latest_version.relative_path
         && latest_version.relative_path != sanitized_relative_path
     {
-        let mut new_relative_path = Default::default();
+        let mut new_relative_path = String::default();
         for candidate in deduped_file_paths(&sanitized_relative_path) {
             if state
                 .database
@@ -188,19 +196,17 @@ async fn internal_update_document(
     };
 
     let new_version = StoredDocumentVersion {
-        vault_id,
         document_id,
         vault_update_id: last_update_id + 1,
         relative_path: new_relative_path,
         content: merged_content,
-        created_date,
         updated_date: chrono::Utc::now(),
-        is_deleted: latest_version.is_deleted,
+        is_deleted: false,
     };
 
     state
         .database
-        .insert_document_version(&new_version, Some(&mut transaction))
+        .insert_document_version(&vault_id, &new_version, Some(&mut transaction))
         .await
         .map_err(server_error)?;
 

frontend/obsidian-plugin/package.json

@@ -14,7 +14,7 @@
 	"license": "MIT",
 	"devDependencies": {
 		"@types/jest": "^29.5.14",
-		"@types/node": "^22.13.5",
+		"@types/node": "^22.13.10",
 		"css-loader": "^7.1.2",
 		"date-fns": "^4.1.0",
 		"file-loader": "^6.2.0",
@@ -23,14 +23,14 @@
 		"mini-css-extract-plugin": "^2.9.2",
 		"obsidian": "1.8.7",
 		"resolve-url-loader": "^5.0.0",
-		"sass": "^1.85.0",
+		"sass": "^1.85.1",
 		"sass-loader": "^16.0.5",
 		"sync-client": "file:../sync-client",
-		"terser-webpack-plugin": "^5.3.11",
+		"terser-webpack-plugin": "^5.3.14",
 		"ts-jest": "^29.2.6",
 		"ts-loader": "^9.5.2",
 		"tslib": "2.8.1",
-		"typescript": "5.7.3",
+		"typescript": "5.8.2",
 		"url": "^0.11.4",
 		"virtual-scroller": "^1.13.1",
 		"webpack": "^5.98.0",

frontend/obsidian-plugin/src/obisidan-event-handler.ts

@@ -9,10 +9,7 @@ export class ObsidianFileEventHandler {
 		if (file instanceof TFile) {
 			this.client.logger.info(`File created: ${file.path}`);
 
-			await this.client.syncer.syncLocallyCreatedFile(
+			await this.client.syncer.syncLocallyCreatedFile(file.path);
-				file.path,
-				new Date(file.stat.ctime)
-			);
 		} else {
 			this.client.logger.debug(`Folder created: ${file.path}, ignored`);
 		}
@@ -34,8 +31,7 @@ export class ObsidianFileEventHandler {
 
 			await this.client.syncer.syncLocallyUpdatedFile({
 				oldPath,
-				relativePath: file.path,
+				relativePath: file.path
-				updateTime: new Date(file.stat.ctime)
 			});
 		} else {
 			this.client.logger.debug(
@@ -53,8 +49,7 @@ export class ObsidianFileEventHandler {
 			this.client.logger.info(`File modified: ${file.path}`);
 
 			await this.client.syncer.syncLocallyUpdatedFile({
-				relativePath: file.path,
+				relativePath: file.path
-				updateTime: new Date(file.stat.ctime)
 			});
 		} else {
 			this.client.logger.debug(`Folder modified: ${file.path}, ignored`);

frontend/obsidian-plugin/src/views/history-view.ts

@@ -60,7 +60,6 @@ export class HistoryView extends ItemView {
 		}
 
 		element.createEl("span", {
-			// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
 			text: entry.relativePath
 		});
 

frontend/obsidian-plugin/src/views/logs-view.ts

@@ -1,6 +1,6 @@
 import type { WorkspaceLeaf } from "obsidian";
 import { ItemView } from "obsidian";
-import type VaultLinkPlugin from "src/vault-link-plugin";
+import type VaultLinkPlugin from "../vault-link-plugin";
 import type { SyncClient } from "sync-client";
 
 export class LogsView extends ItemView {

frontend/obsidian-plugin/src/views/settings-tab.ts

@@ -1,7 +1,7 @@
 import type { App } from "obsidian";
 import { Notice, PluginSettingTab, Setting } from "obsidian";
 
-import type VaultLinkPlugin from "src/vault-link-plugin";
+import type VaultLinkPlugin from "../vault-link-plugin";
 import type { StatusDescription } from "./status-description";
 import { LogsView } from "./logs-view";
 import { HistoryView } from "./history-view";

frontend/obsidian-plugin/src/views/status-bar.ts

@@ -1,5 +1,5 @@
 import type { HistoryStats, SyncClient } from "sync-client";
-import type VaultLinkPlugin from "src/vault-link-plugin";
+import type VaultLinkPlugin from "../vault-link-plugin";
 
 export class StatusBar {
 	private readonly statusBarItem: HTMLElement;

frontend/obsidian-plugin/webpack.config.js

@@ -94,9 +94,6 @@ module.exports = (env, argv) => ({
 		alias: {
 			root: __dirname,
 			src: path.resolve(__dirname, "src")
-		},
-		fallback: {
-			url: require.resolve("url")
 		}
 	},
 	output: {

frontend/package-lock.json

@@ -12,11 +12,11 @@
 			],
 			"devDependencies": {
 				"concurrently": "^9.1.2",
-				"eslint": "9.21.0",
+				"eslint": "9.22.0",
 				"eslint-plugin-unused-imports": "^4.1.4",
-				"npm-check-updates": "^17.1.14",
+				"npm-check-updates": "^17.1.15",
-				"prettier": "^3.5.2",
+				"prettier": "^3.5.3",
-				"typescript-eslint": "8.24.1"
+				"typescript-eslint": "8.26.1"
 			}
 		},
 		"../backend/sync_lib/pkg": {
@@ -39,7 +39,6 @@
 		},
 		"node_modules/@babel/code-frame": {
 			"version": "7.26.2",
-			"dev": true,
 			"license": "MIT",
 			"dependencies": {
 				"@babel/helper-validator-identifier": "^7.25.9",
@@ -179,7 +178,6 @@
 		},
 		"node_modules/@babel/helper-validator-identifier": {
 			"version": "7.25.9",
-			"dev": true,
 			"license": "MIT",
 			"engines": {
 				"node": ">=6.9.0"
@@ -194,12 +192,14 @@
 			}
 		},
 		"node_modules/@babel/helpers": {
-			"version": "7.26.9",
+			"version": "7.26.10",
+			"resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.26.10.tgz",
+			"integrity": "sha512-UPYc3SauzZ3JGgj87GgZ89JVdC5dj0AoetR5Bw6wj4niittNyFh6+eOGonYvJ1ao6B8lEa3Q3klS7ADZ53bc5g==",
 			"dev": true,
 			"license": "MIT",
 			"dependencies": {
 				"@babel/template": "^7.26.9",
-				"@babel/types": "^7.26.9"
+				"@babel/types": "^7.26.10"
 			},
 			"engines": {
 				"node": ">=6.9.0"
@@ -463,7 +463,9 @@
 			}
 		},
 		"node_modules/@babel/types": {
-			"version": "7.26.9",
+			"version": "7.26.10",
+			"resolved": "https://registry.npmjs.org/@babel/types/-/types-7.26.10.tgz",
+			"integrity": "sha512-emqcG3vHrpxUKTrxcblR36dcrcoRDvKmnL/dCL6ZsHaShW80qxCAcNhzQZrpeM765VzEos+xOi4s+r4IXzTwdQ==",
 			"dev": true,
 			"license": "MIT",
 			"dependencies": {
@@ -580,6 +582,16 @@
 				"node": "^18.18.0 || ^20.9.0 || >=21.1.0"
 			}
 		},
+		"node_modules/@eslint/config-helpers": {
+			"version": "0.1.0",
+			"resolved": "https://registry.npmjs.org/@eslint/config-helpers/-/config-helpers-0.1.0.tgz",
+			"integrity": "sha512-kLrdPDJE1ckPo94kmPPf9Hfd0DU0Jw6oKYrhe+pwSC0iTUInmTa+w6fw8sGgcfkFJGNdWOUeOaDM4quW4a7OkA==",
+			"dev": true,
+			"license": "Apache-2.0",
+			"engines": {
+				"node": "^18.18.0 || ^20.9.0 || >=21.1.0"
+			}
+		},
 		"node_modules/@eslint/core": {
 			"version": "0.12.0",
 			"resolved": "https://registry.npmjs.org/@eslint/core/-/core-0.12.0.tgz",
@@ -618,9 +630,9 @@
 			}
 		},
 		"node_modules/@eslint/js": {
-			"version": "9.21.0",
+			"version": "9.22.0",
-			"resolved": "https://registry.npmjs.org/@eslint/js/-/js-9.21.0.tgz",
+			"resolved": "https://registry.npmjs.org/@eslint/js/-/js-9.22.0.tgz",
-			"integrity": "sha512-BqStZ3HX8Yz6LvsF5ByXYrtigrV5AXADWLAGc7PH/1SxOb7/FIYYMszZZWiUou/GB9P2lXWk2SV4d+Z8h0nknw==",
+			"integrity": "sha512-vLFajx9o8d1/oL2ZkpMYbkLv8nDB6yaIwFNt7nI4+I80U/z03SxmfOMsLbvWr3p7C+Wnoh//aOu2pQW8cS0HCQ==",
 			"dev": true,
 			"license": "MIT",
 			"engines": {
@@ -1130,6 +1142,8 @@
 		},
 		"node_modules/@nodelib/fs.scandir": {
 			"version": "2.1.5",
+			"resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz",
+			"integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==",
 			"dev": true,
 			"license": "MIT",
 			"dependencies": {
@@ -1142,6 +1156,8 @@
 		},
 		"node_modules/@nodelib/fs.stat": {
 			"version": "2.0.5",
+			"resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz",
+			"integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==",
 			"dev": true,
 			"license": "MIT",
 			"engines": {
@@ -1150,6 +1166,8 @@
 		},
 		"node_modules/@nodelib/fs.walk": {
 			"version": "1.2.8",
+			"resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz",
+			"integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==",
 			"dev": true,
 			"license": "MIT",
 			"dependencies": {
@@ -1235,7 +1253,6 @@
 		},
 		"node_modules/@redocly/ajv": {
 			"version": "8.11.2",
-			"dev": true,
 			"license": "MIT",
 			"dependencies": {
 				"fast-deep-equal": "^3.1.1",
@@ -1250,17 +1267,14 @@
 		},
 		"node_modules/@redocly/ajv/node_modules/json-schema-traverse": {
 			"version": "1.0.0",
-			"dev": true,
 			"license": "MIT"
 		},
 		"node_modules/@redocly/config": {
 			"version": "0.20.3",
-			"dev": true,
 			"license": "MIT"
 		},
 		"node_modules/@redocly/openapi-core": {
 			"version": "1.29.0",
-			"dev": true,
 			"license": "MIT",
 			"dependencies": {
 				"@redocly/ajv": "^8.11.2",
@@ -1280,7 +1294,6 @@
 		},
 		"node_modules/@redocly/openapi-core/node_modules/brace-expansion": {
 			"version": "2.0.1",
-			"dev": true,
 			"license": "MIT",
 			"dependencies": {
 				"balanced-match": "^1.0.0"
@@ -1288,7 +1301,6 @@
 		},
 		"node_modules/@redocly/openapi-core/node_modules/minimatch": {
 			"version": "5.1.6",
-			"dev": true,
 			"license": "ISC",
 			"dependencies": {
 				"brace-expansion": "^2.0.1"
@@ -1458,9 +1470,9 @@
 			"license": "MIT"
 		},
 		"node_modules/@types/node": {
-			"version": "22.13.5",
+			"version": "22.13.10",
-			"resolved": "https://registry.npmjs.org/@types/node/-/node-22.13.5.tgz",
+			"resolved": "https://registry.npmjs.org/@types/node/-/node-22.13.10.tgz",
-			"integrity": "sha512-+lTU0PxZXn0Dr1NBtC7Y8cR21AJr87dLLU953CWA6pMxxv/UDc7jYAY90upcrie1nRcD6XNG5HOYEDtgW5TxAg==",
+			"integrity": "sha512-I6LPUvlRH+O6VRUqYOcMudhaIdUVWfsjnZavnsraHvpBwaEyMN29ry+0UVJhImYL16xsscu0aske3yA+uPOWfw==",
 			"dev": true,
 			"license": "MIT",
 			"dependencies": {
@@ -1494,15 +1506,17 @@
 			"license": "MIT"
 		},
 		"node_modules/@typescript-eslint/eslint-plugin": {
-			"version": "8.24.1",
+			"version": "8.26.1",
+			"resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.26.1.tgz",
+			"integrity": "sha512-2X3mwqsj9Bd3Ciz508ZUtoQQYpOhU/kWoUqIf49H8Z0+Vbh6UF/y0OEYp0Q0axOGzaBGs7QxRwq0knSQ8khQNA==",
 			"dev": true,
 			"license": "MIT",
 			"dependencies": {
 				"@eslint-community/regexpp": "^4.10.0",
-				"@typescript-eslint/scope-manager": "8.24.1",
+				"@typescript-eslint/scope-manager": "8.26.1",
-				"@typescript-eslint/type-utils": "8.24.1",
+				"@typescript-eslint/type-utils": "8.26.1",
-				"@typescript-eslint/utils": "8.24.1",
+				"@typescript-eslint/utils": "8.26.1",
-				"@typescript-eslint/visitor-keys": "8.24.1",
+				"@typescript-eslint/visitor-keys": "8.26.1",
 				"graphemer": "^1.4.0",
 				"ignore": "^5.3.1",
 				"natural-compare": "^1.4.0",
@@ -1518,18 +1532,20 @@
 			"peerDependencies": {
 				"@typescript-eslint/parser": "^8.0.0 || ^8.0.0-alpha.0",
 				"eslint": "^8.57.0 || ^9.0.0",
-				"typescript": ">=4.8.4 <5.8.0"
+				"typescript": ">=4.8.4 <5.9.0"
 			}
 		},
 		"node_modules/@typescript-eslint/parser": {
-			"version": "8.24.1",
+			"version": "8.26.1",
+			"resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-8.26.1.tgz",
+			"integrity": "sha512-w6HZUV4NWxqd8BdeFf81t07d7/YV9s7TCWrQQbG5uhuvGUAW+fq1usZ1Hmz9UPNLniFnD8GLSsDpjP0hm1S4lQ==",
 			"dev": true,
 			"license": "MIT",
 			"dependencies": {
-				"@typescript-eslint/scope-manager": "8.24.1",
+				"@typescript-eslint/scope-manager": "8.26.1",
-				"@typescript-eslint/types": "8.24.1",
+				"@typescript-eslint/types": "8.26.1",
-				"@typescript-eslint/typescript-estree": "8.24.1",
+				"@typescript-eslint/typescript-estree": "8.26.1",
-				"@typescript-eslint/visitor-keys": "8.24.1",
+				"@typescript-eslint/visitor-keys": "8.26.1",
 				"debug": "^4.3.4"
 			},
 			"engines": {
@@ -1541,16 +1557,18 @@
 			},
 			"peerDependencies": {
 				"eslint": "^8.57.0 || ^9.0.0",
-				"typescript": ">=4.8.4 <5.8.0"
+				"typescript": ">=4.8.4 <5.9.0"
 			}
 		},
 		"node_modules/@typescript-eslint/scope-manager": {
-			"version": "8.24.1",
+			"version": "8.26.1",
+			"resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-8.26.1.tgz",
+			"integrity": "sha512-6EIvbE5cNER8sqBu6V7+KeMZIC1664d2Yjt+B9EWUXrsyWpxx4lEZrmvxgSKRC6gX+efDL/UY9OpPZ267io3mg==",
 			"dev": true,
 			"license": "MIT",
 			"dependencies": {
-				"@typescript-eslint/types": "8.24.1",
+				"@typescript-eslint/types": "8.26.1",
-				"@typescript-eslint/visitor-keys": "8.24.1"
+				"@typescript-eslint/visitor-keys": "8.26.1"
 			},
 			"engines": {
 				"node": "^18.18.0 || ^20.9.0 || >=21.1.0"
@@ -1561,12 +1579,14 @@
 			}
 		},
 		"node_modules/@typescript-eslint/type-utils": {
-			"version": "8.24.1",
+			"version": "8.26.1",
+			"resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-8.26.1.tgz",
+			"integrity": "sha512-Kcj/TagJLwoY/5w9JGEFV0dclQdyqw9+VMndxOJKtoFSjfZhLXhYjzsQEeyza03rwHx2vFEGvrJWJBXKleRvZg==",
 			"dev": true,
 			"license": "MIT",
 			"dependencies": {
-				"@typescript-eslint/typescript-estree": "8.24.1",
+				"@typescript-eslint/typescript-estree": "8.26.1",
-				"@typescript-eslint/utils": "8.24.1",
+				"@typescript-eslint/utils": "8.26.1",
 				"debug": "^4.3.4",
 				"ts-api-utils": "^2.0.1"
 			},
@@ -1579,11 +1599,13 @@
 			},
 			"peerDependencies": {
 				"eslint": "^8.57.0 || ^9.0.0",
-				"typescript": ">=4.8.4 <5.8.0"
+				"typescript": ">=4.8.4 <5.9.0"
 			}
 		},
 		"node_modules/@typescript-eslint/types": {
-			"version": "8.24.1",
+			"version": "8.26.1",
+			"resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-8.26.1.tgz",
+			"integrity": "sha512-n4THUQW27VmQMx+3P+B0Yptl7ydfceUj4ON/AQILAASwgYdZ/2dhfymRMh5egRUrvK5lSmaOm77Ry+lmXPOgBQ==",
 			"dev": true,
 			"license": "MIT",
 			"engines": {
@@ -1595,12 +1617,14 @@
 			}
 		},
 		"node_modules/@typescript-eslint/typescript-estree": {
-			"version": "8.24.1",
+			"version": "8.26.1",
+			"resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-8.26.1.tgz",
+			"integrity": "sha512-yUwPpUHDgdrv1QJ7YQal3cMVBGWfnuCdKbXw1yyjArax3353rEJP1ZA+4F8nOlQ3RfS2hUN/wze3nlY+ZOhvoA==",
 			"dev": true,
 			"license": "MIT",
 			"dependencies": {
-				"@typescript-eslint/types": "8.24.1",
+				"@typescript-eslint/types": "8.26.1",
-				"@typescript-eslint/visitor-keys": "8.24.1",
+				"@typescript-eslint/visitor-keys": "8.26.1",
 				"debug": "^4.3.4",
 				"fast-glob": "^3.3.2",
 				"is-glob": "^4.0.3",
@@ -1616,11 +1640,13 @@
 				"url": "https://opencollective.com/typescript-eslint"
 			},
 			"peerDependencies": {
-				"typescript": ">=4.8.4 <5.8.0"
+				"typescript": ">=4.8.4 <5.9.0"
 			}
 		},
 		"node_modules/@typescript-eslint/typescript-estree/node_modules/brace-expansion": {
 			"version": "2.0.1",
+			"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz",
+			"integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==",
 			"dev": true,
 			"license": "MIT",
 			"dependencies": {
@@ -1629,6 +1655,8 @@
 		},
 		"node_modules/@typescript-eslint/typescript-estree/node_modules/minimatch": {
 			"version": "9.0.5",
+			"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.5.tgz",
+			"integrity": "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==",
 			"dev": true,
 			"license": "ISC",
 			"dependencies": {
@@ -1642,14 +1670,16 @@
 			}
 		},
 		"node_modules/@typescript-eslint/utils": {
-			"version": "8.24.1",
+			"version": "8.26.1",
+			"resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-8.26.1.tgz",
+			"integrity": "sha512-V4Urxa/XtSUroUrnI7q6yUTD3hDtfJ2jzVfeT3VK0ciizfK2q/zGC0iDh1lFMUZR8cImRrep6/q0xd/1ZGPQpg==",
 			"dev": true,
 			"license": "MIT",
 			"dependencies": {
 				"@eslint-community/eslint-utils": "^4.4.0",
-				"@typescript-eslint/scope-manager": "8.24.1",
+				"@typescript-eslint/scope-manager": "8.26.1",
-				"@typescript-eslint/types": "8.24.1",
+				"@typescript-eslint/types": "8.26.1",
-				"@typescript-eslint/typescript-estree": "8.24.1"
+				"@typescript-eslint/typescript-estree": "8.26.1"
 			},
 			"engines": {
 				"node": "^18.18.0 || ^20.9.0 || >=21.1.0"
@@ -1660,15 +1690,17 @@
 			},
 			"peerDependencies": {
 				"eslint": "^8.57.0 || ^9.0.0",
-				"typescript": ">=4.8.4 <5.8.0"
+				"typescript": ">=4.8.4 <5.9.0"
 			}
 		},
 		"node_modules/@typescript-eslint/visitor-keys": {
-			"version": "8.24.1",
+			"version": "8.26.1",
+			"resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-8.26.1.tgz",
+			"integrity": "sha512-AjOC3zfnxd6S4Eiy3jwktJPclqhFHNyd8L6Gycf9WUPoKZpgM5PjkxY1X7uSy61xVpiJDhhk7XT2NVsN3ALTWg==",
 			"dev": true,
 			"license": "MIT",
 			"dependencies": {
-				"@typescript-eslint/types": "8.24.1",
+				"@typescript-eslint/types": "8.26.1",
 				"eslint-visitor-keys": "^4.2.0"
 			},
 			"engines": {
@@ -1909,7 +1941,6 @@
 		},
 		"node_modules/agent-base": {
 			"version": "7.1.3",
-			"dev": true,
 			"license": "MIT",
 			"engines": {
 				"node": ">= 14"
@@ -1976,7 +2007,6 @@
 		},
 		"node_modules/ansi-colors": {
 			"version": "4.1.3",
-			"dev": true,
 			"license": "MIT",
 			"engines": {
 				"node": ">=6"
@@ -2039,7 +2069,6 @@
 		},
 		"node_modules/argparse": {
 			"version": "2.0.1",
-			"dev": true,
 			"license": "Python-2.0"
 		},
 		"node_modules/async": {
@@ -2161,7 +2190,6 @@
 		},
 		"node_modules/balanced-match": {
 			"version": "1.0.2",
-			"dev": true,
 			"license": "MIT"
 		},
 		"node_modules/big.js": {
@@ -2249,7 +2277,6 @@
 		},
 		"node_modules/byte-base64": {
 			"version": "1.1.0",
-			"dev": true,
 			"license": "MIT"
 		},
 		"node_modules/call-bind-apply-helpers": {
@@ -2335,7 +2362,6 @@
 		},
 		"node_modules/change-case": {
 			"version": "5.4.4",
-			"dev": true,
 			"license": "MIT"
 		},
 		"node_modules/char-regex": {
@@ -2445,7 +2471,6 @@
 		},
 		"node_modules/colorette": {
 			"version": "1.4.0",
-			"dev": true,
 			"license": "MIT"
 		},
 		"node_modules/commander": {
@@ -2597,7 +2622,6 @@
 		},
 		"node_modules/debug": {
 			"version": "4.4.0",
-			"dev": true,
 			"license": "MIT",
 			"dependencies": {
 				"ms": "^2.1.3"
@@ -2822,18 +2846,19 @@
 			}
 		},
 		"node_modules/eslint": {
-			"version": "9.21.0",
+			"version": "9.22.0",
-			"resolved": "https://registry.npmjs.org/eslint/-/eslint-9.21.0.tgz",
+			"resolved": "https://registry.npmjs.org/eslint/-/eslint-9.22.0.tgz",
-			"integrity": "sha512-KjeihdFqTPhOMXTt7StsDxriV4n66ueuF/jfPNC3j/lduHwr/ijDwJMsF+wyMJethgiKi5wniIE243vi07d3pg==",
+			"integrity": "sha512-9V/QURhsRN40xuHXWjV64yvrzMjcz7ZyNoF2jJFmy9j/SLk0u1OLSZgXi28MrXjymnjEGSR80WCdab3RGMDveQ==",
 			"dev": true,
 			"license": "MIT",
 			"dependencies": {
 				"@eslint-community/eslint-utils": "^4.2.0",
 				"@eslint-community/regexpp": "^4.12.1",
 				"@eslint/config-array": "^0.19.2",
+				"@eslint/config-helpers": "^0.1.0",
 				"@eslint/core": "^0.12.0",
 				"@eslint/eslintrc": "^3.3.0",
-				"@eslint/js": "9.21.0",
+				"@eslint/js": "9.22.0",
 				"@eslint/plugin-kit": "^0.2.7",
 				"@humanfs/node": "^0.16.6",
 				"@humanwhocodes/module-importer": "^1.0.1",
@@ -2845,7 +2870,7 @@
 				"cross-spawn": "^7.0.6",
 				"debug": "^4.3.2",
 				"escape-string-regexp": "^4.0.0",
-				"eslint-scope": "^8.2.0",
+				"eslint-scope": "^8.3.0",
 				"eslint-visitor-keys": "^4.2.0",
 				"espree": "^10.3.0",
 				"esquery": "^1.5.0",
@@ -2896,7 +2921,9 @@
 			}
 		},
 		"node_modules/eslint-scope": {
-			"version": "8.2.0",
+			"version": "8.3.0",
+			"resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-8.3.0.tgz",
+			"integrity": "sha512-pUNxi75F8MJ/GdeKtVLSbYg4ZI34J6C0C7sbL4YOp2exGwen7ZsuBqKzUhXd0qMQ362yET3z+uPwKeg/0C2XCQ==",
 			"dev": true,
 			"license": "BSD-2-Clause",
 			"dependencies": {
@@ -2991,7 +3018,6 @@
 		},
 		"node_modules/eventemitter3": {
 			"version": "5.0.1",
-			"dev": true,
 			"license": "MIT"
 		},
 		"node_modules/events": {
@@ -3048,11 +3074,12 @@
 		},
 		"node_modules/fast-deep-equal": {
 			"version": "3.1.3",
-			"dev": true,
 			"license": "MIT"
 		},
 		"node_modules/fast-glob": {
 			"version": "3.3.3",
+			"resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.3.tgz",
+			"integrity": "sha512-7MptL8U0cqcFdzIzwOTHoilX9x5BrNqye7Z/LuC7kCMRio1EMSyqRK3BEAUD7sXRq4iT4AzTVuZdhgQ2TCvYLg==",
 			"dev": true,
 			"license": "MIT",
 			"dependencies": {
@@ -3068,6 +3095,8 @@
 		},
 		"node_modules/fast-glob/node_modules/glob-parent": {
 			"version": "5.1.2",
+			"resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz",
+			"integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==",
 			"dev": true,
 			"license": "ISC",
 			"dependencies": {
@@ -3111,7 +3140,9 @@
 			}
 		},
 		"node_modules/fastq": {
-			"version": "1.19.0",
+			"version": "1.19.1",
+			"resolved": "https://registry.npmjs.org/fastq/-/fastq-1.19.1.tgz",
+			"integrity": "sha512-GwLTyxkCXjXbxqIhTsMI2Nui8huMPtnxg7krajPJAjnEG/iiOS7i+zCtWGZR9G0NBKbXKh6X9m9UIsYX/N6vvQ==",
 			"dev": true,
 			"license": "ISC",
 			"dependencies": {
@@ -3128,7 +3159,6 @@
 		},
 		"node_modules/fetch-retry": {
 			"version": "6.0.0",
-			"dev": true,
 			"license": "MIT"
 		},
 		"node_modules/file-entry-cache": {
@@ -3449,7 +3479,6 @@
 		},
 		"node_modules/https-proxy-agent": {
 			"version": "7.0.6",
-			"dev": true,
 			"license": "MIT",
 			"dependencies": {
 				"agent-base": "^7.1.2",
@@ -3536,7 +3565,6 @@
 		},
 		"node_modules/index-to-position": {
 			"version": "0.1.2",
-			"dev": true,
 			"license": "MIT",
 			"engines": {
 				"node": ">=18"
@@ -4271,7 +4299,6 @@
 		},
 		"node_modules/js-levenshtein": {
 			"version": "1.1.6",
-			"dev": true,
 			"license": "MIT",
 			"engines": {
 				"node": ">=0.10.0"
@@ -4279,12 +4306,10 @@
 		},
 		"node_modules/js-tokens": {
 			"version": "4.0.0",
-			"dev": true,
 			"license": "MIT"
 		},
 		"node_modules/js-yaml": {
 			"version": "4.1.0",
-			"dev": true,
 			"license": "MIT",
 			"dependencies": {
 				"argparse": "^2.0.1"
@@ -4497,6 +4522,8 @@
 		},
 		"node_modules/merge2": {
 			"version": "1.4.1",
+			"resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz",
+			"integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==",
 			"dev": true,
 			"license": "MIT",
 			"engines": {
@@ -4631,7 +4658,6 @@
 		},
 		"node_modules/ms": {
 			"version": "2.1.3",
-			"dev": true,
 			"license": "MIT"
 		},
 		"node_modules/nanoid": {
@@ -4686,7 +4712,9 @@
 			}
 		},
 		"node_modules/npm-check-updates": {
-			"version": "17.1.14",
+			"version": "17.1.15",
+			"resolved": "https://registry.npmjs.org/npm-check-updates/-/npm-check-updates-17.1.15.tgz",
+			"integrity": "sha512-miATvKu5rjec/1wxc5TGDjpsucgtCHwRVZorZpDkS6NzdWXfnUWlN4abZddWb7XSijAuBNzzYglIdTm9SbgMVg==",
 			"dev": true,
 			"license": "Apache-2.0",
 			"bin": {
@@ -4758,8 +4786,9 @@
 			}
 		},
 		"node_modules/openapi-fetch": {
-			"version": "0.13.4",
+			"version": "0.13.5",
-			"dev": true,
+			"resolved": "https://registry.npmjs.org/openapi-fetch/-/openapi-fetch-0.13.5.tgz",
+			"integrity": "sha512-AQK8T9GSKFREFlN1DBXTYsLjs7YV2tZcJ7zUWxbjMoQmj8dDSFRrzhLCbHPZWA1TMV3vACqfCxLEZcwf2wxV6Q==",
 			"license": "MIT",
 			"dependencies": {
 				"openapi-typescript-helpers": "^0.0.15"
@@ -4767,7 +4796,6 @@
 		},
 		"node_modules/openapi-typescript": {
 			"version": "7.6.1",
-			"dev": true,
 			"license": "MIT",
 			"dependencies": {
 				"@redocly/openapi-core": "^1.28.0",
@@ -4786,12 +4814,10 @@
 		},
 		"node_modules/openapi-typescript-helpers": {
 			"version": "0.0.15",
-			"dev": true,
 			"license": "MIT"
 		},
 		"node_modules/openapi-typescript/node_modules/parse-json": {
 			"version": "8.1.0",
-			"dev": true,
 			"license": "MIT",
 			"dependencies": {
 				"@babel/code-frame": "^7.22.13",
@@ -4807,7 +4833,6 @@
 		},
 		"node_modules/openapi-typescript/node_modules/supports-color": {
 			"version": "9.4.0",
-			"dev": true,
 			"license": "MIT",
 			"engines": {
 				"node": ">=12"
@@ -4818,7 +4843,6 @@
 		},
 		"node_modules/openapi-typescript/node_modules/type-fest": {
 			"version": "4.35.0",
-			"dev": true,
 			"license": "(MIT OR CC0-1.0)",
 			"engines": {
 				"node": ">=16"
@@ -4873,7 +4897,6 @@
 		},
 		"node_modules/p-queue": {
 			"version": "8.1.0",
-			"dev": true,
 			"license": "MIT",
 			"dependencies": {
 				"eventemitter3": "^5.0.1",
@@ -4888,7 +4911,6 @@
 		},
 		"node_modules/p-timeout": {
 			"version": "6.1.4",
-			"dev": true,
 			"license": "MIT",
 			"engines": {
 				"node": ">=14.16"
@@ -4966,7 +4988,6 @@
 		},
 		"node_modules/picocolors": {
 			"version": "1.1.1",
-			"dev": true,
 			"license": "ISC"
 		},
 		"node_modules/picomatch": {
@@ -5049,7 +5070,6 @@
 		},
 		"node_modules/pluralize": {
 			"version": "8.0.0",
-			"dev": true,
 			"license": "MIT",
 			"engines": {
 				"node": ">=4"
@@ -5163,9 +5183,9 @@
 			}
 		},
 		"node_modules/prettier": {
-			"version": "3.5.2",
+			"version": "3.5.3",
-			"resolved": "https://registry.npmjs.org/prettier/-/prettier-3.5.2.tgz",
+			"resolved": "https://registry.npmjs.org/prettier/-/prettier-3.5.3.tgz",
-			"integrity": "sha512-lc6npv5PH7hVqozBR7lkBNOGXV9vMwROAPlumdBkX0wTbbzPu/U1hk5yL8p2pt4Xoc+2mkT8t/sow2YrV/M5qg==",
+			"integrity": "sha512-QQtaxnoDJeAkDvDKWCLiwIXkTgRhwYDEQCghU9Z6q03iyek/rxRh/2lC3HB7P8sWT2xC/y5JDctPLBIGzHKbhw==",
 			"dev": true,
 			"license": "MIT",
 			"bin": {
@@ -5255,6 +5275,8 @@
 		},
 		"node_modules/queue-microtask": {
 			"version": "1.2.3",
+			"resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz",
+			"integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==",
 			"dev": true,
 			"funding": [
 				{
@@ -5328,7 +5350,6 @@
 		},
 		"node_modules/require-from-string": {
 			"version": "2.0.2",
-			"dev": true,
 			"license": "MIT",
 			"engines": {
 				"node": ">=0.10.0"
@@ -5411,7 +5432,9 @@
 			}
 		},
 		"node_modules/reusify": {
-			"version": "1.0.4",
+			"version": "1.1.0",
+			"resolved": "https://registry.npmjs.org/reusify/-/reusify-1.1.0.tgz",
+			"integrity": "sha512-g6QUff04oZpHs0eG5p83rFLhHeV00ug/Yf9nZM6fLeUrPguBTkTQOdpAWWspMh55TZfVQDPaN3NQJfbVRAxdIw==",
 			"dev": true,
 			"license": "MIT",
 			"engines": {
@@ -5421,6 +5444,8 @@
 		},
 		"node_modules/run-parallel": {
 			"version": "1.2.0",
+			"resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz",
+			"integrity": "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==",
 			"dev": true,
 			"funding": [
 				{
@@ -5469,7 +5494,9 @@
 			"license": "MIT"
 		},
 		"node_modules/sass": {
-			"version": "1.85.0",
+			"version": "1.85.1",
+			"resolved": "https://registry.npmjs.org/sass/-/sass-1.85.1.tgz",
+			"integrity": "sha512-Uk8WpxM5v+0cMR0XjX9KfRIacmSG86RH4DCCZjLU2rFh5tyutt9siAXJ7G+YfxQ99Q6wrRMbMlVl6KqUms71ag==",
 			"dev": true,
 			"license": "MIT",
 			"dependencies": {
@@ -5871,7 +5898,9 @@
 			}
 		},
 		"node_modules/terser-webpack-plugin": {
-			"version": "5.3.11",
+			"version": "5.3.14",
+			"resolved": "https://registry.npmjs.org/terser-webpack-plugin/-/terser-webpack-plugin-5.3.14.tgz",
+			"integrity": "sha512-vkZjpUjb6OMS7dhV+tILUW6BhpDR7P2L/aQSAv+Uwk+m8KATX9EccViHTJR2qDtACKPIYndLGCyl3FMo+r2LMw==",
 			"dev": true,
 			"license": "MIT",
 			"dependencies": {
@@ -6196,8 +6225,9 @@
 			}
 		},
 		"node_modules/typescript": {
-			"version": "5.7.3",
+			"version": "5.8.2",
-			"dev": true,
+			"resolved": "https://registry.npmjs.org/typescript/-/typescript-5.8.2.tgz",
+			"integrity": "sha512-aJn6wq13/afZp/jT9QZmwEjDqqvSGp1VT5GVg+f/t6/oVyrgXM6BY1h9BRh/O5p3PlUPAe+WuiEZOmb/49RqoQ==",
 			"license": "Apache-2.0",
 			"bin": {
 				"tsc": "bin/tsc",
@@ -6208,13 +6238,15 @@
 			}
 		},
 		"node_modules/typescript-eslint": {
-			"version": "8.24.1",
+			"version": "8.26.1",
+			"resolved": "https://registry.npmjs.org/typescript-eslint/-/typescript-eslint-8.26.1.tgz",
+			"integrity": "sha512-t/oIs9mYyrwZGRpDv3g+3K6nZ5uhKEMt2oNmAPwaY4/ye0+EH4nXIPYNtkYFS6QHm+1DFg34DbglYBz5P9Xysg==",
 			"dev": true,
 			"license": "MIT",
 			"dependencies": {
-				"@typescript-eslint/eslint-plugin": "8.24.1",
+				"@typescript-eslint/eslint-plugin": "8.26.1",
-				"@typescript-eslint/parser": "8.24.1",
+				"@typescript-eslint/parser": "8.26.1",
-				"@typescript-eslint/utils": "8.24.1"
+				"@typescript-eslint/utils": "8.26.1"
 			},
 			"engines": {
 				"node": "^18.18.0 || ^20.9.0 || >=21.1.0"
@@ -6225,7 +6257,7 @@
 			},
 			"peerDependencies": {
 				"eslint": "^8.57.0 || ^9.0.0",
-				"typescript": ">=4.8.4 <5.8.0"
+				"typescript": ">=4.8.4 <5.9.0"
 			}
 		},
 		"node_modules/undici-types": {
@@ -6280,7 +6312,6 @@
 		},
 		"node_modules/uri-js-replace": {
 			"version": "1.0.1",
-			"dev": true,
 			"license": "MIT"
 		},
 		"node_modules/url": {
@@ -6311,7 +6342,6 @@
 		},
 		"node_modules/uuid": {
 			"version": "11.1.0",
-			"dev": true,
 			"funding": [
 				"https://github.com/sponsors/broofa",
 				"https://github.com/sponsors/ctavan"
@@ -6480,6 +6510,8 @@
 		},
 		"node_modules/webpack-merge": {
 			"version": "6.0.1",
+			"resolved": "https://registry.npmjs.org/webpack-merge/-/webpack-merge-6.0.1.tgz",
+			"integrity": "sha512-hXXvrjtx2PLYx4qruKl+kyRSLc52V+cCvMxRjmKwoA+CBbbF5GfIBtR6kCvl0fYGqTUPKB+1ktVmTHqMOzgCBg==",
 			"dev": true,
 			"license": "MIT",
 			"dependencies": {
@@ -6643,7 +6675,6 @@
 		},
 		"node_modules/yaml-ast-parser": {
 			"version": "0.0.43",
-			"dev": true,
 			"license": "Apache-2.0"
 		},
 		"node_modules/yargs": {
@@ -6665,7 +6696,6 @@
 		},
 		"node_modules/yargs-parser": {
 			"version": "21.1.1",
-			"dev": true,
 			"license": "ISC",
 			"engines": {
 				"node": ">=12"
@@ -6698,7 +6728,7 @@
 			"license": "MIT",
 			"devDependencies": {
 				"@types/jest": "^29.5.14",
-				"@types/node": "^22.13.5",
+				"@types/node": "^22.13.10",
 				"css-loader": "^7.1.2",
 				"date-fns": "^4.1.0",
 				"file-loader": "^6.2.0",
@@ -6707,14 +6737,14 @@
 				"mini-css-extract-plugin": "^2.9.2",
 				"obsidian": "1.8.7",
 				"resolve-url-loader": "^5.0.0",
-				"sass": "^1.85.0",
+				"sass": "^1.85.1",
 				"sass-loader": "^16.0.5",
 				"sync-client": "file:../sync-client",
-				"terser-webpack-plugin": "^5.3.11",
+				"terser-webpack-plugin": "^5.3.14",
 				"ts-jest": "^29.2.6",
 				"ts-loader": "^9.5.2",
 				"tslib": "2.8.1",
-				"typescript": "5.7.3",
+				"typescript": "5.8.2",
 				"url": "^0.11.4",
 				"virtual-scroller": "^1.13.1",
 				"webpack": "^5.98.0",
@@ -6723,22 +6753,26 @@
 		},
 		"sync-client": {
 			"version": "0.0.0",
-			"devDependencies": {
+			"dependencies": {
-				"@types/jest": "^29.5.14",
-				"@types/node": "^22.13.5",
 				"byte-base64": "^1.1.0",
 				"fetch-retry": "^6.0.0",
-				"jest": "^29.7.0",
+				"openapi-fetch": "0.13.5",
-				"openapi-fetch": "0.13.4",
 				"openapi-typescript": "7.6.1",
 				"p-queue": "^8.1.0",
+				"uuid": "^11.1.0"
+			},
+			"devDependencies": {
+				"@types/jest": "^29.5.14",
+				"@types/node": "^22.13.10",
+				"jest": "^29.7.0",
 				"sync_lib": "file:../../backend/sync_lib/pkg",
 				"ts-jest": "^29.2.6",
 				"ts-loader": "^9.5.2",
 				"tslib": "2.8.1",
-				"typescript": "5.7.3",
+				"typescript": "5.8.2",
 				"webpack": "^5.98.0",
-				"webpack-cli": "^6.0.1"
+				"webpack-cli": "^6.0.1",
+				"webpack-merge": "^6.0.1"
 			}
 		},
 		"test-client": {
@@ -6747,11 +6781,11 @@
 				"test-client": "dist/cli.js"
 			},
 			"devDependencies": {
-				"@types/node": "^22.13.5",
+				"@types/node": "^22.13.10",
 				"sync-client": "file:../sync-client",
 				"ts-loader": "^9.5.2",
 				"tslib": "2.8.1",
-				"typescript": "5.7.3",
+				"typescript": "5.8.2",
 				"uuid": "^11.1.0",
 				"webpack": "^5.98.0",
 				"webpack-cli": "^6.0.1"

frontend/package.json

@@ -21,10 +21,10 @@
 	},
 	"devDependencies": {
 		"concurrently": "^9.1.2",
-		"eslint": "9.21.0",
+		"eslint": "9.22.0",
 		"eslint-plugin-unused-imports": "^4.1.4",
-		"npm-check-updates": "^17.1.14",
+		"npm-check-updates": "^17.1.15",
-		"prettier": "^3.5.2",
+		"prettier": "^3.5.3",
-		"typescript-eslint": "8.24.1"
+		"typescript-eslint": "8.26.1"
 	}
 }

frontend/sync-client/package.json

@@ -1,29 +1,36 @@
 {
 	"name": "sync-client",
-	"version": "0.0.0",
+	"version": "0.0.30",
-	"private": true,
+	"main": "dist/sync-client.node.js",
-	"main": "dist/index.js",
+	"browser": "dist/sync-client.web.js",
 	"types": "dist/types/index.d.ts",
+	"files": [
+		"dist/**/*"
+	],
 	"scripts": {
 		"dev": "webpack watch --mode development",
 		"build": "webpack --mode production",
 		"test": "NODE_OPTIONS=\"$NODE_OPTIONS --experimental-vm-modules\" jest"
 	},
+	"dependencies": {
+		"byte-base64": "^1.1.0",
+		"fetch-retry": "^6.0.0",
+		"openapi-fetch": "0.13.5",
+		"openapi-typescript": "7.6.1",
+		"p-queue": "^8.1.0",
+		"uuid": "^11.1.0"
+	},
 	"devDependencies": {
-		"tslib": "2.8.1",
-		"typescript": "5.7.3",
-		"sync_lib": "file:../../backend/sync_lib/pkg",
 		"@types/jest": "^29.5.14",
-		"@types/node": "^22.13.5",
+		"@types/node": "^22.13.10",
 		"jest": "^29.7.0",
 		"ts-jest": "^29.2.6",
-		"p-queue": "^8.1.0",
-		"fetch-retry": "^6.0.0",
-		"byte-base64": "^1.1.0",
-		"openapi-fetch": "0.13.4",
-		"openapi-typescript": "7.6.1",
 		"ts-loader": "^9.5.2",
+		"tslib": "2.8.1",
+		"typescript": "5.8.2",
 		"webpack": "^5.98.0",
-		"webpack-cli": "^6.0.1"
+		"webpack-cli": "^6.0.1",
+		"webpack-merge": "^6.0.1",
+		"sync_lib": "file:../../backend/sync_lib/pkg"
 	}
 }

frontend/sync-client/src/file-operations/document-locks.ts

@@ -1,6 +1,9 @@
 import type { Logger } from "../tracing/logger";
 import type { RelativePath } from "../persistence/database";
 
+// Manages locks on documents to prevent concurrent modifications
+// allowing the client's FileOperations implementation to be simpler.
+// Locks are granted in a first-in-first-out order.
 export class DocumentLocks {
 	private readonly locked = new Set<RelativePath>();
 	private readonly waiters = new Map<RelativePath, (() => void)[]>();

frontend/sync-client/src/file-operations/file-operations.test.ts

@@ -1,16 +1,27 @@
-import type { FileSystemOperations } from "sync-client";
+import type {
-import type { Database, RelativePath } from "../persistence/database";
+	Database,
+	DocumentRecord,
+	RelativePath
+} from "../persistence/database";
 import { FileOperations } from "./file-operations";
 import { Logger } from "../tracing/logger";
 import { assertSetContainsExactly } from "../utils/assert-set-contains-exactly";
+import type { FileSystemOperations } from "./filesystem-operations";
 
 describe("File operations", () => {
-	class MockDatabase {
+	class MockDatabase implements Partial<Database> {
-		public async updatePath(
+		public getLatestDocumentByRelativePath(
+			_find: RelativePath
+		): DocumentRecord | undefined {
+			// no-op
+			return undefined;
+		}
+
+		public move(
 			_oldRelativePath: RelativePath,
 			_newRelativePath: RelativePath
-		): Promise<void> {
+		): void {
-			// this is called but irrelevant for this mock
+			// no-op
 		}
 	}
 

frontend/sync-client/src/file-operations/file-operations.ts

@@ -1,10 +1,6 @@
-import type { Logger } from "src/tracing/logger";
+import type { Logger } from "../tracing/logger";
 import type { FileSystemOperations } from "./filesystem-operations";
-import type {
+import type { Database, RelativePath } from "../persistence/database";
-	Database,
-	DocumentId,
-	RelativePath
-} from "src/persistence/database";
 import { isBinary, isFileTypeMergable, mergeText } from "sync_lib";
 import { SafeFileSystemOperations } from "./safe-filesystem-operations";
 
@@ -17,7 +13,7 @@ export class FileOperations {
 		private readonly database: Database,
 		fs: FileSystemOperations
 	) {
-		this.fs = new SafeFileSystemOperations(fs);
+		this.fs = new SafeFileSystemOperations(fs, logger);
 	}
 
 	public async listAllFiles(): Promise<RelativePath[]> {
@@ -35,7 +31,7 @@ export class FileOperations {
 
 		const decoder = new TextDecoder("utf-8");
 
-		// Normalize line endings to LF on Windows
+		// Normalize line-endings to LF on Windows
 		let text = decoder.decode(content);
 		text = text.replace(/\r\n/g, "\n");
 
@@ -46,10 +42,6 @@ export class FileOperations {
 		return this.fs.getFileSize(path);
 	}
 
-	public async getModificationTime(path: RelativePath): Promise<Date> {
-		return this.fs.getModificationTime(path);
-	}
-
 	public async exists(path: RelativePath): Promise<boolean> {
 		return this.fs.exists(path);
 	}
@@ -60,18 +52,23 @@ export class FileOperations {
 		path: RelativePath,
 		newContent: Uint8Array
 	): Promise<void> {
+		this.logger.debug(`Creating file: ${path}`);
+
+		await this.fs.write(path, newContent);
+	}
+
+	public async ensureClearPath(path: RelativePath): Promise<void> {
 		if (await this.fs.exists(path)) {
 			const deconflictedPath = await this.deconflictPath(path);
 			this.logger.debug(
 				`Didn't expect ${path} to exist, deconflicting by moving it to '${deconflictedPath}'`
 			);
-			await this.database.updatePath(path, deconflictedPath);
+
+			this.database.move(path, deconflictedPath);
 			await this.fs.rename(path, deconflictedPath);
 		} else {
 			await this.createParentDirectories(path);
 		}
-
-		await this.fs.write(path, newContent);
 	}
 
 	// Update the file at the given path.
@@ -126,40 +123,25 @@ export class FileOperations {
 		return new TextEncoder().encode(resultText);
 	}
 
-	public async remove(path: RelativePath): Promise<void> {
+	public async delete(path: RelativePath): Promise<void> {
-		this.logger.debug(`Deleting file: ${path}`);
+		if (await this.exists(path)) {
-		return this.fs.delete(path);
+			this.logger.debug(`Deleting file: ${path}`);
+			return this.fs.delete(path);
+		} else {
+			this.logger.debug(`No need to delete '${path}', it doesn't exist`);
+		}
 	}
 
 	public async move(
 		oldPath: RelativePath,
-		newPath: RelativePath,
+		newPath: RelativePath
-		documentId?: DocumentId
 	): Promise<void> {
 		if (oldPath === newPath) {
 			return;
 		}
+		await this.ensureClearPath(newPath);
 
-		if (await this.fs.exists(newPath)) {
+		this.database.move(oldPath, newPath);
-			const deconflictedPath = await this.deconflictPath(newPath);
-			this.logger.debug(
-				`Conflict when moving '${oldPath}' to '${newPath}', the latter already exists, deconflicting by moving it to '${deconflictedPath}'`
-			);
-
-			const existingMetadata = this.database.getDocument(newPath);
-			if (
-				existingMetadata === undefined ||
-				existingMetadata.documentId !== documentId
-			) {
-				await this.database.updatePath(newPath, deconflictedPath);
-				await this.fs.rename(newPath, deconflictedPath);
-			} else {
-				await this.database.deleteDocument(newPath);
-			}
-		} else {
-			await this.createParentDirectories(newPath);
-		}
-
 		await this.fs.rename(oldPath, newPath);
 	}
 
@@ -201,17 +183,12 @@ export class FileOperations {
 		);
 		stem = stem.replace(FileOperations.PARENTHESES_REGEX, "");
 
-		// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
+		let newName = path;
-		while (true) {
+		do {
-			const newName =
+			currentCount++;
-				currentCount === 0
+			newName = `${directory}${stem} (${currentCount})${extension}`;
-					? `${directory}${stem}${extension}`
+		} while (await this.fs.exists(newName));
-					: `${directory}${stem} (${currentCount})${extension}`;
+
-			if (await this.fs.exists(newName)) {
+		return newName;
-				currentCount++;
-			} else {
-				return newName;
-			}
-		}
 	}
 }

frontend/sync-client/src/file-operations/filesystem-operations.ts

@@ -1,4 +1,4 @@
-import type { RelativePath } from "src/persistence/database";
+import type { RelativePath } from "../persistence/database";
 
 export interface FileSystemOperations {
 	listAllFiles: () => Promise<RelativePath[]>;
@@ -9,11 +9,8 @@ export interface FileSystemOperations {
 		updater: (currentContent: string) => string
 	) => Promise<string>;
 	getFileSize: (path: RelativePath) => Promise<number>;
-	getModificationTime: (path: RelativePath) => Promise<Date>;
 	exists: (path: RelativePath) => Promise<boolean>;
 	createDirectory: (path: RelativePath) => Promise<void>;
 	delete: (path: RelativePath) => Promise<void>;
-
-	// Must be able to handle renaming to a file that already exists
 	rename: (oldPath: RelativePath, newPath: RelativePath) => Promise<void>;
 }

frontend/sync-client/src/file-operations/safe-filesystem-operations.ts

@@ -1,5 +1,7 @@
-import type { RelativePath } from "src/persistence/database";
+import type { RelativePath } from "../persistence/database";
 import type { FileSystemOperations } from "./filesystem-operations";
+import type { Logger } from "../tracing/logger";
+import { DocumentLocks } from "./document-locks";
 
 export class FileNotFoundError extends Error {
 	public constructor(message: string) {
@@ -9,71 +11,134 @@ export class FileNotFoundError extends Error {
 }
 
 // Decorate FileSystemOperations replacing errors with FileNotFoundError
-// if the accessed file doesn't exist.
+// if the accessed file doesn't exist. It also ensures that there's only
+// ever a single request in-flight for any one file through the use of
+// DocumentLocks.
 export class SafeFileSystemOperations implements FileSystemOperations {
-	public constructor(private readonly fs: FileSystemOperations) {}
+	private readonly locks: DocumentLocks;
+
+	public constructor(
+		private readonly fs: FileSystemOperations,
+		private readonly logger: Logger
+	) {
+		this.locks = new DocumentLocks(logger);
+	}
 
 	public async listAllFiles(): Promise<RelativePath[]> {
 		return this.fs.listAllFiles();
 	}
 
 	public async read(path: RelativePath): Promise<Uint8Array> {
-		return this.safeOperation(path, async () => this.fs.read(path));
+		this.logger.debug(`Reading file: ${path}`);
+		return this.safeOperation(
+			path,
+			this.decorateToHoldLock(path, async () => this.fs.read(path)),
+			"read"
+		);
 	}
 
 	public async write(path: RelativePath, content: Uint8Array): Promise<void> {
-		return this.fs.write(path, content);
+		this.logger.debug(`Writing file: ${path}`);
+		return this.decorateToHoldLock(path, async () =>
+			this.fs.write(path, content)
+		)();
 	}
 
 	public async atomicUpdateText(
 		path: RelativePath,
 		updater: (currentContent: string) => string
 	): Promise<string> {
-		return this.safeOperation(path, async () =>
+		this.logger.debug(`Atomic update of file: ${path}`);
-			this.fs.atomicUpdateText(path, updater)
+		return this.safeOperation(
+			path,
+			this.decorateToHoldLock(path, async () =>
+				this.fs.atomicUpdateText(path, updater)
+			),
+			"atomicUpdateText"
 		);
 	}
 
 	public async getFileSize(path: RelativePath): Promise<number> {
-		return this.safeOperation(path, async () => this.fs.getFileSize(path));
+		this.logger.debug(`Getting file size: ${path}`);
-	}
+		return this.safeOperation(
-
+			path,
-	public async getModificationTime(path: RelativePath): Promise<Date> {
+			this.decorateToHoldLock(path, async () =>
-		return this.safeOperation(path, async () =>
+				this.fs.getFileSize(path)
-			this.fs.getModificationTime(path)
+			),
+			"getFileSize"
 		);
 	}
 
 	public async exists(path: RelativePath): Promise<boolean> {
-		return this.fs.exists(path);
+		this.logger.debug(`Checking if file exists: ${path}`);
+		return this.decorateToHoldLock(path, async () =>
+			this.fs.exists(path)
+		)();
 	}
 
 	public async createDirectory(path: RelativePath): Promise<void> {
-		return this.fs.createDirectory(path);
+		this.logger.debug(`Creating directory: ${path}`);
+		return this.decorateToHoldLock(path, async () =>
+			this.fs.createDirectory(path)
+		)();
 	}
 
 	public async delete(path: RelativePath): Promise<void> {
-		return this.fs.delete(path);
+		this.logger.debug(`Deleting file: ${path}`);
+		return this.decorateToHoldLock(path, async () =>
+			this.fs.delete(path)
+		)();
 	}
 
 	public async rename(
 		oldPath: RelativePath,
 		newPath: RelativePath
 	): Promise<void> {
-		return this.safeOperation(oldPath, async () =>
+		this.logger.debug(`Renaming file: ${oldPath} to ${newPath}`);
-			this.fs.rename(oldPath, newPath)
+		return this.safeOperation(
+			oldPath,
+			this.decorateToHoldLock([oldPath, newPath], async () =>
+				this.fs.rename(oldPath, newPath)
+			),
+			"rename"
 		);
 	}
 
+	private decorateToHoldLock<T>(
+		pathOrPaths: RelativePath | RelativePath[],
+		operation: () => Promise<T>
+	): () => Promise<T> {
+		return async () => {
+			const paths = Array.isArray(pathOrPaths)
+				? pathOrPaths
+				: [pathOrPaths];
+			await Promise.all(
+				paths.map(async (path) => this.locks.waitForDocumentLock(path))
+			);
+			try {
+				return await operation();
+			} finally {
+				await Promise.all(
+					paths.map((path) => {
+						this.locks.unlockDocument(path);
+					})
+				);
+			}
+		};
+	}
+
 	private async safeOperation<T>(
 		path: RelativePath,
-		operation: () => Promise<T>
+		operation: () => Promise<T>,
+		operationName: string
 	): Promise<T> {
 		// Without locking the file, this isn't atomic, however, it's good enough practicaly.
 		// This will only break if the file exists, gets deleted and then immediately
 		// recreated while `operation` is running.
 		if (!(await this.fs.exists(path))) {
-			throw new FileNotFoundError(path);
+			throw new FileNotFoundError(
+				`File not found: ${path} before trying to ${operationName}`
+			);
 		}
 		try {
 			return await operation();
@@ -81,7 +146,9 @@ export class SafeFileSystemOperations implements FileSystemOperations {
 			if (await this.fs.exists(path)) {
 				throw error;
 			} else {
-				throw new FileNotFoundError(path);
+				throw new FileNotFoundError(
+					`File not found: ${path} when trying to ${operationName}`
+				);
 			}
 		}
 	}

frontend/sync-client/src/persistence/database.ts

@@ -1,23 +1,43 @@
+import type { Logger } from "../tracing/logger";
+
 export type VaultUpdateId = number;
 export type DocumentId = string;
 export type RelativePath = string;
 
 export interface DocumentMetadata {
 	parentVersionId: VaultUpdateId;
-	documentId: DocumentId;
 	hash: string;
 }
 
-import type { Logger } from "src/tracing/logger";
+export interface StoredDocumentMetadata {
+	relativePath: RelativePath;
+	documentId: DocumentId;
+	parentVersionId: VaultUpdateId;
+	hash: string;
+}
 
 export interface StoredDatabase {
-	documents: Record<RelativePath, DocumentMetadata>;
+	documents: StoredDocumentMetadata[];
 	lastSeenUpdateId: VaultUpdateId | undefined;
 }
 
-export class Database {
+/**
-	private documents = new Map<RelativePath, DocumentMetadata>();
+ * Represents a document in the database.
+ *
+ * It is mutable and its content should always represent the latest
+ * state of the document on disk based on the update events we have seen.
+ */
+export interface DocumentRecord {
+	relativePath: RelativePath;
+	documentId: DocumentId;
+	metadata: DocumentMetadata | undefined;
+	isDeleted: boolean;
+	updates: Promise<void>[];
+	parallelVersion: number;
+}
 
+export class Database {
+	private documents: DocumentRecord[];
 	private lastSeenUpdateId: VaultUpdateId | undefined;
 
 	public constructor(
@@ -26,16 +46,21 @@ export class Database {
 		private readonly saveData: (data: StoredDatabase) => Promise<void>
 	) {
 		initialState ??= {};
-		if (initialState.documents) {
-			for (const [relativePath, metadata] of Object.entries(
-				initialState.documents
-			)) {
-				this.documents.set(relativePath, metadata);
-			}
-		}
-		this.ensureConsistency();
 
-		this.logger.debug(`Loaded ${this.documents.size} documents`);
+		this.documents =
+			initialState.documents?.map(
+				({ relativePath, documentId, ...metadata }) => ({
+					relativePath,
+					documentId,
+					metadata,
+					isDeleted: false,
+					updates: [],
+					parallelVersion: 0
+				})
+			) ?? [];
+
+		this.ensureConsistency();
+		this.logger.debug(`Loaded ${this.documents.length} documents`);
 
 		this.lastSeenUpdateId = initialState.lastSeenUpdateId;
 		this.logger.debug(
@@ -43,109 +68,213 @@ export class Database {
 		);
 	}
 
-	public getDocuments(): Map<RelativePath, DocumentMetadata> {
+	public get length(): number {
-		return this.documents;
+		return this.documents.length;
+	}
+
+	public get resolvedDocuments(): DocumentRecord[] {
+		const paths = new Map<string, DocumentRecord[]>();
+		this.documents
+			.filter(({ metadata }) => metadata !== undefined)
+			.forEach((record) =>
+				paths.set(record.relativePath, [
+					record,
+					...(paths.get(record.relativePath) ?? [])
+				])
+			);
+
+		return Array.from(paths.values()).map((records) => {
+			records.sort(
+				(a, b) => b.parallelVersion - a.parallelVersion // descending
+			);
+
+			if (
+				records.length > 1 &&
+				records.some((current, i) =>
+					i === 0
+						? false
+						: records[i - 1].parallelVersion ===
+							current.parallelVersion
+				)
+			) {
+				throw new Error(
+					`Multiple documents with the same parallel version and path at ${records[0].relativePath}`
+				);
+			}
+			return records[0];
+		});
 	}
 
 	public getLastSeenUpdateId(): VaultUpdateId | undefined {
 		return this.lastSeenUpdateId;
 	}
 
-	public async setLastSeenUpdateId(
+	public setLastSeenUpdateId(value: VaultUpdateId | undefined): void {
-		value: VaultUpdateId | undefined
-	): Promise<void> {
 		this.lastSeenUpdateId = value;
-		await this.save();
+		this.save();
 	}
 
-	public async resetSyncState(): Promise<void> {
+	public resetSyncState(): void {
-		this.documents = new Map();
+		this.documents = [];
 		this.lastSeenUpdateId = 0;
-		await this.save();
+		this.save();
+	}
+
+	public updateDocumentMetadata(
+		metadata: {
+			parentVersionId: VaultUpdateId;
+			hash: string;
+		},
+		toUpdate: DocumentRecord
+	): void {
+		if (!this.documents.includes(toUpdate)) {
+			throw new Error("Document not found in database");
+		}
+
+		toUpdate.metadata = metadata;
+
+		this.save();
+	}
+
+	public removeDocumentPromise(promise: Promise<void>): void {
+		const entry = this.documents.find(({ updates }) =>
+			updates.includes(promise)
+		);
+
+		if (entry === undefined) {
+			throw new Error("Document not found by update promise");
+		}
+
+		entry.updates = entry.updates.filter((update) => update !== promise);
+		// No need to save as Promises don't get serialized
+	}
+
+	public getLatestDocumentByRelativePath(
+		find: RelativePath
+	): DocumentRecord | undefined {
+		const candidates = this.documents.filter(
+			({ relativePath }) => relativePath === find
+		);
+		candidates.sort((a, b) => b.parallelVersion - a.parallelVersion); // descending
+		return candidates[0];
+	}
+
+	public async getResolvedDocumentByRelativePath(
+		relativePath: RelativePath,
+		promise: Promise<void>
+	): Promise<DocumentRecord> {
+		const entry = this.getLatestDocumentByRelativePath(relativePath);
+
+		if (entry === undefined) {
+			throw new Error(
+				`Document not found by relative path: ${relativePath}, ${JSON.stringify(
+					this.documents,
+					null,
+					2
+				)}`
+			);
+		}
+
+		const currentPromises = entry.updates;
+		entry.updates = [...currentPromises, promise];
+		await Promise.all(currentPromises);
+
+		return entry;
+	}
+
+	public createNewPendingDocument(
+		documentId: DocumentId,
+		relativePath: RelativePath,
+		promise: Promise<void>
+	): DocumentRecord {
+		const previousEntry =
+			this.getLatestDocumentByRelativePath(relativePath);
+
+		const entry = {
+			relativePath,
+			documentId,
+			metadata: undefined,
+			isDeleted: false,
+			updates: [promise],
+			parallelVersion:
+				previousEntry?.parallelVersion === undefined
+					? 0
+					: previousEntry.parallelVersion + 1
+		};
+
+		this.documents.push(entry);
+		this.save();
+
+		return entry;
 	}
 
 	public getDocumentByDocumentId(
-		documentId: DocumentId
+		find: DocumentId
-	): [RelativePath, DocumentMetadata] | undefined {
+	): DocumentRecord | undefined {
-		return [...this.documents.entries()].find(
+		return this.documents.find(({ documentId }) => documentId === find);
-			([_, metadata]) => metadata.documentId === documentId
-		);
 	}
 
-	public async setDocument({
+	public move(
-		documentId,
-		relativePath,
-		parentVersionId,
-		hash
-	}: {
-		documentId: DocumentId;
-		relativePath: RelativePath;
-		parentVersionId: VaultUpdateId;
-		hash: string;
-	}): Promise<void> {
-		this.documents.set(relativePath, {
-			documentId,
-			parentVersionId,
-			hash
-		});
-		await this.save();
-	}
-
-	public async removeDocument(relativePath: RelativePath): Promise<void> {
-		this.documents.delete(relativePath);
-		await this.save();
-	}
-
-	public getDocument(
-		relativePath: RelativePath
-	): DocumentMetadata | undefined {
-		return this.documents.get(relativePath);
-	}
-
-	public async deleteDocument(relativePath: RelativePath): Promise<void> {
-		this.documents.delete(relativePath);
-		await this.save();
-	}
-
-	public async updatePath(
 		oldRelativePath: RelativePath,
 		newRelativePath: RelativePath
-	): Promise<void> {
+	): void {
-		const document = this.documents.get(oldRelativePath);
+		const oldDocument =
-		if (!document) {
+			this.getLatestDocumentByRelativePath(oldRelativePath);
+
+		if (oldDocument === undefined) {
+			return;
+		}
+
+		const newDocument =
+			this.getLatestDocumentByRelativePath(newRelativePath);
+		if (newDocument?.isDeleted === false) {
 			throw new Error(
-				`Cannot update physical path for document that does not exist: ${oldRelativePath}`
+				`Document already exists at new location: ${newRelativePath}`
 			);
 		}
 
-		if (this.documents.has(newRelativePath)) {
+		oldDocument.relativePath = newRelativePath;
-			throw new Error(
+		// We're in a strange state where the target of the move has just got deleted,
-				`Cannot update physical path to path that is already in use: ${newRelativePath}`
+		// however, its metadata might already have a bunch of updates queued up for
-			);
+		// the document at the new location. We need to keep these updates.
-		}
+		oldDocument.parallelVersion =
+			newDocument !== undefined ? newDocument.parallelVersion + 1 : 0;
 
-		this.documents.delete(oldRelativePath);
+		this.save();
-		this.documents.set(newRelativePath, document);
-
-		await this.save();
 	}
 
-	private async save(): Promise<void> {
+	public delete(relativePath: RelativePath): void {
+		const candidate = this.getLatestDocumentByRelativePath(relativePath);
+		if (candidate === undefined) {
+			throw new Error(
+				`Document not found by relative path: ${relativePath}`
+			);
+		}
+		candidate.isDeleted = true;
+	}
+
+	private save(): void {
 		this.ensureConsistency();
-		await this.saveData({
+		void this.saveData({
-			documents: Object.fromEntries(this.documents.entries()),
+			documents: this.resolvedDocuments.map(
+				({ relativePath, documentId, metadata }) => ({
+					documentId,
+					relativePath,
+					// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+					...metadata! // resolvedDocuments only returns docs with metadata set
+				})
+			),
 			lastSeenUpdateId: this.lastSeenUpdateId
 		});
 	}
 
 	private ensureConsistency(): void {
-		const allMetadata = Array.from(this.documents.entries());
+		const idToPath = new Map<string, string[]>();
-		const idToPath = new Map<string, Array<string>>();
 
-		allMetadata.forEach(([name, metadata]) => {
+		this.resolvedDocuments.forEach(({ relativePath, documentId }) => {
-			idToPath.set(metadata.documentId, [
+			idToPath.set(documentId, [
-				...(idToPath.get(metadata.documentId) ?? []),
+				...(idToPath.get(documentId) ?? []),
-				name
+				relativePath
 			]);
 		});
 

frontend/sync-client/src/persistence/settings.ts

@@ -1,5 +1,5 @@
-import type { Logger } from "src/tracing/logger";
+import type { Logger } from "../tracing/logger";
-import { LogLevel } from "src/tracing/logger";
+import { LogLevel } from "../tracing/logger";
 
 export interface SyncSettings {
 	remoteUri: string;

frontend/sync-client/src/services/connected-state.ts

@@ -0,0 +1,51 @@
+import type { Settings } from "../persistence/settings";
+import type { Logger } from "../tracing/logger";
+import { createPromise } from "../utils/create-promise";
+import { retriedFetchFactory } from "../utils/retried-fetch";
+
+export class ConnectedState {
+	private resolveIsSyncEnabled: (() => void) | undefined;
+	private syncIsEnabled: Promise<void> | undefined;
+
+	public constructor(
+		settings: Settings,
+		private readonly logger: Logger
+	) {
+		settings.addOnSettingsChangeHandlers((newSettings, oldSettings) => {
+			if (!oldSettings.isSyncEnabled && newSettings.isSyncEnabled) {
+				this.handleComingOnline();
+			} else if (
+				oldSettings.isSyncEnabled &&
+				!newSettings.isSyncEnabled
+			) {
+				this.handleGoingOffline();
+			}
+		});
+	}
+
+	public getFetchImplementation(
+		fetch: typeof globalThis.fetch,
+		{ doRetries = true }: { doRetries: boolean } = { doRetries: true }
+	): typeof globalThis.fetch {
+		const retriedFetch = doRetries
+			? retriedFetchFactory(this.logger, fetch)
+			: fetch;
+
+		return async (input: RequestInfo | URL): Promise<Response> => {
+			if (this.syncIsEnabled !== undefined) {
+				await this.syncIsEnabled;
+			}
+			return retriedFetch(input);
+		};
+	}
+
+	private handleComingOnline(): void {
+		this.logger.debug("Sync is enabled");
+		this.resolveIsSyncEnabled?.();
+	}
+
+	private handleGoingOffline(): void {
+		this.logger.debug("Sync is disabled");
+		[this.syncIsEnabled, this.resolveIsSyncEnabled] = createPromise();
+	}
+}

frontend/sync-client/src/services/sync-service.ts

@@ -6,20 +6,22 @@ import type {
 	RelativePath,
 	VaultUpdateId
 } from "../persistence/database";
-import type { Logger } from "src/tracing/logger";
+import type { Logger } from "../tracing/logger";
-import { retriedFetchFactory } from "src/utils/retried-fetch";
+import type { Settings } from "../persistence/settings";
-import type { Settings } from "src/persistence/settings";
+import type { ConnectedState } from "./connected-state";
 
 export interface CheckConnectionResult {
 	isSuccessful: boolean;
 	message: string;
 }
+
 export class SyncService {
 	private client!: Client<paths>;
 	private clientWithoutRetries!: Client<paths>;
 	private _fetchImplementation: typeof globalThis.fetch = globalThis.fetch;
 
 	public constructor(
+		private readonly connectedState: ConnectedState,
 		private readonly settings: Settings,
 		private readonly logger: Logger
 	) {
@@ -52,17 +54,19 @@ export class SyncService {
 	}
 
 	public async create({
+		documentId,
 		relativePath,
-		contentBytes,
+		contentBytes
-		createdDate
 	}: {
+		documentId?: DocumentId;
 		relativePath: RelativePath;
 		contentBytes: Uint8Array;
-		createdDate: Date;
+	}): Promise<components["schemas"]["DocumentVersionWithoutContent"]> {
-	}): Promise<components["schemas"]["DocumentUpdateResponse"]> {
 		const formData = new FormData();
+		if (documentId !== undefined) {
+			formData.append("document_id", documentId);
+		}
 		formData.append("relative_path", relativePath);
-		formData.append("created_date", createdDate.toISOString());
 		formData.append("content", new Blob([contentBytes]));
 
 		const response = await this.client.POST(
@@ -100,18 +104,18 @@ export class SyncService {
 		parentVersionId,
 		documentId,
 		relativePath,
-		contentBytes,
+		contentBytes
-		createdDate
 	}: {
 		parentVersionId: VaultUpdateId;
 		documentId: DocumentId;
 		relativePath: RelativePath;
 		contentBytes: Uint8Array;
-		createdDate: Date;
 	}): Promise<components["schemas"]["DocumentUpdateResponse"]> {
+		this.logger.debug(
+			`Updating document ${documentId} with parent version ${parentVersionId} and relative path ${relativePath}`
+		);
 		const formData = new FormData();
 		formData.append("parent_version_id", parentVersionId.toString());
-		formData.append("created_date", createdDate.toISOString());
 		formData.append("relative_path", relativePath);
 		formData.append("content", new Blob([contentBytes]));
 
@@ -149,13 +153,11 @@ export class SyncService {
 
 	public async delete({
 		documentId,
-		relativePath,
+		relativePath
-		createdDate
 	}: {
 		documentId: DocumentId;
 		relativePath: RelativePath;
-		createdDate: Date;
+	}): Promise<components["schemas"]["DocumentVersionWithoutContent"]> {
-	}): Promise<void> {
 		const response = await this.client.DELETE(
 			"/vaults/{vault_id}/documents/{document_id}",
 			{
@@ -169,7 +171,6 @@ export class SyncService {
 					}
 				},
 				body: {
-					createdDate: createdDate.toISOString(),
 					relativePath
 				}
 			}
@@ -295,11 +296,17 @@ export class SyncService {
 	private createClient(remoteUri: string): void {
 		this.client = createClient<paths>({
 			baseUrl: remoteUri,
-			fetch: retriedFetchFactory(this.logger, this._fetchImplementation)
+			fetch: this.connectedState.getFetchImplementation(
+				this._fetchImplementation
+			)
 		});
 
 		this.clientWithoutRetries = createClient<paths>({
-			baseUrl: remoteUri
+			baseUrl: remoteUri,
+			fetch: this.connectedState.getFetchImplementation(
+				this._fetchImplementation,
+				{ doRetries: false }
+			)
 		});
 	}
 }

frontend/sync-client/src/services/types.ts

@@ -274,12 +274,13 @@ export interface paths {
 				};
 			};
 			responses: {
-				/** @description no content */
 				200: {
 					headers: {
 						[name: string]: unknown;
 					};
-					content?: never;
+					content: {
+						"application/json": components["schemas"]["DocumentVersionWithoutContent"];
+					};
 				};
 				default: {
 					headers: {
@@ -451,26 +452,25 @@ export interface components {
 		Array_of_uint8: number[];
 		CreateDocumentVersion: {
 			contentBase64: string;
-			/** Format: date-time */
+			/**
-			createdDate: string;
+			 * Format: uuid
+			 * @description The client can decide the document id (if it wishes to) in order to help with syncing. If the client does not provide a document id, the server will generate one. If the client provides a document id it must not already exist in the database.
+			 */
+			documentId?: string | null;
 			relativePath: string;
 		};
 		CreateDocumentVersionMultipart: {
 			content: components["schemas"]["Array_of_uint8"];
-			/** Format: date-time */
+			/** Format: uuid */
-			created_date: string;
+			document_id?: string | null;
 			relative_path: string;
 		};
 		DeleteDocumentVersion: {
-			/** Format: date-time */
-			createdDate: string;
 			relativePath: string;
 		};
-		/** @description Response to a update document request. */
+		/** @description Response to an update document request. */
 		DocumentUpdateResponse:
 			| {
-					/** Format: date-time */
-					createdDate: string;
 					/** Format: uuid */
 					documentId: string;
 					isDeleted: boolean;
@@ -479,14 +479,11 @@ export interface components {
 					type: "FastForwardUpdate";
 					/** Format: date-time */
 					updatedDate: string;
-					vaultId: string;
 					/** Format: int64 */
 					vaultUpdateId: number;
 			  }
 			| {
 					contentBase64: string;
-					/** Format: date-time */
-					createdDate: string;
 					/** Format: uuid */
 					documentId: string;
 					isDeleted: boolean;
@@ -495,34 +492,27 @@ export interface components {
 					type: "MergingUpdate";
 					/** Format: date-time */
 					updatedDate: string;
-					vaultId: string;
 					/** Format: int64 */
 					vaultUpdateId: number;
 			  };
 		DocumentVersion: {
 			contentBase64: string;
-			/** Format: date-time */
-			createdDate: string;
 			/** Format: uuid */
 			documentId: string;
 			isDeleted: boolean;
 			relativePath: string;
 			/** Format: date-time */
 			updatedDate: string;
-			vaultId: string;
 			/** Format: int64 */
 			vaultUpdateId: number;
 		};
 		DocumentVersionWithoutContent: {
-			/** Format: date-time */
-			createdDate: string;
 			/** Format: uuid */
 			documentId: string;
 			isDeleted: boolean;
 			relativePath: string;
 			/** Format: date-time */
 			updatedDate: string;
-			vaultId: string;
 			/** Format: int64 */
 			vaultUpdateId: number;
 		};
@@ -587,16 +577,12 @@ export interface components {
 		};
 		UpdateDocumentVersion: {
 			contentBase64: string;
-			/** Format: date-time */
-			createdDate: string;
 			/** Format: int64 */
 			parentVersionId: number;
 			relativePath: string;
 		};
 		UpdateDocumentVersionMultipart: {
 			content: components["schemas"]["Array_of_uint8"];
-			/** Format: date-time */
-			createdDate: string;
 			/** Format: int64 */
 			parentVersionId: number;
 			relativePath: string;

frontend/sync-client/src/sync-client.ts

@@ -12,6 +12,7 @@ import { SyncService } from "./services/sync-service";
 import { Syncer } from "./sync-operations/syncer";
 import type { FileSystemOperations } from "./file-operations/filesystem-operations";
 import { FileOperations } from "./file-operations/file-operations";
+import { ConnectedState } from "./services/connected-state";
 
 export class SyncClient {
 	private remoteListenerIntervalId: NodeJS.Timeout | null = null;
@@ -42,7 +43,7 @@ export class SyncClient {
 	}
 
 	public get documentCount(): number {
-		return this._database.getDocuments().size;
+		return this._database.length;
 	}
 
 	public set fetchImplementation(fetch: typeof globalThis.fetch) {
@@ -90,7 +91,9 @@ export class SyncClient {
 			}
 		);
 
-		const syncService = new SyncService(settings, logger);
+		const connectedState = new ConnectedState(settings, logger);
+
+		const syncService = new SyncService(connectedState, settings, logger);
 
 		const syncer = new Syncer(
 			logger,
@@ -117,18 +120,13 @@ export class SyncClient {
 		);
 
 		settings.addOnSettingsChangeHandlers((newSettings, oldSettings) => {
-			client.registerRemoteEventListener(
+			if (
-				newSettings.fetchChangesUpdateIntervalMs
+				newSettings.fetchChangesUpdateIntervalMs !==
-			);
+				oldSettings.fetchChangesUpdateIntervalMs
-
+			) {
-			if (!oldSettings.isSyncEnabled && newSettings.isSyncEnabled) {
+				client.registerRemoteEventListener(
-				syncer
+					newSettings.fetchChangesUpdateIntervalMs
-					.scheduleSyncForOfflineChanges()
+				);
-					.catch((_error: unknown) => {
-						logger.error(
-							"Failed to schedule sync for offline changes"
-						);
-					});
 			}
 		});
 
@@ -148,7 +146,7 @@ export class SyncClient {
 		this.stop();
 		await this._syncer.reset();
 		this._history.reset();
-		await this._database.resetSyncState();
+		this._database.resetSyncState();
 		this.logger.reset();
 	}
 

frontend/sync-client/src/sync-operations/syncer.ts

@@ -1,15 +1,17 @@
 import type { Database, RelativePath } from "../persistence/database";
-
+import type { SyncService } from "../services/sync-service";
-import type { SyncService } from "src/services/sync-service";
+import type { Logger } from "../tracing/logger";
-import type { Logger } from "src/tracing/logger";
+import type { SyncHistory } from "../tracing/sync-history";
-import type { SyncHistory } from "src/tracing/sync-history";
 import PQueue from "p-queue";
-import { hash } from "src/utils/hash";
+import { hash } from "../utils/hash";
-import type { components } from "src/services/types";
+import { v4 as uuidv4 } from "uuid";
-import type { Settings } from "src/persistence/settings";
+import type { components } from "../services/types";
-import type { FileOperations } from "src/file-operations/file-operations";
+import type { Settings } from "../persistence/settings";
-import { findMatchingFileBasedOnHash } from "src/utils/find-matching-file-based-on-hash";
+import type { FileOperations } from "../file-operations/file-operations";
+import { findMatchingFile } from "../utils/find-matching-file";
 import { UnrestrictedSyncer } from "./unrestricted-syncer";
+import { FileNotFoundError } from "../file-operations/safe-filesystem-operations";
+import { createPromise } from "../utils/create-promise";
 
 export class Syncer {
 	private readonly remainingOperationsListeners: ((
@@ -18,17 +20,15 @@ export class Syncer {
 
 	private readonly syncQueue: PQueue;
 
-	private runningScheduleSyncForOfflineChanges: Promise<void> | undefined =
+	private runningScheduleSyncForOfflineChanges: Promise<void> | undefined;
-		undefined;
+	private runningApplyRemoteChangesLocally: Promise<void> | undefined;
-	private runningApplyRemoteChangesLocally: Promise<void> | undefined =
-		undefined;
 
 	private readonly internalSyncer: UnrestrictedSyncer;
 
 	public constructor(
 		private readonly logger: Logger,
 		private readonly database: Database,
-		private readonly settings: Settings,
+		settings: Settings,
 		private readonly syncService: SyncService,
 		private readonly operations: FileOperations,
 		history: SyncHistory
@@ -45,7 +45,9 @@ export class Syncer {
 		});
 
 		this.syncQueue.on("active", () => {
-			this.emitRemainingOperationsChange(this.syncQueue.size);
+			this.remainingOperationsListeners.forEach((listener) => {
+				listener(this.syncQueue.size);
+			});
 		});
 
 		this.internalSyncer = new UnrestrictedSyncer(
@@ -65,48 +67,131 @@ export class Syncer {
 	}
 
 	public async syncLocallyCreatedFile(
-		relativePath: RelativePath,
+		relativePath: RelativePath
-		updateTime: Date
 	): Promise<void> {
-		await this.syncQueue.add(async () =>
+		if (
-			this.internalSyncer.unrestrictedSyncLocallyCreatedFile(
+			this.database.getLatestDocumentByRelativePath(relativePath)
-				relativePath,
+				?.isDeleted === false
-				updateTime
+		) {
-			)
+			this.logger.debug(
-		);
+				`Document ${relativePath} already exists in the database, skipping`
-	}
+			);
+			return;
+		}
 
-	public async syncLocallyUpdatedFile(args: {
+		const [promise, resolve, reject] = createPromise();
-		oldPath?: RelativePath;
-		relativePath: RelativePath;
-		updateTime: Date;
-	}): Promise<void> {
-		await this.syncQueue.add(async () =>
-			this.internalSyncer.unrestrictedSyncLocallyUpdatedFile(args)
-		);
-	}
 
-	public async waitForSyncQueue(): Promise<void> {
+		const document = this.database.createNewPendingDocument(
-		return this.syncQueue.onEmpty();
+			uuidv4(),
+			relativePath,
+			promise
+		);
+
+		try {
+			await this.syncQueue.add(async () =>
+				this.internalSyncer.unrestrictedSyncLocallyCreatedFile(document)
+			);
+
+			resolve();
+		} catch (e) {
+			reject(e);
+		} finally {
+			this.database.removeDocumentPromise(promise);
+		}
 	}
 
 	public async syncLocallyDeletedFile(
 		relativePath: RelativePath
 	): Promise<void> {
-		await this.syncQueue.add(async () =>
+		// We have to have a record of the delete in case there's an in-flight update for the same
-			this.internalSyncer.unrestrictedSyncLocallyDeletedFile(relativePath)
+		// document which finishes after the delete has succeeded and would introduce a phantom metadata record.
+		this.database.delete(relativePath);
+
+		const [promise, resolve, reject] = createPromise();
+
+		const document = await this.database.getResolvedDocumentByRelativePath(
+			relativePath,
+			promise
 		);
+
+		try {
+			await this.syncQueue.add(async () =>
+				this.internalSyncer.unrestrictedSyncLocallyDeletedFile(document)
+			);
+
+			resolve();
+		} catch (e) {
+			reject(e);
+		} finally {
+			this.database.removeDocumentPromise(promise);
+		}
 	}
 
-	public async scheduleSyncForOfflineChanges(): Promise<void> {
+	public async syncLocallyUpdatedFile({
-		if (!this.settings.getSettings().isSyncEnabled) {
+		oldPath,
+		relativePath
+	}: {
+		oldPath?: RelativePath;
+		relativePath: RelativePath;
+	}): Promise<void> {
+		if (
+			oldPath !== undefined &&
+			(this.database.getLatestDocumentByRelativePath(relativePath) ===
+				undefined ||
+				this.database.getLatestDocumentByRelativePath(relativePath)
+					?.isDeleted === true)
+		) {
+			if (oldPath === relativePath) {
+				throw new Error(
+					`Old path and new path are the same: ${oldPath}`
+				);
+			}
+
+			this.database.move(oldPath, relativePath);
+		}
+
+		let document =
+			this.database.getLatestDocumentByRelativePath(relativePath);
+
+		if (document === undefined) {
 			this.logger.debug(
-				`Syncing is disabled, not uploading local changes`
+				`Cannot find document ${relativePath} in the database, skipping`
 			);
 			return;
 		}
 
-		if (this.runningScheduleSyncForOfflineChanges != null) {
+		if (document.isDeleted) {
+			this.logger.debug(
+				`Document ${relativePath} has been deleted locally, skipping`
+			);
+			return;
+		}
+
+		const [promise, resolve, reject] = createPromise();
+
+		document = await this.database.getResolvedDocumentByRelativePath(
+			relativePath,
+			promise
+		);
+
+		try {
+			await this.syncQueue.add(async () =>
+				this.internalSyncer.unrestrictedSyncLocallyUpdatedFile({
+					oldPath,
+					document
+				})
+			);
+
+			resolve();
+		} catch (e) {
+			reject(e);
+		} finally {
+			this.database.removeDocumentPromise(promise);
+		}
+	}
+
+	public async scheduleSyncForOfflineChanges(): Promise<void> {
+		if (this.runningScheduleSyncForOfflineChanges !== undefined) {
 			this.logger.debug("Uploading local changes is already in progress");
 			return this.runningScheduleSyncForOfflineChanges;
 		}
@@ -127,13 +212,6 @@ export class Syncer {
 	}
 
 	public async applyRemoteChangesLocally(): Promise<void> {
-		if (!this.settings.getSettings().isSyncEnabled) {
-			this.logger.debug(
-				`Syncing is disabled, not fetching remote changes`
-			);
-			return;
-		}
-
 		if (this.runningApplyRemoteChangesLocally != null) {
 			this.logger.debug(
 				"Applying remote changes locally is already in progress"
@@ -154,6 +232,10 @@ export class Syncer {
 		}
 	}
 
+	public async waitForSyncQueue(): Promise<void> {
+		return this.syncQueue.onEmpty();
+	}
+
 	public async reset(): Promise<void> {
 		this.syncQueue.clear();
 		await this.syncQueue.onEmpty();
@@ -163,115 +245,15 @@ export class Syncer {
 		this.internalSyncer.reset();
 	}
 
-	private async syncRemotelyUpdatedFile(
-		remoteVersion: components["schemas"]["DocumentVersionWithoutContent"]
-	): Promise<void> {
-		await this.syncQueue.add(async () =>
-			this.internalSyncer.unrestrictedSyncRemotelyUpdatedFile(
-				remoteVersion
-			)
-		);
-	}
-
-	private async internalScheduleSyncForOfflineChanges(): Promise<void> {
-		const allLocalFiles = await this.operations.listAllFiles();
-
-		// This includes renamed files for now
-		let locallyPossiblyDeletedFiles = [
-			...this.database.getDocuments().entries()
-		].filter(([path, _]) => !allLocalFiles.includes(path));
-
-		await Promise.all(
-			allLocalFiles.map(async (relativePath) =>
-				this.syncQueue.add(async () => {
-					const metadata = this.database.getDocument(relativePath);
-
-					if (metadata) {
-						this.logger.debug(
-							`Document ${relativePath} might have been updated locally, scheduling sync to validate and update it`
-						);
-						return this.internalSyncer.unrestrictedSyncLocallyUpdatedFile(
-							{
-								relativePath,
-								updateTime:
-									await this.operations.getModificationTime(
-										relativePath
-									)
-							}
-						);
-					}
-
-					// Perhaps the file has been moved. Let's check by looking at the deleted files
-					const contentBytes =
-						await this.operations.read(relativePath);
-					const contentHash = hash(contentBytes);
-
-					// todo: make this smarter so that offline files can be renamed & edited at the same time
-					const originalFile = findMatchingFileBasedOnHash(
-						contentHash,
-						locallyPossiblyDeletedFiles
-					);
-					if (originalFile !== undefined) {
-						// `originalFile` hasn't been deleted but it got moved instead
-						locallyPossiblyDeletedFiles =
-							locallyPossiblyDeletedFiles.filter(
-								(item) => item[0] !== originalFile[0]
-							);
-
-						this.logger.debug(
-							`Document '${originalFile[0]}' was not found under its current path in the database but was found under a different path (${relativePath}), scheduling sync to move it`
-						);
-						return this.internalSyncer.unrestrictedSyncLocallyUpdatedFile(
-							{
-								oldPath: originalFile[0],
-								relativePath: relativePath,
-								updateTime:
-									await this.operations.getModificationTime(
-										relativePath
-									),
-								optimisations: {
-									contentBytes,
-									contentHash
-								}
-							}
-						);
-					}
-
-					this.logger.debug(
-						`Document ${relativePath} not found in database, scheduling sync to create it`
-					);
-					return this.internalSyncer.unrestrictedSyncLocallyCreatedFile(
-						relativePath,
-						await this.operations.getModificationTime(relativePath)
-					);
-				})
-			)
-		);
-
-		await Promise.all(
-			locallyPossiblyDeletedFiles.map(async ([relativePath, _]) => {
-				this.logger.debug(
-					`Document ${relativePath} has been deleted locally, scheduling sync to delete it`
-				);
-
-				if (await this.operations.exists(relativePath)) {
-					this.logger.debug(
-						`Document ${relativePath} actually exists locally, skipping`
-					);
-					return Promise.resolve();
-				}
-
-				// We're outside of the pqueue, so we need to call the public wrapper
-				return this.syncLocallyDeletedFile(relativePath);
-			})
-		);
-	}
-
 	private async internalApplyRemoteChangesLocally(): Promise<void> {
-		const remote = await this.syncService.getAll(
+		const remote = await this.syncQueue.add(async () =>
-			this.database.getLastSeenUpdateId()
+			this.syncService.getAll(this.database.getLastSeenUpdateId())
 		);
 
+		if (!remote) {
+			throw new Error("Failed to fetch remote changes");
+		}
+
 		if (remote.latestDocuments.length === 0) {
 			this.logger.debug("No remote changes to apply");
 			return;
@@ -280,9 +262,7 @@ export class Syncer {
 		this.logger.info("Applying remote changes locally");
 
 		await Promise.all(
-			remote.latestDocuments.map(async (remoteDocument) =>
+			remote.latestDocuments.map(this.syncRemotelyUpdatedFile.bind(this))
-				this.syncRemotelyUpdatedFile(remoteDocument)
-			)
 		);
 
 		const lastSeenUpdateId = this.database.getLastSeenUpdateId();
@@ -290,13 +270,124 @@ export class Syncer {
 			lastSeenUpdateId === undefined ||
 			remote.lastUpdateId > lastSeenUpdateId
 		) {
-			await this.database.setLastSeenUpdateId(remote.lastUpdateId);
+			this.database.setLastSeenUpdateId(remote.lastUpdateId);
 		}
 	}
 
-	private emitRemainingOperationsChange(remainingOperations: number): void {
+	private async syncRemotelyUpdatedFile(
-		this.remainingOperationsListeners.forEach((listener) => {
+		remoteVersion: components["schemas"]["DocumentVersionWithoutContent"]
-			listener(remainingOperations);
+	): Promise<void> {
-		});
+		let document = this.database.getDocumentByDocumentId(
+			remoteVersion.documentId
+		);
+
+		const [promise, resolve, reject] = createPromise();
+
+		if (document === undefined) {
+			await this.syncQueue.add(async () =>
+				this.internalSyncer.unrestrictedSyncRemotelyUpdatedFile(
+					remoteVersion
+				)
+			);
+		} else {
+			document = await this.database.getResolvedDocumentByRelativePath(
+				document.relativePath,
+				promise
+			);
+
+			try {
+				await this.syncQueue.add(async () =>
+					this.internalSyncer.unrestrictedSyncRemotelyUpdatedFile(
+						remoteVersion,
+						document
+					)
+				);
+
+				resolve();
+			} catch (e) {
+				reject(e);
+			} finally {
+				this.database.removeDocumentPromise(promise);
+			}
+		}
+	}
+
+	private async internalScheduleSyncForOfflineChanges(): Promise<void> {
+		const allLocalFiles = await this.operations.listAllFiles();
+
+		let locallyPossiblyDeletedFiles = [
+			...this.database.resolvedDocuments
+		].filter(({ relativePath }) => !allLocalFiles.includes(relativePath));
+
+		const updates = Promise.all(
+			allLocalFiles.map(async (relativePath) => {
+				if (
+					this.database.getLatestDocumentByRelativePath(relativePath)
+						?.metadata !== undefined
+				) {
+					this.logger.debug(
+						`Document ${relativePath} might have been updated locally, scheduling sync to validate and update it`
+					);
+
+					return this.syncLocallyUpdatedFile({
+						relativePath
+					});
+				}
+
+				// Perhaps the file has been moved; let's check by looking at the deleted files
+				const contentHash = await this.syncQueue.add(async () => {
+					const contentBytes =
+						await this.operations.read(relativePath); // this can throw FileNotFoundError
+					return hash(contentBytes);
+				});
+
+				if (contentHash == undefined) {
+					// The file was deleted before we had a chance to read it, no need to sync it here
+					return;
+				}
+
+				const originalFile = findMatchingFile(
+					contentHash,
+					locallyPossiblyDeletedFiles
+				);
+				if (originalFile !== undefined) {
+					// `originalFile` hasn't been deleted but it got moved instead
+					locallyPossiblyDeletedFiles =
+						locallyPossiblyDeletedFiles.filter(
+							(item) =>
+								item.relativePath !== originalFile.relativePath
+						);
+
+					this.logger.debug(
+						`Document '${originalFile.relativePath}' was not found under its current path in the database but was found under a different path (${relativePath}), scheduling sync to move it`
+					);
+
+					// We're outside of the pqueue, so we need to call the public wrapper
+					return this.syncLocallyUpdatedFile({
+						oldPath: originalFile.relativePath,
+						relativePath
+					});
+				}
+
+				this.logger.debug(
+					`Document ${relativePath} not found in database, scheduling sync to create it`
+				);
+				// We're outside of the pqueue, so we need to call the public wrapper
+				return this.syncLocallyCreatedFile(relativePath);
+			})
+		);
+
+		const deletes = Promise.all(
+			locallyPossiblyDeletedFiles.map(async ({ relativePath }) => {
+				this.logger.debug(
+					`Document ${relativePath} has been deleted locally, scheduling sync to delete it`
+				);
+
+				// We're outside of the pqueue, so we need to call the public wrapper
+				return this.syncLocallyDeletedFile(relativePath);
+			})
+		);
+
+		await Promise.all([updates, deletes]);
 	}
 }

frontend/sync-client/src/sync-operations/unrestricted-syncer.ts

@@ -1,19 +1,24 @@
-import type { Database, RelativePath } from "../persistence/database";
+import type {
+	Database,
+	DocumentRecord,
+	RelativePath
+} from "../persistence/database";
 
-import type { SyncService } from "src/services/sync-service";
+import type { SyncService } from "../services/sync-service";
-import type { Logger } from "src/tracing/logger";
+import type { Logger } from "../tracing/logger";
-import type { SyncHistory } from "src/tracing/sync-history";
+import type { SyncHistory } from "../tracing/sync-history";
-import { SyncSource, SyncStatus, SyncType } from "src/tracing/sync-history";
+import { SyncSource, SyncStatus, SyncType } from "../tracing/sync-history";
-import { hash } from "src/utils/hash";
+import { EMPTY_HASH, hash } from "../utils/hash";
-import type { components } from "src/services/types";
+import type { components } from "../services/types";
-import { deserialize } from "src/utils/deserialize";
+import { deserialize } from "../utils/deserialize";
-import type { Settings } from "src/persistence/settings";
+import type { Settings } from "../persistence/settings";
-import type { FileOperations } from "src/file-operations/file-operations";
+import type { FileOperations } from "../file-operations/file-operations";
-import { FileNotFoundError } from "src/file-operations/safe-filesystem-operations";
+import { FileNotFoundError } from "../file-operations/safe-filesystem-operations";
-import { DocumentLocks } from "./document-locks";
+import { DocumentLocks } from "../file-operations/document-locks";
+import { createPromise } from "../utils/create-promise";
 
 export class UnrestrictedSyncer {
-	private readonly locks = new DocumentLocks();
+	private readonly locks: DocumentLocks;
 
 	public constructor(
 		private readonly logger: Logger,
@@ -22,507 +27,375 @@ export class UnrestrictedSyncer {
 		private readonly syncService: SyncService,
 		private readonly operations: FileOperations,
 		private readonly history: SyncHistory
-	) {}
+	) {
+		this.locks = new DocumentLocks(logger);
+	}
 
 	public async unrestrictedSyncLocallyCreatedFile(
-		relativePath: RelativePath,
+		document: DocumentRecord
-		updateTime: Date,
-		optimisations?: {
-			contentBytes?: Uint8Array;
-			contentHash?: string;
-		}
 	): Promise<void> {
-		await this.executeWhileHoldingFileLock(
+		return this.executeSync(
-			[relativePath],
+			document.relativePath,
 			SyncType.CREATE,
 			SyncSource.PUSH,
 			async () => {
-				if (
+				const contentBytes = await this.operations.read(
-					(await this.operations.getFileSize(relativePath)) / // this can throw FileNotFoundError
+					document.relativePath
-						1024 /
+				); // this can throw FileNotFoundError
-						1024 >
+				const contentHash = hash(contentBytes);
-					this.settings.getSettings().maxFileSizeMB
-				) {
-					this.history.addHistoryEntry({
-						status: SyncStatus.ERROR,
-						relativePath,
-						message: `File size exceeds the maximum file size limit of ${
-							this.settings.getSettings().maxFileSizeMB
-						}MB`,
-						type: SyncType.CREATE
-					});
-					return;
-				}
-
-				const contentBytes =
-					optimisations?.contentBytes ??
-					(await this.operations.read(relativePath)); // this can throw FileNotFoundError
-				let contentHash =
-					optimisations?.contentHash ?? hash(contentBytes);
-
-				const localMetadata = this.database.getDocument(relativePath);
-				if (localMetadata) {
-					this.logger.debug(
-						`Document metadata already exists for ${relativePath}, it must have been downloaded from the server`
-					);
-
-					if (localMetadata.hash === contentHash) {
-						this.history.addHistoryEntry({
-							status: SyncStatus.NO_OP,
-							relativePath,
-							message: `File hash matches with last synced version, no need to sync`,
-							type: SyncType.UPDATE
-						});
-						return;
-					}
-				}
 
 				const response = await this.syncService.create({
-					relativePath,
+					documentId: document.documentId,
-					contentBytes,
+					relativePath: document.relativePath,
-					createdDate: updateTime
+					contentBytes
 				});
 
 				this.history.addHistoryEntry({
 					status: SyncStatus.SUCCESS,
 					source: SyncSource.PUSH,
-					relativePath,
+					relativePath: document.relativePath,
 					message: `Successfully uploaded locally created file`,
 					type: SyncType.CREATE
 				});
 
-				// The response can't have a different relative path than the one we sent
+				this.database.updateDocumentMetadata(
-				// because the relative path is the key when finding existing documents
+					{
-				// when a create request is sent.
+						parentVersionId: response.vaultUpdateId,
+						hash: contentHash
+					},
+					document
+				);
 
-				if (response.type === "MergingUpdate") {
+				this.tryIncrementVaultUpdateId(response.vaultUpdateId);
-					const responseBytes = deserialize(response.contentBase64);
+			}
-					contentHash = hash(responseBytes);
+		);
+	}
 
-					await this.operations.write(
+	public async unrestrictedSyncLocallyDeletedFile(
-						relativePath,
+		document: DocumentRecord
-						contentBytes,
+	): Promise<void> {
-						responseBytes
+		await this.executeSync(
-					);
+			document.relativePath,
-					this.history.addHistoryEntry({
+			SyncType.DELETE,
-						status: SyncStatus.SUCCESS,
+			SyncSource.PUSH,
-						source: SyncSource.PULL,
+			async () => {
-						relativePath,
+				const response = await this.syncService.delete({
-						message: `The file we created locally has already existed remotely, so we have merged them`,
+					documentId: document.documentId,
-						type: SyncType.UPDATE
+					relativePath: document.relativePath
-					});
-				}
-
-				await this.database.setDocument({
-					documentId: response.documentId,
-					relativePath: response.relativePath,
-					parentVersionId: response.vaultUpdateId,
-					hash: contentHash
 				});
 
-				await this.tryIncrementVaultUpdateId(response.vaultUpdateId);
+				this.history.addHistoryEntry({
+					status: SyncStatus.SUCCESS,
+					source: SyncSource.PUSH,
+					relativePath: document.relativePath,
+					message: `Successfully deleted locally deleted file on the remote server`,
+					type: SyncType.DELETE
+				});
+
+				this.database.updateDocumentMetadata(
+					{
+						parentVersionId: response.vaultUpdateId,
+						hash: EMPTY_HASH
+					},
+					document
+				);
 			}
 		);
 	}
 
 	public async unrestrictedSyncLocallyUpdatedFile({
 		oldPath,
-		relativePath,
+		document,
-		updateTime,
+		force = false
-		optimisations
 	}: {
 		oldPath?: RelativePath;
-		relativePath: RelativePath;
+		force?: boolean;
-		updateTime: Date;
+		document: DocumentRecord;
-		optimisations?: {
-			contentBytes?: Uint8Array;
-			contentHash?: string;
-		};
 	}): Promise<void> {
-		await this.executeWhileHoldingFileLock(
+		await this.executeSync(
-			[oldPath, relativePath].filter((path) => path !== undefined),
+			document.relativePath,
 			SyncType.UPDATE,
 			SyncSource.PUSH,
 			async () => {
-				// Check the new path first in case the metadata has been already moved
+				const originalRelativePath = document.relativePath;
-				let localMetadata = this.database.getDocument(relativePath);
-				let metadataPath = relativePath;
 
-				if (localMetadata === undefined && oldPath !== undefined) {
+				if (document.metadata === undefined || document.isDeleted) {
-					localMetadata = this.database.getDocument(oldPath);
+					this.logger.debug(
-					metadataPath = oldPath;
+						`Document ${document.relativePath} has been already deleted, no need to update it`
-				}
+					);
-
-				if (!localMetadata) {
-					// It's fine, a subsequent sync operation must have dealt with this
 					return;
 				}
 
-				if (
+				const contentBytes = await this.operations.read(
-					(await this.operations.getFileSize(relativePath)) / // this can throw FileNotFoundError
+					document.relativePath
-						1024 /
+				); // this can throw FileNotFoundError
-						1024 >
+				let contentHash = hash(contentBytes);
-					this.settings.getSettings().maxFileSizeMB
-				) {
-					this.history.addHistoryEntry({
-						status: SyncStatus.ERROR,
-						relativePath,
-						message: `File size exceeds the maximum file size limit of ${
-							this.settings.getSettings().maxFileSizeMB
-						}MB`,
-						type: SyncType.CREATE
-					});
-					return;
-				}
-
-				const contentBytes =
-					optimisations?.contentBytes ??
-					(await this.operations.read(relativePath)); // this can throw FileNotFoundError
-
-				let contentHash =
-					optimisations?.contentHash ?? hash(contentBytes);
 
 				if (
-					localMetadata.hash === contentHash &&
+					document.metadata.hash === contentHash &&
-					oldPath === undefined
+					oldPath === undefined &&
+					!force
 				) {
-					this.history.addHistoryEntry({
+					this.logger.debug(
-						status: SyncStatus.NO_OP,
+						`File hash of ${document.relativePath} matches with last synced version and the path hasn't changed; no need to sync`
-						relativePath,
+					);
-						message: `File hash matches with last synced version, no need to sync`,
-						type: SyncType.UPDATE
-					});
 					return;
 				}
 
 				const response = await this.syncService.put({
-					documentId: localMetadata.documentId,
+					documentId: document.documentId,
-					parentVersionId: localMetadata.parentVersionId,
+					parentVersionId: document.metadata.parentVersionId,
-					relativePath,
+					relativePath: document.relativePath,
-					contentBytes,
+					contentBytes
-					createdDate: updateTime
 				});
 
+				// `document` is mutable and reflects the latest state in the local database
+				// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
+				if (document.isDeleted) {
+					this.logger.info(
+						`Document ${document.relativePath} has been deleted before we could finish updating it`
+					);
+					return;
+				}
+
+				// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
+				if (document.metadata === undefined) {
+					throw new Error(
+						`Document ${document.relativePath} no longer has metadata after updating it, this cannot happen`
+					);
+				}
+
+				if (
+					document.metadata.parentVersionId >= response.vaultUpdateId
+				) {
+					this.logger.debug(
+						`Document ${document.relativePath} is already more up to date than the fetched version`
+					);
+					return;
+				}
+
 				this.history.addHistoryEntry({
 					status: SyncStatus.SUCCESS,
 					source: SyncSource.PUSH,
-					relativePath,
+					relativePath: document.relativePath,
 					message: `Successfully uploaded locally updated file to the remote server`,
 					type: SyncType.UPDATE
 				});
 
 				if (response.isDeleted) {
-					await this.operations.remove(oldPath ?? relativePath);
-					await this.database.removeDocument(oldPath ?? relativePath);
-					await this.tryIncrementVaultUpdateId(
-						response.vaultUpdateId
-					);
-
 					this.history.addHistoryEntry({
 						status: SyncStatus.SUCCESS,
 						source: SyncSource.PULL,
-						relativePath,
+						relativePath: document.relativePath,
 						message:
 							"The file we tried to update had been deleted remotely, therefore, we have deleted it locally",
 						type: SyncType.DELETE
 					});
 
+					this.database.delete(document.relativePath);
+					this.database.updateDocumentMetadata(
+						{
+							parentVersionId: response.vaultUpdateId,
+							hash: EMPTY_HASH
+						},
+						document
+					);
+
+					await this.operations.delete(document.relativePath);
+
+					this.tryIncrementVaultUpdateId(response.vaultUpdateId);
+
 					return;
 				}
 
-				if (
+				let actualPath = document.relativePath;
-					response.relativePath != relativePath &&
+
-					response.relativePath != oldPath
+				if (response.relativePath != originalRelativePath) {
-				) {
+					actualPath = response.relativePath;
-					await this.locks.waitForDocumentLock(response.relativePath);
+					await this.operations.move(
+						document.relativePath,
+						response.relativePath
+					); // this can throw FileNotFoundError
 				}
 
-				try {
+				this.database.updateDocumentMetadata(
-					if (response.relativePath != relativePath) {
+					{
-						// TODO: this can fail, that's bad
-						await this.operations.move(
-							// this can throw FileNotFoundError
-							relativePath,
-							response.relativePath,
-							response.documentId
-						);
-					}
-
-					if (response.type === "MergingUpdate") {
-						const responseBytes = deserialize(
-							response.contentBase64
-						);
-						contentHash = hash(responseBytes);
-
-						await this.operations.write(
-							response.relativePath,
-							contentBytes,
-							responseBytes
-						);
-
-						this.history.addHistoryEntry({
-							status: SyncStatus.SUCCESS,
-							source: SyncSource.PULL,
-							relativePath,
-							message: `The file we updated had been updated remotely, so we downloaded the merged version`,
-							type: SyncType.UPDATE
-						});
-					}
-
-					if (metadataPath !== response.relativePath) {
-						await this.database.updatePath(
-							metadataPath,
-							response.relativePath
-						);
-					}
-					await this.database.setDocument({
-						documentId: localMetadata.documentId,
-						relativePath: response.relativePath,
 						parentVersionId: response.vaultUpdateId,
 						hash: contentHash
-					});
+					},
+					document
+				);
 
-					await this.tryIncrementVaultUpdateId(
+				if (response.type === "MergingUpdate") {
-						response.vaultUpdateId
+					const responseBytes = deserialize(response.contentBase64);
+					contentHash = hash(responseBytes);
+
+					await this.operations.write(
+						actualPath,
+						contentBytes,
+						responseBytes
 					);
-				} finally {
-					if (
-						response.relativePath != relativePath &&
-						response.relativePath != oldPath
-					) {
-						this.locks.unlockDocument(response.relativePath);
-					}
-				}
-			}
-		);
-	}
 
-	public async unrestrictedSyncLocallyDeletedFile(
-		relativePath: RelativePath
-	): Promise<void> {
-		await this.executeWhileHoldingFileLock(
-			[relativePath],
-			SyncType.DELETE,
-			SyncSource.PUSH,
-			async () => {
-				const localMetadata = this.database.getDocument(relativePath);
-				if (!localMetadata) {
 					this.history.addHistoryEntry({
-						status: SyncStatus.NO_OP,
+						status: SyncStatus.SUCCESS,
-						relativePath,
+						source: SyncSource.PULL,
-						message: `Locally deleted file hasn't been uploaded yet, so there's no need to delete it on the remote server`,
+						relativePath: document.relativePath,
-						type: SyncType.DELETE
+						message: `The file we updated had been updated remotely, so we downloaded the merged version`,
+						type: SyncType.UPDATE
 					});
-					return;
 				}
 
-				await this.syncService.delete({
+				this.tryIncrementVaultUpdateId(response.vaultUpdateId);
-					documentId: localMetadata.documentId,
-					relativePath,
-					createdDate: new Date() // We got the event now, so it must have been deleted just now
-				});
-
-				this.history.addHistoryEntry({
-					status: SyncStatus.SUCCESS,
-					source: SyncSource.PUSH,
-					relativePath,
-					message: `Successfully deleted locally deleted file on the remote server`,
-					type: SyncType.DELETE
-				});
-
-				await this.database.removeDocument(relativePath);
 			}
 		);
 	}
 
 	public async unrestrictedSyncRemotelyUpdatedFile(
-		remoteVersion: components["schemas"]["DocumentVersionWithoutContent"]
+		remoteVersion: components["schemas"]["DocumentVersionWithoutContent"],
+		document?: DocumentRecord
 	): Promise<void> {
-		await this.executeWhileHoldingFileLock(
+		await this.executeSync(
-			[remoteVersion.relativePath],
+			remoteVersion.relativePath,
 			SyncType.UPDATE,
 			SyncSource.PULL,
 			async () => {
-				let localMetadata = this.database.getDocumentByDocumentId(
+				if (document?.metadata !== undefined) {
-					remoteVersion.documentId
+					// If the file exists locally, let's pretend the user has updated it
-				);
+					// and deal with remote update/deletion within `unrestrictedSyncLocallyUpdatedFile`
-
+					if (
-				if (
+						document.metadata.parentVersionId >=
-					localMetadata &&
+						remoteVersion.vaultUpdateId
-					localMetadata[0] !== remoteVersion.relativePath
+					) {
-				) {
+						this.logger.debug(
-					await this.locks.waitForDocumentLock(localMetadata[0]);
+							`Document ${remoteVersion.relativePath} is already more up to date than the fetched version`
-				}
+						);
-				// Waiting for the new lock might take a while so we need to fetch the database
-				// entry again in case it's changed.
-				localMetadata = this.database.getDocumentByDocumentId(
-					remoteVersion.documentId
-				);
-
-				if (!localMetadata) {
-					if (remoteVersion.isDeleted) {
-						this.history.addHistoryEntry({
-							status: SyncStatus.NO_OP,
-							source: SyncSource.PULL,
-							relativePath: remoteVersion.relativePath,
-							message: `Remotely deleted file hasn't been synced yet, so there's no need to delete it locally`,
-							type: SyncType.DELETE
-						});
 						return;
 					}
 
-					const content = (
+					return this.unrestrictedSyncLocallyUpdatedFile({
-						await this.syncService.get({
+						document,
-							documentId: remoteVersion.documentId
+						force: true
-						})
+					});
-					).contentBase64;
+				} else if (remoteVersion.isDeleted) {
-					const contentBytes = deserialize(content);
+					// Either the doc hasn't made it to us before and therefore we don't need to delete it,
-
+					// or we already have it, in which case the preceeding if will deal with it
-					await this.operations.create(
+					this.logger.debug(
-						remoteVersion.relativePath,
+						`Document ${remoteVersion.relativePath} has been deleted remotely, no need to sync`
-						contentBytes
 					);
-					await this.database.setDocument({
+					return;
-						documentId: remoteVersion.documentId,
+				}
-						relativePath: remoteVersion.relativePath,
+
+				const content = (
+					await this.syncService.get({
+						documentId: remoteVersion.documentId
+					})
+				).contentBase64;
+
+				document = this.database.getDocumentByDocumentId(
+					remoteVersion.documentId
+				);
+
+				if (document?.isDeleted === true) {
+					this.logger.info(
+						`Document ${remoteVersion.relativePath} has been deleted locally before we could finish updating it`
+					);
+					return;
+				}
+
+				if (
+					(document?.metadata?.parentVersionId ?? -1) >=
+					remoteVersion.vaultUpdateId
+				) {
+					this.logger.debug(
+						`Document ${remoteVersion.relativePath} is already more up to date than the fetched version`
+					);
+					return;
+				}
+
+				const contentBytes = deserialize(content);
+
+				await this.operations.ensureClearPath(
+					remoteVersion.relativePath
+				);
+
+				const [promise, resolve] = createPromise();
+				this.database.updateDocumentMetadata(
+					{
 						parentVersionId: remoteVersion.vaultUpdateId,
 						hash: hash(contentBytes)
-					});
+					},
-					this.history.addHistoryEntry({
+					this.database.createNewPendingDocument(
-						status: SyncStatus.SUCCESS,
+						remoteVersion.documentId,
-						source: SyncSource.PULL,
+						remoteVersion.relativePath,
-						relativePath: remoteVersion.relativePath,
+						promise
-						message: `Successfully downloaded remote file which hadn't existed locally`,
+					)
-						type: SyncType.CREATE
+				);
-					});
-					return;
-				}
 
-				const [relativePath, metadata] = localMetadata;
+				await this.operations.create(
+					remoteVersion.relativePath,
+					contentBytes
+				);
 
-				if (remoteVersion.vaultUpdateId <= metadata.parentVersionId) {
+				resolve();
-					this.logger.debug(
+				this.database.removeDocumentPromise(promise);
-						`Document ${relativePath} is already up to date`
-					);
-					return;
-				}
 
-				try {
+				this.history.addHistoryEntry({
-					if (remoteVersion.isDeleted) {
+					status: SyncStatus.SUCCESS,
-						await this.operations.remove(relativePath);
+					source: SyncSource.PULL,
-						await this.database.removeDocument(relativePath);
+					relativePath: remoteVersion.relativePath,
-
+					message: `Successfully downloaded remote file which hadn't existed locally`,
-						this.history.addHistoryEntry({
+					type: SyncType.CREATE
-							status: SyncStatus.SUCCESS,
+				});
-							source: SyncSource.PULL,
-							relativePath: remoteVersion.relativePath,
-							message: `Successfully deleted remotely deleted file locally`,
-							type: SyncType.DELETE
-						});
-					} else {
-						// TODO: this can fail, that's bad
-						const currentContent =
-							await this.operations.read(relativePath); // this can throw FileNotFoundError
-						const currentHash = hash(currentContent);
-
-						if (currentHash !== metadata.hash) {
-							this.logger.info(
-								`Document ${relativePath} has been updated both remotely and locally, letting the local file update event handle it`
-							);
-							return;
-						}
-
-						const content = (
-							await this.syncService.get({
-								documentId: remoteVersion.documentId
-							})
-						).contentBase64;
-						const contentBytes = deserialize(content);
-						const contentHash = hash(contentBytes);
-
-						if (relativePath !== remoteVersion.relativePath) {
-							// TODO: this can fail, that's bad
-							await this.operations.move(
-								// this can throw FileNotFoundError
-								relativePath,
-								remoteVersion.relativePath,
-								remoteVersion.documentId
-							);
-
-							await this.database.updatePath(
-								relativePath,
-								remoteVersion.relativePath
-							);
-						}
-
-						await this.operations.write(
-							remoteVersion.relativePath,
-							currentContent,
-							contentBytes
-						);
-						await this.database.setDocument({
-							documentId: remoteVersion.documentId,
-							relativePath: remoteVersion.relativePath,
-							parentVersionId: remoteVersion.vaultUpdateId,
-							hash: contentHash
-						});
-
-						this.history.addHistoryEntry({
-							status: SyncStatus.SUCCESS,
-							source: SyncSource.PULL,
-							relativePath: remoteVersion.relativePath,
-							message: `Successfully updated remotely updated file locally`,
-							type: SyncType.UPDATE
-						});
-					}
-				} finally {
-					if (relativePath !== remoteVersion.relativePath) {
-						this.locks.unlockDocument(relativePath);
-					}
-				}
 			}
 		);
 	}
 
-	public async executeWhileHoldingFileLock(
+	public async executeSync<T>(
-		lockedPaths: RelativePath[],
+		relativePath: RelativePath,
 		syncType: SyncType,
 		syncSource: SyncSource,
-		fn: () => Promise<void>
+		fn: () => Promise<T>
-	): Promise<void> {
+	): Promise<T | undefined> {
-		const relativePath = lockedPaths[lockedPaths.length - 1];
-
-		if (!this.settings.getSettings().isSyncEnabled) {
-			this.logger.info(
-				`Syncing is disabled, not syncing ${relativePath}`
-			);
-			return;
-		}
 		if (!this.operations.isFileEligibleForSync(relativePath)) {
-			this.logger.info(
+			this.history.addHistoryEntry({
-				`File ${relativePath} is not eligible for syncing`
+				status: SyncStatus.ERROR,
-			);
+				relativePath,
+				message: `File ${relativePath} is not eligible for syncing`,
+				type: syncType
+			});
 			return;
 		}
+
 		this.logger.debug(
 			`Syncing ${relativePath} (${syncSource} - ${syncType})`
 		);
 
-		await Promise.all(
-			lockedPaths.map(this.locks.waitForDocumentLock.bind(this.locks))
-		);
 		try {
-			await fn();
+			if (
+				(await this.operations.exists(relativePath)) &&
+				(await this.operations.getFileSize(relativePath)) / // this can throw FileNotFoundError
+					1024 /
+					1024 >
+					this.settings.getSettings().maxFileSizeMB
+			) {
+				this.history.addHistoryEntry({
+					status: SyncStatus.ERROR,
+					relativePath,
+					message: `File size exceeds the maximum file size limit of ${
+						this.settings.getSettings().maxFileSizeMB
+					}MB`,
+					type: syncType
+				});
+				return;
+			}
+
+			return await fn();
 		} catch (e) {
 			if (e instanceof FileNotFoundError) {
 				// A subsequent sync operation must have been creating to deal with this
-				this.history.addHistoryEntry({
+				this.logger.info(
-					status: SyncStatus.NO_OP,
+					`Skip ${syncSource.toLocaleLowerCase()} file because it no longer exists when trying to ${syncType.toLocaleLowerCase()} it`
-					relativePath,
+				);
-					message: `Skip ${syncSource.toLocaleLowerCase()} file because it no longer exists when trying to ${syncType.toLocaleLowerCase()} it`,
-					type: syncType,
-					source: syncSource
-				});
 			} else {
 				this.history.addHistoryEntry({
 					status: SyncStatus.ERROR,
@@ -533,8 +406,6 @@ export class UnrestrictedSyncer {
 				});
 				throw e;
 			}
-		} finally {
-			lockedPaths.forEach(this.locks.unlockDocument.bind(this.locks));
 		}
 	}
 
@@ -542,11 +413,9 @@ export class UnrestrictedSyncer {
 		this.locks.reset();
 	}
 
-	private async tryIncrementVaultUpdateId(
+	private tryIncrementVaultUpdateId(responseVaultUpdateId: number): void {
-		responseVaultUpdateId: number
-	): Promise<void> {
 		if (this.database.getLastSeenUpdateId() === responseVaultUpdateId - 1) {
-			await this.database.setLastSeenUpdateId(responseVaultUpdateId);
+			this.database.setLastSeenUpdateId(responseVaultUpdateId);
 		}
 	}
 }

frontend/sync-client/src/tracing/sync-history.ts

@@ -1,4 +1,4 @@
-import type { RelativePath } from "src/persistence/database";
+import type { RelativePath } from "../persistence/database";
 import type { Logger } from "./logger";
 
 export interface CommonHistoryEntry {

frontend/sync-client/src/utils/create-promise.ts

@@ -0,0 +1,15 @@
+export function createPromise<T = void>(): [
+	Promise<T>,
+	(value: T) => void,
+	(error: unknown) => void
+] {
+	let resolve: undefined | ((resolved: T) => void) = undefined;
+	let reject: undefined | ((error: unknown) => void) = undefined;
+
+	const creationPromise = new Promise<T>(
+		(resolve_, reject_) => ((resolve = resolve_), (reject = reject_))
+	);
+
+	// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+	return [creationPromise, resolve!, reject!];
+}

frontend/sync-client/src/utils/find-matching-file-based-on-hash.ts

@@ -1,13 +0,0 @@
-import type { DocumentMetadata, RelativePath } from "src/persistence/database";
-import { EMPTY_HASH } from "./hash";
-
-export function findMatchingFileBasedOnHash(
-	contentHash: string,
-	candidates: [RelativePath, DocumentMetadata][]
-): [RelativePath, DocumentMetadata] | undefined {
-	if (contentHash === EMPTY_HASH) {
-		return undefined;
-	}
-
-	return candidates.find(([_, metadata]) => metadata.hash === contentHash);
-}

frontend/sync-client/src/utils/find-matching-file.ts

@@ -0,0 +1,14 @@
+import type { DocumentRecord } from "../persistence/database";
+import { EMPTY_HASH } from "./hash";
+
+// TODO: make this smarter so that offline files can be renamed & edited at the same time
+export function findMatchingFile(
+	contentHash: string,
+	candidates: DocumentRecord[]
+): DocumentRecord | undefined {
+	if (contentHash === EMPTY_HASH) {
+		return undefined;
+	}
+
+	return candidates.find(({ metadata }) => metadata?.hash === contentHash);
+}

frontend/sync-client/src/utils/hash.ts

@@ -6,7 +6,7 @@ export function hash(content: Uint8Array): string {
 		result = (result << 5) - result + content[i];
 		result |= 0; // Convert to 32bit integer
 	}
-	return Math.abs(result).toString(16);
+	return Math.abs(result).toString(16).padStart(8, "0");
 }
 
 export const EMPTY_HASH = hash(new Uint8Array(0));

frontend/sync-client/src/utils/retried-fetch.ts

@@ -1,6 +1,6 @@
 import * as fetchRetryFactory from "fetch-retry";
 import type { RequestInitRetryParams } from "fetch-retry";
-import type { Logger } from "src/tracing/logger";
+import type { Logger } from "../tracing/logger";
 
 function getUrlFromInput(input: RequestInfo | URL): string {
 	if (input instanceof URL) {
@@ -31,7 +31,6 @@ export function retriedFetchFactory(
 				}
 				return false;
 			},
-			retries: 6,
 			retryDelay: (attempt) => Math.pow(1.5, attempt) * 500,
 			...init
 		});

frontend/sync-client/tsconfig.json

@@ -1,12 +1,15 @@
 {
 	"compilerOptions": {
-		"baseUrl": ".",
 		"module": "ESNext",
 		"target": "ESNext",
 		"strict": true,
-		"moduleResolution": "bundler",
 		"allowSyntheticDefaultImports": true,
-		"lib": ["DOM", "ESNext"]
+		"moduleResolution": "bundler",
+		"lib": [
+			"DOM" // to get "fetch"
+		],
+		"declaration": true,
+		"declarationDir": "./dist/types"
 	},
 	"exclude": ["./dist"]
 }

frontend/sync-client/webpack.config.js

@@ -1,25 +1,13 @@
 const path = require("path");
+const { merge } = require("webpack-merge");
 
-module.exports = (_env, _argv) => ({
+const common = {
 	entry: "./src/index.ts",
-	devtool: "source-map",
-	target: "node",
 	module: {
 		rules: [
 			{
 				test: /\.ts$/,
-				use: [
+				use: ["ts-loader"]
-					{
-						loader: "ts-loader",
-						options: {
-							compilerOptions: {
-								declaration: true,
-								declarationDir: "./dist/types"
-							},
-							transpileOnly: false
-						}
-					}
-				]
 			},
 			{
 				test: /\.wasm$/,
@@ -28,22 +16,40 @@ module.exports = (_env, _argv) => ({
 		]
 	},
 	optimization: {
+		// the consuming project should take care of minification
 		minimize: false
 	},
 	resolve: {
-		extensions: [".ts", ".js"],
+		extensions: [".ts"],
 		alias: {
 			root: __dirname,
 			src: path.resolve(__dirname, "src")
 		}
 	},
-	output: {
+	performance: {
-		clean: true,
+		hints: false // it's a library, no need to warn about its size
-		filename: "index.js",
-		library: {
-			name: "SyncClient",
-			type: "umd"
-		},
-		path: path.resolve(__dirname, "dist")
 	}
-});
+};
+
+module.exports = [
+	merge(common, {
+		target: "web",
+		output: {
+			path: path.resolve(__dirname, "dist"),
+			filename: "sync-client.web.js",
+			library: {
+				name: "SyncClient",
+				type: "umd"
+			},
+			globalObject: "this"
+		}
+	}),
+	merge(common, {
+		target: "node",
+		output: {
+			path: path.resolve(__dirname, "dist"),
+			filename: "sync-client.node.js",
+			libraryTarget: "commonjs2"
+		}
+	})
+];

frontend/test-client/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "test-client",
-	"version": "0.0.0",
+	"version": "0.0.30",
 	"private": true,
 	"bin": {
 		"test-client": "./dist/cli.js"
@@ -11,11 +11,11 @@
 		"test": "jest --passWithNoTests"
 	},
 	"devDependencies": {
-		"@types/node": "^22.13.5",
+		"@types/node": "^22.13.10",
 		"sync-client": "file:../sync-client",
 		"ts-loader": "^9.5.2",
 		"tslib": "2.8.1",
-		"typescript": "5.7.3",
+		"typescript": "5.8.2",
 		"uuid": "^11.1.0",
 		"webpack": "^5.98.0",
 		"webpack-cli": "^6.0.1"

frontend/test-client/src/agent/mock-agent.ts

@@ -18,9 +18,10 @@ export class MockAgent extends MockClient {
 		initialSettings: Partial<SyncSettings>,
 		public readonly name: string,
 		private readonly doDeletes: boolean,
+		useSlowFileEvents: boolean,
 		private readonly jitterScaleInSeconds: number
 	) {
-		super(initialSettings);
+		super(initialSettings, useSlowFileEvents);
 	}
 
 	public async init(): Promise<void> {
@@ -46,27 +47,33 @@ export class MockAgent extends MockClient {
 				? "(online) "
 				: "(offline)";
 			const formatted = `[${this.name} ${state}] ${logLine.timestamp.toISOString()} ${logLine.level} ${logLine.message}`;
+
+			// HACK: we have to ensure the file has been synced if we want to change it offline without data loss
+			const historyEntry = /.*History entry: (.*.md).*/.exec(
+				logLine.message
+			);
+
+			if (historyEntry) {
+				this.doNotTouchWhileOffline =
+					this.doNotTouchWhileOffline.filter(
+						(file) => file !== historyEntry[1]
+					);
+			}
 			switch (logLine.level) {
 				case LogLevel.ERROR:
 					console.error(formatted);
-					// Let's not ignore errors
+
-					process.exit(1);
+					if (!this.useSlowFileEvents) {
+						// Let's not ignore errors
+						// eslint-disable-next-line @typescript-eslint/no-floating-promises
+						sleep(100).then(() => process.exit(1));
+					}
+
 					break;
 				case LogLevel.WARNING:
 					console.warn(formatted);
 					break;
 				case LogLevel.INFO:
-					// HACK: we have to ensure the file has been synced if we want to change it offline without data loss
-					const result = /.*History entry: (.*.md).*/.exec(
-						logLine.message
-					);
-					if (result) {
-						this.doNotTouchWhileOffline =
-							this.doNotTouchWhileOffline.filter(
-								(file) => file !== result[1]
-							);
-					}
-
 					console.info(formatted);
 					break;
 				case LogLevel.DEBUG:
@@ -84,11 +91,10 @@ export class MockAgent extends MockClient {
 			this.changeFetchChangesUpdateIntervalMsAction.bind(this)
 		];
 
-		if (
+		if (this.client.settings.getSettings().isSyncEnabled) {
-			this.client.settings.getSettings().isSyncEnabled &&
+			if (this.doNotTouchWhileOffline.length === 0) {
-			this.doNotTouchWhileOffline.length === 0
+				options.push(this.disableSyncAction.bind(this));
-		) {
+			}
-			options.push(this.disableSyncAction.bind(this));
 		} else {
 			options.push(this.enableSyncAction.bind(this));
 		}
@@ -186,6 +192,14 @@ export class MockAgent extends MockClient {
 	}
 
 	public assertAllContentIsPresentOnce(): void {
+		if (this.useSlowFileEvents) {
+			this.client.logger.info(
+				// We can't ensure that we have seen every single update
+				`Skipping content check for ${this.name} because slow file events are enabled`
+			);
+			return;
+		}
+
 		for (const content of this.writtenContents) {
 			const found = Array.from(this.localFiles.keys()).filter((key) => {
 				return new TextDecoder()
@@ -215,7 +229,7 @@ export class MockAgent extends MockClient {
 				);
 				assert(
 					fileContent.split(content).length == 2,
-					`Content ${content} (of ${this.name}) found more than once in file ${file}. File content:\n${fileContent}`
+					`Content ${content} (of ${this.name}) found more than once in '${file}'. File content:\n${fileContent}`
 				);
 			}
 		}
@@ -237,10 +251,7 @@ export class MockAgent extends MockClient {
 			`Decided to create file ${file} with content ${content}`
 		);
 
-		return this.create(
+		return this.create(file, new TextEncoder().encode(` ${content} `));
-			file,
-			new TextEncoder().encode(`   |${content}|   `)
-		);
 	}
 
 	private async changeFetchChangesUpdateIntervalMsAction(): Promise<void> {
@@ -314,7 +325,7 @@ export class MockAgent extends MockClient {
 			`Decided to update file ${file} with ${content}`
 		);
 		this.doNotTouchWhileOffline.push(file);
-		await this.atomicUpdateText(file, (old) => old + `   |${content}|   `);
+		await this.atomicUpdateText(file, (old) => old + ` ${content} `);
 	}
 
 	private async deleteFileAction(files: RelativePath[]): Promise<void> {

frontend/test-client/src/agent/mock-client.ts

@@ -1,9 +1,10 @@
-import type {
+import { assert } from "../utils/assert";
-	RelativePath,
+import {
-	FileSystemOperations,
+	type RelativePath,
-	SyncSettings
+	type FileSystemOperations,
+	type SyncSettings,
+	SyncClient
 } from "sync-client";
-import { SyncClient } from "sync-client";
 
 export class MockClient implements FileSystemOperations {
 	protected readonly localFiles = new Map<string, Uint8Array>();
@@ -11,7 +12,8 @@ export class MockClient implements FileSystemOperations {
 	protected data: object | undefined = undefined;
 
 	public constructor(
-		private readonly initialSettings: Partial<SyncSettings>
+		private readonly initialSettings: Partial<SyncSettings>,
+		protected readonly useSlowFileEvents: boolean
 	) {}
 
 	public async init(): Promise<void> {
@@ -22,9 +24,10 @@ export class MockClient implements FileSystemOperations {
 
 		await Promise.all(
 			Object.keys(this.initialSettings).map(async (key) => {
+				const settingKey = key as keyof SyncSettings; // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion
 				return this.client.settings.setSetting(
-					key as keyof SyncSettings, // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion
+					settingKey,
-					this.initialSettings[key as keyof SyncSettings] // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion
+					this.initialSettings[settingKey]! // eslint-disable-line @typescript-eslint/no-non-null-assertion
 				);
 			})
 		);
@@ -46,13 +49,6 @@ export class MockClient implements FileSystemOperations {
 		return (await this.read(path)).length;
 	}
 
-	public async getModificationTime(path: RelativePath): Promise<Date> {
-		if (!this.localFiles.has(path)) {
-			throw new Error(`File ${path} does not exist`);
-		}
-		return new Date();
-	}
-
 	public async exists(path: RelativePath): Promise<boolean> {
 		return this.localFiles.has(path);
 	}
@@ -68,7 +64,10 @@ export class MockClient implements FileSystemOperations {
 			`Creating file ${path} with content ${new TextDecoder().decode(newContent)}`
 		);
 		this.localFiles.set(path, newContent);
-		void this.client.syncer.syncLocallyCreatedFile(path, new Date());
+
+		this.runCallback(() => {
+			void this.client.syncer.syncLocallyCreatedFile(path);
+		});
 	}
 
 	public async createDirectory(_path: RelativePath): Promise<void> {
@@ -88,28 +87,51 @@ export class MockClient implements FileSystemOperations {
 		const newContentUint8Array = new TextEncoder().encode(newContent);
 		this.localFiles.set(path, newContentUint8Array);
 
+		if (!this.useSlowFileEvents) {
+			const existingParts = currentContent
+				.split(" ")
+				.map((part) => part.trim());
+			const newParts = newContent.split(" ").map((part) => part.trim());
+			existingParts.forEach((part) =>
+				// all changes should be additive
+				{
+					assert(
+						newParts.includes(part),
+						`Part ${part} not found in new content`
+					);
+				}
+			);
+		}
+
 		this.client.logger.info(
 			`Updated file ${path} with:\n  current content: ${currentContent}\n  new content: ${newContent}`
 		);
 
-		void this.client.syncer.syncLocallyUpdatedFile({
+		this.runCallback(() => {
-			relativePath: path,
+			void this.client.syncer.syncLocallyUpdatedFile({
-			updateTime: new Date()
+				relativePath: path
+			});
 		});
 
 		return newContent;
 	}
 
 	public async write(path: RelativePath, content: Uint8Array): Promise<void> {
+		const hasExisted = this.localFiles.has(path);
 		this.localFiles.set(path, content);
 
 		this.client.logger.info(
 			`Updated file ${path} with:\n  new content: ${new TextDecoder().decode(content)}`
 		);
 
-		void this.client.syncer.syncLocallyUpdatedFile({
+		this.runCallback(() => {
-			relativePath: path,
+			if (hasExisted) {
-			updateTime: new Date()
+				void this.client.syncer.syncLocallyUpdatedFile({
+					relativePath: path
+				});
+			} else {
+				void this.client.syncer.syncLocallyCreatedFile(path);
+			}
 		});
 	}
 
@@ -118,7 +140,10 @@ export class MockClient implements FileSystemOperations {
 			`Deleting file: ${path} with:\n  content ${new TextDecoder().decode(this.localFiles.get(path))}`
 		);
 		this.localFiles.delete(path);
-		void this.client.syncer.syncLocallyDeletedFile(path);
+
+		this.runCallback(() => {
+			void this.client.syncer.syncLocallyDeletedFile(path);
+		});
 	}
 
 	public async rename(
@@ -138,10 +163,20 @@ export class MockClient implements FileSystemOperations {
 			`Renamed file: ${oldPath} -> ${newPath} with:\n  content ${new TextDecoder().decode(file)}`
 		);
 
-		void this.client.syncer.syncLocallyUpdatedFile({
+		this.runCallback(() => {
-			oldPath,
+			void this.client.syncer.syncLocallyUpdatedFile({
-			relativePath: newPath,
+				oldPath,
-			updateTime: new Date()
+				relativePath: newPath
+			});
 		});
 	}
+
+	private runCallback(callback: () => void): void {
+		if (this.useSlowFileEvents) {
+			// we aren't the best client and it takes some time to notice changes
+			setTimeout(callback, 100);
+		} else {
+			callback();
+		}
+	}
 }

frontend/test-client/src/cli.ts

@@ -3,20 +3,26 @@ import { MockAgent } from "./agent/mock-agent";
 import { sleep } from "./utils/sleep";
 import { v4 as uuidv4 } from "uuid";
 
+let slowFileEvents = false;
+
 async function runTest({
 	agentCount,
 	concurrency,
 	iterations,
 	doDeletes,
+	useSlowFileEvents,
 	jitterScaleInSeconds
 }: {
 	agentCount: number;
 	concurrency: number;
 	iterations: number;
 	doDeletes: boolean;
+	useSlowFileEvents: boolean;
 	jitterScaleInSeconds: number;
 }): Promise<void> {
-	const settings = `with ${agentCount} agents, concurrency ${concurrency}, iterations ${iterations}, doDeletes ${doDeletes}, jitterScaleInSeconds ${jitterScaleInSeconds}`;
+	slowFileEvents = useSlowFileEvents;
+
+	const settings = `with ${agentCount} agents, concurrency ${concurrency}, iterations ${iterations}, doDeletes ${doDeletes}, jitterScaleInSeconds ${jitterScaleInSeconds}, useSlowFileEvents ${useSlowFileEvents}`;
 	console.info(`Running test ${settings}`);
 
 	const initialSettings: Partial<SyncSettings> = {
@@ -34,6 +40,7 @@ async function runTest({
 				initialSettings,
 				`agent-${i}`,
 				doDeletes,
+				useSlowFileEvents,
 				jitterScaleInSeconds
 			)
 		);
@@ -52,12 +59,24 @@ async function runTest({
 
 		// Each agent can have unpushed changes which might conflict with eachother so each has to resolve the conflicts & push, and
 		for (const client of clients) {
-			await client.finish();
+			try {
+				await client.finish();
+			} catch (err) {
+				if (!slowFileEvents) {
+					throw err;
+				}
+			}
 		}
 
 		// then we need a second pass to ensure that all agents pull the same state.
 		for (const client of clients) {
-			await client.finish();
+			try {
+				await client.finish();
+			} catch (err) {
+				if (!slowFileEvents) {
+					throw err;
+				}
+			}
 		}
 
 		console.info("Agents finished successfully");
@@ -78,41 +97,49 @@ async function runTest({
 			console.info(`Content check for ${client.name} passed`);
 		});
 
-		console.info(`Test passed with ${settings}`);
+		console.info(`Test passed ${settings}`);
 	} catch (err) {
-		console.error(`Test failed with ${settings}`);
+		console.error(`Test failed ${settings}`);
 		throw err;
 	}
 }
 
 async function runTests(): Promise<void> {
-	const agentCounts = [2, 10];
+	for (const useSlowFileEvents of [false, true]) {
-	const jitterScaleInSeconds = [0.5, 3, 0];
+		for (const concurrency of [
-	const concurrencies = [1, 16];
+			16,
-	const iterations = [50, 300];
+			1 // test with concurrency 1 to check for deadlocks
-	const doDeletes = [false, true];
+		]) {
-
+			for (const doDeletes of [true, false]) {
-	for (const agentCount of agentCounts) {
+				await runTest({
-		for (const concurrency of concurrencies) {
+					agentCount: 3,
-			for (const jitter of jitterScaleInSeconds) {
+					concurrency,
-				for (const iteration of iterations) {
+					iterations: 100,
-					for (const deleteFiles of doDeletes) {
+					doDeletes,
-						while (true) {
+					useSlowFileEvents,
-							await runTest({
+					jitterScaleInSeconds: 0.75
-								agentCount,
+				});
-								concurrency,
-								iterations: iteration,
-								doDeletes: deleteFiles,
-								jitterScaleInSeconds: jitter
-							});
-						}
-					}
-				}
 			}
 		}
 	}
 }
 
+process.on("uncaughtException", (error) => {
+	if (slowFileEvents) {
+		return;
+	}
+	console.error("Uncaught Exception:", error);
+	process.exit(1);
+});
+
+process.on("unhandledRejection", (reason, _promise) => {
+	if (slowFileEvents) {
+		return;
+	}
+	console.error("Unhandled Rejection:", reason);
+	process.exit(1);
+});
+
 runTests()
 	.then(() => {
 		process.exit(0);

frontend/test-client/webpack.config.js

@@ -12,8 +12,7 @@ module.exports = {
 		rules: [
 			{
 				test: /\.ts$/,
-				use: "ts-loader",
+				use: "ts-loader"
-				exclude: /node_modules/
 			}
 		]
 	},

scripts/bump-version.sh

@@ -27,20 +27,20 @@ cd backend
 cargo set-version --bump patch
 
 echo "Bumping frontend versions"
-cd ../plugin
+cd ../frontend
-npm version patch
+npm version patch --workspaces
 
 echo "Updating frontend dependencies to match the new backend versions"
 cd ../backend/sync_lib
 wasm-pack build --target web --features console_error_panic_hook
 
-cd ../../plugin
+cd ../../frontend
 npm install
 
 cd ..
-cp plugin/manifest.json manifest.json  # for BRAT, otherwise it wouldn't update
+cp frontend/obsidian-plugin/manifest.json manifest.json  # for BRAT, otherwise it wouldn't update
 
-# Commit and tag
+Commit and tag
 git add .
 TAG=$(node -p "require('./plugin/package.json').version")
 git commit -m "Bump versions to $TAG"

scripts/clean-up.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+rm -rf backend/databases
+rm -rf logs

scripts/e2e.sh

@@ -0,0 +1,79 @@
+#!/bin/bash
+
+set -e
+set -o pipefail
+
+# Check if the argument is provided
+if [ $# -eq 0 ]; then
+    echo "Usage: $0 <number_of_processes>"
+    exit 1
+fi
+
+# Get the number of processes from the first argument
+process_count=$1
+
+mkdir -p logs
+
+cd frontend
+npm run build
+
+pids=()
+for i in $(seq 1 $process_count); do
+    node test-client/dist/cli.js > "../logs/log_${i}.log" 2>&1 &
+    pids+=($!)
+done
+
+cd -
+
+print_failed_log() {
+    for i in $(seq 1 $process_count); do
+        if [ -n "${pids[$i-1]}" ] && ! kill -0 ${pids[$i-1]} 2>/dev/null; then
+            # Get the exit code of the process
+            wait ${pids[$i-1]}
+            exit_code=$?
+            
+            # Only consider non-zero exit codes as failures
+            if [ $exit_code -ne 0 ]; then
+                echo "Process ${pids[$i-1]} failed with exit code $exit_code. Log file: $(pwd)/logs/log_${i}.log"
+                return 0
+            else
+                echo "Process ${pids[$i-1]} completed successfully with exit code 0"
+                # Mark this PID as processed by setting it to empty
+                pids[$i-1]=""
+            fi
+        fi
+    done
+    return 1
+}
+
+echo "Monitoring $process_count processes"
+
+# Monitor processes
+while true; do
+    if print_failed_log; then
+        # Kill remaining processes
+        for pid in "${pids[@]}"; do
+            if [ -n "$pid" ]; then
+                kill $pid 2>/dev/null || true
+            fi
+        done
+        exit 1
+    fi
+
+    # Check if all processes have completed
+    all_done=true
+    for pid in "${pids[@]}"; do
+        if [ -n "$pid" ] && kill -0 $pid 2>/dev/null; then
+            all_done=false
+            break
+        fi
+    done
+
+    if $all_done; then
+        echo "All processes completed successfully"
+        exit 0
+    fi
+
+    sleep 0.2
+done
+

scripts/update-api-types.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+npm install -g openapi-typescript
+openapi-typescript http://localhost:3030/api.json --output frontend/sync-client/src/services/types.ts