diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 35e1842..5acccc1 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -17,11 +17,14 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Setup + - name: Setup Node.js environment + uses: actions/setup-node@v4.2.0 + with: + node-version: "22.x" + check-latest: true + + - name: Setup rust run: | - rustup install nightly - rustup default nightly - rustup component add clippy rustfmt cargo install sqlx-cli cd backend sqlx database create --database-url sqlite://db.sqlite3 @@ -44,7 +47,7 @@ jobs: cd backend cargo test --verbose cd sync_lib - # wasm-pack test --node # todo: fix this in CI + wasm-pack test --node - name: Lint frontend run: | diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml new file mode 100644 index 0000000..dd2fe5d --- /dev/null +++ b/.github/workflows/e2e.yml @@ -0,0 +1,45 @@ +name: E2E tests + +on: + push: + branches: ["master"] + pull_request: + branches: ["master"] + +env: + CARGO_TERM_COLOR: always + RUSTFLAGS: "-Dwarnings" + +jobs: + build: + runs-on: self-hosted + + steps: + - uses: actions/checkout@v4 + + - name: Setup Node.js environment + uses: actions/setup-node@v4.2.0 + with: + node-version: "22.x" + check-latest: true + + - name: Setup rust + run: | + cargo install sqlx-cli wasm-pack + cd backend + sqlx database create --database-url sqlite://db.sqlite3 + sqlx migrate run --source sync_server/src/database/migrations --database-url sqlite://db.sqlite3 + + - name: Build wasm + run: | + cd backend + wasm-pack build --target web sync_lib + + - name: E2E tests + run: | + cd backend + RUST_BACKTRACE=1 cargo run -p sync_server & + cd ../frontend + npm ci + cd .. + scripts/e2e.sh 32 diff --git a/.github/workflows/publish-docker.yml b/.github/workflows/publish-docker.yml index 993c03d..14516a6 100644 --- a/.github/workflows/publish-docker.yml +++ b/.github/workflows/publish-docker.yml @@ -7,8 +7,9 @@ name: Publish server Docker image on: push: - tags: - - "*" + branches: ["master"] + pull_request: + branches: ["master"] env: # Use docker.io for Docker Hub if empty @@ -17,8 +18,9 @@ env: IMAGE_NAME: ${{ github.repository }} jobs: - build-docker: - runs-on: ubuntu-latest + publish-docker: + runs-on: self-hosted + permissions: contents: read packages: write @@ -33,7 +35,7 @@ jobs: # Install the cosign tool except on PR # https://github.com/sigstore/cosign-installer - name: Install cosign - if: github.event_name != 'pull_request' + if: ${{ github.ref_type == 'tag' }} uses: sigstore/cosign-installer@59acb6260d9c0ba8f4a2f9d9b48431a222b68e20 #v3.5.0 with: cosign-release: "v2.2.4" @@ -47,7 +49,7 @@ jobs: # Login against a Docker registry except on PR # https://github.com/docker/login-action - name: Log into registry ${{ env.REGISTRY }} - if: github.event_name != 'pull_request' + if: ${{ github.ref_type == 'tag' }} uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0 with: registry: ${{ env.REGISTRY }} @@ -69,7 +71,7 @@ jobs: uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v5.0.0 with: context: backend - push: ${{ github.event_name != 'pull_request' }} + push: ${{ github.ref_type == 'tag' }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} cache-from: type=gha @@ -81,7 +83,7 @@ jobs: # transparency data even for private images, pass --force to cosign below. # https://github.com/sigstore/cosign - name: Sign the published Docker image - if: ${{ github.event_name != 'pull_request' }} + if: ${{ github.ref_type == 'tag' }} env: # https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions#using-an-intermediate-environment-variable TAGS: ${{ steps.meta.outputs.tags }} diff --git a/.github/workflows/publish-plugin.yml b/.github/workflows/publish-plugin.yml index f1c816f..19bcc78 100644 --- a/.github/workflows/publish-plugin.yml +++ b/.github/workflows/publish-plugin.yml @@ -2,29 +2,27 @@ name: Publish Obsidian plugin on: push: - tags: - - "*" + tags: ["*"] env: CARGO_TERM_COLOR: always jobs: - build-plugin: + publish-plugin: runs-on: self-hosted steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - - name: Use Node.js - uses: actions/setup-node@v3 + - name: Setup Node.js environment + uses: actions/setup-node@v4.2.0 with: - node-version: "18.x" + node-version: "22.x" + check-latest: true - name: Build wasm run: | cd backend - rustup install nightly - rustup default nightly cargo install wasm-pack wasm-pack build --target web sync_lib diff --git a/.gitignore b/.gitignore index 41188af..d2a8367 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ backend/target frontend/*/dist backend/db.sqlite3* +backend/databases backend/config.yml *.log diff --git a/README.md b/README.md index 51440c3..6ad857e 100644 --- a/README.md +++ b/README.md @@ -1,65 +1,56 @@ -## VaultLink self-hosted Obsidian sync plugin +# VaultLink self-hosted Obsidian plugin for file syncing [![Check](https://github.com/schmelczer/vault-link/actions/workflows/check.yml/badge.svg)](https://github.com/schmelczer/vault-link/actions/workflows/check.yml) +[![E2E tests](https://github.com/schmelczer/vault-link/actions/workflows/e2e.yml/badge.svg)](https://github.com/schmelczer/vault-link/actions/workflows/e2e.yml) [![Publish server Docker image](https://github.com/schmelczer/vault-link/actions/workflows/publish-docker.yml/badge.svg)](https://github.com/schmelczer/vault-link/actions/workflows/publish-docker.yml) [![Publish Obsidian plugin](https://github.com/schmelczer/vault-link/actions/workflows/publish-plugin.yml/badge.svg)](https://github.com/schmelczer/vault-link/actions/workflows/publish-plugin.yml) -## Install [nvm](https://github.com/nvm-sh/nvm) +## Develop + +### Install [nvm](https://github.com/nvm-sh/nvm) - `curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh | bash` -- `nvm install 20` -- `nvm use 20` -- Optionally set the system-wide default: `nvm alias default 20` +- `nvm install 22` +- `nvm use 22` +- Optionally set the system-wide default: `nvm alias default 22` -## Set up Rust +### Set up Rust - Install [`rustup`](https://rustup.rs): `curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh` -- `sudo apt install llvm -y` -- `rustup self update` -- `rustup update` -- `rustup install nightly` -- `rustup default nightly` -- `rustup component add llvm-tools-preview` -- `cargo install cargo-generate cargo-fuzz cargo-insta rustfilt cargo-binutils` - Install [`wasm-pack`](https://rustwasm.github.io/wasm-pack/installer): `curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh` - `cargo install cargo-insta sqlx-cli cargo-edit` - -## Publish new version +### Install Obsidian on Linux ```sh -./bump-version.sh patch -``` - - -## Update HTTP API TS bindings - -```sh -npm install -g openapi-typescript -openapi-typescript http://localhost:3030/api.json --output frontend/sync-client/src/services/types.ts -``` - -``` - -todo: enable -[workspace.lints.clippy] -single_call_fn = { level = "allow", priority = 1 } -absolute_paths = { level = "allow", priority = 1 } -arithmetic_side_effects = { level = "allow", priority = 1 } -similar_names = { level = "allow", priority = 1 } -self_named_module_files = { level = "allow", priority = 1 } -single_char_lifetime_names = { level = "allow", priority = 1 } -missing_docs_in_private_items = { level = "allow", priority = 1 } -question_mark_used = { level = "allow", priority = 1 } -implicit_return = { level = "allow", priority = 1 } -pedantic = { level = "warn", priority = 0 } -cargo = { level = "warn", priority = 0 } - -``` - apt install flatpak flatpak remote-add --if-not-exists flathub https://dl.flathub.org/repo/flathub.flatpakrepo flatpak install flathub md.obsidian.Obsidian flatpak run md.obsidian.Obsidian +``` + +### Scripts + +#### Update HTTP API TS bindings + +```sh +scripts/update-api-types.sh +``` + +#### Publish new version + +```sh +scripts/bump-version.sh patch +``` + + +#### Run E2E tests + +```sh +scripts/e2e.sh +``` + +And to clean up the logs & database files, run `scripts/clean-up.sh` +``` diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 3cce7cf..8eddadf 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -105,12 +105,6 @@ dependencies = [ "backtrace", ] -[[package]] -name = "arbitrary" -version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dde20b3d026af13f561bdd0f15edf01fc734f0dafcedbaf42bba506a9517f223" - [[package]] name = "async-trait" version = "0.1.85" @@ -381,8 +375,6 @@ version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f34d93e62b03caf570cccc334cbc6c2fceca82f39211051345108adcba3eebdc" dependencies = [ - "jobserver", - "libc", "shlex", ] @@ -1228,15 +1220,6 @@ version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674" -[[package]] -name = "jobserver" -version = "0.1.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0" -dependencies = [ - "libc", -] - [[package]] name = "js-sys" version = "0.3.76" @@ -1290,16 +1273,6 @@ version = "0.2.167" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09d6582e104315a817dff97f75133544b2e094ee22447d2acf4a74e189ba06fc" -[[package]] -name = "libfuzzer-sys" -version = "0.4.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b9569d2f74e257076d8c6bfa73fb505b46b851e51ddaecc825944aa3bed17fa" -dependencies = [ - "arbitrary", - "cc", -] - [[package]] name = "libm" version = "0.2.11" @@ -1781,14 +1754,6 @@ dependencies = [ "test-case", ] -[[package]] -name = "reconcile-fuzz" -version = "0.0.30" -dependencies = [ - "libfuzzer-sys", - "reconcile", -] - [[package]] name = "redox_syscall" version = "0.5.7" diff --git a/backend/Cargo.toml b/backend/Cargo.toml index cfb865a..5c3768a 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -2,7 +2,6 @@ resolver = "2" members = [ "reconcile", - "fuzz", "sync_server", "sync_lib" ] @@ -57,3 +56,19 @@ uninlined_format_args = "warn" unnested_or_patterns = "warn" unused_self = "warn" verbose_file_reads = "warn" + +cast_possible_truncation = { level = "allow", priority = 1 } +doc_link_with_quotes = { level = "allow", priority = 1 } +cast_sign_loss = { level = "allow", priority = 1 } +cast_possible_wrap = { level = "allow", priority = 1 } +struct_field_names = { level = "allow", priority = 1 } +single_call_fn = { level = "allow", priority = 1 } +absolute_paths = { level = "allow", priority = 1 } +arithmetic_side_effects = { level = "allow", priority = 1 } +similar_names = { level = "allow", priority = 1 } +self_named_module_files = { level = "allow", priority = 1 } +single_char_lifetime_names = { level = "allow", priority = 1 } +missing_docs_in_private_items = { level = "allow", priority = 1 } +question_mark_used = { level = "allow", priority = 1 } +implicit_return = { level = "allow", priority = 1 } +pedantic = { level = "warn", priority = 0 } diff --git a/backend/Dockerfile b/backend/Dockerfile index 8d2fdc4..24388c7 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -3,8 +3,6 @@ FROM rust:1.83 AS builder WORKDIR /usr/src/backend RUN apt update && apt install -y musl-tools -RUN rustup install nightly && rustup default nightly -RUN rustup target add x86_64-unknown-linux-musl RUN cargo install sqlx-cli COPY . . @@ -23,7 +21,7 @@ RUN apk add --no-cache curl COPY --from=builder /usr/src/backend/target/x86_64-unknown-linux-musl/release/sync_server /app/sync_server -VOLUME /data +VOLUME /data/databases EXPOSE 3000/tcp WORKDIR /data diff --git a/backend/fuzz/.gitignore b/backend/fuzz/.gitignore deleted file mode 100644 index 1a45eee..0000000 --- a/backend/fuzz/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -target -corpus -artifacts -coverage diff --git a/backend/fuzz/Cargo.toml b/backend/fuzz/Cargo.toml deleted file mode 100644 index d764ba4..0000000 --- a/backend/fuzz/Cargo.toml +++ /dev/null @@ -1,25 +0,0 @@ -[package] -name = "reconcile-fuzz" -version.workspace = true -edition.workspace = true -authors.workspace = true -license.workspace = true -repository.workspace = true -publish = false - -[package.metadata] -cargo-fuzz = true - -[dependencies] -libfuzzer-sys = "0.4" -reconcile = { path = "../reconcile" } - -[[bin]] -name = "reconcile" -path = "fuzz_targets/reconcile.rs" -test = false -doc = false -bench = false - -[lints] -workspace = true diff --git a/backend/fuzz/fuzz_targets/reconcile.rs b/backend/fuzz/fuzz_targets/reconcile.rs deleted file mode 100644 index b30d9f5..0000000 --- a/backend/fuzz/fuzz_targets/reconcile.rs +++ /dev/null @@ -1,8 +0,0 @@ -#![no_main] - -use libfuzzer_sys::fuzz_target; - -fuzz_target!(|texts: (String, String, String)| { - let (original, left, right) = texts; - let _ = reconcile::reconcile(&original, &left, &right); -}); diff --git a/backend/reconcile/src/diffs/myers.rs b/backend/reconcile/src/diffs/myers.rs index e2f4498..9692c22 100644 --- a/backend/reconcile/src/diffs/myers.rs +++ b/backend/reconcile/src/diffs/myers.rs @@ -38,7 +38,7 @@ use crate::{ /// execution time permitted before it bails and falls back to an approximation. pub fn diff(old: &[Token], new: &[Token]) -> Vec> where - T: PartialEq + Clone, + T: PartialEq + Clone + std::fmt::Debug, { let max_d = (old.len() + new.len()).div_ceil(2) + 1; let mut vb = V::new(max_d); @@ -99,7 +99,6 @@ impl IndexMut for V { } } -#[inline(always)] fn split_at(range: Range, at: usize) -> (Range, Range) { (range.start..at, at..range.end) } @@ -124,7 +123,7 @@ fn find_middle_snake( vb: &mut V, ) -> Option<(usize, usize)> where - T: PartialEq + Clone, + T: PartialEq + Clone + std::fmt::Debug, { let n = old_range.len(); let m = new_range.len(); @@ -230,7 +229,7 @@ fn conquer( vb: &mut V, result: &mut Vec>, ) where - T: PartialEq + Clone, + T: PartialEq + Clone + std::fmt::Debug, { // Check for common prefix let common_prefix_len = common_prefix_len(old, old_range.clone(), new, new_range.clone()); diff --git a/backend/reconcile/src/diffs/raw_operation.rs b/backend/reconcile/src/diffs/raw_operation.rs index 280460f..0df48f5 100644 --- a/backend/reconcile/src/diffs/raw_operation.rs +++ b/backend/reconcile/src/diffs/raw_operation.rs @@ -3,7 +3,7 @@ use crate::tokenizer::token::Token; #[derive(Debug, Clone, PartialEq)] pub enum RawOperation where - T: PartialEq + Clone, + T: PartialEq + Clone + std::fmt::Debug, { Insert(Vec>), Delete(Vec>), @@ -12,13 +12,13 @@ where impl RawOperation where - T: PartialEq + Clone, + T: PartialEq + Clone + std::fmt::Debug, { pub fn tokens(&self) -> &Vec> { match self { - RawOperation::Insert(tokens) => tokens, - RawOperation::Delete(tokens) => tokens, - RawOperation::Equal(tokens) => tokens, + RawOperation::Insert(tokens) + | RawOperation::Delete(tokens) + | RawOperation::Equal(tokens) => tokens, } } diff --git a/backend/reconcile/src/operation_transformation.rs b/backend/reconcile/src/operation_transformation.rs index ef9a5e8..a71bc65 100644 --- a/backend/reconcile/src/operation_transformation.rs +++ b/backend/reconcile/src/operation_transformation.rs @@ -37,7 +37,7 @@ pub fn reconcile_with_tokenizer( tokenizer: &Tokenizer, ) -> String where - T: PartialEq + Clone, + T: PartialEq + Clone + std::fmt::Debug, { let left_operations = EditedText::from_strings_with_tokenizer(original, left, tokenizer); let right_operations = EditedText::from_strings_with_tokenizer(original, right, tokenizer); @@ -73,7 +73,8 @@ mod test { "original_1 edit_1 original_3", ); - // One deleted a large range, the other deleted subranges and inserted as well + // One deleted a large range, the other deleted subranges and inserted as + // well test_merge_both_ways( "original_1 original_2 original_3 original_4 original_5", "original_1 original_5", @@ -120,9 +121,6 @@ mod test { "hi, my friend!", ); - // test_merge_both_ways("hello world", "world !", "hi hello world", "hi world - // !"); - test_merge_both_ways( "both delete the same word", "both the same word", @@ -147,7 +145,33 @@ mod test { ); } - #[ignore = "it's too slow"] + #[test] + fn test_reconcile_idempotent_inserts() { + // Both inserted the same prefix; this should get deduped + test_merge_both_ways( + "hi ", + "hi there ", + "hi there my friend ", + "hi there my friend ", + ); + + // The prefix of the 2nd appears on the 1st so it shouldn't get duplicated + test_merge_both_ways( + "hi ", + "hi there you ", + "hi there my friend ", + "hi there my friend you ", + ); + + test_merge_both_ways("a", "a b c", "a b c d", "a b c d"); + + test_merge_both_ways( + " |7ca2b36d-6ee7-49eb-8eb1-d77e4cc1a001| ", + " |7ca2b36d-6ee7-49eb-8eb1-d77e4cc1a001| |cd9195cc-103a-4f13-90c8-4fba0ba421ee| |d39156cc-cfd6-42a8-b70a-75020896069d| |fbad794c-9c47-41f2-a343-490284ecb5a0| |dup| ", + " |7ca2b36d-6ee7-49eb-8eb1-d77e4cc1a001| |cd9195cc-103a-4f13-90c8-4fba0ba421ee| |dup| ", + " |7ca2b36d-6ee7-49eb-8eb1-d77e4cc1a001| |cd9195cc-103a-4f13-90c8-4fba0ba421ee| |d39156cc-cfd6-42a8-b70a-75020896069d| |fbad794c-9c47-41f2-a343-490284ecb5a0| |dup| |dup| "); + } + #[test_matrix( [ "pride_and_prejudice.txt", "romeo_and_juliet.txt", diff --git a/backend/reconcile/src/operation_transformation/edited_text.rs b/backend/reconcile/src/operation_transformation/edited_text.rs index 32bda1b..8a7013e 100644 --- a/backend/reconcile/src/operation_transformation/edited_text.rs +++ b/backend/reconcile/src/operation_transformation/edited_text.rs @@ -25,7 +25,7 @@ use crate::{ #[derive(Debug, Clone, PartialEq, Default)] pub struct EditedText<'a, T> where - T: PartialEq + Clone, + T: PartialEq + Clone + std::fmt::Debug, { text: &'a str, operations: Vec>, @@ -46,7 +46,7 @@ impl<'a> EditedText<'a, String> { impl<'a, T> EditedText<'a, T> where - T: PartialEq + Clone, + T: PartialEq + Clone + std::fmt::Debug, { /// Create an `EditedText` from the given original (old) and updated (new) /// strings. The returned `EditedText` represents the changes from the @@ -65,7 +65,6 @@ where Self::new( original, - // Self::cook_operations(diff), Self::cook_operations(Self::elongate_operations(diff)).collect(), ) } @@ -191,7 +190,7 @@ where pub fn merge(self, other: Self) -> Self { debug_assert_eq!( self.text, other.text, - "EditedText-s must be derived from the same text to be mergable" + "`EditedText`-s must be derived from the same text to be mergable" ); let mut left_merge_context = MergeContext::default(); @@ -207,9 +206,21 @@ where |(operation, _)| { ( operation.order, - // Operations on left and right must come in the same order so that + // Operations on the left and right must come in the same order so that // inserts can be merged with other inserts and deletes with deletes. usize::from(matches!(operation.operation, Operation::Delete { .. })), + // Make sure that the ordering is deterministic regardless which text + // is left or right. + match &operation.operation { + Operation::Insert { text, .. } => text + .iter() + .map(super::super::tokenizer::token::Token::original) + .collect::(), + Operation::Delete { + deleted_character_count, + .. + } => deleted_character_count.to_string(), + }, ) }, ) @@ -232,6 +243,7 @@ where } /// Apply the operations to the text and return the resulting text. + #[must_use] pub fn apply(&self) -> String { let mut builder: StringBuilder<'_> = StringBuilder::new(self.text); @@ -282,7 +294,7 @@ mod tests { let original = "hello world! ..."; let left = "Hello world! I'm Andras."; let right = "Hello world! How are you?"; - let expected = "Hello world! I'm Andras.How are you?"; + let expected = "Hello world! How are you? I'm Andras."; let operations_1 = EditedText::from_strings(original, left); let operations_2 = EditedText::from_strings(original, right); diff --git a/backend/reconcile/src/operation_transformation/merge_context.rs b/backend/reconcile/src/operation_transformation/merge_context.rs index 0bc3c34..980389d 100644 --- a/backend/reconcile/src/operation_transformation/merge_context.rs +++ b/backend/reconcile/src/operation_transformation/merge_context.rs @@ -5,7 +5,7 @@ use crate::operation_transformation::Operation; #[derive(Clone)] pub struct MergeContext where - T: PartialEq + Clone, + T: PartialEq + Clone + std::fmt::Debug, { last_operation: Option>, pub shift: i64, @@ -13,7 +13,7 @@ where impl Default for MergeContext where - T: PartialEq + Clone, + T: PartialEq + Clone + std::fmt::Debug, { fn default() -> Self { MergeContext { @@ -25,7 +25,7 @@ where impl Debug for MergeContext where - T: PartialEq + Clone, + T: PartialEq + Clone + std::fmt::Debug, { fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { f.debug_struct("MergeContext") @@ -37,7 +37,7 @@ where impl MergeContext where - T: PartialEq + Clone, + T: PartialEq + Clone + std::fmt::Debug, { pub fn last_operation(&self) -> Option<&Operation> { self.last_operation.as_ref() } diff --git a/backend/reconcile/src/operation_transformation/operation.rs b/backend/reconcile/src/operation_transformation/operation.rs index c19265b..d0d285b 100644 --- a/backend/reconcile/src/operation_transformation/operation.rs +++ b/backend/reconcile/src/operation_transformation/operation.rs @@ -1,7 +1,5 @@ -use core::{ - fmt::{Debug, Display}, - ops::Range, -}; +use core::fmt::{Debug, Display}; +use std::ops::Range; #[cfg(feature = "serde")] use serde::{Deserialize, Serialize}; @@ -9,7 +7,10 @@ use serde::{Deserialize, Serialize}; use super::merge_context::MergeContext; use crate::{ Token, - utils::{find_common_overlap::find_common_overlap, string_builder::StringBuilder}, + utils::{ + find_longest_prefix_contained_within::find_longest_prefix_contained_within, + string_builder::StringBuilder, + }, }; /// Represents a change that can be applied to a text document. @@ -19,7 +20,7 @@ use crate::{ #[derive(Clone, PartialEq)] pub enum Operation where - T: PartialEq + Clone, + T: PartialEq + Clone + std::fmt::Debug, { Insert { index: usize, @@ -37,7 +38,7 @@ where impl Operation where - T: PartialEq + Clone, + T: PartialEq + Clone + std::fmt::Debug, { /// Creates an insert operation with the given index and text. /// If the text is empty (meaning that the operation would be a no-op), @@ -81,15 +82,8 @@ where }) } - /// Tries to apply the operation to the given `ropey::Rope` text, returning - /// the modified text. - /// - /// # Errors - /// - /// Returns a `SyncLibError::OperationApplicationError` if the operation - /// cannot be applied. - /// - /// # Panics + /// Applies the operation to the given `StringBuilder`, returning the + /// modified `StringBuilder`. /// /// When compiled in debug mode, panics if a delete operation is attempted /// on a range of text that does not match the text to be deleted. @@ -114,7 +108,7 @@ where builder.delete(self.range()); } - }; + } builder } @@ -122,8 +116,7 @@ where /// Returns the index of the first character that the operation affects. pub fn start_index(&self) -> usize { match self { - Operation::Insert { index, .. } => *index, - Operation::Delete { index, .. } => *index, + Operation::Insert { index, .. } | Operation::Delete { index, .. } => *index, } } @@ -137,6 +130,7 @@ where } /// Returns the range of indices of characters that the operation affects. + #[allow(clippy::range_plus_one)] pub fn range(&self) -> Range { self.start_index()..self.end_index() + 1 } /// Returns the number of affected characters. It is always greater than 0 @@ -212,17 +206,20 @@ where .. }), ) => { - let offset_in_tokens = find_common_overlap(previous_inserted_text, &text); - let trimmed_length_in_tokens = previous_inserted_text.len() - offset_in_tokens; - let trimmed_length = previous_inserted_text + // In case the current insert's prefix appears in the previously inserted text, + // we can trim the current insert to only include the non-overlapping part. + // This way, we don't end up duplicating text. + let offset_in_tokens = + find_longest_prefix_contained_within(previous_inserted_text, &text); + let offset_in_length = text .iter() - .skip(offset_in_tokens) + .take(offset_in_tokens) .map(Token::get_original_length) .sum::(); let trimmed_operation = - Operation::create_insert(index, text[trimmed_length_in_tokens..].to_vec()); + Operation::create_insert(index, text[offset_in_tokens..].to_vec()); - affecting_context.shift -= trimmed_length as i64; + affecting_context.shift -= offset_in_length as i64; produced_context.shift += trimmed_operation .as_ref() .map(Operation::len) @@ -297,7 +294,7 @@ where impl Display for Operation where - T: PartialEq + Clone, + T: PartialEq + Clone + std::fmt::Debug, { fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { match self { @@ -341,7 +338,7 @@ where impl Debug for Operation where - T: PartialEq + Clone, + T: PartialEq + Clone + std::fmt::Debug, { fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { write!(f, "{self}") } } @@ -353,7 +350,7 @@ mod tests { use super::*; #[test] - #[should_panic] + #[should_panic(expected = "Shifted index must be non-negative")] fn test_shifting_error() { insta::assert_debug_snapshot!( Operation::create_insert(1, vec!["hi".into()]) diff --git a/backend/reconcile/src/operation_transformation/snapshots/reconcile__operation_transformation__edited_text__tests__calculate_operations.snap b/backend/reconcile/src/operation_transformation/snapshots/reconcile__operation_transformation__edited_text__tests__calculate_operations.snap index e8a0487..0630f98 100644 --- a/backend/reconcile/src/operation_transformation/snapshots/reconcile__operation_transformation__edited_text__tests__calculate_operations.snap +++ b/backend/reconcile/src/operation_transformation/snapshots/reconcile__operation_transformation__edited_text__tests__calculate_operations.snap @@ -8,19 +8,19 @@ EditedText { operations: [ OrderedOperation { order: 0, - operation: , + operation: , }, OrderedOperation { order: 0, - operation: , + operation: , }, OrderedOperation { - order: 21, - operation: , + order: 20, + operation: , }, OrderedOperation { - order: 21, - operation: , + order: 20, + operation: , }, ], } diff --git a/backend/reconcile/src/tokenizer/snapshots/reconcile__tokenizer__word_tokenizer__tests__with_snapshots-2.snap b/backend/reconcile/src/tokenizer/snapshots/reconcile__tokenizer__word_tokenizer__tests__with_snapshots-2.snap new file mode 100644 index 0000000..892e524 --- /dev/null +++ b/backend/reconcile/src/tokenizer/snapshots/reconcile__tokenizer__word_tokenizer__tests__with_snapshots-2.snap @@ -0,0 +1,6 @@ +--- +source: reconcile/src/tokenizer/word_tokenizer.rs +expression: "word_tokenizer(\"\")" +snapshot_kind: text +--- +[] diff --git a/backend/reconcile/src/tokenizer/snapshots/reconcile__tokenizer__word_tokenizer__tests__with_snapshots-3.snap b/backend/reconcile/src/tokenizer/snapshots/reconcile__tokenizer__word_tokenizer__tests__with_snapshots-3.snap new file mode 100644 index 0000000..58d749e --- /dev/null +++ b/backend/reconcile/src/tokenizer/snapshots/reconcile__tokenizer__word_tokenizer__tests__with_snapshots-3.snap @@ -0,0 +1,15 @@ +--- +source: reconcile/src/tokenizer/word_tokenizer.rs +expression: "word_tokenizer(\" what? \")" +snapshot_kind: text +--- +[ + Token { + normalised: "what?", + original: " what?", + }, + Token { + normalised: "", + original: " ", + }, +] diff --git a/backend/reconcile/src/tokenizer/snapshots/reconcile__tokenizer__word_tokenizer__tests__with_snapshots-4.snap b/backend/reconcile/src/tokenizer/snapshots/reconcile__tokenizer__word_tokenizer__tests__with_snapshots-4.snap new file mode 100644 index 0000000..4c28a7f --- /dev/null +++ b/backend/reconcile/src/tokenizer/snapshots/reconcile__tokenizer__word_tokenizer__tests__with_snapshots-4.snap @@ -0,0 +1,23 @@ +--- +source: reconcile/src/tokenizer/word_tokenizer.rs +expression: "word_tokenizer(\" hello, \\nwhere are you?\")" +snapshot_kind: text +--- +[ + Token { + normalised: "hello,", + original: " hello,", + }, + Token { + normalised: "where", + original: " \nwhere", + }, + Token { + normalised: "are", + original: " are", + }, + Token { + normalised: "you?", + original: " you?", + }, +] diff --git a/backend/reconcile/src/tokenizer/snapshots/reconcile__tokenizer__word_tokenizer__tests__with_snapshots.snap b/backend/reconcile/src/tokenizer/snapshots/reconcile__tokenizer__word_tokenizer__tests__with_snapshots.snap new file mode 100644 index 0000000..206c7fe --- /dev/null +++ b/backend/reconcile/src/tokenizer/snapshots/reconcile__tokenizer__word_tokenizer__tests__with_snapshots.snap @@ -0,0 +1,15 @@ +--- +source: reconcile/src/tokenizer/word_tokenizer.rs +expression: "word_tokenizer(\"Hi there!\")" +snapshot_kind: text +--- +[ + Token { + normalised: "Hi", + original: "Hi", + }, + Token { + normalised: "there!", + original: " there!", + }, +] diff --git a/backend/reconcile/src/tokenizer/token.rs b/backend/reconcile/src/tokenizer/token.rs index f723a2c..ab521a7 100644 --- a/backend/reconcile/src/tokenizer/token.rs +++ b/backend/reconcile/src/tokenizer/token.rs @@ -8,24 +8,19 @@ use serde::{Deserialize, Serialize}; #[derive(Debug, Clone)] pub struct Token where - T: PartialEq + Clone, + T: PartialEq + Clone + std::fmt::Debug, { normalised: T, original: String, } impl From<&str> for Token { - fn from(s: &str) -> Self { - Token { - normalised: s.to_owned(), - original: s.to_owned(), - } - } + fn from(s: &str) -> Self { Token::new(s.trim().to_owned(), s.to_owned()) } } impl Token where - T: PartialEq + Clone, + T: PartialEq + Clone + std::fmt::Debug, { pub fn new(normalised: T, original: String) -> Self { Token { @@ -43,7 +38,7 @@ where impl PartialEq for Token where - T: PartialEq + Clone, + T: PartialEq + Clone + std::fmt::Debug, { fn eq(&self, other: &Self) -> bool { self.normalised == other.normalised } } diff --git a/backend/reconcile/src/tokenizer/word_tokenizer.rs b/backend/reconcile/src/tokenizer/word_tokenizer.rs index 3449cba..37d748b 100644 --- a/backend/reconcile/src/tokenizer/word_tokenizer.rs +++ b/backend/reconcile/src/tokenizer/word_tokenizer.rs @@ -1,7 +1,48 @@ use super::token::Token; +/// Splits on whitespace keeping the leading whitespace. +/// +/// +/// ## Example +/// +/// "Hi there!" -> ["Hi", " there!"] pub fn word_tokenizer(text: &str) -> Vec> { - text.split_inclusive(char::is_whitespace) - .map(|s| Token::new(s.to_owned(), s.to_owned())) - .collect() + let mut result: Vec> = Vec::new(); + + let mut last_whitespace = 0; + let mut previous_char_is_whitespace = true; + + for (i, c) in text.char_indices() { + let is_current_char_whitespace = c.is_whitespace(); + if !previous_char_is_whitespace && is_current_char_whitespace { + result.push(text[last_whitespace..i].into()); + last_whitespace = i; + } + + previous_char_is_whitespace = is_current_char_whitespace; + } + + if last_whitespace < text.len() { + result.push(text[last_whitespace..].into()); + } + + result +} + +#[cfg(test)] +mod tests { + use insta::assert_debug_snapshot; + + use super::*; + + #[test] + fn test_with_snapshots() { + assert_debug_snapshot!(word_tokenizer("Hi there!")); + + assert_debug_snapshot!(word_tokenizer("")); + + assert_debug_snapshot!(word_tokenizer(" what? ")); + + assert_debug_snapshot!(word_tokenizer(" hello, \nwhere are you?")); + } } diff --git a/backend/reconcile/src/utils.rs b/backend/reconcile/src/utils.rs index c0c3c33..8461b5f 100644 --- a/backend/reconcile/src/utils.rs +++ b/backend/reconcile/src/utils.rs @@ -1,6 +1,6 @@ pub mod common_prefix_len; pub mod common_suffix_len; -pub mod find_common_overlap; +pub mod find_longest_prefix_contained_within; pub mod merge_iters; pub mod ordered_operation; pub mod side; diff --git a/backend/reconcile/src/utils/find_common_overlap.rs b/backend/reconcile/src/utils/find_common_overlap.rs deleted file mode 100644 index ac586b8..0000000 --- a/backend/reconcile/src/utils/find_common_overlap.rs +++ /dev/null @@ -1,71 +0,0 @@ -use crate::Token; - -/// Given two lists of tokens, returns the offset in the first (old) list from -/// which the two lists have the same tokens until the end of the first list. -/// Thus, the suffix of the old list from the offset to the end is equal to a -/// prefix of the new list. -/// -/// If there is no overlap, the function returns the maxmium offset, the length -/// of the old list. -/// -/// ## Example -/// -/// ```not_rust -/// old: [0, 1, 9, 0, 2, 5] -/// new: [9, 0, 2, 5, 1] -/// ``` -/// > results in an offset of 2 -pub fn find_common_overlap(old: &[Token], new: &[Token]) -> usize -where - T: PartialEq + Clone, -{ - let minimum_offset = old.len().saturating_sub(new.len()); - for offset in minimum_offset..old.len() { - if old.iter().skip(offset).zip(new.iter()).all(|(a, b)| a == b) { - return offset; - } - } - - old.len() -} - -#[cfg(test)] -mod tests { - use pretty_assertions::assert_eq; - - use super::*; - - #[test] - fn test_common_overlap() { - assert_eq!(find_common_overlap(&["".into()], &["".into()]), 0); - - assert_eq!( - find_common_overlap( - &["a".into(), "b".into(), "c".into()], - &["b".into(), "c".into(), "a".into()] - ), - 1 - ); - - assert_eq!( - find_common_overlap( - &["a".into(), "a".into(), "a".into()], - &["a".into(), "b".into(), "c".into()] - ), - 2 - ); - - assert_eq!( - find_common_overlap( - &["a".into(), "b".into(), "c".into()], - &["d".into(), "e".into(), "a".into()] - ), - 3 - ); - - assert_eq!( - find_common_overlap(&["a".into(), "a".into()], &["a".into()]), - 1 - ); - } -} diff --git a/backend/reconcile/src/utils/find_longest_prefix_contained_within.rs b/backend/reconcile/src/utils/find_longest_prefix_contained_within.rs new file mode 100644 index 0000000..eb4b826 --- /dev/null +++ b/backend/reconcile/src/utils/find_longest_prefix_contained_within.rs @@ -0,0 +1,103 @@ +use crate::Token; + +/// Given two lists of tokens, returns `length` where `old` list somewhere +/// within contains the `length` prefix of the `new` list. +/// +/// ## Example +/// +/// ```not_rust +/// old: [0, 1, 9, 0, 2, 5] +/// new: [9, 0, 2, 5, 1] +/// ``` +/// > results in an length of 4 +/// +/// +/// ```not_rust +/// old: [0, 1, 9, 0, 2, 5] +/// new: [0, 2] +/// ``` +/// > results in an length of 2 +/// +/// ```not_rust +/// old: [0, 1, 9, 0, 2, 5] +/// new: [0, 4] +/// ``` +/// > results in an length of 1 +pub fn find_longest_prefix_contained_within(old: &[Token], new: &[Token]) -> usize +where + T: PartialEq + Clone + std::fmt::Debug, +{ + let max_possible = new.len().min(old.len()); + + for len in (1..=max_possible).rev() { + let prefix = &new[..len]; + if old.windows(len).any(|window| window == prefix) { + return len; + } + } + + 0 +} + +#[cfg(test)] +mod tests { + use pretty_assertions::assert_eq; + + use super::*; + + #[test] + fn test_common_overlap() { + assert_eq!( + find_longest_prefix_contained_within(&["".into()], &["".into()]), + 1 + ); + + assert_eq!( + find_longest_prefix_contained_within( + &["a".into(), "b".into(), "c".into()], + &["b".into(), "c".into(), "a".into()] + ), + 2 + ); + + assert_eq!( + find_longest_prefix_contained_within( + &["a".into(), "b".into(), "c".into()], + &["b".into(), "c".into()] + ), + 2 + ); + + assert_eq!( + find_longest_prefix_contained_within( + &["a".into(), "b".into(), "c".into()], + &["b".into()] + ), + 1 + ); + + assert_eq!( + find_longest_prefix_contained_within( + &["a".into(), "b".into(), "c".into(), "b".into(), "a".into()], + &["b".into(), "a".into()] + ), + 2 + ); + + assert_eq!( + find_longest_prefix_contained_within( + &["a".into(), "a".into(), "a".into()], + &["a".into(), "b".into(), "c".into()] + ), + 1 + ); + + assert_eq!( + find_longest_prefix_contained_within( + &["a".into(), "b".into(), "c".into()], + &["d".into(), "e".into(), "a".into()] + ), + 0 + ); + } +} diff --git a/backend/reconcile/src/utils/merge_iters.rs b/backend/reconcile/src/utils/merge_iters.rs index c7b7334..2730c33 100644 --- a/backend/reconcile/src/utils/merge_iters.rs +++ b/backend/reconcile/src/utils/merge_iters.rs @@ -46,8 +46,7 @@ where }; match order { - Some(Ordering::Less) | None => self.left.next(), - Some(Ordering::Equal) => self.left.next(), + Some(Ordering::Less | Ordering::Equal) | None => self.left.next(), Some(Ordering::Greater) => self.right.next(), } } diff --git a/backend/reconcile/src/utils/ordered_operation.rs b/backend/reconcile/src/utils/ordered_operation.rs index 17229d2..116b637 100644 --- a/backend/reconcile/src/utils/ordered_operation.rs +++ b/backend/reconcile/src/utils/ordered_operation.rs @@ -7,7 +7,7 @@ use crate::operation_transformation::Operation; #[derive(Debug, Clone, PartialEq)] pub struct OrderedOperation where - T: PartialEq + Clone, + T: PartialEq + Clone + std::fmt::Debug, { pub order: usize, pub operation: Operation, diff --git a/backend/rust-toolchain.toml b/backend/rust-toolchain.toml new file mode 100644 index 0000000..8e46664 --- /dev/null +++ b/backend/rust-toolchain.toml @@ -0,0 +1,4 @@ +[toolchain] +channel = "nightly-2025-03-14" +targets = [ "x86_64-unknown-linux-gnu", "x86_64-unknown-linux-musl" ] +profile = "default" diff --git a/backend/sync_lib/src/lib.rs b/backend/sync_lib/src/lib.rs index 16e01f2..6f27e05 100644 --- a/backend/sync_lib/src/lib.rs +++ b/backend/sync_lib/src/lib.rs @@ -18,7 +18,20 @@ pub mod errors; /// Encode binary data for easy transport over HTTP. Inverse of /// `base64_to_bytes`. +/// +/// # Arguments +/// +/// - `input`: The binary data to encode. +/// +/// # Returns +/// +/// The base64-encoded string. +/// +/// # Panics +/// +/// If the input is not valid UTF-8. #[wasm_bindgen(js_name = bytesToBase64)] +#[must_use] pub fn bytes_to_base64(input: &[u8]) -> String { set_panic_hook(); @@ -26,6 +39,19 @@ pub fn bytes_to_base64(input: &[u8]) -> String { } /// Inverse of `bytes_to_base64`. +/// Decode base64-encoded data into binary data. +/// +/// # Arguments +/// +/// - `input`: The base64-encoded string. +/// +/// # Returns +/// +/// The decoded binary data. +/// +/// # Errors +/// +/// If the input is not valid base64. #[wasm_bindgen(js_name = base64ToBytes)] pub fn base64_to_bytes(input: &str) -> Result, SyncLibError> { set_panic_hook(); @@ -36,7 +62,22 @@ pub fn base64_to_bytes(input: &str) -> Result, SyncLibError> { /// Merge two documents with a common parent. Relies on `reconcile::reconcile` /// for texts and returns the right document as-is if either of the updated /// documents is binary. +/// +/// # Arguments +/// +/// - `parent`: The common parent document. +/// - `left`: The left document updated by one user. +/// - `right`: The right document updated by another user. +/// +/// # Returns +/// +/// The merged document. +/// +/// # Panics +/// +/// If any of the input documents are not valid UTF-8 strings. #[wasm_bindgen] +#[must_use] pub fn merge(parent: &[u8], left: &[u8], right: &[u8]) -> Vec { set_panic_hook(); @@ -54,6 +95,7 @@ pub fn merge(parent: &[u8], left: &[u8], right: &[u8]) -> Vec { /// WASM wrapper around `reconcile::reconcile` for text merging. #[wasm_bindgen(js_name = mergeText)] +#[must_use] pub fn merge_text(parent: &str, left: &str, right: &str) -> String { set_panic_hook(); @@ -63,10 +105,11 @@ pub fn merge_text(parent: &str, left: &str, right: &str) -> String { /// Heuristically determine if the given data is a binary or a text file's /// content. #[wasm_bindgen(js_name = isBinary)] +#[must_use] pub fn is_binary(data: &[u8]) -> bool { set_panic_hook(); - if data.iter().any(|&b| b == 0) { + if data.contains(&0) { // Even though the NUL character is valid in UTF-8, it's highly suspicious in // human-readable text. return true; @@ -77,6 +120,7 @@ pub fn is_binary(data: &[u8]) -> bool { /// We don't want to support merging structured data like JSON, YAML, etc. #[wasm_bindgen(js_name = isFileTypeMergable)] +#[must_use] pub fn is_file_type_mergable(path_or_file_name: &str) -> bool { set_panic_hook(); diff --git a/backend/sync_lib/tests/web.rs b/backend/sync_lib/tests/web.rs index ffea18d..e45cbea 100644 --- a/backend/sync_lib/tests/web.rs +++ b/backend/sync_lib/tests/web.rs @@ -26,7 +26,8 @@ fn test_base64_to_bytes_error() { fn merge_text() { let left = b"hello "; let right = b"world"; - assert_eq!(merge(b"", left, right), b"hello world".to_vec()); + let result = merge(b"", left, right); + assert_eq!(result, b"hello world"); } #[wasm_bindgen_test(unsupported = test)] diff --git a/backend/sync_server/src/config.rs b/backend/sync_server/src/config.rs index 829375d..862dd0e 100644 --- a/backend/sync_server/src/config.rs +++ b/backend/sync_server/src/config.rs @@ -42,9 +42,12 @@ impl Config { } pub async fn load_from_file(path: &Path) -> Result { - let contents = fs::read_to_string(path) - .await - .with_context(|| format!("Cannot load configuration from disk from ({path:?})"))?; + let contents = fs::read_to_string(path).await.with_context(|| { + format!( + "Cannot load configuration from disk from {}", + path.display() + ) + })?; let config = serde_yaml::from_str(&contents).context("Failed to parse configuration")?; diff --git a/backend/sync_server/src/config/database_config.rs b/backend/sync_server/src/config/database_config.rs index effcfde..b3d2fad 100644 --- a/backend/sync_server/src/config/database_config.rs +++ b/backend/sync_server/src/config/database_config.rs @@ -1,31 +1,33 @@ +use std::path::PathBuf; + use log::debug; use serde::{Deserialize, Serialize}; -use crate::consts::{DEFAULT_MAX_CONNECTIONS, DEFAULT_SQLITE_URL}; +use crate::consts::{DEFAULT_DATABASES_DIRECTORY_PATH, DEFAULT_MAX_CONNECTIONS}; #[derive(Debug, Deserialize, Serialize, Clone)] pub struct DatabaseConfig { - #[serde(default = "default_sqlite_url")] - pub sqlite_url: String, + #[serde(default = "default_databases_directory_path")] + pub databases_directory_path: PathBuf, #[serde(default = "default_max_connections")] pub max_connections: u32, } -fn default_sqlite_url() -> String { - debug!("Using default sqlite url: {}", DEFAULT_SQLITE_URL); - DEFAULT_SQLITE_URL.to_owned() +fn default_databases_directory_path() -> PathBuf { + debug!("Using default databases directory path: {DEFAULT_DATABASES_DIRECTORY_PATH:?}"); + PathBuf::from(DEFAULT_DATABASES_DIRECTORY_PATH) } fn default_max_connections() -> u32 { - debug!("Using default max connections: {}", DEFAULT_MAX_CONNECTIONS); + debug!("Using default max connections: {DEFAULT_MAX_CONNECTIONS}"); DEFAULT_MAX_CONNECTIONS } impl Default for DatabaseConfig { fn default() -> Self { Self { - sqlite_url: default_sqlite_url(), + databases_directory_path: default_databases_directory_path(), max_connections: default_max_connections(), } } diff --git a/backend/sync_server/src/config/server_config.rs b/backend/sync_server/src/config/server_config.rs index 88b1f48..8d7c63e 100644 --- a/backend/sync_server/src/config/server_config.rs +++ b/backend/sync_server/src/config/server_config.rs @@ -15,20 +15,17 @@ pub struct ServerConfig { } fn default_host() -> String { - debug!("Using default server host: {}", DEFAULT_HOST); + debug!("Using default server host: {DEFAULT_HOST}"); DEFAULT_HOST.to_owned() } fn default_port() -> u16 { - debug!("Using default server port: {}", DEFAULT_PORT); + debug!("Using default server port: {DEFAULT_PORT}"); DEFAULT_PORT } fn default_max_body_size_mb() -> usize { - debug!( - "Using default max body size (MB): {}", - DEFAULT_MAX_BODY_SIZE_MB - ); + debug!("Using default max body size (MB): {DEFAULT_MAX_BODY_SIZE_MB}"); DEFAULT_MAX_BODY_SIZE_MB } diff --git a/backend/sync_server/src/consts.rs b/backend/sync_server/src/consts.rs index 2b727f5..f38012d 100644 --- a/backend/sync_server/src/consts.rs +++ b/backend/sync_server/src/consts.rs @@ -1,5 +1,5 @@ pub const CONFIG_PATH: &str = "config.yml"; -pub const DEFAULT_SQLITE_URL: &str = "db.sqlite3"; +pub const DEFAULT_DATABASES_DIRECTORY_PATH: &str = "databases"; pub const DEFAULT_HOST: &str = "127.0.0.1"; pub const DEFAULT_PORT: u16 = 3000; pub const DEFAULT_MAX_CONNECTIONS: u32 = 12; diff --git a/backend/sync_server/src/database.rs b/backend/sync_server/src/database.rs index 5230562..882bd0a 100644 --- a/backend/sync_server/src/database.rs +++ b/backend/sync_server/src/database.rs @@ -1,4 +1,5 @@ -use core::{str::FromStr as _, time::Duration}; +use core::time::Duration; +use std::{collections::HashMap, sync::Arc}; use anyhow::{Context as _, Result}; use models::{ @@ -7,19 +8,66 @@ use models::{ use sqlx::{sqlite::SqliteConnectOptions, types::chrono::Utc}; pub mod models; use sqlx::{Pool, Sqlite, sqlite::SqlitePoolOptions}; +use tokio::sync::Mutex; +use uuid::fmt::Hyphenated; use crate::config::database_config::DatabaseConfig; #[derive(Clone, Debug)] pub struct Database { - connection_pool: Pool, + config: DatabaseConfig, + connection_pools: Arc>>>, } pub type Transaction<'a> = sqlx::Transaction<'a, Sqlite>; impl Database { pub async fn try_new(config: &DatabaseConfig) -> Result { - let connection_options = SqliteConnectOptions::from_str(&config.sqlite_url)? + tokio::fs::create_dir_all(&config.databases_directory_path) + .await + .with_context(|| { + format!( + "Failed to create databases directory: {}", + config.databases_directory_path.to_string_lossy() + ) + })?; + + let mut connection_pools = std::collections::HashMap::new(); + + let mut entries = tokio::fs::read_dir(&config.databases_directory_path).await?; + while let Some(entry) = entries.next_entry().await? { + if !entry.file_name().to_string_lossy().ends_with(".sqlite") { + continue; + } + + let vault: VaultId = entry + .file_name() + .to_string_lossy() + .trim_end_matches(".sqlite") + .to_owned(); + + connection_pools.insert( + vault.clone(), + Self::create_vault_database(config, &vault).await?, + ); + } + + Ok(Self { + config: config.clone(), + connection_pools: Arc::new(Mutex::new(connection_pools)), + }) + } + + async fn create_vault_database( + config: &DatabaseConfig, + vault: &VaultId, + ) -> Result> { + let file_name = config + .databases_directory_path + .join(format!("{vault}.sqlite")); + + let connection_options = SqliteConnectOptions::new() + .filename(file_name.clone()) .create_if_missing(true) .busy_timeout(Duration::from_secs(3600)) .journal_mode(sqlx::sqlite::SqliteJournalMode::Wal); @@ -29,18 +77,11 @@ impl Database { .test_before_acquire(true) .connect_with(connection_options) .await - .with_context(|| { - format!( - "Cannot connect to database with url: {}", - &config.sqlite_url - ) - })?; + .with_context(|| format!("Cannot open database at {}", file_name.display()))?; Self::run_migrations(&pool).await?; - Ok(Self { - connection_pool: pool, - }) + Ok(pool) } async fn run_migrations(pool: &Pool) -> Result<()> { @@ -50,17 +91,38 @@ impl Database { .context("Cannot check for pending migrations") } + async fn get_connection_pool(&mut self, vault: &VaultId) -> Result> { + let mut pools = self.connection_pools.lock().await; + if !pools.contains_key(vault) { + let pool = Self::create_vault_database(&self.config, vault).await?; + pools.insert(vault.clone(), pool); + } + + let pool = pools + .get(vault) + .expect("Pool was just inserted or already exists"); + + Ok(pool.clone()) + } + /// Attempting to write from this transaction might result in a /// database locked error. Use this transaction for read-only operations. - pub async fn create_readonly_transaction(&self) -> Result> { - self.connection_pool + pub async fn create_readonly_transaction( + &mut self, + vault: &VaultId, + ) -> Result> { + self.get_connection_pool(vault) + .await? .begin() .await .context("Cannot create transaction") } - pub async fn create_write_transaction(&self) -> Result> { - let mut transaction = self.create_readonly_transaction().await?; + pub async fn create_write_transaction( + &mut self, + vault: &VaultId, + ) -> Result> { + let mut transaction = self.create_readonly_transaction(vault).await?; // sqlx doesn't support immediate transactions for sqlite: https://github.com/launchbadge/sqlx/issues/481 sqlx::query!("END; BEGIN IMMEDIATE;") @@ -72,7 +134,7 @@ impl Database { /// Return the latest state of all documents in the vault pub async fn get_latest_documents( - &self, + &mut self, vault: &VaultId, transaction: Option<&mut Transaction<'_>>, ) -> Result> { @@ -80,24 +142,22 @@ impl Database { DocumentVersionWithoutContent, r#" select - vault_id, vault_update_id, - document_id as "document_id: uuid::Uuid", + document_id as "document_id: Hyphenated", relative_path, - created_date as "created_date: chrono::DateTime", updated_date as "updated_date: chrono::DateTime", is_deleted from latest_document_versions - where vault_id = ? order by vault_update_id desc "#, - vault, ); if let Some(transaction) = transaction { query.fetch_all(&mut **transaction).await } else { - query.fetch_all(&self.connection_pool).await + query + .fetch_all(&self.get_connection_pool(vault).await?) + .await } .context("Cannot fetch latest documents") } @@ -105,7 +165,7 @@ impl Database { /// Return the latest state of all documents (including deleted) in the /// vault which have changed since the given update id pub async fn get_latest_documents_since( - &self, + &mut self, vault: &VaultId, vault_update_id: VaultUpdateId, transaction: Option<&mut Transaction<'_>>, @@ -114,25 +174,24 @@ impl Database { DocumentVersionWithoutContent, r#" select - vault_id, vault_update_id, - document_id as "document_id: uuid::Uuid", + document_id as "document_id: Hyphenated", relative_path, - created_date as "created_date: chrono::DateTime", updated_date as "updated_date: chrono::DateTime", is_deleted from latest_document_versions - where vault_id = ? and vault_update_id > ? + where vault_update_id > ? order by vault_update_id desc "#, - vault, vault_update_id ); if let Some(transaction) = transaction { query.fetch_all(&mut **transaction).await } else { - query.fetch_all(&self.connection_pool).await + query + .fetch_all(&self.get_connection_pool(vault).await?) + .await } .with_context(|| { format!("Cannot fetch latest documents since vault_update_id {vault_update_id}") @@ -140,7 +199,7 @@ impl Database { } pub async fn get_max_update_id_in_vault( - &self, + &mut self, vault: &VaultId, transaction: Option<&mut Transaction<'_>>, ) -> Result { @@ -148,22 +207,22 @@ impl Database { r#" select coalesce(max(vault_update_id), 0) as max_vault_update_id from documents - where vault_id = ? "#, - vault ); if let Some(transaction) = transaction { query.fetch_one(&mut **transaction).await } else { - query.fetch_one(&self.connection_pool).await + query + .fetch_one(&self.get_connection_pool(vault).await?) + .await } .map(|row| row.max_vault_update_id) .context("Cannot fetch max update id in vault") } pub async fn get_latest_document_by_path( - &self, + &mut self, vault: &VaultId, relative_path: &str, transaction: Option<&mut Transaction<'_>>, @@ -172,68 +231,67 @@ impl Database { StoredDocumentVersion, r#" select - vault_id, vault_update_id, - document_id as "document_id: uuid::Uuid", + document_id as "document_id: Hyphenated", relative_path, - created_date as "created_date: chrono::DateTime", updated_date as "updated_date: chrono::DateTime", content, is_deleted from latest_document_versions - where vault_id = ? and relative_path = ? + where relative_path = ? order by vault_update_id desc -- `latest_document_versions` only contains a single latest version of each document, however, -- multiple documents can have the same `relative_path`, if they have been deleted. That's -- why we only care about the latest version of the document with the given relative path. limit 1 "#, - vault, relative_path ); if let Some(transaction) = transaction { query.fetch_optional(&mut **transaction).await } else { - query.fetch_optional(&self.connection_pool).await + query + .fetch_optional(&self.get_connection_pool(vault).await?) + .await } .context("Cannot fetch latest document version") } pub async fn get_latest_document( - &self, + &mut self, vault: &VaultId, document_id: &DocumentId, transaction: Option<&mut Transaction<'_>>, ) -> Result> { + let document_id = document_id.as_hyphenated(); let query = sqlx::query_as!( StoredDocumentVersion, r#" select - vault_id, vault_update_id, - document_id as "document_id: uuid::Uuid", + document_id as "document_id: Hyphenated", relative_path, - created_date as "created_date: chrono::DateTime", updated_date as "updated_date: chrono::DateTime", content, is_deleted from latest_document_versions - where vault_id = ? and document_id = ? + where document_id = ? "#, - vault, document_id ); if let Some(transaction) = transaction { query.fetch_optional(&mut **transaction).await } else { - query.fetch_optional(&self.connection_pool).await + query + .fetch_optional(&self.get_connection_pool(vault).await?) + .await } .context("Cannot fetch latest document version") } pub async fn get_document_version( - &self, + &mut self, vault: &VaultId, vault_update_id: VaultUpdateId, transaction: Option<&mut Transaction<'_>>, @@ -242,52 +300,49 @@ impl Database { StoredDocumentVersion, r#" select - vault_id, vault_update_id, - document_id as "document_id: uuid::Uuid", + document_id as "document_id: Hyphenated", relative_path, - created_date as "created_date: chrono::DateTime", updated_date as "updated_date: chrono::DateTime", content, is_deleted from documents - where vault_id = ? and vault_update_id = ?"#, - vault, + where vault_update_id = ?"#, vault_update_id ); if let Some(transaction) = transaction { query.fetch_optional(&mut **transaction).await } else { - query.fetch_optional(&self.connection_pool).await + query + .fetch_optional(&self.get_connection_pool(vault).await?) + .await } .context("Cannot fetch document version") } pub async fn insert_document_version( - &self, + &mut self, + vault: &VaultId, version: &StoredDocumentVersion, transaction: Option<&mut Transaction<'_>>, ) -> Result<()> { + let document_id = version.document_id.as_hyphenated(); let query = sqlx::query!( r#" insert into documents ( - vault_id, vault_update_id, document_id, relative_path, - created_date, updated_date, content, is_deleted ) - values (?, ?, ?, ?, ?, ?, ?, ?) + values (?, ?, ?, ?, ?, ?) "#, - version.vault_id, version.vault_update_id, - version.document_id, + document_id, version.relative_path, - version.created_date, version.updated_date, version.content, version.is_deleted @@ -296,7 +351,7 @@ impl Database { if let Some(transaction) = transaction { query.execute(&mut **transaction).await } else { - query.execute(&self.connection_pool).await + query.execute(&self.get_connection_pool(vault).await?).await } .context("Cannot insert document version")?; diff --git a/backend/sync_server/src/database/migrations/20241207143519_bootstrap.sql b/backend/sync_server/src/database/migrations/20241207143519_bootstrap.sql index 360b34d..4a9f31b 100644 --- a/backend/sync_server/src/database/migrations/20241207143519_bootstrap.sql +++ b/backend/sync_server/src/database/migrations/20241207143519_bootstrap.sql @@ -1,25 +1,21 @@ CREATE TABLE IF NOT EXISTS documents ( - vault_id TEXT NOT NULL, - vault_update_id INTEGER NOT NULL, + vault_update_id INTEGER NOT NULL PRIMARY KEY, document_id TEXT NOT NULL, relative_path TEXT NOT NULL, - created_date TIMESTAMP NOT NULL, updated_date TIMESTAMP NOT NULL, content BLOB NOT NULL, - is_deleted BOOLEAN NOT NULL, - PRIMARY KEY (vault_id, vault_update_id) + is_deleted BOOLEAN NOT NULL ); CREATE VIEW IF NOT EXISTS latest_document_versions AS SELECT d.* FROM documents d INNER JOIN ( - SELECT vault_id, MAX(vault_update_id) AS max_version_id + SELECT MAX(vault_update_id) AS max_version_id FROM documents - GROUP BY vault_id, document_id + GROUP BY document_id ) max_versions -ON d.vault_id = max_versions.vault_id -AND d.vault_update_id = max_versions.max_version_id; +ON d.vault_update_id = max_versions.max_version_id; CREATE INDEX IF NOT EXISTS idx_documents_vault_id_relative_path -ON documents (vault_id, relative_path); +ON documents (relative_path); diff --git a/backend/sync_server/src/database/models.rs b/backend/sync_server/src/database/models.rs index d8f743a..a837e93 100644 --- a/backend/sync_server/src/database/models.rs +++ b/backend/sync_server/src/database/models.rs @@ -9,30 +9,24 @@ pub type DocumentId = uuid::Uuid; #[derive(Debug, Clone)] pub struct StoredDocumentVersion { - pub vault_id: VaultId, pub vault_update_id: VaultUpdateId, pub document_id: DocumentId, pub relative_path: String, - pub created_date: DateTime, pub updated_date: DateTime, pub content: Vec, pub is_deleted: bool, } impl PartialEq for StoredDocumentVersion { - fn eq(&self, other: &Self) -> bool { - self.vault_id == other.vault_id && self.vault_update_id == other.vault_update_id - } + fn eq(&self, other: &Self) -> bool { self.vault_update_id == other.vault_update_id } } #[derive(Debug, Clone, Serialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct DocumentVersionWithoutContent { - pub vault_id: VaultId, pub vault_update_id: VaultUpdateId, pub document_id: DocumentId, pub relative_path: String, - pub created_date: DateTime, pub updated_date: DateTime, pub is_deleted: bool, } @@ -40,11 +34,9 @@ pub struct DocumentVersionWithoutContent { impl From for DocumentVersionWithoutContent { fn from(value: StoredDocumentVersion) -> Self { Self { - vault_id: value.vault_id, vault_update_id: value.vault_update_id, document_id: value.document_id, relative_path: value.relative_path, - created_date: value.created_date, updated_date: value.updated_date, is_deleted: value.is_deleted, } @@ -54,11 +46,9 @@ impl From for DocumentVersionWithoutContent { #[derive(Debug, Clone, Serialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct DocumentVersion { - pub vault_id: VaultId, pub vault_update_id: VaultUpdateId, pub document_id: DocumentId, pub relative_path: String, - pub created_date: DateTime, pub updated_date: DateTime, pub content_base64: String, pub is_deleted: bool, @@ -67,11 +57,9 @@ pub struct DocumentVersion { impl From for DocumentVersion { fn from(value: StoredDocumentVersion) -> Self { Self { - vault_id: value.vault_id, vault_update_id: value.vault_update_id, document_id: value.document_id, relative_path: value.relative_path, - created_date: value.created_date, updated_date: value.updated_date, content_base64: bytes_to_base64(&value.content), is_deleted: value.is_deleted, diff --git a/backend/sync_server/src/errors.rs b/backend/sync_server/src/errors.rs index aa109c8..5aec9c3 100644 --- a/backend/sync_server/src/errors.rs +++ b/backend/sync_server/src/errors.rs @@ -33,12 +33,12 @@ pub enum SyncServerError { impl SyncServerError { pub fn serialize(&self) -> SerializedError { match self { - Self::InitError(error) => error.into(), - Self::ClientError(error) => error.into(), - Self::ServerError(error) => error.into(), - Self::NotFound(error) => error.into(), - Self::Unauthorized(error) => error.into(), - Self::PermissionDeniedError(error) => error.into(), + Self::InitError(error) + | Self::ClientError(error) + | Self::ServerError(error) + | Self::NotFound(error) + | Self::Unauthorized(error) + | Self::PermissionDeniedError(error) => error.into(), } } } @@ -48,9 +48,10 @@ impl IntoResponse for SyncServerError { let body = Json(self.serialize()); match self { - Self::InitError(_) => (StatusCode::INTERNAL_SERVER_ERROR, body).into_response(), + Self::InitError(_) | Self::ServerError(_) => { + (StatusCode::INTERNAL_SERVER_ERROR, body).into_response() + } Self::ClientError(_) => (StatusCode::BAD_REQUEST, body).into_response(), - Self::ServerError(_) => (StatusCode::INTERNAL_SERVER_ERROR, body).into_response(), Self::NotFound(_) => (StatusCode::NOT_FOUND, body).into_response(), Self::Unauthorized(_) => (StatusCode::UNAUTHORIZED, body).into_response(), Self::PermissionDeniedError(_) => (StatusCode::FORBIDDEN, body).into_response(), diff --git a/backend/sync_server/src/server.rs b/backend/sync_server/src/server.rs index bf62fec..511187e 100644 --- a/backend/sync_server/src/server.rs +++ b/backend/sync_server/src/server.rs @@ -16,6 +16,7 @@ use axum::{ extract::{DefaultBodyLimit, Request}, http::{self, HeaderValue, Method}, response::IntoResponse, + routing::IntoMakeService, }; use log::{error, info}; use tokio::signal; @@ -30,7 +31,10 @@ use tower_http::{ }; use tracing::{Level, info_span}; -use crate::errors::{SerializedError, not_found_error}; +use crate::{ + config::server_config::ServerConfig, + errors::{SerializedError, not_found_error}, +}; mod app_state; mod auth; mod create_document; @@ -52,24 +56,9 @@ pub async fn create_server() -> Result<()> { .await .context("Failed to initialise app state")?; - let address = format!( - "{}:{}", - &app_state.config.server.host, &app_state.config.server.port - ); - - let mut api = OpenApi { - info: Info { - title: "VaultLink sync server".to_owned(), - summary: Some( - "Simple API for syncing documents between concurrent clients.".to_owned(), - ), - description: Some(include_str!("../README.md").to_owned()), - version: env!("CARGO_PKG_VERSION").to_owned(), - ..Info::default() - }, - ..OpenApi::default() - }; + let server_config = app_state.config.server.clone(); + let mut api = create_open_api(); let app = ApiRouter::new() .api_route("/ping", get(ping::ping)) .api_route( @@ -140,11 +129,42 @@ pub async fn create_server() -> Result<()> { .allow_methods([Method::GET, Method::POST, Method::PUT, Method::DELETE]), ) .with_state(app_state) - .finish_api_with(&mut api, api_docs) + .finish_api_with(&mut api, add_api_docs_error_example) .layer(Extension(Arc::new(api))) // https://github.com/tamasfe/aide/blob/507f4a8822bc0c13cbda0f589da1e0f4cbcdb812/examples/example-axum/src/main.rs#L39 .fallback(handler_404) .into_make_service(); + start_server(app, &server_config).await +} + +async fn serve_api(Extension(api): Extension>) -> impl IntoResponse { Json(api) } + +fn create_open_api() -> OpenApi { + OpenApi { + info: Info { + title: "VaultLink sync server".to_owned(), + summary: Some( + "Simple API for syncing documents between concurrent clients.".to_owned(), + ), + description: Some(include_str!("../README.md").to_owned()), + version: env!("CARGO_PKG_VERSION").to_owned(), + ..Info::default() + }, + ..OpenApi::default() + } +} + +fn add_api_docs_error_example(api: TransformOpenApi<'_>) -> TransformOpenApi<'_> { + api.default_response_with::, _>(|res| { + res.example(SerializedError { + message: "An error has occurred".to_owned(), + causes: vec![], + }) + }) +} + +async fn start_server(app: IntoMakeService, config: &ServerConfig) -> Result<()> { + let address = format!("{}:{}", config.host, config.port); let listener = tokio::net::TcpListener::bind(address.clone()) .await .with_context(|| format!("Failed to bind to address: {address}"))?; @@ -163,17 +183,6 @@ pub async fn create_server() -> Result<()> { .context("Failed to start server") } -async fn serve_api(Extension(api): Extension>) -> impl IntoResponse { Json(api) } - -fn api_docs(api: TransformOpenApi<'_>) -> TransformOpenApi<'_> { - api.default_response_with::, _>(|res| { - res.example(SerializedError { - message: "An error has occurred".to_owned(), - causes: vec![], - }) - }) -} - async fn shutdown_signal() { let ctrl_c = async { signal::ctrl_c() @@ -193,8 +202,8 @@ async fn shutdown_signal() { let terminate = std::future::pending::<()>(); tokio::select! { - _ = ctrl_c => {}, - _ = terminate => {}, + () = ctrl_c => {}, + () = terminate => {}, } } diff --git a/backend/sync_server/src/server/create_document.rs b/backend/sync_server/src/server/create_document.rs index a256793..89f5478 100644 --- a/backend/sync_server/src/server/create_document.rs +++ b/backend/sync_server/src/server/create_document.rs @@ -6,7 +6,6 @@ use axum_extra::{ headers::{Authorization, authorization::Bearer}, }; use axum_jsonschema::Json; -use chrono::{DateTime, Utc}; use schemars::JsonSchema; use serde::Deserialize; use sync_lib::base64_to_bytes; @@ -17,7 +16,7 @@ use super::{ requests::{CreateDocumentVersion, CreateDocumentVersionMultipart}, }; use crate::{ - database::models::{DocumentVersionWithoutContent, StoredDocumentVersion, VaultId}, + database::models::{DocumentId, DocumentVersionWithoutContent, StoredDocumentVersion, VaultId}, errors::{SyncServerError, client_error, server_error}, utils::sanitize_path, }; @@ -44,8 +43,8 @@ pub async fn create_document_multipart( auth_header, state, vault_id, + request.document_id, request.relative_path, - request.created_date, request.content.contents.to_vec(), ) .await @@ -69,8 +68,8 @@ pub async fn create_document_json( auth_header, state, vault_id, + request.document_id, request.relative_path, - request.created_date, content_bytes, ) .await @@ -78,20 +77,39 @@ pub async fn create_document_json( async fn internal_create_document( auth_header: Authorization, - state: AppState, + mut state: AppState, vault_id: VaultId, + document_id: Option, relative_path: String, - created_date: DateTime, content: Vec, ) -> Result, SyncServerError> { auth(&state, auth_header.token())?; let mut transaction = state .database - .create_write_transaction() + .create_write_transaction(&vault_id) .await .map_err(server_error)?; + let document_id = match document_id { + Some(document_id) => { + let existing_version = state + .database + .get_latest_document(&vault_id, &document_id, Some(&mut transaction)) + .await + .map_err(server_error)?; + + if existing_version.is_some() { + return Err(client_error(anyhow::anyhow!( + "Document with the same ID already exists" + ))); + } + + document_id + } + None => uuid::Uuid::new_v4(), + }; + let last_update_id = state .database .get_max_update_id_in_vault(&vault_id, Some(&mut transaction)) @@ -101,19 +119,17 @@ async fn internal_create_document( let sanitized_relative_path = sanitize_path(&relative_path); let new_version = StoredDocumentVersion { - vault_id, vault_update_id: last_update_id + 1, - document_id: uuid::Uuid::new_v4(), + document_id, relative_path: sanitized_relative_path, content, - created_date, updated_date: chrono::Utc::now(), is_deleted: false, }; state .database - .insert_document_version(&new_version, Some(&mut transaction)) + .insert_document_version(&vault_id, &new_version, Some(&mut transaction)) .await .map_err(server_error)?; diff --git a/backend/sync_server/src/server/delete_document.rs b/backend/sync_server/src/server/delete_document.rs index a9d307b..75f90d2 100644 --- a/backend/sync_server/src/server/delete_document.rs +++ b/backend/sync_server/src/server/delete_document.rs @@ -10,7 +10,7 @@ use serde::Deserialize; use super::{app_state::AppState, auth::auth, requests::DeleteDocumentVersion}; use crate::{ - database::models::{DocumentId, StoredDocumentVersion, VaultId}, + database::models::{DocumentId, DocumentVersionWithoutContent, StoredDocumentVersion, VaultId}, errors::{SyncServerError, server_error}, utils::sanitize_path, }; @@ -29,14 +29,14 @@ pub async fn delete_document( vault_id, document_id, }): Path, - State(state): State, + State(mut state): State, Json(request): Json, -) -> Result<(), SyncServerError> { +) -> Result, SyncServerError> { auth(&state, auth_header.token())?; let mut transaction = state .database - .create_write_transaction() + .create_write_transaction(&vault_id) .await .map_err(server_error)?; @@ -47,19 +47,17 @@ pub async fn delete_document( .map_err(server_error)?; let new_version = StoredDocumentVersion { - vault_id, vault_update_id: last_update_id + 1, document_id, relative_path: sanitize_path(&request.relative_path), content: vec![], - created_date: request.created_date, updated_date: chrono::Utc::now(), is_deleted: true, }; state .database - .insert_document_version(&new_version, Some(&mut transaction)) + .insert_document_version(&vault_id, &new_version, Some(&mut transaction)) .await .map_err(server_error)?; @@ -69,5 +67,5 @@ pub async fn delete_document( .context("Failed to commit successful transaction") .map_err(server_error)?; - Ok(()) + Ok(Json(new_version.into())) } diff --git a/backend/sync_server/src/server/fetch_document_version.rs b/backend/sync_server/src/server/fetch_document_version.rs index c643160..a2b157e 100644 --- a/backend/sync_server/src/server/fetch_document_version.rs +++ b/backend/sync_server/src/server/fetch_document_version.rs @@ -30,7 +30,7 @@ pub async fn fetch_document_version( document_id, vault_update_id, }): Path, - State(state): State, + State(mut state): State, ) -> Result, SyncServerError> { auth(&state, auth_header.token())?; @@ -39,12 +39,14 @@ pub async fn fetch_document_version( .get_document_version(&vault_id, vault_update_id, None) .await .map_err(server_error)? - .map(Ok) - .unwrap_or_else(|| { - Err(not_found_error(anyhow!( - "Document with vault update id `{vault_update_id}` not found", - ))) - })?; + .map_or_else( + || { + Err(not_found_error(anyhow!( + "Document with vault update id `{vault_update_id}` not found", + ))) + }, + Ok, + )?; if result.document_id != document_id { return Err(not_found_error(anyhow!( diff --git a/backend/sync_server/src/server/fetch_document_version_content.rs b/backend/sync_server/src/server/fetch_document_version_content.rs index 68e3825..203f0af 100644 --- a/backend/sync_server/src/server/fetch_document_version_content.rs +++ b/backend/sync_server/src/server/fetch_document_version_content.rs @@ -32,7 +32,7 @@ pub async fn fetch_document_version_content( document_id, vault_update_id, }): Path, - State(state): State, + State(mut state): State, ) -> Result { auth(&state, auth_header.token())?; @@ -41,12 +41,14 @@ pub async fn fetch_document_version_content( .get_document_version(&vault_id, vault_update_id, None) .await .map_err(server_error)? - .map(Ok) - .unwrap_or_else(|| { - Err(not_found_error(anyhow!( - "Document with vault update id `{vault_update_id}` not found", - ))) - })?; + .map_or_else( + || { + Err(not_found_error(anyhow!( + "Document with vault update id `{vault_update_id}` not found", + ))) + }, + Ok, + )?; if result.document_id != document_id { return Err(not_found_error(anyhow!( diff --git a/backend/sync_server/src/server/fetch_latest_document_version.rs b/backend/sync_server/src/server/fetch_latest_document_version.rs index a53f270..331730e 100644 --- a/backend/sync_server/src/server/fetch_latest_document_version.rs +++ b/backend/sync_server/src/server/fetch_latest_document_version.rs @@ -28,7 +28,7 @@ pub async fn fetch_latest_document_version( vault_id, document_id, }): Path, - State(state): State, + State(mut state): State, ) -> Result, SyncServerError> { auth(&state, auth_header.token())?; @@ -37,12 +37,14 @@ pub async fn fetch_latest_document_version( .get_latest_document(&vault_id, &document_id, None) .await .map_err(server_error)? - .map(Ok) - .unwrap_or_else(|| { - Err(not_found_error(anyhow!( - "Document with id `{document_id}` not found", - ))) - })?; + .map_or_else( + || { + Err(not_found_error(anyhow!( + "Document with id `{document_id}` not found", + ))) + }, + Ok, + )?; Ok(Json(latest_version.into())) } diff --git a/backend/sync_server/src/server/fetch_latest_documents.rs b/backend/sync_server/src/server/fetch_latest_documents.rs index b19c3de..b7ff09b 100644 --- a/backend/sync_server/src/server/fetch_latest_documents.rs +++ b/backend/sync_server/src/server/fetch_latest_documents.rs @@ -30,7 +30,7 @@ pub async fn fetch_latest_documents( TypedHeader(auth_header): TypedHeader>, Path(PathParams { vault_id }): Path, Query(QueryParams { since_update_id }): Query, - State(state): State, + State(mut state): State, ) -> Result, SyncServerError> { auth(&state, auth_header.token())?; diff --git a/backend/sync_server/src/server/requests.rs b/backend/sync_server/src/server/requests.rs index 1720f96..3c88826 100644 --- a/backend/sync_server/src/server/requests.rs +++ b/backend/sync_server/src/server/requests.rs @@ -1,24 +1,27 @@ use aide_axum_typed_multipart::FieldData; use axum::body::Bytes; use axum_typed_multipart::TryFromMultipart; -use chrono::{DateTime, Utc}; use schemars::JsonSchema; use serde::{self, Deserialize}; -use crate::database::models::VaultUpdateId; +use crate::database::models::{DocumentId, VaultUpdateId}; #[derive(Debug, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct CreateDocumentVersion { + /// The client can decide the document id (if it wishes to) in order + /// to help with syncing. If the client does not provide a document id, + /// the server will generate one. If the client provides a document id + /// it must not already exist in the database. + pub document_id: Option, pub relative_path: String, - pub created_date: DateTime, pub content_base64: String, } #[derive(Debug, TryFromMultipart, JsonSchema)] pub struct CreateDocumentVersionMultipart { + pub document_id: Option, pub relative_path: String, - pub created_date: DateTime, #[form_data(limit = "unlimited")] pub content: FieldData, } @@ -28,7 +31,6 @@ pub struct CreateDocumentVersionMultipart { pub struct UpdateDocumentVersion { pub parent_version_id: VaultUpdateId, pub relative_path: String, - pub created_date: DateTime, pub content_base64: String, } @@ -37,7 +39,6 @@ pub struct UpdateDocumentVersion { pub struct UpdateDocumentVersionMultipart { pub parent_version_id: VaultUpdateId, pub relative_path: String, - pub created_date: DateTime, #[form_data(limit = "unlimited")] pub content: FieldData, } @@ -46,5 +47,4 @@ pub struct UpdateDocumentVersionMultipart { #[serde(rename_all = "camelCase")] pub struct DeleteDocumentVersion { pub relative_path: String, - pub created_date: DateTime, } diff --git a/backend/sync_server/src/server/update_document.rs b/backend/sync_server/src/server/update_document.rs index 414180b..a9b9c13 100644 --- a/backend/sync_server/src/server/update_document.rs +++ b/backend/sync_server/src/server/update_document.rs @@ -6,7 +6,6 @@ use axum_extra::{ headers::{Authorization, authorization::Bearer}, }; use axum_jsonschema::Json; -use chrono::{DateTime, Utc}; use log::info; use schemars::JsonSchema; use serde::Deserialize; @@ -50,7 +49,6 @@ pub async fn update_document_multipart( document_id, request.parent_version_id, request.relative_path, - request.created_date, request.content.contents.to_vec(), ) .await @@ -77,21 +75,19 @@ pub async fn update_document_json( document_id, request.parent_version_id, request.relative_path, - request.created_date, content_bytes, ) .await } -#[allow(clippy::too_many_arguments)] +#[allow(clippy::too_many_arguments, clippy::too_many_lines)] async fn internal_update_document( auth_header: Authorization, - state: AppState, + mut state: AppState, vault_id: VaultId, document_id: DocumentId, parent_version_id: VaultUpdateId, relative_path: String, - created_date: DateTime, content: Vec, ) -> Result, SyncServerError> { auth(&state, auth_header.token())?; @@ -114,7 +110,7 @@ async fn internal_update_document( let mut transaction = state .database - .create_write_transaction() + .create_write_transaction(&vault_id) .await .map_err(server_error)?; @@ -138,6 +134,18 @@ async fn internal_update_document( Ok, )?; + if latest_version.is_deleted { + transaction + .rollback() + .await + .context("Failed to roll back transaction") + .map_err(server_error)?; + + return Ok(Json(DocumentUpdateResponse::FastForwardUpdate( + latest_version.into(), + ))); + } + let sanitized_relative_path = sanitize_path(&relative_path); // Return the latest version if the content and path are the same as the latest @@ -168,7 +176,7 @@ async fn internal_update_document( let new_relative_path = if parent_document.relative_path == latest_version.relative_path && latest_version.relative_path != sanitized_relative_path { - let mut new_relative_path = Default::default(); + let mut new_relative_path = String::default(); for candidate in deduped_file_paths(&sanitized_relative_path) { if state .database @@ -188,19 +196,17 @@ async fn internal_update_document( }; let new_version = StoredDocumentVersion { - vault_id, document_id, vault_update_id: last_update_id + 1, relative_path: new_relative_path, content: merged_content, - created_date, updated_date: chrono::Utc::now(), - is_deleted: latest_version.is_deleted, + is_deleted: false, }; state .database - .insert_document_version(&new_version, Some(&mut transaction)) + .insert_document_version(&vault_id, &new_version, Some(&mut transaction)) .await .map_err(server_error)?; diff --git a/frontend/obsidian-plugin/package.json b/frontend/obsidian-plugin/package.json index 30fad71..7fcd9c0 100644 --- a/frontend/obsidian-plugin/package.json +++ b/frontend/obsidian-plugin/package.json @@ -14,7 +14,7 @@ "license": "MIT", "devDependencies": { "@types/jest": "^29.5.14", - "@types/node": "^22.13.5", + "@types/node": "^22.13.10", "css-loader": "^7.1.2", "date-fns": "^4.1.0", "file-loader": "^6.2.0", @@ -23,14 +23,14 @@ "mini-css-extract-plugin": "^2.9.2", "obsidian": "1.8.7", "resolve-url-loader": "^5.0.0", - "sass": "^1.85.0", + "sass": "^1.85.1", "sass-loader": "^16.0.5", "sync-client": "file:../sync-client", - "terser-webpack-plugin": "^5.3.11", + "terser-webpack-plugin": "^5.3.14", "ts-jest": "^29.2.6", "ts-loader": "^9.5.2", "tslib": "2.8.1", - "typescript": "5.7.3", + "typescript": "5.8.2", "url": "^0.11.4", "virtual-scroller": "^1.13.1", "webpack": "^5.98.0", diff --git a/frontend/obsidian-plugin/src/obisidan-event-handler.ts b/frontend/obsidian-plugin/src/obisidan-event-handler.ts index b2d58b7..9fb5dba 100644 --- a/frontend/obsidian-plugin/src/obisidan-event-handler.ts +++ b/frontend/obsidian-plugin/src/obisidan-event-handler.ts @@ -9,10 +9,7 @@ export class ObsidianFileEventHandler { if (file instanceof TFile) { this.client.logger.info(`File created: ${file.path}`); - await this.client.syncer.syncLocallyCreatedFile( - file.path, - new Date(file.stat.ctime) - ); + await this.client.syncer.syncLocallyCreatedFile(file.path); } else { this.client.logger.debug(`Folder created: ${file.path}, ignored`); } @@ -34,8 +31,7 @@ export class ObsidianFileEventHandler { await this.client.syncer.syncLocallyUpdatedFile({ oldPath, - relativePath: file.path, - updateTime: new Date(file.stat.ctime) + relativePath: file.path }); } else { this.client.logger.debug( @@ -53,8 +49,7 @@ export class ObsidianFileEventHandler { this.client.logger.info(`File modified: ${file.path}`); await this.client.syncer.syncLocallyUpdatedFile({ - relativePath: file.path, - updateTime: new Date(file.stat.ctime) + relativePath: file.path }); } else { this.client.logger.debug(`Folder modified: ${file.path}, ignored`); diff --git a/frontend/obsidian-plugin/src/views/history-view.ts b/frontend/obsidian-plugin/src/views/history-view.ts index 56c8d53..d253b27 100644 --- a/frontend/obsidian-plugin/src/views/history-view.ts +++ b/frontend/obsidian-plugin/src/views/history-view.ts @@ -60,7 +60,6 @@ export class HistoryView extends ItemView { } element.createEl("span", { - // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment text: entry.relativePath }); diff --git a/frontend/obsidian-plugin/src/views/logs-view.ts b/frontend/obsidian-plugin/src/views/logs-view.ts index 66aa30c..22def86 100644 --- a/frontend/obsidian-plugin/src/views/logs-view.ts +++ b/frontend/obsidian-plugin/src/views/logs-view.ts @@ -1,6 +1,6 @@ import type { WorkspaceLeaf } from "obsidian"; import { ItemView } from "obsidian"; -import type VaultLinkPlugin from "src/vault-link-plugin"; +import type VaultLinkPlugin from "../vault-link-plugin"; import type { SyncClient } from "sync-client"; export class LogsView extends ItemView { diff --git a/frontend/obsidian-plugin/src/views/settings-tab.ts b/frontend/obsidian-plugin/src/views/settings-tab.ts index b3a39dd..d37a26d 100644 --- a/frontend/obsidian-plugin/src/views/settings-tab.ts +++ b/frontend/obsidian-plugin/src/views/settings-tab.ts @@ -1,7 +1,7 @@ import type { App } from "obsidian"; import { Notice, PluginSettingTab, Setting } from "obsidian"; -import type VaultLinkPlugin from "src/vault-link-plugin"; +import type VaultLinkPlugin from "../vault-link-plugin"; import type { StatusDescription } from "./status-description"; import { LogsView } from "./logs-view"; import { HistoryView } from "./history-view"; diff --git a/frontend/obsidian-plugin/src/views/status-bar.ts b/frontend/obsidian-plugin/src/views/status-bar.ts index 9ecd7b8..ba7d733 100644 --- a/frontend/obsidian-plugin/src/views/status-bar.ts +++ b/frontend/obsidian-plugin/src/views/status-bar.ts @@ -1,5 +1,5 @@ import type { HistoryStats, SyncClient } from "sync-client"; -import type VaultLinkPlugin from "src/vault-link-plugin"; +import type VaultLinkPlugin from "../vault-link-plugin"; export class StatusBar { private readonly statusBarItem: HTMLElement; diff --git a/frontend/obsidian-plugin/webpack.config.js b/frontend/obsidian-plugin/webpack.config.js index 66c35d6..75c87fb 100644 --- a/frontend/obsidian-plugin/webpack.config.js +++ b/frontend/obsidian-plugin/webpack.config.js @@ -94,9 +94,6 @@ module.exports = (env, argv) => ({ alias: { root: __dirname, src: path.resolve(__dirname, "src") - }, - fallback: { - url: require.resolve("url") } }, output: { diff --git a/frontend/package-lock.json b/frontend/package-lock.json index e41f20a..fea89cd 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -12,11 +12,11 @@ ], "devDependencies": { "concurrently": "^9.1.2", - "eslint": "9.21.0", + "eslint": "9.22.0", "eslint-plugin-unused-imports": "^4.1.4", - "npm-check-updates": "^17.1.14", - "prettier": "^3.5.2", - "typescript-eslint": "8.24.1" + "npm-check-updates": "^17.1.15", + "prettier": "^3.5.3", + "typescript-eslint": "8.26.1" } }, "../backend/sync_lib/pkg": { @@ -39,7 +39,6 @@ }, "node_modules/@babel/code-frame": { "version": "7.26.2", - "dev": true, "license": "MIT", "dependencies": { "@babel/helper-validator-identifier": "^7.25.9", @@ -179,7 +178,6 @@ }, "node_modules/@babel/helper-validator-identifier": { "version": "7.25.9", - "dev": true, "license": "MIT", "engines": { "node": ">=6.9.0" @@ -194,12 +192,14 @@ } }, "node_modules/@babel/helpers": { - "version": "7.26.9", + "version": "7.26.10", + "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.26.10.tgz", + "integrity": "sha512-UPYc3SauzZ3JGgj87GgZ89JVdC5dj0AoetR5Bw6wj4niittNyFh6+eOGonYvJ1ao6B8lEa3Q3klS7ADZ53bc5g==", "dev": true, "license": "MIT", "dependencies": { "@babel/template": "^7.26.9", - "@babel/types": "^7.26.9" + "@babel/types": "^7.26.10" }, "engines": { "node": ">=6.9.0" @@ -463,7 +463,9 @@ } }, "node_modules/@babel/types": { - "version": "7.26.9", + "version": "7.26.10", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.26.10.tgz", + "integrity": "sha512-emqcG3vHrpxUKTrxcblR36dcrcoRDvKmnL/dCL6ZsHaShW80qxCAcNhzQZrpeM765VzEos+xOi4s+r4IXzTwdQ==", "dev": true, "license": "MIT", "dependencies": { @@ -580,6 +582,16 @@ "node": "^18.18.0 || ^20.9.0 || >=21.1.0" } }, + "node_modules/@eslint/config-helpers": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/@eslint/config-helpers/-/config-helpers-0.1.0.tgz", + "integrity": "sha512-kLrdPDJE1ckPo94kmPPf9Hfd0DU0Jw6oKYrhe+pwSC0iTUInmTa+w6fw8sGgcfkFJGNdWOUeOaDM4quW4a7OkA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + } + }, "node_modules/@eslint/core": { "version": "0.12.0", "resolved": "https://registry.npmjs.org/@eslint/core/-/core-0.12.0.tgz", @@ -618,9 +630,9 @@ } }, "node_modules/@eslint/js": { - "version": "9.21.0", - "resolved": "https://registry.npmjs.org/@eslint/js/-/js-9.21.0.tgz", - "integrity": "sha512-BqStZ3HX8Yz6LvsF5ByXYrtigrV5AXADWLAGc7PH/1SxOb7/FIYYMszZZWiUou/GB9P2lXWk2SV4d+Z8h0nknw==", + "version": "9.22.0", + "resolved": "https://registry.npmjs.org/@eslint/js/-/js-9.22.0.tgz", + "integrity": "sha512-vLFajx9o8d1/oL2ZkpMYbkLv8nDB6yaIwFNt7nI4+I80U/z03SxmfOMsLbvWr3p7C+Wnoh//aOu2pQW8cS0HCQ==", "dev": true, "license": "MIT", "engines": { @@ -1130,6 +1142,8 @@ }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", + "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", + "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==", "dev": true, "license": "MIT", "dependencies": { @@ -1142,6 +1156,8 @@ }, "node_modules/@nodelib/fs.stat": { "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz", + "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==", "dev": true, "license": "MIT", "engines": { @@ -1150,6 +1166,8 @@ }, "node_modules/@nodelib/fs.walk": { "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz", + "integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==", "dev": true, "license": "MIT", "dependencies": { @@ -1235,7 +1253,6 @@ }, "node_modules/@redocly/ajv": { "version": "8.11.2", - "dev": true, "license": "MIT", "dependencies": { "fast-deep-equal": "^3.1.1", @@ -1250,17 +1267,14 @@ }, "node_modules/@redocly/ajv/node_modules/json-schema-traverse": { "version": "1.0.0", - "dev": true, "license": "MIT" }, "node_modules/@redocly/config": { "version": "0.20.3", - "dev": true, "license": "MIT" }, "node_modules/@redocly/openapi-core": { "version": "1.29.0", - "dev": true, "license": "MIT", "dependencies": { "@redocly/ajv": "^8.11.2", @@ -1280,7 +1294,6 @@ }, "node_modules/@redocly/openapi-core/node_modules/brace-expansion": { "version": "2.0.1", - "dev": true, "license": "MIT", "dependencies": { "balanced-match": "^1.0.0" @@ -1288,7 +1301,6 @@ }, "node_modules/@redocly/openapi-core/node_modules/minimatch": { "version": "5.1.6", - "dev": true, "license": "ISC", "dependencies": { "brace-expansion": "^2.0.1" @@ -1458,9 +1470,9 @@ "license": "MIT" }, "node_modules/@types/node": { - "version": "22.13.5", - "resolved": "https://registry.npmjs.org/@types/node/-/node-22.13.5.tgz", - "integrity": "sha512-+lTU0PxZXn0Dr1NBtC7Y8cR21AJr87dLLU953CWA6pMxxv/UDc7jYAY90upcrie1nRcD6XNG5HOYEDtgW5TxAg==", + "version": "22.13.10", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.13.10.tgz", + "integrity": "sha512-I6LPUvlRH+O6VRUqYOcMudhaIdUVWfsjnZavnsraHvpBwaEyMN29ry+0UVJhImYL16xsscu0aske3yA+uPOWfw==", "dev": true, "license": "MIT", "dependencies": { @@ -1494,15 +1506,17 @@ "license": "MIT" }, "node_modules/@typescript-eslint/eslint-plugin": { - "version": "8.24.1", + "version": "8.26.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.26.1.tgz", + "integrity": "sha512-2X3mwqsj9Bd3Ciz508ZUtoQQYpOhU/kWoUqIf49H8Z0+Vbh6UF/y0OEYp0Q0axOGzaBGs7QxRwq0knSQ8khQNA==", "dev": true, "license": "MIT", "dependencies": { "@eslint-community/regexpp": "^4.10.0", - "@typescript-eslint/scope-manager": "8.24.1", - "@typescript-eslint/type-utils": "8.24.1", - "@typescript-eslint/utils": "8.24.1", - "@typescript-eslint/visitor-keys": "8.24.1", + "@typescript-eslint/scope-manager": "8.26.1", + "@typescript-eslint/type-utils": "8.26.1", + "@typescript-eslint/utils": "8.26.1", + "@typescript-eslint/visitor-keys": "8.26.1", "graphemer": "^1.4.0", "ignore": "^5.3.1", "natural-compare": "^1.4.0", @@ -1518,18 +1532,20 @@ "peerDependencies": { "@typescript-eslint/parser": "^8.0.0 || ^8.0.0-alpha.0", "eslint": "^8.57.0 || ^9.0.0", - "typescript": ">=4.8.4 <5.8.0" + "typescript": ">=4.8.4 <5.9.0" } }, "node_modules/@typescript-eslint/parser": { - "version": "8.24.1", + "version": "8.26.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-8.26.1.tgz", + "integrity": "sha512-w6HZUV4NWxqd8BdeFf81t07d7/YV9s7TCWrQQbG5uhuvGUAW+fq1usZ1Hmz9UPNLniFnD8GLSsDpjP0hm1S4lQ==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/scope-manager": "8.24.1", - "@typescript-eslint/types": "8.24.1", - "@typescript-eslint/typescript-estree": "8.24.1", - "@typescript-eslint/visitor-keys": "8.24.1", + "@typescript-eslint/scope-manager": "8.26.1", + "@typescript-eslint/types": "8.26.1", + "@typescript-eslint/typescript-estree": "8.26.1", + "@typescript-eslint/visitor-keys": "8.26.1", "debug": "^4.3.4" }, "engines": { @@ -1541,16 +1557,18 @@ }, "peerDependencies": { "eslint": "^8.57.0 || ^9.0.0", - "typescript": ">=4.8.4 <5.8.0" + "typescript": ">=4.8.4 <5.9.0" } }, "node_modules/@typescript-eslint/scope-manager": { - "version": "8.24.1", + "version": "8.26.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-8.26.1.tgz", + "integrity": "sha512-6EIvbE5cNER8sqBu6V7+KeMZIC1664d2Yjt+B9EWUXrsyWpxx4lEZrmvxgSKRC6gX+efDL/UY9OpPZ267io3mg==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/types": "8.24.1", - "@typescript-eslint/visitor-keys": "8.24.1" + "@typescript-eslint/types": "8.26.1", + "@typescript-eslint/visitor-keys": "8.26.1" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -1561,12 +1579,14 @@ } }, "node_modules/@typescript-eslint/type-utils": { - "version": "8.24.1", + "version": "8.26.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-8.26.1.tgz", + "integrity": "sha512-Kcj/TagJLwoY/5w9JGEFV0dclQdyqw9+VMndxOJKtoFSjfZhLXhYjzsQEeyza03rwHx2vFEGvrJWJBXKleRvZg==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/typescript-estree": "8.24.1", - "@typescript-eslint/utils": "8.24.1", + "@typescript-eslint/typescript-estree": "8.26.1", + "@typescript-eslint/utils": "8.26.1", "debug": "^4.3.4", "ts-api-utils": "^2.0.1" }, @@ -1579,11 +1599,13 @@ }, "peerDependencies": { "eslint": "^8.57.0 || ^9.0.0", - "typescript": ">=4.8.4 <5.8.0" + "typescript": ">=4.8.4 <5.9.0" } }, "node_modules/@typescript-eslint/types": { - "version": "8.24.1", + "version": "8.26.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-8.26.1.tgz", + "integrity": "sha512-n4THUQW27VmQMx+3P+B0Yptl7ydfceUj4ON/AQILAASwgYdZ/2dhfymRMh5egRUrvK5lSmaOm77Ry+lmXPOgBQ==", "dev": true, "license": "MIT", "engines": { @@ -1595,12 +1617,14 @@ } }, "node_modules/@typescript-eslint/typescript-estree": { - "version": "8.24.1", + "version": "8.26.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-8.26.1.tgz", + "integrity": "sha512-yUwPpUHDgdrv1QJ7YQal3cMVBGWfnuCdKbXw1yyjArax3353rEJP1ZA+4F8nOlQ3RfS2hUN/wze3nlY+ZOhvoA==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/types": "8.24.1", - "@typescript-eslint/visitor-keys": "8.24.1", + "@typescript-eslint/types": "8.26.1", + "@typescript-eslint/visitor-keys": "8.26.1", "debug": "^4.3.4", "fast-glob": "^3.3.2", "is-glob": "^4.0.3", @@ -1616,11 +1640,13 @@ "url": "https://opencollective.com/typescript-eslint" }, "peerDependencies": { - "typescript": ">=4.8.4 <5.8.0" + "typescript": ">=4.8.4 <5.9.0" } }, "node_modules/@typescript-eslint/typescript-estree/node_modules/brace-expansion": { "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", "dev": true, "license": "MIT", "dependencies": { @@ -1629,6 +1655,8 @@ }, "node_modules/@typescript-eslint/typescript-estree/node_modules/minimatch": { "version": "9.0.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.5.tgz", + "integrity": "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==", "dev": true, "license": "ISC", "dependencies": { @@ -1642,14 +1670,16 @@ } }, "node_modules/@typescript-eslint/utils": { - "version": "8.24.1", + "version": "8.26.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-8.26.1.tgz", + "integrity": "sha512-V4Urxa/XtSUroUrnI7q6yUTD3hDtfJ2jzVfeT3VK0ciizfK2q/zGC0iDh1lFMUZR8cImRrep6/q0xd/1ZGPQpg==", "dev": true, "license": "MIT", "dependencies": { "@eslint-community/eslint-utils": "^4.4.0", - "@typescript-eslint/scope-manager": "8.24.1", - "@typescript-eslint/types": "8.24.1", - "@typescript-eslint/typescript-estree": "8.24.1" + "@typescript-eslint/scope-manager": "8.26.1", + "@typescript-eslint/types": "8.26.1", + "@typescript-eslint/typescript-estree": "8.26.1" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -1660,15 +1690,17 @@ }, "peerDependencies": { "eslint": "^8.57.0 || ^9.0.0", - "typescript": ">=4.8.4 <5.8.0" + "typescript": ">=4.8.4 <5.9.0" } }, "node_modules/@typescript-eslint/visitor-keys": { - "version": "8.24.1", + "version": "8.26.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-8.26.1.tgz", + "integrity": "sha512-AjOC3zfnxd6S4Eiy3jwktJPclqhFHNyd8L6Gycf9WUPoKZpgM5PjkxY1X7uSy61xVpiJDhhk7XT2NVsN3ALTWg==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/types": "8.24.1", + "@typescript-eslint/types": "8.26.1", "eslint-visitor-keys": "^4.2.0" }, "engines": { @@ -1909,7 +1941,6 @@ }, "node_modules/agent-base": { "version": "7.1.3", - "dev": true, "license": "MIT", "engines": { "node": ">= 14" @@ -1976,7 +2007,6 @@ }, "node_modules/ansi-colors": { "version": "4.1.3", - "dev": true, "license": "MIT", "engines": { "node": ">=6" @@ -2039,7 +2069,6 @@ }, "node_modules/argparse": { "version": "2.0.1", - "dev": true, "license": "Python-2.0" }, "node_modules/async": { @@ -2161,7 +2190,6 @@ }, "node_modules/balanced-match": { "version": "1.0.2", - "dev": true, "license": "MIT" }, "node_modules/big.js": { @@ -2249,7 +2277,6 @@ }, "node_modules/byte-base64": { "version": "1.1.0", - "dev": true, "license": "MIT" }, "node_modules/call-bind-apply-helpers": { @@ -2335,7 +2362,6 @@ }, "node_modules/change-case": { "version": "5.4.4", - "dev": true, "license": "MIT" }, "node_modules/char-regex": { @@ -2445,7 +2471,6 @@ }, "node_modules/colorette": { "version": "1.4.0", - "dev": true, "license": "MIT" }, "node_modules/commander": { @@ -2597,7 +2622,6 @@ }, "node_modules/debug": { "version": "4.4.0", - "dev": true, "license": "MIT", "dependencies": { "ms": "^2.1.3" @@ -2822,18 +2846,19 @@ } }, "node_modules/eslint": { - "version": "9.21.0", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-9.21.0.tgz", - "integrity": "sha512-KjeihdFqTPhOMXTt7StsDxriV4n66ueuF/jfPNC3j/lduHwr/ijDwJMsF+wyMJethgiKi5wniIE243vi07d3pg==", + "version": "9.22.0", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-9.22.0.tgz", + "integrity": "sha512-9V/QURhsRN40xuHXWjV64yvrzMjcz7ZyNoF2jJFmy9j/SLk0u1OLSZgXi28MrXjymnjEGSR80WCdab3RGMDveQ==", "dev": true, "license": "MIT", "dependencies": { "@eslint-community/eslint-utils": "^4.2.0", "@eslint-community/regexpp": "^4.12.1", "@eslint/config-array": "^0.19.2", + "@eslint/config-helpers": "^0.1.0", "@eslint/core": "^0.12.0", "@eslint/eslintrc": "^3.3.0", - "@eslint/js": "9.21.0", + "@eslint/js": "9.22.0", "@eslint/plugin-kit": "^0.2.7", "@humanfs/node": "^0.16.6", "@humanwhocodes/module-importer": "^1.0.1", @@ -2845,7 +2870,7 @@ "cross-spawn": "^7.0.6", "debug": "^4.3.2", "escape-string-regexp": "^4.0.0", - "eslint-scope": "^8.2.0", + "eslint-scope": "^8.3.0", "eslint-visitor-keys": "^4.2.0", "espree": "^10.3.0", "esquery": "^1.5.0", @@ -2896,7 +2921,9 @@ } }, "node_modules/eslint-scope": { - "version": "8.2.0", + "version": "8.3.0", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-8.3.0.tgz", + "integrity": "sha512-pUNxi75F8MJ/GdeKtVLSbYg4ZI34J6C0C7sbL4YOp2exGwen7ZsuBqKzUhXd0qMQ362yET3z+uPwKeg/0C2XCQ==", "dev": true, "license": "BSD-2-Clause", "dependencies": { @@ -2991,7 +3018,6 @@ }, "node_modules/eventemitter3": { "version": "5.0.1", - "dev": true, "license": "MIT" }, "node_modules/events": { @@ -3048,11 +3074,12 @@ }, "node_modules/fast-deep-equal": { "version": "3.1.3", - "dev": true, "license": "MIT" }, "node_modules/fast-glob": { "version": "3.3.3", + "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.3.tgz", + "integrity": "sha512-7MptL8U0cqcFdzIzwOTHoilX9x5BrNqye7Z/LuC7kCMRio1EMSyqRK3BEAUD7sXRq4iT4AzTVuZdhgQ2TCvYLg==", "dev": true, "license": "MIT", "dependencies": { @@ -3068,6 +3095,8 @@ }, "node_modules/fast-glob/node_modules/glob-parent": { "version": "5.1.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", + "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", "dev": true, "license": "ISC", "dependencies": { @@ -3111,7 +3140,9 @@ } }, "node_modules/fastq": { - "version": "1.19.0", + "version": "1.19.1", + "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.19.1.tgz", + "integrity": "sha512-GwLTyxkCXjXbxqIhTsMI2Nui8huMPtnxg7krajPJAjnEG/iiOS7i+zCtWGZR9G0NBKbXKh6X9m9UIsYX/N6vvQ==", "dev": true, "license": "ISC", "dependencies": { @@ -3128,7 +3159,6 @@ }, "node_modules/fetch-retry": { "version": "6.0.0", - "dev": true, "license": "MIT" }, "node_modules/file-entry-cache": { @@ -3449,7 +3479,6 @@ }, "node_modules/https-proxy-agent": { "version": "7.0.6", - "dev": true, "license": "MIT", "dependencies": { "agent-base": "^7.1.2", @@ -3536,7 +3565,6 @@ }, "node_modules/index-to-position": { "version": "0.1.2", - "dev": true, "license": "MIT", "engines": { "node": ">=18" @@ -4271,7 +4299,6 @@ }, "node_modules/js-levenshtein": { "version": "1.1.6", - "dev": true, "license": "MIT", "engines": { "node": ">=0.10.0" @@ -4279,12 +4306,10 @@ }, "node_modules/js-tokens": { "version": "4.0.0", - "dev": true, "license": "MIT" }, "node_modules/js-yaml": { "version": "4.1.0", - "dev": true, "license": "MIT", "dependencies": { "argparse": "^2.0.1" @@ -4497,6 +4522,8 @@ }, "node_modules/merge2": { "version": "1.4.1", + "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", + "integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==", "dev": true, "license": "MIT", "engines": { @@ -4631,7 +4658,6 @@ }, "node_modules/ms": { "version": "2.1.3", - "dev": true, "license": "MIT" }, "node_modules/nanoid": { @@ -4686,7 +4712,9 @@ } }, "node_modules/npm-check-updates": { - "version": "17.1.14", + "version": "17.1.15", + "resolved": "https://registry.npmjs.org/npm-check-updates/-/npm-check-updates-17.1.15.tgz", + "integrity": "sha512-miATvKu5rjec/1wxc5TGDjpsucgtCHwRVZorZpDkS6NzdWXfnUWlN4abZddWb7XSijAuBNzzYglIdTm9SbgMVg==", "dev": true, "license": "Apache-2.0", "bin": { @@ -4758,8 +4786,9 @@ } }, "node_modules/openapi-fetch": { - "version": "0.13.4", - "dev": true, + "version": "0.13.5", + "resolved": "https://registry.npmjs.org/openapi-fetch/-/openapi-fetch-0.13.5.tgz", + "integrity": "sha512-AQK8T9GSKFREFlN1DBXTYsLjs7YV2tZcJ7zUWxbjMoQmj8dDSFRrzhLCbHPZWA1TMV3vACqfCxLEZcwf2wxV6Q==", "license": "MIT", "dependencies": { "openapi-typescript-helpers": "^0.0.15" @@ -4767,7 +4796,6 @@ }, "node_modules/openapi-typescript": { "version": "7.6.1", - "dev": true, "license": "MIT", "dependencies": { "@redocly/openapi-core": "^1.28.0", @@ -4786,12 +4814,10 @@ }, "node_modules/openapi-typescript-helpers": { "version": "0.0.15", - "dev": true, "license": "MIT" }, "node_modules/openapi-typescript/node_modules/parse-json": { "version": "8.1.0", - "dev": true, "license": "MIT", "dependencies": { "@babel/code-frame": "^7.22.13", @@ -4807,7 +4833,6 @@ }, "node_modules/openapi-typescript/node_modules/supports-color": { "version": "9.4.0", - "dev": true, "license": "MIT", "engines": { "node": ">=12" @@ -4818,7 +4843,6 @@ }, "node_modules/openapi-typescript/node_modules/type-fest": { "version": "4.35.0", - "dev": true, "license": "(MIT OR CC0-1.0)", "engines": { "node": ">=16" @@ -4873,7 +4897,6 @@ }, "node_modules/p-queue": { "version": "8.1.0", - "dev": true, "license": "MIT", "dependencies": { "eventemitter3": "^5.0.1", @@ -4888,7 +4911,6 @@ }, "node_modules/p-timeout": { "version": "6.1.4", - "dev": true, "license": "MIT", "engines": { "node": ">=14.16" @@ -4966,7 +4988,6 @@ }, "node_modules/picocolors": { "version": "1.1.1", - "dev": true, "license": "ISC" }, "node_modules/picomatch": { @@ -5049,7 +5070,6 @@ }, "node_modules/pluralize": { "version": "8.0.0", - "dev": true, "license": "MIT", "engines": { "node": ">=4" @@ -5163,9 +5183,9 @@ } }, "node_modules/prettier": { - "version": "3.5.2", - "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.5.2.tgz", - "integrity": "sha512-lc6npv5PH7hVqozBR7lkBNOGXV9vMwROAPlumdBkX0wTbbzPu/U1hk5yL8p2pt4Xoc+2mkT8t/sow2YrV/M5qg==", + "version": "3.5.3", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.5.3.tgz", + "integrity": "sha512-QQtaxnoDJeAkDvDKWCLiwIXkTgRhwYDEQCghU9Z6q03iyek/rxRh/2lC3HB7P8sWT2xC/y5JDctPLBIGzHKbhw==", "dev": true, "license": "MIT", "bin": { @@ -5255,6 +5275,8 @@ }, "node_modules/queue-microtask": { "version": "1.2.3", + "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", + "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==", "dev": true, "funding": [ { @@ -5328,7 +5350,6 @@ }, "node_modules/require-from-string": { "version": "2.0.2", - "dev": true, "license": "MIT", "engines": { "node": ">=0.10.0" @@ -5411,7 +5432,9 @@ } }, "node_modules/reusify": { - "version": "1.0.4", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.1.0.tgz", + "integrity": "sha512-g6QUff04oZpHs0eG5p83rFLhHeV00ug/Yf9nZM6fLeUrPguBTkTQOdpAWWspMh55TZfVQDPaN3NQJfbVRAxdIw==", "dev": true, "license": "MIT", "engines": { @@ -5421,6 +5444,8 @@ }, "node_modules/run-parallel": { "version": "1.2.0", + "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", + "integrity": "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==", "dev": true, "funding": [ { @@ -5469,7 +5494,9 @@ "license": "MIT" }, "node_modules/sass": { - "version": "1.85.0", + "version": "1.85.1", + "resolved": "https://registry.npmjs.org/sass/-/sass-1.85.1.tgz", + "integrity": "sha512-Uk8WpxM5v+0cMR0XjX9KfRIacmSG86RH4DCCZjLU2rFh5tyutt9siAXJ7G+YfxQ99Q6wrRMbMlVl6KqUms71ag==", "dev": true, "license": "MIT", "dependencies": { @@ -5871,7 +5898,9 @@ } }, "node_modules/terser-webpack-plugin": { - "version": "5.3.11", + "version": "5.3.14", + "resolved": "https://registry.npmjs.org/terser-webpack-plugin/-/terser-webpack-plugin-5.3.14.tgz", + "integrity": "sha512-vkZjpUjb6OMS7dhV+tILUW6BhpDR7P2L/aQSAv+Uwk+m8KATX9EccViHTJR2qDtACKPIYndLGCyl3FMo+r2LMw==", "dev": true, "license": "MIT", "dependencies": { @@ -6196,8 +6225,9 @@ } }, "node_modules/typescript": { - "version": "5.7.3", - "dev": true, + "version": "5.8.2", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.8.2.tgz", + "integrity": "sha512-aJn6wq13/afZp/jT9QZmwEjDqqvSGp1VT5GVg+f/t6/oVyrgXM6BY1h9BRh/O5p3PlUPAe+WuiEZOmb/49RqoQ==", "license": "Apache-2.0", "bin": { "tsc": "bin/tsc", @@ -6208,13 +6238,15 @@ } }, "node_modules/typescript-eslint": { - "version": "8.24.1", + "version": "8.26.1", + "resolved": "https://registry.npmjs.org/typescript-eslint/-/typescript-eslint-8.26.1.tgz", + "integrity": "sha512-t/oIs9mYyrwZGRpDv3g+3K6nZ5uhKEMt2oNmAPwaY4/ye0+EH4nXIPYNtkYFS6QHm+1DFg34DbglYBz5P9Xysg==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/eslint-plugin": "8.24.1", - "@typescript-eslint/parser": "8.24.1", - "@typescript-eslint/utils": "8.24.1" + "@typescript-eslint/eslint-plugin": "8.26.1", + "@typescript-eslint/parser": "8.26.1", + "@typescript-eslint/utils": "8.26.1" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -6225,7 +6257,7 @@ }, "peerDependencies": { "eslint": "^8.57.0 || ^9.0.0", - "typescript": ">=4.8.4 <5.8.0" + "typescript": ">=4.8.4 <5.9.0" } }, "node_modules/undici-types": { @@ -6280,7 +6312,6 @@ }, "node_modules/uri-js-replace": { "version": "1.0.1", - "dev": true, "license": "MIT" }, "node_modules/url": { @@ -6311,7 +6342,6 @@ }, "node_modules/uuid": { "version": "11.1.0", - "dev": true, "funding": [ "https://github.com/sponsors/broofa", "https://github.com/sponsors/ctavan" @@ -6480,6 +6510,8 @@ }, "node_modules/webpack-merge": { "version": "6.0.1", + "resolved": "https://registry.npmjs.org/webpack-merge/-/webpack-merge-6.0.1.tgz", + "integrity": "sha512-hXXvrjtx2PLYx4qruKl+kyRSLc52V+cCvMxRjmKwoA+CBbbF5GfIBtR6kCvl0fYGqTUPKB+1ktVmTHqMOzgCBg==", "dev": true, "license": "MIT", "dependencies": { @@ -6643,7 +6675,6 @@ }, "node_modules/yaml-ast-parser": { "version": "0.0.43", - "dev": true, "license": "Apache-2.0" }, "node_modules/yargs": { @@ -6665,7 +6696,6 @@ }, "node_modules/yargs-parser": { "version": "21.1.1", - "dev": true, "license": "ISC", "engines": { "node": ">=12" @@ -6698,7 +6728,7 @@ "license": "MIT", "devDependencies": { "@types/jest": "^29.5.14", - "@types/node": "^22.13.5", + "@types/node": "^22.13.10", "css-loader": "^7.1.2", "date-fns": "^4.1.0", "file-loader": "^6.2.0", @@ -6707,14 +6737,14 @@ "mini-css-extract-plugin": "^2.9.2", "obsidian": "1.8.7", "resolve-url-loader": "^5.0.0", - "sass": "^1.85.0", + "sass": "^1.85.1", "sass-loader": "^16.0.5", "sync-client": "file:../sync-client", - "terser-webpack-plugin": "^5.3.11", + "terser-webpack-plugin": "^5.3.14", "ts-jest": "^29.2.6", "ts-loader": "^9.5.2", "tslib": "2.8.1", - "typescript": "5.7.3", + "typescript": "5.8.2", "url": "^0.11.4", "virtual-scroller": "^1.13.1", "webpack": "^5.98.0", @@ -6723,22 +6753,26 @@ }, "sync-client": { "version": "0.0.0", - "devDependencies": { - "@types/jest": "^29.5.14", - "@types/node": "^22.13.5", + "dependencies": { "byte-base64": "^1.1.0", "fetch-retry": "^6.0.0", - "jest": "^29.7.0", - "openapi-fetch": "0.13.4", + "openapi-fetch": "0.13.5", "openapi-typescript": "7.6.1", "p-queue": "^8.1.0", + "uuid": "^11.1.0" + }, + "devDependencies": { + "@types/jest": "^29.5.14", + "@types/node": "^22.13.10", + "jest": "^29.7.0", "sync_lib": "file:../../backend/sync_lib/pkg", "ts-jest": "^29.2.6", "ts-loader": "^9.5.2", "tslib": "2.8.1", - "typescript": "5.7.3", + "typescript": "5.8.2", "webpack": "^5.98.0", - "webpack-cli": "^6.0.1" + "webpack-cli": "^6.0.1", + "webpack-merge": "^6.0.1" } }, "test-client": { @@ -6747,11 +6781,11 @@ "test-client": "dist/cli.js" }, "devDependencies": { - "@types/node": "^22.13.5", + "@types/node": "^22.13.10", "sync-client": "file:../sync-client", "ts-loader": "^9.5.2", "tslib": "2.8.1", - "typescript": "5.7.3", + "typescript": "5.8.2", "uuid": "^11.1.0", "webpack": "^5.98.0", "webpack-cli": "^6.0.1" diff --git a/frontend/package.json b/frontend/package.json index 301e28f..24c4638 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -21,10 +21,10 @@ }, "devDependencies": { "concurrently": "^9.1.2", - "eslint": "9.21.0", + "eslint": "9.22.0", "eslint-plugin-unused-imports": "^4.1.4", - "npm-check-updates": "^17.1.14", - "prettier": "^3.5.2", - "typescript-eslint": "8.24.1" + "npm-check-updates": "^17.1.15", + "prettier": "^3.5.3", + "typescript-eslint": "8.26.1" } } diff --git a/frontend/sync-client/package.json b/frontend/sync-client/package.json index 94340f3..f08b5f5 100644 --- a/frontend/sync-client/package.json +++ b/frontend/sync-client/package.json @@ -1,29 +1,36 @@ { "name": "sync-client", - "version": "0.0.0", - "private": true, - "main": "dist/index.js", + "version": "0.0.30", + "main": "dist/sync-client.node.js", + "browser": "dist/sync-client.web.js", "types": "dist/types/index.d.ts", + "files": [ + "dist/**/*" + ], "scripts": { "dev": "webpack watch --mode development", "build": "webpack --mode production", "test": "NODE_OPTIONS=\"$NODE_OPTIONS --experimental-vm-modules\" jest" }, + "dependencies": { + "byte-base64": "^1.1.0", + "fetch-retry": "^6.0.0", + "openapi-fetch": "0.13.5", + "openapi-typescript": "7.6.1", + "p-queue": "^8.1.0", + "uuid": "^11.1.0" + }, "devDependencies": { - "tslib": "2.8.1", - "typescript": "5.7.3", - "sync_lib": "file:../../backend/sync_lib/pkg", "@types/jest": "^29.5.14", - "@types/node": "^22.13.5", + "@types/node": "^22.13.10", "jest": "^29.7.0", "ts-jest": "^29.2.6", - "p-queue": "^8.1.0", - "fetch-retry": "^6.0.0", - "byte-base64": "^1.1.0", - "openapi-fetch": "0.13.4", - "openapi-typescript": "7.6.1", "ts-loader": "^9.5.2", + "tslib": "2.8.1", + "typescript": "5.8.2", "webpack": "^5.98.0", - "webpack-cli": "^6.0.1" + "webpack-cli": "^6.0.1", + "webpack-merge": "^6.0.1", + "sync_lib": "file:../../backend/sync_lib/pkg" } -} +} \ No newline at end of file diff --git a/frontend/sync-client/src/file-operations/document-locks.ts b/frontend/sync-client/src/file-operations/document-locks.ts index 3dc7ec2..522ed02 100644 --- a/frontend/sync-client/src/file-operations/document-locks.ts +++ b/frontend/sync-client/src/file-operations/document-locks.ts @@ -1,6 +1,9 @@ import type { Logger } from "../tracing/logger"; import type { RelativePath } from "../persistence/database"; +// Manages locks on documents to prevent concurrent modifications +// allowing the client's FileOperations implementation to be simpler. +// Locks are granted in a first-in-first-out order. export class DocumentLocks { private readonly locked = new Set(); private readonly waiters = new Map void)[]>(); diff --git a/frontend/sync-client/src/file-operations/file-operations.test.ts b/frontend/sync-client/src/file-operations/file-operations.test.ts index 2e7c57b..43308f8 100644 --- a/frontend/sync-client/src/file-operations/file-operations.test.ts +++ b/frontend/sync-client/src/file-operations/file-operations.test.ts @@ -1,16 +1,27 @@ -import type { FileSystemOperations } from "sync-client"; -import type { Database, RelativePath } from "../persistence/database"; +import type { + Database, + DocumentRecord, + RelativePath +} from "../persistence/database"; import { FileOperations } from "./file-operations"; import { Logger } from "../tracing/logger"; import { assertSetContainsExactly } from "../utils/assert-set-contains-exactly"; +import type { FileSystemOperations } from "./filesystem-operations"; describe("File operations", () => { - class MockDatabase { - public async updatePath( + class MockDatabase implements Partial { + public getLatestDocumentByRelativePath( + _find: RelativePath + ): DocumentRecord | undefined { + // no-op + return undefined; + } + + public move( _oldRelativePath: RelativePath, _newRelativePath: RelativePath - ): Promise { - // this is called but irrelevant for this mock + ): void { + // no-op } } diff --git a/frontend/sync-client/src/file-operations/file-operations.ts b/frontend/sync-client/src/file-operations/file-operations.ts index 9977d60..b198caa 100644 --- a/frontend/sync-client/src/file-operations/file-operations.ts +++ b/frontend/sync-client/src/file-operations/file-operations.ts @@ -1,10 +1,6 @@ -import type { Logger } from "src/tracing/logger"; +import type { Logger } from "../tracing/logger"; import type { FileSystemOperations } from "./filesystem-operations"; -import type { - Database, - DocumentId, - RelativePath -} from "src/persistence/database"; +import type { Database, RelativePath } from "../persistence/database"; import { isBinary, isFileTypeMergable, mergeText } from "sync_lib"; import { SafeFileSystemOperations } from "./safe-filesystem-operations"; @@ -17,7 +13,7 @@ export class FileOperations { private readonly database: Database, fs: FileSystemOperations ) { - this.fs = new SafeFileSystemOperations(fs); + this.fs = new SafeFileSystemOperations(fs, logger); } public async listAllFiles(): Promise { @@ -35,7 +31,7 @@ export class FileOperations { const decoder = new TextDecoder("utf-8"); - // Normalize line endings to LF on Windows + // Normalize line-endings to LF on Windows let text = decoder.decode(content); text = text.replace(/\r\n/g, "\n"); @@ -46,10 +42,6 @@ export class FileOperations { return this.fs.getFileSize(path); } - public async getModificationTime(path: RelativePath): Promise { - return this.fs.getModificationTime(path); - } - public async exists(path: RelativePath): Promise { return this.fs.exists(path); } @@ -60,18 +52,23 @@ export class FileOperations { path: RelativePath, newContent: Uint8Array ): Promise { + this.logger.debug(`Creating file: ${path}`); + + await this.fs.write(path, newContent); + } + + public async ensureClearPath(path: RelativePath): Promise { if (await this.fs.exists(path)) { const deconflictedPath = await this.deconflictPath(path); this.logger.debug( `Didn't expect ${path} to exist, deconflicting by moving it to '${deconflictedPath}'` ); - await this.database.updatePath(path, deconflictedPath); + + this.database.move(path, deconflictedPath); await this.fs.rename(path, deconflictedPath); } else { await this.createParentDirectories(path); } - - await this.fs.write(path, newContent); } // Update the file at the given path. @@ -126,40 +123,25 @@ export class FileOperations { return new TextEncoder().encode(resultText); } - public async remove(path: RelativePath): Promise { - this.logger.debug(`Deleting file: ${path}`); - return this.fs.delete(path); + public async delete(path: RelativePath): Promise { + if (await this.exists(path)) { + this.logger.debug(`Deleting file: ${path}`); + return this.fs.delete(path); + } else { + this.logger.debug(`No need to delete '${path}', it doesn't exist`); + } } public async move( oldPath: RelativePath, - newPath: RelativePath, - documentId?: DocumentId + newPath: RelativePath ): Promise { if (oldPath === newPath) { return; } + await this.ensureClearPath(newPath); - if (await this.fs.exists(newPath)) { - const deconflictedPath = await this.deconflictPath(newPath); - this.logger.debug( - `Conflict when moving '${oldPath}' to '${newPath}', the latter already exists, deconflicting by moving it to '${deconflictedPath}'` - ); - - const existingMetadata = this.database.getDocument(newPath); - if ( - existingMetadata === undefined || - existingMetadata.documentId !== documentId - ) { - await this.database.updatePath(newPath, deconflictedPath); - await this.fs.rename(newPath, deconflictedPath); - } else { - await this.database.deleteDocument(newPath); - } - } else { - await this.createParentDirectories(newPath); - } - + this.database.move(oldPath, newPath); await this.fs.rename(oldPath, newPath); } @@ -201,17 +183,12 @@ export class FileOperations { ); stem = stem.replace(FileOperations.PARENTHESES_REGEX, ""); - // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition - while (true) { - const newName = - currentCount === 0 - ? `${directory}${stem}${extension}` - : `${directory}${stem} (${currentCount})${extension}`; - if (await this.fs.exists(newName)) { - currentCount++; - } else { - return newName; - } - } + let newName = path; + do { + currentCount++; + newName = `${directory}${stem} (${currentCount})${extension}`; + } while (await this.fs.exists(newName)); + + return newName; } } diff --git a/frontend/sync-client/src/file-operations/filesystem-operations.ts b/frontend/sync-client/src/file-operations/filesystem-operations.ts index 9ea577f..3cab9d2 100644 --- a/frontend/sync-client/src/file-operations/filesystem-operations.ts +++ b/frontend/sync-client/src/file-operations/filesystem-operations.ts @@ -1,4 +1,4 @@ -import type { RelativePath } from "src/persistence/database"; +import type { RelativePath } from "../persistence/database"; export interface FileSystemOperations { listAllFiles: () => Promise; @@ -9,11 +9,8 @@ export interface FileSystemOperations { updater: (currentContent: string) => string ) => Promise; getFileSize: (path: RelativePath) => Promise; - getModificationTime: (path: RelativePath) => Promise; exists: (path: RelativePath) => Promise; createDirectory: (path: RelativePath) => Promise; delete: (path: RelativePath) => Promise; - - // Must be able to handle renaming to a file that already exists rename: (oldPath: RelativePath, newPath: RelativePath) => Promise; } diff --git a/frontend/sync-client/src/file-operations/safe-filesystem-operations.ts b/frontend/sync-client/src/file-operations/safe-filesystem-operations.ts index e493d12..c13611e 100644 --- a/frontend/sync-client/src/file-operations/safe-filesystem-operations.ts +++ b/frontend/sync-client/src/file-operations/safe-filesystem-operations.ts @@ -1,5 +1,7 @@ -import type { RelativePath } from "src/persistence/database"; +import type { RelativePath } from "../persistence/database"; import type { FileSystemOperations } from "./filesystem-operations"; +import type { Logger } from "../tracing/logger"; +import { DocumentLocks } from "./document-locks"; export class FileNotFoundError extends Error { public constructor(message: string) { @@ -9,71 +11,134 @@ export class FileNotFoundError extends Error { } // Decorate FileSystemOperations replacing errors with FileNotFoundError -// if the accessed file doesn't exist. +// if the accessed file doesn't exist. It also ensures that there's only +// ever a single request in-flight for any one file through the use of +// DocumentLocks. export class SafeFileSystemOperations implements FileSystemOperations { - public constructor(private readonly fs: FileSystemOperations) {} + private readonly locks: DocumentLocks; + + public constructor( + private readonly fs: FileSystemOperations, + private readonly logger: Logger + ) { + this.locks = new DocumentLocks(logger); + } public async listAllFiles(): Promise { return this.fs.listAllFiles(); } public async read(path: RelativePath): Promise { - return this.safeOperation(path, async () => this.fs.read(path)); + this.logger.debug(`Reading file: ${path}`); + return this.safeOperation( + path, + this.decorateToHoldLock(path, async () => this.fs.read(path)), + "read" + ); } public async write(path: RelativePath, content: Uint8Array): Promise { - return this.fs.write(path, content); + this.logger.debug(`Writing file: ${path}`); + return this.decorateToHoldLock(path, async () => + this.fs.write(path, content) + )(); } public async atomicUpdateText( path: RelativePath, updater: (currentContent: string) => string ): Promise { - return this.safeOperation(path, async () => - this.fs.atomicUpdateText(path, updater) + this.logger.debug(`Atomic update of file: ${path}`); + return this.safeOperation( + path, + this.decorateToHoldLock(path, async () => + this.fs.atomicUpdateText(path, updater) + ), + "atomicUpdateText" ); } public async getFileSize(path: RelativePath): Promise { - return this.safeOperation(path, async () => this.fs.getFileSize(path)); - } - - public async getModificationTime(path: RelativePath): Promise { - return this.safeOperation(path, async () => - this.fs.getModificationTime(path) + this.logger.debug(`Getting file size: ${path}`); + return this.safeOperation( + path, + this.decorateToHoldLock(path, async () => + this.fs.getFileSize(path) + ), + "getFileSize" ); } public async exists(path: RelativePath): Promise { - return this.fs.exists(path); + this.logger.debug(`Checking if file exists: ${path}`); + return this.decorateToHoldLock(path, async () => + this.fs.exists(path) + )(); } public async createDirectory(path: RelativePath): Promise { - return this.fs.createDirectory(path); + this.logger.debug(`Creating directory: ${path}`); + return this.decorateToHoldLock(path, async () => + this.fs.createDirectory(path) + )(); } public async delete(path: RelativePath): Promise { - return this.fs.delete(path); + this.logger.debug(`Deleting file: ${path}`); + return this.decorateToHoldLock(path, async () => + this.fs.delete(path) + )(); } public async rename( oldPath: RelativePath, newPath: RelativePath ): Promise { - return this.safeOperation(oldPath, async () => - this.fs.rename(oldPath, newPath) + this.logger.debug(`Renaming file: ${oldPath} to ${newPath}`); + return this.safeOperation( + oldPath, + this.decorateToHoldLock([oldPath, newPath], async () => + this.fs.rename(oldPath, newPath) + ), + "rename" ); } + private decorateToHoldLock( + pathOrPaths: RelativePath | RelativePath[], + operation: () => Promise + ): () => Promise { + return async () => { + const paths = Array.isArray(pathOrPaths) + ? pathOrPaths + : [pathOrPaths]; + await Promise.all( + paths.map(async (path) => this.locks.waitForDocumentLock(path)) + ); + try { + return await operation(); + } finally { + await Promise.all( + paths.map((path) => { + this.locks.unlockDocument(path); + }) + ); + } + }; + } + private async safeOperation( path: RelativePath, - operation: () => Promise + operation: () => Promise, + operationName: string ): Promise { // Without locking the file, this isn't atomic, however, it's good enough practicaly. // This will only break if the file exists, gets deleted and then immediately // recreated while `operation` is running. if (!(await this.fs.exists(path))) { - throw new FileNotFoundError(path); + throw new FileNotFoundError( + `File not found: ${path} before trying to ${operationName}` + ); } try { return await operation(); @@ -81,7 +146,9 @@ export class SafeFileSystemOperations implements FileSystemOperations { if (await this.fs.exists(path)) { throw error; } else { - throw new FileNotFoundError(path); + throw new FileNotFoundError( + `File not found: ${path} when trying to ${operationName}` + ); } } } diff --git a/frontend/sync-client/src/persistence/database.ts b/frontend/sync-client/src/persistence/database.ts index a013d9a..9f003d4 100644 --- a/frontend/sync-client/src/persistence/database.ts +++ b/frontend/sync-client/src/persistence/database.ts @@ -1,23 +1,43 @@ +import type { Logger } from "../tracing/logger"; + export type VaultUpdateId = number; export type DocumentId = string; export type RelativePath = string; export interface DocumentMetadata { parentVersionId: VaultUpdateId; - documentId: DocumentId; hash: string; } -import type { Logger } from "src/tracing/logger"; +export interface StoredDocumentMetadata { + relativePath: RelativePath; + documentId: DocumentId; + parentVersionId: VaultUpdateId; + hash: string; +} export interface StoredDatabase { - documents: Record; + documents: StoredDocumentMetadata[]; lastSeenUpdateId: VaultUpdateId | undefined; } -export class Database { - private documents = new Map(); +/** + * Represents a document in the database. + * + * It is mutable and its content should always represent the latest + * state of the document on disk based on the update events we have seen. + */ +export interface DocumentRecord { + relativePath: RelativePath; + documentId: DocumentId; + metadata: DocumentMetadata | undefined; + isDeleted: boolean; + updates: Promise[]; + parallelVersion: number; +} +export class Database { + private documents: DocumentRecord[]; private lastSeenUpdateId: VaultUpdateId | undefined; public constructor( @@ -26,16 +46,21 @@ export class Database { private readonly saveData: (data: StoredDatabase) => Promise ) { initialState ??= {}; - if (initialState.documents) { - for (const [relativePath, metadata] of Object.entries( - initialState.documents - )) { - this.documents.set(relativePath, metadata); - } - } - this.ensureConsistency(); - this.logger.debug(`Loaded ${this.documents.size} documents`); + this.documents = + initialState.documents?.map( + ({ relativePath, documentId, ...metadata }) => ({ + relativePath, + documentId, + metadata, + isDeleted: false, + updates: [], + parallelVersion: 0 + }) + ) ?? []; + + this.ensureConsistency(); + this.logger.debug(`Loaded ${this.documents.length} documents`); this.lastSeenUpdateId = initialState.lastSeenUpdateId; this.logger.debug( @@ -43,109 +68,213 @@ export class Database { ); } - public getDocuments(): Map { - return this.documents; + public get length(): number { + return this.documents.length; + } + + public get resolvedDocuments(): DocumentRecord[] { + const paths = new Map(); + this.documents + .filter(({ metadata }) => metadata !== undefined) + .forEach((record) => + paths.set(record.relativePath, [ + record, + ...(paths.get(record.relativePath) ?? []) + ]) + ); + + return Array.from(paths.values()).map((records) => { + records.sort( + (a, b) => b.parallelVersion - a.parallelVersion // descending + ); + + if ( + records.length > 1 && + records.some((current, i) => + i === 0 + ? false + : records[i - 1].parallelVersion === + current.parallelVersion + ) + ) { + throw new Error( + `Multiple documents with the same parallel version and path at ${records[0].relativePath}` + ); + } + return records[0]; + }); } public getLastSeenUpdateId(): VaultUpdateId | undefined { return this.lastSeenUpdateId; } - public async setLastSeenUpdateId( - value: VaultUpdateId | undefined - ): Promise { + public setLastSeenUpdateId(value: VaultUpdateId | undefined): void { this.lastSeenUpdateId = value; - await this.save(); + this.save(); } - public async resetSyncState(): Promise { - this.documents = new Map(); + public resetSyncState(): void { + this.documents = []; this.lastSeenUpdateId = 0; - await this.save(); + this.save(); + } + + public updateDocumentMetadata( + metadata: { + parentVersionId: VaultUpdateId; + hash: string; + }, + toUpdate: DocumentRecord + ): void { + if (!this.documents.includes(toUpdate)) { + throw new Error("Document not found in database"); + } + + toUpdate.metadata = metadata; + + this.save(); + } + + public removeDocumentPromise(promise: Promise): void { + const entry = this.documents.find(({ updates }) => + updates.includes(promise) + ); + + if (entry === undefined) { + throw new Error("Document not found by update promise"); + } + + entry.updates = entry.updates.filter((update) => update !== promise); + // No need to save as Promises don't get serialized + } + + public getLatestDocumentByRelativePath( + find: RelativePath + ): DocumentRecord | undefined { + const candidates = this.documents.filter( + ({ relativePath }) => relativePath === find + ); + candidates.sort((a, b) => b.parallelVersion - a.parallelVersion); // descending + return candidates[0]; + } + + public async getResolvedDocumentByRelativePath( + relativePath: RelativePath, + promise: Promise + ): Promise { + const entry = this.getLatestDocumentByRelativePath(relativePath); + + if (entry === undefined) { + throw new Error( + `Document not found by relative path: ${relativePath}, ${JSON.stringify( + this.documents, + null, + 2 + )}` + ); + } + + const currentPromises = entry.updates; + entry.updates = [...currentPromises, promise]; + await Promise.all(currentPromises); + + return entry; + } + + public createNewPendingDocument( + documentId: DocumentId, + relativePath: RelativePath, + promise: Promise + ): DocumentRecord { + const previousEntry = + this.getLatestDocumentByRelativePath(relativePath); + + const entry = { + relativePath, + documentId, + metadata: undefined, + isDeleted: false, + updates: [promise], + parallelVersion: + previousEntry?.parallelVersion === undefined + ? 0 + : previousEntry.parallelVersion + 1 + }; + + this.documents.push(entry); + this.save(); + + return entry; } public getDocumentByDocumentId( - documentId: DocumentId - ): [RelativePath, DocumentMetadata] | undefined { - return [...this.documents.entries()].find( - ([_, metadata]) => metadata.documentId === documentId - ); + find: DocumentId + ): DocumentRecord | undefined { + return this.documents.find(({ documentId }) => documentId === find); } - public async setDocument({ - documentId, - relativePath, - parentVersionId, - hash - }: { - documentId: DocumentId; - relativePath: RelativePath; - parentVersionId: VaultUpdateId; - hash: string; - }): Promise { - this.documents.set(relativePath, { - documentId, - parentVersionId, - hash - }); - await this.save(); - } - - public async removeDocument(relativePath: RelativePath): Promise { - this.documents.delete(relativePath); - await this.save(); - } - - public getDocument( - relativePath: RelativePath - ): DocumentMetadata | undefined { - return this.documents.get(relativePath); - } - - public async deleteDocument(relativePath: RelativePath): Promise { - this.documents.delete(relativePath); - await this.save(); - } - - public async updatePath( + public move( oldRelativePath: RelativePath, newRelativePath: RelativePath - ): Promise { - const document = this.documents.get(oldRelativePath); - if (!document) { + ): void { + const oldDocument = + this.getLatestDocumentByRelativePath(oldRelativePath); + + if (oldDocument === undefined) { + return; + } + + const newDocument = + this.getLatestDocumentByRelativePath(newRelativePath); + if (newDocument?.isDeleted === false) { throw new Error( - `Cannot update physical path for document that does not exist: ${oldRelativePath}` + `Document already exists at new location: ${newRelativePath}` ); } - if (this.documents.has(newRelativePath)) { - throw new Error( - `Cannot update physical path to path that is already in use: ${newRelativePath}` - ); - } + oldDocument.relativePath = newRelativePath; + // We're in a strange state where the target of the move has just got deleted, + // however, its metadata might already have a bunch of updates queued up for + // the document at the new location. We need to keep these updates. + oldDocument.parallelVersion = + newDocument !== undefined ? newDocument.parallelVersion + 1 : 0; - this.documents.delete(oldRelativePath); - this.documents.set(newRelativePath, document); - - await this.save(); + this.save(); } - private async save(): Promise { + public delete(relativePath: RelativePath): void { + const candidate = this.getLatestDocumentByRelativePath(relativePath); + if (candidate === undefined) { + throw new Error( + `Document not found by relative path: ${relativePath}` + ); + } + candidate.isDeleted = true; + } + + private save(): void { this.ensureConsistency(); - await this.saveData({ - documents: Object.fromEntries(this.documents.entries()), + void this.saveData({ + documents: this.resolvedDocuments.map( + ({ relativePath, documentId, metadata }) => ({ + documentId, + relativePath, + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + ...metadata! // resolvedDocuments only returns docs with metadata set + }) + ), lastSeenUpdateId: this.lastSeenUpdateId }); } private ensureConsistency(): void { - const allMetadata = Array.from(this.documents.entries()); - const idToPath = new Map>(); + const idToPath = new Map(); - allMetadata.forEach(([name, metadata]) => { - idToPath.set(metadata.documentId, [ - ...(idToPath.get(metadata.documentId) ?? []), - name + this.resolvedDocuments.forEach(({ relativePath, documentId }) => { + idToPath.set(documentId, [ + ...(idToPath.get(documentId) ?? []), + relativePath ]); }); diff --git a/frontend/sync-client/src/persistence/settings.ts b/frontend/sync-client/src/persistence/settings.ts index 29a77ff..dbeb8c1 100644 --- a/frontend/sync-client/src/persistence/settings.ts +++ b/frontend/sync-client/src/persistence/settings.ts @@ -1,5 +1,5 @@ -import type { Logger } from "src/tracing/logger"; -import { LogLevel } from "src/tracing/logger"; +import type { Logger } from "../tracing/logger"; +import { LogLevel } from "../tracing/logger"; export interface SyncSettings { remoteUri: string; diff --git a/frontend/sync-client/src/services/connected-state.ts b/frontend/sync-client/src/services/connected-state.ts new file mode 100644 index 0000000..4b62b79 --- /dev/null +++ b/frontend/sync-client/src/services/connected-state.ts @@ -0,0 +1,51 @@ +import type { Settings } from "../persistence/settings"; +import type { Logger } from "../tracing/logger"; +import { createPromise } from "../utils/create-promise"; +import { retriedFetchFactory } from "../utils/retried-fetch"; + +export class ConnectedState { + private resolveIsSyncEnabled: (() => void) | undefined; + private syncIsEnabled: Promise | undefined; + + public constructor( + settings: Settings, + private readonly logger: Logger + ) { + settings.addOnSettingsChangeHandlers((newSettings, oldSettings) => { + if (!oldSettings.isSyncEnabled && newSettings.isSyncEnabled) { + this.handleComingOnline(); + } else if ( + oldSettings.isSyncEnabled && + !newSettings.isSyncEnabled + ) { + this.handleGoingOffline(); + } + }); + } + + public getFetchImplementation( + fetch: typeof globalThis.fetch, + { doRetries = true }: { doRetries: boolean } = { doRetries: true } + ): typeof globalThis.fetch { + const retriedFetch = doRetries + ? retriedFetchFactory(this.logger, fetch) + : fetch; + + return async (input: RequestInfo | URL): Promise => { + if (this.syncIsEnabled !== undefined) { + await this.syncIsEnabled; + } + return retriedFetch(input); + }; + } + + private handleComingOnline(): void { + this.logger.debug("Sync is enabled"); + this.resolveIsSyncEnabled?.(); + } + + private handleGoingOffline(): void { + this.logger.debug("Sync is disabled"); + [this.syncIsEnabled, this.resolveIsSyncEnabled] = createPromise(); + } +} diff --git a/frontend/sync-client/src/services/sync-service.ts b/frontend/sync-client/src/services/sync-service.ts index 56e8a69..74954cf 100644 --- a/frontend/sync-client/src/services/sync-service.ts +++ b/frontend/sync-client/src/services/sync-service.ts @@ -6,20 +6,22 @@ import type { RelativePath, VaultUpdateId } from "../persistence/database"; -import type { Logger } from "src/tracing/logger"; -import { retriedFetchFactory } from "src/utils/retried-fetch"; -import type { Settings } from "src/persistence/settings"; +import type { Logger } from "../tracing/logger"; +import type { Settings } from "../persistence/settings"; +import type { ConnectedState } from "./connected-state"; export interface CheckConnectionResult { isSuccessful: boolean; message: string; } + export class SyncService { private client!: Client; private clientWithoutRetries!: Client; private _fetchImplementation: typeof globalThis.fetch = globalThis.fetch; public constructor( + private readonly connectedState: ConnectedState, private readonly settings: Settings, private readonly logger: Logger ) { @@ -52,17 +54,19 @@ export class SyncService { } public async create({ + documentId, relativePath, - contentBytes, - createdDate + contentBytes }: { + documentId?: DocumentId; relativePath: RelativePath; contentBytes: Uint8Array; - createdDate: Date; - }): Promise { + }): Promise { const formData = new FormData(); + if (documentId !== undefined) { + formData.append("document_id", documentId); + } formData.append("relative_path", relativePath); - formData.append("created_date", createdDate.toISOString()); formData.append("content", new Blob([contentBytes])); const response = await this.client.POST( @@ -100,18 +104,18 @@ export class SyncService { parentVersionId, documentId, relativePath, - contentBytes, - createdDate + contentBytes }: { parentVersionId: VaultUpdateId; documentId: DocumentId; relativePath: RelativePath; contentBytes: Uint8Array; - createdDate: Date; }): Promise { + this.logger.debug( + `Updating document ${documentId} with parent version ${parentVersionId} and relative path ${relativePath}` + ); const formData = new FormData(); formData.append("parent_version_id", parentVersionId.toString()); - formData.append("created_date", createdDate.toISOString()); formData.append("relative_path", relativePath); formData.append("content", new Blob([contentBytes])); @@ -149,13 +153,11 @@ export class SyncService { public async delete({ documentId, - relativePath, - createdDate + relativePath }: { documentId: DocumentId; relativePath: RelativePath; - createdDate: Date; - }): Promise { + }): Promise { const response = await this.client.DELETE( "/vaults/{vault_id}/documents/{document_id}", { @@ -169,7 +171,6 @@ export class SyncService { } }, body: { - createdDate: createdDate.toISOString(), relativePath } } @@ -295,11 +296,17 @@ export class SyncService { private createClient(remoteUri: string): void { this.client = createClient({ baseUrl: remoteUri, - fetch: retriedFetchFactory(this.logger, this._fetchImplementation) + fetch: this.connectedState.getFetchImplementation( + this._fetchImplementation + ) }); this.clientWithoutRetries = createClient({ - baseUrl: remoteUri + baseUrl: remoteUri, + fetch: this.connectedState.getFetchImplementation( + this._fetchImplementation, + { doRetries: false } + ) }); } } diff --git a/frontend/sync-client/src/services/types.ts b/frontend/sync-client/src/services/types.ts index 79d4b5f..e8a954f 100644 --- a/frontend/sync-client/src/services/types.ts +++ b/frontend/sync-client/src/services/types.ts @@ -274,12 +274,13 @@ export interface paths { }; }; responses: { - /** @description no content */ 200: { headers: { [name: string]: unknown; }; - content?: never; + content: { + "application/json": components["schemas"]["DocumentVersionWithoutContent"]; + }; }; default: { headers: { @@ -451,26 +452,25 @@ export interface components { Array_of_uint8: number[]; CreateDocumentVersion: { contentBase64: string; - /** Format: date-time */ - createdDate: string; + /** + * Format: uuid + * @description The client can decide the document id (if it wishes to) in order to help with syncing. If the client does not provide a document id, the server will generate one. If the client provides a document id it must not already exist in the database. + */ + documentId?: string | null; relativePath: string; }; CreateDocumentVersionMultipart: { content: components["schemas"]["Array_of_uint8"]; - /** Format: date-time */ - created_date: string; + /** Format: uuid */ + document_id?: string | null; relative_path: string; }; DeleteDocumentVersion: { - /** Format: date-time */ - createdDate: string; relativePath: string; }; - /** @description Response to a update document request. */ + /** @description Response to an update document request. */ DocumentUpdateResponse: | { - /** Format: date-time */ - createdDate: string; /** Format: uuid */ documentId: string; isDeleted: boolean; @@ -479,14 +479,11 @@ export interface components { type: "FastForwardUpdate"; /** Format: date-time */ updatedDate: string; - vaultId: string; /** Format: int64 */ vaultUpdateId: number; } | { contentBase64: string; - /** Format: date-time */ - createdDate: string; /** Format: uuid */ documentId: string; isDeleted: boolean; @@ -495,34 +492,27 @@ export interface components { type: "MergingUpdate"; /** Format: date-time */ updatedDate: string; - vaultId: string; /** Format: int64 */ vaultUpdateId: number; }; DocumentVersion: { contentBase64: string; - /** Format: date-time */ - createdDate: string; /** Format: uuid */ documentId: string; isDeleted: boolean; relativePath: string; /** Format: date-time */ updatedDate: string; - vaultId: string; /** Format: int64 */ vaultUpdateId: number; }; DocumentVersionWithoutContent: { - /** Format: date-time */ - createdDate: string; /** Format: uuid */ documentId: string; isDeleted: boolean; relativePath: string; /** Format: date-time */ updatedDate: string; - vaultId: string; /** Format: int64 */ vaultUpdateId: number; }; @@ -587,16 +577,12 @@ export interface components { }; UpdateDocumentVersion: { contentBase64: string; - /** Format: date-time */ - createdDate: string; /** Format: int64 */ parentVersionId: number; relativePath: string; }; UpdateDocumentVersionMultipart: { content: components["schemas"]["Array_of_uint8"]; - /** Format: date-time */ - createdDate: string; /** Format: int64 */ parentVersionId: number; relativePath: string; diff --git a/frontend/sync-client/src/sync-client.ts b/frontend/sync-client/src/sync-client.ts index 302daf3..dfd366c 100644 --- a/frontend/sync-client/src/sync-client.ts +++ b/frontend/sync-client/src/sync-client.ts @@ -12,6 +12,7 @@ import { SyncService } from "./services/sync-service"; import { Syncer } from "./sync-operations/syncer"; import type { FileSystemOperations } from "./file-operations/filesystem-operations"; import { FileOperations } from "./file-operations/file-operations"; +import { ConnectedState } from "./services/connected-state"; export class SyncClient { private remoteListenerIntervalId: NodeJS.Timeout | null = null; @@ -42,7 +43,7 @@ export class SyncClient { } public get documentCount(): number { - return this._database.getDocuments().size; + return this._database.length; } public set fetchImplementation(fetch: typeof globalThis.fetch) { @@ -90,7 +91,9 @@ export class SyncClient { } ); - const syncService = new SyncService(settings, logger); + const connectedState = new ConnectedState(settings, logger); + + const syncService = new SyncService(connectedState, settings, logger); const syncer = new Syncer( logger, @@ -117,18 +120,13 @@ export class SyncClient { ); settings.addOnSettingsChangeHandlers((newSettings, oldSettings) => { - client.registerRemoteEventListener( - newSettings.fetchChangesUpdateIntervalMs - ); - - if (!oldSettings.isSyncEnabled && newSettings.isSyncEnabled) { - syncer - .scheduleSyncForOfflineChanges() - .catch((_error: unknown) => { - logger.error( - "Failed to schedule sync for offline changes" - ); - }); + if ( + newSettings.fetchChangesUpdateIntervalMs !== + oldSettings.fetchChangesUpdateIntervalMs + ) { + client.registerRemoteEventListener( + newSettings.fetchChangesUpdateIntervalMs + ); } }); @@ -148,7 +146,7 @@ export class SyncClient { this.stop(); await this._syncer.reset(); this._history.reset(); - await this._database.resetSyncState(); + this._database.resetSyncState(); this.logger.reset(); } diff --git a/frontend/sync-client/src/sync-operations/syncer.ts b/frontend/sync-client/src/sync-operations/syncer.ts index dcb476d..70ba88d 100644 --- a/frontend/sync-client/src/sync-operations/syncer.ts +++ b/frontend/sync-client/src/sync-operations/syncer.ts @@ -1,15 +1,17 @@ import type { Database, RelativePath } from "../persistence/database"; - -import type { SyncService } from "src/services/sync-service"; -import type { Logger } from "src/tracing/logger"; -import type { SyncHistory } from "src/tracing/sync-history"; +import type { SyncService } from "../services/sync-service"; +import type { Logger } from "../tracing/logger"; +import type { SyncHistory } from "../tracing/sync-history"; import PQueue from "p-queue"; -import { hash } from "src/utils/hash"; -import type { components } from "src/services/types"; -import type { Settings } from "src/persistence/settings"; -import type { FileOperations } from "src/file-operations/file-operations"; -import { findMatchingFileBasedOnHash } from "src/utils/find-matching-file-based-on-hash"; +import { hash } from "../utils/hash"; +import { v4 as uuidv4 } from "uuid"; +import type { components } from "../services/types"; +import type { Settings } from "../persistence/settings"; +import type { FileOperations } from "../file-operations/file-operations"; +import { findMatchingFile } from "../utils/find-matching-file"; import { UnrestrictedSyncer } from "./unrestricted-syncer"; +import { FileNotFoundError } from "../file-operations/safe-filesystem-operations"; +import { createPromise } from "../utils/create-promise"; export class Syncer { private readonly remainingOperationsListeners: (( @@ -18,17 +20,15 @@ export class Syncer { private readonly syncQueue: PQueue; - private runningScheduleSyncForOfflineChanges: Promise | undefined = - undefined; - private runningApplyRemoteChangesLocally: Promise | undefined = - undefined; + private runningScheduleSyncForOfflineChanges: Promise | undefined; + private runningApplyRemoteChangesLocally: Promise | undefined; private readonly internalSyncer: UnrestrictedSyncer; public constructor( private readonly logger: Logger, private readonly database: Database, - private readonly settings: Settings, + settings: Settings, private readonly syncService: SyncService, private readonly operations: FileOperations, history: SyncHistory @@ -45,7 +45,9 @@ export class Syncer { }); this.syncQueue.on("active", () => { - this.emitRemainingOperationsChange(this.syncQueue.size); + this.remainingOperationsListeners.forEach((listener) => { + listener(this.syncQueue.size); + }); }); this.internalSyncer = new UnrestrictedSyncer( @@ -65,48 +67,131 @@ export class Syncer { } public async syncLocallyCreatedFile( - relativePath: RelativePath, - updateTime: Date + relativePath: RelativePath ): Promise { - await this.syncQueue.add(async () => - this.internalSyncer.unrestrictedSyncLocallyCreatedFile( - relativePath, - updateTime - ) - ); - } + if ( + this.database.getLatestDocumentByRelativePath(relativePath) + ?.isDeleted === false + ) { + this.logger.debug( + `Document ${relativePath} already exists in the database, skipping` + ); + return; + } - public async syncLocallyUpdatedFile(args: { - oldPath?: RelativePath; - relativePath: RelativePath; - updateTime: Date; - }): Promise { - await this.syncQueue.add(async () => - this.internalSyncer.unrestrictedSyncLocallyUpdatedFile(args) - ); - } + const [promise, resolve, reject] = createPromise(); - public async waitForSyncQueue(): Promise { - return this.syncQueue.onEmpty(); + const document = this.database.createNewPendingDocument( + uuidv4(), + relativePath, + promise + ); + + try { + await this.syncQueue.add(async () => + this.internalSyncer.unrestrictedSyncLocallyCreatedFile(document) + ); + + resolve(); + } catch (e) { + reject(e); + } finally { + this.database.removeDocumentPromise(promise); + } } public async syncLocallyDeletedFile( relativePath: RelativePath ): Promise { - await this.syncQueue.add(async () => - this.internalSyncer.unrestrictedSyncLocallyDeletedFile(relativePath) + // We have to have a record of the delete in case there's an in-flight update for the same + // document which finishes after the delete has succeeded and would introduce a phantom metadata record. + this.database.delete(relativePath); + + const [promise, resolve, reject] = createPromise(); + + const document = await this.database.getResolvedDocumentByRelativePath( + relativePath, + promise ); + + try { + await this.syncQueue.add(async () => + this.internalSyncer.unrestrictedSyncLocallyDeletedFile(document) + ); + + resolve(); + } catch (e) { + reject(e); + } finally { + this.database.removeDocumentPromise(promise); + } } - public async scheduleSyncForOfflineChanges(): Promise { - if (!this.settings.getSettings().isSyncEnabled) { + public async syncLocallyUpdatedFile({ + oldPath, + relativePath + }: { + oldPath?: RelativePath; + relativePath: RelativePath; + }): Promise { + if ( + oldPath !== undefined && + (this.database.getLatestDocumentByRelativePath(relativePath) === + undefined || + this.database.getLatestDocumentByRelativePath(relativePath) + ?.isDeleted === true) + ) { + if (oldPath === relativePath) { + throw new Error( + `Old path and new path are the same: ${oldPath}` + ); + } + + this.database.move(oldPath, relativePath); + } + + let document = + this.database.getLatestDocumentByRelativePath(relativePath); + + if (document === undefined) { this.logger.debug( - `Syncing is disabled, not uploading local changes` + `Cannot find document ${relativePath} in the database, skipping` ); return; } - if (this.runningScheduleSyncForOfflineChanges != null) { + if (document.isDeleted) { + this.logger.debug( + `Document ${relativePath} has been deleted locally, skipping` + ); + return; + } + + const [promise, resolve, reject] = createPromise(); + + document = await this.database.getResolvedDocumentByRelativePath( + relativePath, + promise + ); + + try { + await this.syncQueue.add(async () => + this.internalSyncer.unrestrictedSyncLocallyUpdatedFile({ + oldPath, + document + }) + ); + + resolve(); + } catch (e) { + reject(e); + } finally { + this.database.removeDocumentPromise(promise); + } + } + + public async scheduleSyncForOfflineChanges(): Promise { + if (this.runningScheduleSyncForOfflineChanges !== undefined) { this.logger.debug("Uploading local changes is already in progress"); return this.runningScheduleSyncForOfflineChanges; } @@ -127,13 +212,6 @@ export class Syncer { } public async applyRemoteChangesLocally(): Promise { - if (!this.settings.getSettings().isSyncEnabled) { - this.logger.debug( - `Syncing is disabled, not fetching remote changes` - ); - return; - } - if (this.runningApplyRemoteChangesLocally != null) { this.logger.debug( "Applying remote changes locally is already in progress" @@ -154,6 +232,10 @@ export class Syncer { } } + public async waitForSyncQueue(): Promise { + return this.syncQueue.onEmpty(); + } + public async reset(): Promise { this.syncQueue.clear(); await this.syncQueue.onEmpty(); @@ -163,115 +245,15 @@ export class Syncer { this.internalSyncer.reset(); } - private async syncRemotelyUpdatedFile( - remoteVersion: components["schemas"]["DocumentVersionWithoutContent"] - ): Promise { - await this.syncQueue.add(async () => - this.internalSyncer.unrestrictedSyncRemotelyUpdatedFile( - remoteVersion - ) - ); - } - - private async internalScheduleSyncForOfflineChanges(): Promise { - const allLocalFiles = await this.operations.listAllFiles(); - - // This includes renamed files for now - let locallyPossiblyDeletedFiles = [ - ...this.database.getDocuments().entries() - ].filter(([path, _]) => !allLocalFiles.includes(path)); - - await Promise.all( - allLocalFiles.map(async (relativePath) => - this.syncQueue.add(async () => { - const metadata = this.database.getDocument(relativePath); - - if (metadata) { - this.logger.debug( - `Document ${relativePath} might have been updated locally, scheduling sync to validate and update it` - ); - return this.internalSyncer.unrestrictedSyncLocallyUpdatedFile( - { - relativePath, - updateTime: - await this.operations.getModificationTime( - relativePath - ) - } - ); - } - - // Perhaps the file has been moved. Let's check by looking at the deleted files - const contentBytes = - await this.operations.read(relativePath); - const contentHash = hash(contentBytes); - - // todo: make this smarter so that offline files can be renamed & edited at the same time - const originalFile = findMatchingFileBasedOnHash( - contentHash, - locallyPossiblyDeletedFiles - ); - if (originalFile !== undefined) { - // `originalFile` hasn't been deleted but it got moved instead - locallyPossiblyDeletedFiles = - locallyPossiblyDeletedFiles.filter( - (item) => item[0] !== originalFile[0] - ); - - this.logger.debug( - `Document '${originalFile[0]}' was not found under its current path in the database but was found under a different path (${relativePath}), scheduling sync to move it` - ); - return this.internalSyncer.unrestrictedSyncLocallyUpdatedFile( - { - oldPath: originalFile[0], - relativePath: relativePath, - updateTime: - await this.operations.getModificationTime( - relativePath - ), - optimisations: { - contentBytes, - contentHash - } - } - ); - } - - this.logger.debug( - `Document ${relativePath} not found in database, scheduling sync to create it` - ); - return this.internalSyncer.unrestrictedSyncLocallyCreatedFile( - relativePath, - await this.operations.getModificationTime(relativePath) - ); - }) - ) - ); - - await Promise.all( - locallyPossiblyDeletedFiles.map(async ([relativePath, _]) => { - this.logger.debug( - `Document ${relativePath} has been deleted locally, scheduling sync to delete it` - ); - - if (await this.operations.exists(relativePath)) { - this.logger.debug( - `Document ${relativePath} actually exists locally, skipping` - ); - return Promise.resolve(); - } - - // We're outside of the pqueue, so we need to call the public wrapper - return this.syncLocallyDeletedFile(relativePath); - }) - ); - } - private async internalApplyRemoteChangesLocally(): Promise { - const remote = await this.syncService.getAll( - this.database.getLastSeenUpdateId() + const remote = await this.syncQueue.add(async () => + this.syncService.getAll(this.database.getLastSeenUpdateId()) ); + if (!remote) { + throw new Error("Failed to fetch remote changes"); + } + if (remote.latestDocuments.length === 0) { this.logger.debug("No remote changes to apply"); return; @@ -280,9 +262,7 @@ export class Syncer { this.logger.info("Applying remote changes locally"); await Promise.all( - remote.latestDocuments.map(async (remoteDocument) => - this.syncRemotelyUpdatedFile(remoteDocument) - ) + remote.latestDocuments.map(this.syncRemotelyUpdatedFile.bind(this)) ); const lastSeenUpdateId = this.database.getLastSeenUpdateId(); @@ -290,13 +270,124 @@ export class Syncer { lastSeenUpdateId === undefined || remote.lastUpdateId > lastSeenUpdateId ) { - await this.database.setLastSeenUpdateId(remote.lastUpdateId); + this.database.setLastSeenUpdateId(remote.lastUpdateId); } } - private emitRemainingOperationsChange(remainingOperations: number): void { - this.remainingOperationsListeners.forEach((listener) => { - listener(remainingOperations); - }); + private async syncRemotelyUpdatedFile( + remoteVersion: components["schemas"]["DocumentVersionWithoutContent"] + ): Promise { + let document = this.database.getDocumentByDocumentId( + remoteVersion.documentId + ); + + const [promise, resolve, reject] = createPromise(); + + if (document === undefined) { + await this.syncQueue.add(async () => + this.internalSyncer.unrestrictedSyncRemotelyUpdatedFile( + remoteVersion + ) + ); + } else { + document = await this.database.getResolvedDocumentByRelativePath( + document.relativePath, + promise + ); + + try { + await this.syncQueue.add(async () => + this.internalSyncer.unrestrictedSyncRemotelyUpdatedFile( + remoteVersion, + document + ) + ); + + resolve(); + } catch (e) { + reject(e); + } finally { + this.database.removeDocumentPromise(promise); + } + } + } + + private async internalScheduleSyncForOfflineChanges(): Promise { + const allLocalFiles = await this.operations.listAllFiles(); + + let locallyPossiblyDeletedFiles = [ + ...this.database.resolvedDocuments + ].filter(({ relativePath }) => !allLocalFiles.includes(relativePath)); + + const updates = Promise.all( + allLocalFiles.map(async (relativePath) => { + if ( + this.database.getLatestDocumentByRelativePath(relativePath) + ?.metadata !== undefined + ) { + this.logger.debug( + `Document ${relativePath} might have been updated locally, scheduling sync to validate and update it` + ); + + return this.syncLocallyUpdatedFile({ + relativePath + }); + } + + // Perhaps the file has been moved; let's check by looking at the deleted files + const contentHash = await this.syncQueue.add(async () => { + const contentBytes = + await this.operations.read(relativePath); // this can throw FileNotFoundError + return hash(contentBytes); + }); + + if (contentHash == undefined) { + // The file was deleted before we had a chance to read it, no need to sync it here + return; + } + + const originalFile = findMatchingFile( + contentHash, + locallyPossiblyDeletedFiles + ); + if (originalFile !== undefined) { + // `originalFile` hasn't been deleted but it got moved instead + locallyPossiblyDeletedFiles = + locallyPossiblyDeletedFiles.filter( + (item) => + item.relativePath !== originalFile.relativePath + ); + + this.logger.debug( + `Document '${originalFile.relativePath}' was not found under its current path in the database but was found under a different path (${relativePath}), scheduling sync to move it` + ); + + // We're outside of the pqueue, so we need to call the public wrapper + return this.syncLocallyUpdatedFile({ + oldPath: originalFile.relativePath, + relativePath + }); + } + + this.logger.debug( + `Document ${relativePath} not found in database, scheduling sync to create it` + ); + // We're outside of the pqueue, so we need to call the public wrapper + return this.syncLocallyCreatedFile(relativePath); + }) + ); + + const deletes = Promise.all( + locallyPossiblyDeletedFiles.map(async ({ relativePath }) => { + this.logger.debug( + `Document ${relativePath} has been deleted locally, scheduling sync to delete it` + ); + + // We're outside of the pqueue, so we need to call the public wrapper + return this.syncLocallyDeletedFile(relativePath); + }) + ); + + await Promise.all([updates, deletes]); } } diff --git a/frontend/sync-client/src/sync-operations/unrestricted-syncer.ts b/frontend/sync-client/src/sync-operations/unrestricted-syncer.ts index 86ed308..fe268f4 100644 --- a/frontend/sync-client/src/sync-operations/unrestricted-syncer.ts +++ b/frontend/sync-client/src/sync-operations/unrestricted-syncer.ts @@ -1,19 +1,24 @@ -import type { Database, RelativePath } from "../persistence/database"; +import type { + Database, + DocumentRecord, + RelativePath +} from "../persistence/database"; -import type { SyncService } from "src/services/sync-service"; -import type { Logger } from "src/tracing/logger"; -import type { SyncHistory } from "src/tracing/sync-history"; -import { SyncSource, SyncStatus, SyncType } from "src/tracing/sync-history"; -import { hash } from "src/utils/hash"; -import type { components } from "src/services/types"; -import { deserialize } from "src/utils/deserialize"; -import type { Settings } from "src/persistence/settings"; -import type { FileOperations } from "src/file-operations/file-operations"; -import { FileNotFoundError } from "src/file-operations/safe-filesystem-operations"; -import { DocumentLocks } from "./document-locks"; +import type { SyncService } from "../services/sync-service"; +import type { Logger } from "../tracing/logger"; +import type { SyncHistory } from "../tracing/sync-history"; +import { SyncSource, SyncStatus, SyncType } from "../tracing/sync-history"; +import { EMPTY_HASH, hash } from "../utils/hash"; +import type { components } from "../services/types"; +import { deserialize } from "../utils/deserialize"; +import type { Settings } from "../persistence/settings"; +import type { FileOperations } from "../file-operations/file-operations"; +import { FileNotFoundError } from "../file-operations/safe-filesystem-operations"; +import { DocumentLocks } from "../file-operations/document-locks"; +import { createPromise } from "../utils/create-promise"; export class UnrestrictedSyncer { - private readonly locks = new DocumentLocks(); + private readonly locks: DocumentLocks; public constructor( private readonly logger: Logger, @@ -22,507 +27,375 @@ export class UnrestrictedSyncer { private readonly syncService: SyncService, private readonly operations: FileOperations, private readonly history: SyncHistory - ) {} + ) { + this.locks = new DocumentLocks(logger); + } public async unrestrictedSyncLocallyCreatedFile( - relativePath: RelativePath, - updateTime: Date, - optimisations?: { - contentBytes?: Uint8Array; - contentHash?: string; - } + document: DocumentRecord ): Promise { - await this.executeWhileHoldingFileLock( - [relativePath], + return this.executeSync( + document.relativePath, SyncType.CREATE, SyncSource.PUSH, async () => { - if ( - (await this.operations.getFileSize(relativePath)) / // this can throw FileNotFoundError - 1024 / - 1024 > - this.settings.getSettings().maxFileSizeMB - ) { - this.history.addHistoryEntry({ - status: SyncStatus.ERROR, - relativePath, - message: `File size exceeds the maximum file size limit of ${ - this.settings.getSettings().maxFileSizeMB - }MB`, - type: SyncType.CREATE - }); - return; - } - - const contentBytes = - optimisations?.contentBytes ?? - (await this.operations.read(relativePath)); // this can throw FileNotFoundError - let contentHash = - optimisations?.contentHash ?? hash(contentBytes); - - const localMetadata = this.database.getDocument(relativePath); - if (localMetadata) { - this.logger.debug( - `Document metadata already exists for ${relativePath}, it must have been downloaded from the server` - ); - - if (localMetadata.hash === contentHash) { - this.history.addHistoryEntry({ - status: SyncStatus.NO_OP, - relativePath, - message: `File hash matches with last synced version, no need to sync`, - type: SyncType.UPDATE - }); - return; - } - } + const contentBytes = await this.operations.read( + document.relativePath + ); // this can throw FileNotFoundError + const contentHash = hash(contentBytes); const response = await this.syncService.create({ - relativePath, - contentBytes, - createdDate: updateTime + documentId: document.documentId, + relativePath: document.relativePath, + contentBytes }); this.history.addHistoryEntry({ status: SyncStatus.SUCCESS, source: SyncSource.PUSH, - relativePath, + relativePath: document.relativePath, message: `Successfully uploaded locally created file`, type: SyncType.CREATE }); - // The response can't have a different relative path than the one we sent - // because the relative path is the key when finding existing documents - // when a create request is sent. + this.database.updateDocumentMetadata( + { + parentVersionId: response.vaultUpdateId, + hash: contentHash + }, + document + ); - if (response.type === "MergingUpdate") { - const responseBytes = deserialize(response.contentBase64); - contentHash = hash(responseBytes); + this.tryIncrementVaultUpdateId(response.vaultUpdateId); + } + ); + } - await this.operations.write( - relativePath, - contentBytes, - responseBytes - ); - this.history.addHistoryEntry({ - status: SyncStatus.SUCCESS, - source: SyncSource.PULL, - relativePath, - message: `The file we created locally has already existed remotely, so we have merged them`, - type: SyncType.UPDATE - }); - } - - await this.database.setDocument({ - documentId: response.documentId, - relativePath: response.relativePath, - parentVersionId: response.vaultUpdateId, - hash: contentHash + public async unrestrictedSyncLocallyDeletedFile( + document: DocumentRecord + ): Promise { + await this.executeSync( + document.relativePath, + SyncType.DELETE, + SyncSource.PUSH, + async () => { + const response = await this.syncService.delete({ + documentId: document.documentId, + relativePath: document.relativePath }); - await this.tryIncrementVaultUpdateId(response.vaultUpdateId); + this.history.addHistoryEntry({ + status: SyncStatus.SUCCESS, + source: SyncSource.PUSH, + relativePath: document.relativePath, + message: `Successfully deleted locally deleted file on the remote server`, + type: SyncType.DELETE + }); + + this.database.updateDocumentMetadata( + { + parentVersionId: response.vaultUpdateId, + hash: EMPTY_HASH + }, + document + ); } ); } public async unrestrictedSyncLocallyUpdatedFile({ oldPath, - relativePath, - updateTime, - optimisations + document, + force = false }: { oldPath?: RelativePath; - relativePath: RelativePath; - updateTime: Date; - optimisations?: { - contentBytes?: Uint8Array; - contentHash?: string; - }; + force?: boolean; + document: DocumentRecord; }): Promise { - await this.executeWhileHoldingFileLock( - [oldPath, relativePath].filter((path) => path !== undefined), + await this.executeSync( + document.relativePath, SyncType.UPDATE, SyncSource.PUSH, async () => { - // Check the new path first in case the metadata has been already moved - let localMetadata = this.database.getDocument(relativePath); - let metadataPath = relativePath; + const originalRelativePath = document.relativePath; - if (localMetadata === undefined && oldPath !== undefined) { - localMetadata = this.database.getDocument(oldPath); - metadataPath = oldPath; - } - - if (!localMetadata) { - // It's fine, a subsequent sync operation must have dealt with this + if (document.metadata === undefined || document.isDeleted) { + this.logger.debug( + `Document ${document.relativePath} has been already deleted, no need to update it` + ); return; } - if ( - (await this.operations.getFileSize(relativePath)) / // this can throw FileNotFoundError - 1024 / - 1024 > - this.settings.getSettings().maxFileSizeMB - ) { - this.history.addHistoryEntry({ - status: SyncStatus.ERROR, - relativePath, - message: `File size exceeds the maximum file size limit of ${ - this.settings.getSettings().maxFileSizeMB - }MB`, - type: SyncType.CREATE - }); - return; - } - - const contentBytes = - optimisations?.contentBytes ?? - (await this.operations.read(relativePath)); // this can throw FileNotFoundError - - let contentHash = - optimisations?.contentHash ?? hash(contentBytes); + const contentBytes = await this.operations.read( + document.relativePath + ); // this can throw FileNotFoundError + let contentHash = hash(contentBytes); if ( - localMetadata.hash === contentHash && - oldPath === undefined + document.metadata.hash === contentHash && + oldPath === undefined && + !force ) { - this.history.addHistoryEntry({ - status: SyncStatus.NO_OP, - relativePath, - message: `File hash matches with last synced version, no need to sync`, - type: SyncType.UPDATE - }); + this.logger.debug( + `File hash of ${document.relativePath} matches with last synced version and the path hasn't changed; no need to sync` + ); return; } const response = await this.syncService.put({ - documentId: localMetadata.documentId, - parentVersionId: localMetadata.parentVersionId, - relativePath, - contentBytes, - createdDate: updateTime + documentId: document.documentId, + parentVersionId: document.metadata.parentVersionId, + relativePath: document.relativePath, + contentBytes }); + // `document` is mutable and reflects the latest state in the local database + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition + if (document.isDeleted) { + this.logger.info( + `Document ${document.relativePath} has been deleted before we could finish updating it` + ); + return; + } + + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition + if (document.metadata === undefined) { + throw new Error( + `Document ${document.relativePath} no longer has metadata after updating it, this cannot happen` + ); + } + + if ( + document.metadata.parentVersionId >= response.vaultUpdateId + ) { + this.logger.debug( + `Document ${document.relativePath} is already more up to date than the fetched version` + ); + return; + } + this.history.addHistoryEntry({ status: SyncStatus.SUCCESS, source: SyncSource.PUSH, - relativePath, + relativePath: document.relativePath, message: `Successfully uploaded locally updated file to the remote server`, type: SyncType.UPDATE }); if (response.isDeleted) { - await this.operations.remove(oldPath ?? relativePath); - await this.database.removeDocument(oldPath ?? relativePath); - await this.tryIncrementVaultUpdateId( - response.vaultUpdateId - ); - this.history.addHistoryEntry({ status: SyncStatus.SUCCESS, source: SyncSource.PULL, - relativePath, + relativePath: document.relativePath, message: "The file we tried to update had been deleted remotely, therefore, we have deleted it locally", type: SyncType.DELETE }); + this.database.delete(document.relativePath); + this.database.updateDocumentMetadata( + { + parentVersionId: response.vaultUpdateId, + hash: EMPTY_HASH + }, + document + ); + + await this.operations.delete(document.relativePath); + + this.tryIncrementVaultUpdateId(response.vaultUpdateId); + return; } - if ( - response.relativePath != relativePath && - response.relativePath != oldPath - ) { - await this.locks.waitForDocumentLock(response.relativePath); + let actualPath = document.relativePath; + + if (response.relativePath != originalRelativePath) { + actualPath = response.relativePath; + await this.operations.move( + document.relativePath, + response.relativePath + ); // this can throw FileNotFoundError } - try { - if (response.relativePath != relativePath) { - // TODO: this can fail, that's bad - await this.operations.move( - // this can throw FileNotFoundError - relativePath, - response.relativePath, - response.documentId - ); - } - - if (response.type === "MergingUpdate") { - const responseBytes = deserialize( - response.contentBase64 - ); - contentHash = hash(responseBytes); - - await this.operations.write( - response.relativePath, - contentBytes, - responseBytes - ); - - this.history.addHistoryEntry({ - status: SyncStatus.SUCCESS, - source: SyncSource.PULL, - relativePath, - message: `The file we updated had been updated remotely, so we downloaded the merged version`, - type: SyncType.UPDATE - }); - } - - if (metadataPath !== response.relativePath) { - await this.database.updatePath( - metadataPath, - response.relativePath - ); - } - await this.database.setDocument({ - documentId: localMetadata.documentId, - relativePath: response.relativePath, + this.database.updateDocumentMetadata( + { parentVersionId: response.vaultUpdateId, hash: contentHash - }); + }, + document + ); - await this.tryIncrementVaultUpdateId( - response.vaultUpdateId + if (response.type === "MergingUpdate") { + const responseBytes = deserialize(response.contentBase64); + contentHash = hash(responseBytes); + + await this.operations.write( + actualPath, + contentBytes, + responseBytes ); - } finally { - if ( - response.relativePath != relativePath && - response.relativePath != oldPath - ) { - this.locks.unlockDocument(response.relativePath); - } - } - } - ); - } - public async unrestrictedSyncLocallyDeletedFile( - relativePath: RelativePath - ): Promise { - await this.executeWhileHoldingFileLock( - [relativePath], - SyncType.DELETE, - SyncSource.PUSH, - async () => { - const localMetadata = this.database.getDocument(relativePath); - if (!localMetadata) { this.history.addHistoryEntry({ - status: SyncStatus.NO_OP, - relativePath, - message: `Locally deleted file hasn't been uploaded yet, so there's no need to delete it on the remote server`, - type: SyncType.DELETE + status: SyncStatus.SUCCESS, + source: SyncSource.PULL, + relativePath: document.relativePath, + message: `The file we updated had been updated remotely, so we downloaded the merged version`, + type: SyncType.UPDATE }); - return; } - await this.syncService.delete({ - documentId: localMetadata.documentId, - relativePath, - createdDate: new Date() // We got the event now, so it must have been deleted just now - }); - - this.history.addHistoryEntry({ - status: SyncStatus.SUCCESS, - source: SyncSource.PUSH, - relativePath, - message: `Successfully deleted locally deleted file on the remote server`, - type: SyncType.DELETE - }); - - await this.database.removeDocument(relativePath); + this.tryIncrementVaultUpdateId(response.vaultUpdateId); } ); } public async unrestrictedSyncRemotelyUpdatedFile( - remoteVersion: components["schemas"]["DocumentVersionWithoutContent"] + remoteVersion: components["schemas"]["DocumentVersionWithoutContent"], + document?: DocumentRecord ): Promise { - await this.executeWhileHoldingFileLock( - [remoteVersion.relativePath], + await this.executeSync( + remoteVersion.relativePath, SyncType.UPDATE, SyncSource.PULL, async () => { - let localMetadata = this.database.getDocumentByDocumentId( - remoteVersion.documentId - ); - - if ( - localMetadata && - localMetadata[0] !== remoteVersion.relativePath - ) { - await this.locks.waitForDocumentLock(localMetadata[0]); - } - // Waiting for the new lock might take a while so we need to fetch the database - // entry again in case it's changed. - localMetadata = this.database.getDocumentByDocumentId( - remoteVersion.documentId - ); - - if (!localMetadata) { - if (remoteVersion.isDeleted) { - this.history.addHistoryEntry({ - status: SyncStatus.NO_OP, - source: SyncSource.PULL, - relativePath: remoteVersion.relativePath, - message: `Remotely deleted file hasn't been synced yet, so there's no need to delete it locally`, - type: SyncType.DELETE - }); + if (document?.metadata !== undefined) { + // If the file exists locally, let's pretend the user has updated it + // and deal with remote update/deletion within `unrestrictedSyncLocallyUpdatedFile` + if ( + document.metadata.parentVersionId >= + remoteVersion.vaultUpdateId + ) { + this.logger.debug( + `Document ${remoteVersion.relativePath} is already more up to date than the fetched version` + ); return; } - const content = ( - await this.syncService.get({ - documentId: remoteVersion.documentId - }) - ).contentBase64; - const contentBytes = deserialize(content); - - await this.operations.create( - remoteVersion.relativePath, - contentBytes + return this.unrestrictedSyncLocallyUpdatedFile({ + document, + force: true + }); + } else if (remoteVersion.isDeleted) { + // Either the doc hasn't made it to us before and therefore we don't need to delete it, + // or we already have it, in which case the preceeding if will deal with it + this.logger.debug( + `Document ${remoteVersion.relativePath} has been deleted remotely, no need to sync` ); - await this.database.setDocument({ - documentId: remoteVersion.documentId, - relativePath: remoteVersion.relativePath, + return; + } + + const content = ( + await this.syncService.get({ + documentId: remoteVersion.documentId + }) + ).contentBase64; + + document = this.database.getDocumentByDocumentId( + remoteVersion.documentId + ); + + if (document?.isDeleted === true) { + this.logger.info( + `Document ${remoteVersion.relativePath} has been deleted locally before we could finish updating it` + ); + return; + } + + if ( + (document?.metadata?.parentVersionId ?? -1) >= + remoteVersion.vaultUpdateId + ) { + this.logger.debug( + `Document ${remoteVersion.relativePath} is already more up to date than the fetched version` + ); + return; + } + + const contentBytes = deserialize(content); + + await this.operations.ensureClearPath( + remoteVersion.relativePath + ); + + const [promise, resolve] = createPromise(); + this.database.updateDocumentMetadata( + { parentVersionId: remoteVersion.vaultUpdateId, hash: hash(contentBytes) - }); - this.history.addHistoryEntry({ - status: SyncStatus.SUCCESS, - source: SyncSource.PULL, - relativePath: remoteVersion.relativePath, - message: `Successfully downloaded remote file which hadn't existed locally`, - type: SyncType.CREATE - }); - return; - } + }, + this.database.createNewPendingDocument( + remoteVersion.documentId, + remoteVersion.relativePath, + promise + ) + ); - const [relativePath, metadata] = localMetadata; + await this.operations.create( + remoteVersion.relativePath, + contentBytes + ); - if (remoteVersion.vaultUpdateId <= metadata.parentVersionId) { - this.logger.debug( - `Document ${relativePath} is already up to date` - ); - return; - } + resolve(); + this.database.removeDocumentPromise(promise); - try { - if (remoteVersion.isDeleted) { - await this.operations.remove(relativePath); - await this.database.removeDocument(relativePath); - - this.history.addHistoryEntry({ - status: SyncStatus.SUCCESS, - source: SyncSource.PULL, - relativePath: remoteVersion.relativePath, - message: `Successfully deleted remotely deleted file locally`, - type: SyncType.DELETE - }); - } else { - // TODO: this can fail, that's bad - const currentContent = - await this.operations.read(relativePath); // this can throw FileNotFoundError - const currentHash = hash(currentContent); - - if (currentHash !== metadata.hash) { - this.logger.info( - `Document ${relativePath} has been updated both remotely and locally, letting the local file update event handle it` - ); - return; - } - - const content = ( - await this.syncService.get({ - documentId: remoteVersion.documentId - }) - ).contentBase64; - const contentBytes = deserialize(content); - const contentHash = hash(contentBytes); - - if (relativePath !== remoteVersion.relativePath) { - // TODO: this can fail, that's bad - await this.operations.move( - // this can throw FileNotFoundError - relativePath, - remoteVersion.relativePath, - remoteVersion.documentId - ); - - await this.database.updatePath( - relativePath, - remoteVersion.relativePath - ); - } - - await this.operations.write( - remoteVersion.relativePath, - currentContent, - contentBytes - ); - await this.database.setDocument({ - documentId: remoteVersion.documentId, - relativePath: remoteVersion.relativePath, - parentVersionId: remoteVersion.vaultUpdateId, - hash: contentHash - }); - - this.history.addHistoryEntry({ - status: SyncStatus.SUCCESS, - source: SyncSource.PULL, - relativePath: remoteVersion.relativePath, - message: `Successfully updated remotely updated file locally`, - type: SyncType.UPDATE - }); - } - } finally { - if (relativePath !== remoteVersion.relativePath) { - this.locks.unlockDocument(relativePath); - } - } + this.history.addHistoryEntry({ + status: SyncStatus.SUCCESS, + source: SyncSource.PULL, + relativePath: remoteVersion.relativePath, + message: `Successfully downloaded remote file which hadn't existed locally`, + type: SyncType.CREATE + }); } ); } - public async executeWhileHoldingFileLock( - lockedPaths: RelativePath[], + public async executeSync( + relativePath: RelativePath, syncType: SyncType, syncSource: SyncSource, - fn: () => Promise - ): Promise { - const relativePath = lockedPaths[lockedPaths.length - 1]; - - if (!this.settings.getSettings().isSyncEnabled) { - this.logger.info( - `Syncing is disabled, not syncing ${relativePath}` - ); - return; - } + fn: () => Promise + ): Promise { if (!this.operations.isFileEligibleForSync(relativePath)) { - this.logger.info( - `File ${relativePath} is not eligible for syncing` - ); + this.history.addHistoryEntry({ + status: SyncStatus.ERROR, + relativePath, + message: `File ${relativePath} is not eligible for syncing`, + type: syncType + }); return; } + this.logger.debug( `Syncing ${relativePath} (${syncSource} - ${syncType})` ); - await Promise.all( - lockedPaths.map(this.locks.waitForDocumentLock.bind(this.locks)) - ); try { - await fn(); + if ( + (await this.operations.exists(relativePath)) && + (await this.operations.getFileSize(relativePath)) / // this can throw FileNotFoundError + 1024 / + 1024 > + this.settings.getSettings().maxFileSizeMB + ) { + this.history.addHistoryEntry({ + status: SyncStatus.ERROR, + relativePath, + message: `File size exceeds the maximum file size limit of ${ + this.settings.getSettings().maxFileSizeMB + }MB`, + type: syncType + }); + return; + } + + return await fn(); } catch (e) { if (e instanceof FileNotFoundError) { // A subsequent sync operation must have been creating to deal with this - this.history.addHistoryEntry({ - status: SyncStatus.NO_OP, - relativePath, - message: `Skip ${syncSource.toLocaleLowerCase()} file because it no longer exists when trying to ${syncType.toLocaleLowerCase()} it`, - type: syncType, - source: syncSource - }); + this.logger.info( + `Skip ${syncSource.toLocaleLowerCase()} file because it no longer exists when trying to ${syncType.toLocaleLowerCase()} it` + ); } else { this.history.addHistoryEntry({ status: SyncStatus.ERROR, @@ -533,8 +406,6 @@ export class UnrestrictedSyncer { }); throw e; } - } finally { - lockedPaths.forEach(this.locks.unlockDocument.bind(this.locks)); } } @@ -542,11 +413,9 @@ export class UnrestrictedSyncer { this.locks.reset(); } - private async tryIncrementVaultUpdateId( - responseVaultUpdateId: number - ): Promise { + private tryIncrementVaultUpdateId(responseVaultUpdateId: number): void { if (this.database.getLastSeenUpdateId() === responseVaultUpdateId - 1) { - await this.database.setLastSeenUpdateId(responseVaultUpdateId); + this.database.setLastSeenUpdateId(responseVaultUpdateId); } } } diff --git a/frontend/sync-client/src/tracing/sync-history.ts b/frontend/sync-client/src/tracing/sync-history.ts index ea87bca..ec8841e 100644 --- a/frontend/sync-client/src/tracing/sync-history.ts +++ b/frontend/sync-client/src/tracing/sync-history.ts @@ -1,4 +1,4 @@ -import type { RelativePath } from "src/persistence/database"; +import type { RelativePath } from "../persistence/database"; import type { Logger } from "./logger"; export interface CommonHistoryEntry { diff --git a/frontend/sync-client/src/utils/create-promise.ts b/frontend/sync-client/src/utils/create-promise.ts new file mode 100644 index 0000000..056c169 --- /dev/null +++ b/frontend/sync-client/src/utils/create-promise.ts @@ -0,0 +1,15 @@ +export function createPromise(): [ + Promise, + (value: T) => void, + (error: unknown) => void +] { + let resolve: undefined | ((resolved: T) => void) = undefined; + let reject: undefined | ((error: unknown) => void) = undefined; + + const creationPromise = new Promise( + (resolve_, reject_) => ((resolve = resolve_), (reject = reject_)) + ); + + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + return [creationPromise, resolve!, reject!]; +} diff --git a/frontend/sync-client/src/utils/find-matching-file-based-on-hash.ts b/frontend/sync-client/src/utils/find-matching-file-based-on-hash.ts deleted file mode 100644 index 6a247f5..0000000 --- a/frontend/sync-client/src/utils/find-matching-file-based-on-hash.ts +++ /dev/null @@ -1,13 +0,0 @@ -import type { DocumentMetadata, RelativePath } from "src/persistence/database"; -import { EMPTY_HASH } from "./hash"; - -export function findMatchingFileBasedOnHash( - contentHash: string, - candidates: [RelativePath, DocumentMetadata][] -): [RelativePath, DocumentMetadata] | undefined { - if (contentHash === EMPTY_HASH) { - return undefined; - } - - return candidates.find(([_, metadata]) => metadata.hash === contentHash); -} diff --git a/frontend/sync-client/src/utils/find-matching-file.ts b/frontend/sync-client/src/utils/find-matching-file.ts new file mode 100644 index 0000000..10545f2 --- /dev/null +++ b/frontend/sync-client/src/utils/find-matching-file.ts @@ -0,0 +1,14 @@ +import type { DocumentRecord } from "../persistence/database"; +import { EMPTY_HASH } from "./hash"; + +// TODO: make this smarter so that offline files can be renamed & edited at the same time +export function findMatchingFile( + contentHash: string, + candidates: DocumentRecord[] +): DocumentRecord | undefined { + if (contentHash === EMPTY_HASH) { + return undefined; + } + + return candidates.find(({ metadata }) => metadata?.hash === contentHash); +} diff --git a/frontend/sync-client/src/utils/hash.ts b/frontend/sync-client/src/utils/hash.ts index 10f20d1..cd965db 100644 --- a/frontend/sync-client/src/utils/hash.ts +++ b/frontend/sync-client/src/utils/hash.ts @@ -6,7 +6,7 @@ export function hash(content: Uint8Array): string { result = (result << 5) - result + content[i]; result |= 0; // Convert to 32bit integer } - return Math.abs(result).toString(16); + return Math.abs(result).toString(16).padStart(8, "0"); } export const EMPTY_HASH = hash(new Uint8Array(0)); diff --git a/frontend/sync-client/src/utils/retried-fetch.ts b/frontend/sync-client/src/utils/retried-fetch.ts index e4c47f0..a3856f8 100644 --- a/frontend/sync-client/src/utils/retried-fetch.ts +++ b/frontend/sync-client/src/utils/retried-fetch.ts @@ -1,6 +1,6 @@ import * as fetchRetryFactory from "fetch-retry"; import type { RequestInitRetryParams } from "fetch-retry"; -import type { Logger } from "src/tracing/logger"; +import type { Logger } from "../tracing/logger"; function getUrlFromInput(input: RequestInfo | URL): string { if (input instanceof URL) { @@ -31,7 +31,6 @@ export function retriedFetchFactory( } return false; }, - retries: 6, retryDelay: (attempt) => Math.pow(1.5, attempt) * 500, ...init }); diff --git a/frontend/sync-client/tsconfig.json b/frontend/sync-client/tsconfig.json index 6db72fc..ee31a31 100644 --- a/frontend/sync-client/tsconfig.json +++ b/frontend/sync-client/tsconfig.json @@ -1,12 +1,15 @@ { "compilerOptions": { - "baseUrl": ".", "module": "ESNext", "target": "ESNext", "strict": true, - "moduleResolution": "bundler", "allowSyntheticDefaultImports": true, - "lib": ["DOM", "ESNext"] + "moduleResolution": "bundler", + "lib": [ + "DOM" // to get "fetch" + ], + "declaration": true, + "declarationDir": "./dist/types" }, "exclude": ["./dist"] } diff --git a/frontend/sync-client/webpack.config.js b/frontend/sync-client/webpack.config.js index cd2c051..3f91304 100644 --- a/frontend/sync-client/webpack.config.js +++ b/frontend/sync-client/webpack.config.js @@ -1,25 +1,13 @@ const path = require("path"); +const { merge } = require("webpack-merge"); -module.exports = (_env, _argv) => ({ +const common = { entry: "./src/index.ts", - devtool: "source-map", - target: "node", module: { rules: [ { test: /\.ts$/, - use: [ - { - loader: "ts-loader", - options: { - compilerOptions: { - declaration: true, - declarationDir: "./dist/types" - }, - transpileOnly: false - } - } - ] + use: ["ts-loader"] }, { test: /\.wasm$/, @@ -28,22 +16,40 @@ module.exports = (_env, _argv) => ({ ] }, optimization: { + // the consuming project should take care of minification minimize: false }, resolve: { - extensions: [".ts", ".js"], + extensions: [".ts"], alias: { root: __dirname, src: path.resolve(__dirname, "src") } }, - output: { - clean: true, - filename: "index.js", - library: { - name: "SyncClient", - type: "umd" - }, - path: path.resolve(__dirname, "dist") + performance: { + hints: false // it's a library, no need to warn about its size } -}); +}; + +module.exports = [ + merge(common, { + target: "web", + output: { + path: path.resolve(__dirname, "dist"), + filename: "sync-client.web.js", + library: { + name: "SyncClient", + type: "umd" + }, + globalObject: "this" + } + }), + merge(common, { + target: "node", + output: { + path: path.resolve(__dirname, "dist"), + filename: "sync-client.node.js", + libraryTarget: "commonjs2" + } + }) +]; diff --git a/frontend/test-client/package.json b/frontend/test-client/package.json index 676da96..b11a4c4 100644 --- a/frontend/test-client/package.json +++ b/frontend/test-client/package.json @@ -1,6 +1,6 @@ { "name": "test-client", - "version": "0.0.0", + "version": "0.0.30", "private": true, "bin": { "test-client": "./dist/cli.js" @@ -11,13 +11,13 @@ "test": "jest --passWithNoTests" }, "devDependencies": { - "@types/node": "^22.13.5", + "@types/node": "^22.13.10", "sync-client": "file:../sync-client", "ts-loader": "^9.5.2", "tslib": "2.8.1", - "typescript": "5.7.3", + "typescript": "5.8.2", "uuid": "^11.1.0", "webpack": "^5.98.0", "webpack-cli": "^6.0.1" } -} +} \ No newline at end of file diff --git a/frontend/test-client/src/agent/mock-agent.ts b/frontend/test-client/src/agent/mock-agent.ts index 2163a50..7713b52 100644 --- a/frontend/test-client/src/agent/mock-agent.ts +++ b/frontend/test-client/src/agent/mock-agent.ts @@ -18,9 +18,10 @@ export class MockAgent extends MockClient { initialSettings: Partial, public readonly name: string, private readonly doDeletes: boolean, + useSlowFileEvents: boolean, private readonly jitterScaleInSeconds: number ) { - super(initialSettings); + super(initialSettings, useSlowFileEvents); } public async init(): Promise { @@ -46,27 +47,33 @@ export class MockAgent extends MockClient { ? "(online) " : "(offline)"; const formatted = `[${this.name} ${state}] ${logLine.timestamp.toISOString()} ${logLine.level} ${logLine.message}`; + + // HACK: we have to ensure the file has been synced if we want to change it offline without data loss + const historyEntry = /.*History entry: (.*.md).*/.exec( + logLine.message + ); + + if (historyEntry) { + this.doNotTouchWhileOffline = + this.doNotTouchWhileOffline.filter( + (file) => file !== historyEntry[1] + ); + } switch (logLine.level) { case LogLevel.ERROR: console.error(formatted); - // Let's not ignore errors - process.exit(1); + + if (!this.useSlowFileEvents) { + // Let's not ignore errors + // eslint-disable-next-line @typescript-eslint/no-floating-promises + sleep(100).then(() => process.exit(1)); + } + break; case LogLevel.WARNING: console.warn(formatted); break; case LogLevel.INFO: - // HACK: we have to ensure the file has been synced if we want to change it offline without data loss - const result = /.*History entry: (.*.md).*/.exec( - logLine.message - ); - if (result) { - this.doNotTouchWhileOffline = - this.doNotTouchWhileOffline.filter( - (file) => file !== result[1] - ); - } - console.info(formatted); break; case LogLevel.DEBUG: @@ -84,11 +91,10 @@ export class MockAgent extends MockClient { this.changeFetchChangesUpdateIntervalMsAction.bind(this) ]; - if ( - this.client.settings.getSettings().isSyncEnabled && - this.doNotTouchWhileOffline.length === 0 - ) { - options.push(this.disableSyncAction.bind(this)); + if (this.client.settings.getSettings().isSyncEnabled) { + if (this.doNotTouchWhileOffline.length === 0) { + options.push(this.disableSyncAction.bind(this)); + } } else { options.push(this.enableSyncAction.bind(this)); } @@ -186,6 +192,14 @@ export class MockAgent extends MockClient { } public assertAllContentIsPresentOnce(): void { + if (this.useSlowFileEvents) { + this.client.logger.info( + // We can't ensure that we have seen every single update + `Skipping content check for ${this.name} because slow file events are enabled` + ); + return; + } + for (const content of this.writtenContents) { const found = Array.from(this.localFiles.keys()).filter((key) => { return new TextDecoder() @@ -215,7 +229,7 @@ export class MockAgent extends MockClient { ); assert( fileContent.split(content).length == 2, - `Content ${content} (of ${this.name}) found more than once in file ${file}. File content:\n${fileContent}` + `Content ${content} (of ${this.name}) found more than once in '${file}'. File content:\n${fileContent}` ); } } @@ -237,10 +251,7 @@ export class MockAgent extends MockClient { `Decided to create file ${file} with content ${content}` ); - return this.create( - file, - new TextEncoder().encode(` |${content}| `) - ); + return this.create(file, new TextEncoder().encode(` ${content} `)); } private async changeFetchChangesUpdateIntervalMsAction(): Promise { @@ -314,7 +325,7 @@ export class MockAgent extends MockClient { `Decided to update file ${file} with ${content}` ); this.doNotTouchWhileOffline.push(file); - await this.atomicUpdateText(file, (old) => old + ` |${content}| `); + await this.atomicUpdateText(file, (old) => old + ` ${content} `); } private async deleteFileAction(files: RelativePath[]): Promise { diff --git a/frontend/test-client/src/agent/mock-client.ts b/frontend/test-client/src/agent/mock-client.ts index e627eb7..7e4e14c 100644 --- a/frontend/test-client/src/agent/mock-client.ts +++ b/frontend/test-client/src/agent/mock-client.ts @@ -1,9 +1,10 @@ -import type { - RelativePath, - FileSystemOperations, - SyncSettings +import { assert } from "../utils/assert"; +import { + type RelativePath, + type FileSystemOperations, + type SyncSettings, + SyncClient } from "sync-client"; -import { SyncClient } from "sync-client"; export class MockClient implements FileSystemOperations { protected readonly localFiles = new Map(); @@ -11,7 +12,8 @@ export class MockClient implements FileSystemOperations { protected data: object | undefined = undefined; public constructor( - private readonly initialSettings: Partial + private readonly initialSettings: Partial, + protected readonly useSlowFileEvents: boolean ) {} public async init(): Promise { @@ -22,9 +24,10 @@ export class MockClient implements FileSystemOperations { await Promise.all( Object.keys(this.initialSettings).map(async (key) => { + const settingKey = key as keyof SyncSettings; // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion return this.client.settings.setSetting( - key as keyof SyncSettings, // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion - this.initialSettings[key as keyof SyncSettings] // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion + settingKey, + this.initialSettings[settingKey]! // eslint-disable-line @typescript-eslint/no-non-null-assertion ); }) ); @@ -46,13 +49,6 @@ export class MockClient implements FileSystemOperations { return (await this.read(path)).length; } - public async getModificationTime(path: RelativePath): Promise { - if (!this.localFiles.has(path)) { - throw new Error(`File ${path} does not exist`); - } - return new Date(); - } - public async exists(path: RelativePath): Promise { return this.localFiles.has(path); } @@ -68,7 +64,10 @@ export class MockClient implements FileSystemOperations { `Creating file ${path} with content ${new TextDecoder().decode(newContent)}` ); this.localFiles.set(path, newContent); - void this.client.syncer.syncLocallyCreatedFile(path, new Date()); + + this.runCallback(() => { + void this.client.syncer.syncLocallyCreatedFile(path); + }); } public async createDirectory(_path: RelativePath): Promise { @@ -88,28 +87,51 @@ export class MockClient implements FileSystemOperations { const newContentUint8Array = new TextEncoder().encode(newContent); this.localFiles.set(path, newContentUint8Array); + if (!this.useSlowFileEvents) { + const existingParts = currentContent + .split(" ") + .map((part) => part.trim()); + const newParts = newContent.split(" ").map((part) => part.trim()); + existingParts.forEach((part) => + // all changes should be additive + { + assert( + newParts.includes(part), + `Part ${part} not found in new content` + ); + } + ); + } + this.client.logger.info( `Updated file ${path} with:\n current content: ${currentContent}\n new content: ${newContent}` ); - void this.client.syncer.syncLocallyUpdatedFile({ - relativePath: path, - updateTime: new Date() + this.runCallback(() => { + void this.client.syncer.syncLocallyUpdatedFile({ + relativePath: path + }); }); return newContent; } public async write(path: RelativePath, content: Uint8Array): Promise { + const hasExisted = this.localFiles.has(path); this.localFiles.set(path, content); this.client.logger.info( `Updated file ${path} with:\n new content: ${new TextDecoder().decode(content)}` ); - void this.client.syncer.syncLocallyUpdatedFile({ - relativePath: path, - updateTime: new Date() + this.runCallback(() => { + if (hasExisted) { + void this.client.syncer.syncLocallyUpdatedFile({ + relativePath: path + }); + } else { + void this.client.syncer.syncLocallyCreatedFile(path); + } }); } @@ -118,7 +140,10 @@ export class MockClient implements FileSystemOperations { `Deleting file: ${path} with:\n content ${new TextDecoder().decode(this.localFiles.get(path))}` ); this.localFiles.delete(path); - void this.client.syncer.syncLocallyDeletedFile(path); + + this.runCallback(() => { + void this.client.syncer.syncLocallyDeletedFile(path); + }); } public async rename( @@ -138,10 +163,20 @@ export class MockClient implements FileSystemOperations { `Renamed file: ${oldPath} -> ${newPath} with:\n content ${new TextDecoder().decode(file)}` ); - void this.client.syncer.syncLocallyUpdatedFile({ - oldPath, - relativePath: newPath, - updateTime: new Date() + this.runCallback(() => { + void this.client.syncer.syncLocallyUpdatedFile({ + oldPath, + relativePath: newPath + }); }); } + + private runCallback(callback: () => void): void { + if (this.useSlowFileEvents) { + // we aren't the best client and it takes some time to notice changes + setTimeout(callback, 100); + } else { + callback(); + } + } } diff --git a/frontend/test-client/src/cli.ts b/frontend/test-client/src/cli.ts index 7f8b29a..ea7ebbd 100644 --- a/frontend/test-client/src/cli.ts +++ b/frontend/test-client/src/cli.ts @@ -3,20 +3,26 @@ import { MockAgent } from "./agent/mock-agent"; import { sleep } from "./utils/sleep"; import { v4 as uuidv4 } from "uuid"; +let slowFileEvents = false; + async function runTest({ agentCount, concurrency, iterations, doDeletes, + useSlowFileEvents, jitterScaleInSeconds }: { agentCount: number; concurrency: number; iterations: number; doDeletes: boolean; + useSlowFileEvents: boolean; jitterScaleInSeconds: number; }): Promise { - const settings = `with ${agentCount} agents, concurrency ${concurrency}, iterations ${iterations}, doDeletes ${doDeletes}, jitterScaleInSeconds ${jitterScaleInSeconds}`; + slowFileEvents = useSlowFileEvents; + + const settings = `with ${agentCount} agents, concurrency ${concurrency}, iterations ${iterations}, doDeletes ${doDeletes}, jitterScaleInSeconds ${jitterScaleInSeconds}, useSlowFileEvents ${useSlowFileEvents}`; console.info(`Running test ${settings}`); const initialSettings: Partial = { @@ -34,6 +40,7 @@ async function runTest({ initialSettings, `agent-${i}`, doDeletes, + useSlowFileEvents, jitterScaleInSeconds ) ); @@ -52,12 +59,24 @@ async function runTest({ // Each agent can have unpushed changes which might conflict with eachother so each has to resolve the conflicts & push, and for (const client of clients) { - await client.finish(); + try { + await client.finish(); + } catch (err) { + if (!slowFileEvents) { + throw err; + } + } } // then we need a second pass to ensure that all agents pull the same state. for (const client of clients) { - await client.finish(); + try { + await client.finish(); + } catch (err) { + if (!slowFileEvents) { + throw err; + } + } } console.info("Agents finished successfully"); @@ -78,41 +97,49 @@ async function runTest({ console.info(`Content check for ${client.name} passed`); }); - console.info(`Test passed with ${settings}`); + console.info(`Test passed ${settings}`); } catch (err) { - console.error(`Test failed with ${settings}`); + console.error(`Test failed ${settings}`); throw err; } } async function runTests(): Promise { - const agentCounts = [2, 10]; - const jitterScaleInSeconds = [0.5, 3, 0]; - const concurrencies = [1, 16]; - const iterations = [50, 300]; - const doDeletes = [false, true]; - - for (const agentCount of agentCounts) { - for (const concurrency of concurrencies) { - for (const jitter of jitterScaleInSeconds) { - for (const iteration of iterations) { - for (const deleteFiles of doDeletes) { - while (true) { - await runTest({ - agentCount, - concurrency, - iterations: iteration, - doDeletes: deleteFiles, - jitterScaleInSeconds: jitter - }); - } - } - } + for (const useSlowFileEvents of [false, true]) { + for (const concurrency of [ + 16, + 1 // test with concurrency 1 to check for deadlocks + ]) { + for (const doDeletes of [true, false]) { + await runTest({ + agentCount: 3, + concurrency, + iterations: 100, + doDeletes, + useSlowFileEvents, + jitterScaleInSeconds: 0.75 + }); } } } } +process.on("uncaughtException", (error) => { + if (slowFileEvents) { + return; + } + console.error("Uncaught Exception:", error); + process.exit(1); +}); + +process.on("unhandledRejection", (reason, _promise) => { + if (slowFileEvents) { + return; + } + console.error("Unhandled Rejection:", reason); + process.exit(1); +}); + runTests() .then(() => { process.exit(0); diff --git a/frontend/test-client/webpack.config.js b/frontend/test-client/webpack.config.js index aa42a7d..b2324b9 100644 --- a/frontend/test-client/webpack.config.js +++ b/frontend/test-client/webpack.config.js @@ -12,8 +12,7 @@ module.exports = { rules: [ { test: /\.ts$/, - use: "ts-loader", - exclude: /node_modules/ + use: "ts-loader" } ] }, diff --git a/frontend/manifest.json b/manifest.json similarity index 100% rename from frontend/manifest.json rename to manifest.json diff --git a/bump-version.sh b/scripts/bump-version.sh similarity index 84% rename from bump-version.sh rename to scripts/bump-version.sh index b17842b..c8de483 100755 --- a/bump-version.sh +++ b/scripts/bump-version.sh @@ -27,20 +27,20 @@ cd backend cargo set-version --bump patch echo "Bumping frontend versions" -cd ../plugin -npm version patch +cd ../frontend +npm version patch --workspaces echo "Updating frontend dependencies to match the new backend versions" cd ../backend/sync_lib wasm-pack build --target web --features console_error_panic_hook -cd ../../plugin +cd ../../frontend npm install cd .. -cp plugin/manifest.json manifest.json # for BRAT, otherwise it wouldn't update +cp frontend/obsidian-plugin/manifest.json manifest.json # for BRAT, otherwise it wouldn't update -# Commit and tag +Commit and tag git add . TAG=$(node -p "require('./plugin/package.json').version") git commit -m "Bump versions to $TAG" diff --git a/scripts/clean-up.sh b/scripts/clean-up.sh new file mode 100755 index 0000000..85c12d1 --- /dev/null +++ b/scripts/clean-up.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +rm -rf backend/databases +rm -rf logs diff --git a/scripts/e2e.sh b/scripts/e2e.sh new file mode 100755 index 0000000..fa06d82 --- /dev/null +++ b/scripts/e2e.sh @@ -0,0 +1,79 @@ +#!/bin/bash + +set -e +set -o pipefail + +# Check if the argument is provided +if [ $# -eq 0 ]; then + echo "Usage: $0 " + exit 1 +fi + +# Get the number of processes from the first argument +process_count=$1 + +mkdir -p logs + +cd frontend +npm run build + +pids=() +for i in $(seq 1 $process_count); do + node test-client/dist/cli.js > "../logs/log_${i}.log" 2>&1 & + pids+=($!) +done + +cd - + +print_failed_log() { + for i in $(seq 1 $process_count); do + if [ -n "${pids[$i-1]}" ] && ! kill -0 ${pids[$i-1]} 2>/dev/null; then + # Get the exit code of the process + wait ${pids[$i-1]} + exit_code=$? + + # Only consider non-zero exit codes as failures + if [ $exit_code -ne 0 ]; then + echo "Process ${pids[$i-1]} failed with exit code $exit_code. Log file: $(pwd)/logs/log_${i}.log" + return 0 + else + echo "Process ${pids[$i-1]} completed successfully with exit code 0" + # Mark this PID as processed by setting it to empty + pids[$i-1]="" + fi + fi + done + return 1 +} + +echo "Monitoring $process_count processes" + +# Monitor processes +while true; do + if print_failed_log; then + # Kill remaining processes + for pid in "${pids[@]}"; do + if [ -n "$pid" ]; then + kill $pid 2>/dev/null || true + fi + done + exit 1 + fi + + # Check if all processes have completed + all_done=true + for pid in "${pids[@]}"; do + if [ -n "$pid" ] && kill -0 $pid 2>/dev/null; then + all_done=false + break + fi + done + + if $all_done; then + echo "All processes completed successfully" + exit 0 + fi + + sleep 0.2 +done + diff --git a/scripts/update-api-types.sh b/scripts/update-api-types.sh new file mode 100755 index 0000000..d0aa235 --- /dev/null +++ b/scripts/update-api-types.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +npm install -g openapi-typescript +openapi-typescript http://localhost:3030/api.json --output frontend/sync-client/src/services/types.ts