andras/reconcile
1
0
Fork 0
Code Issues Pull requests 5 Projects Releases Packages Wiki Activity Actions

Fix syncing when network latency is present (#4)

Browse source 
* WIP

* Add debug

* Dedupe inserts

* Add deterministic ordering

* Fix whitespaces

* Update insta

* Add integration test script

* Rename

* Add test

* Working for non-deletes

* omg it mostly works for deletes

* Isdeleted fix

* remove created dates

* update api

* Take document id

* No max attempt

* works

* Use string uuids

* .

* working!!!! (hopefully)

* Improve bundling

* Add module

* lint

* .

* lint

* Fix CI

* use toolchain

* clean up

* Add useSlowFileEvents

* Delete fuzz

* Fix CI

* use docker

* fix script

* clean up

* Clean up

* change node version

* Build docker image on every commit

* fix ci

* 1 db per vault

* Add scritps folder

* Bump versions

* Lint

* .

* Fix tests for real

* Style

* .

* try

* Consistent ordering

* Fix tests

* hmm

* .

* Clean up diff

* Fixes

* .

* Fix version bump

* .

* .

* .
This commit is contained in:
Andras Schmelczer 2025-03-16 20:13:49 +00:00 • committed by GitHub
parent bcf48c428d
commit 8b8f1d91d9
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
91 changed files with 2252 additions and 1586 deletions
Download patch file Download diff file Expand all files Collapse all files

13
.github/workflows/check.yml vendored
View file

@ -17,11 +17,14 @@ jobs:
steps:
- uses: actions/checkout@v4
- name: Setup
- name: Setup Node.js environment
uses: actions/setup-node@v4.2.0
with:
node-version: "22.x"
check-latest: true
- name: Setup rust
run: |
rustup install nightly
rustup default nightly
rustup component add clippy rustfmt
cargo install sqlx-cli
cd backend
sqlx database create --database-url sqlite://db.sqlite3
@ -44,7 +47,7 @@ jobs:
cd backend
cargo test --verbose
cd sync_lib
# wasm-pack test --node # todo: fix this in CI
wasm-pack test --node
- name: Lint frontend
run: |

45
.github/workflows/e2e.yml vendored Normal file
View file

@ -0,0 +1,45 @@
name: E2E tests
on:
push:
branches: ["master"]
pull_request:
branches: ["master"]
env:
CARGO_TERM_COLOR: always
RUSTFLAGS: "-Dwarnings"
jobs:
build:
runs-on: self-hosted
steps:
- uses: actions/checkout@v4
- name: Setup Node.js environment
uses: actions/setup-node@v4.2.0
with:
node-version: "22.x"
check-latest: true
- name: Setup rust
run: |
cargo install sqlx-cli wasm-pack
cd backend
sqlx database create --database-url sqlite://db.sqlite3
sqlx migrate run --source sync_server/src/database/migrations --database-url sqlite://db.sqlite3
- name: Build wasm
run: |
cd backend
wasm-pack build --target web sync_lib
- name: E2E tests
run: |
cd backend
RUST_BACKTRACE=1 cargo run -p sync_server &
cd ../frontend
npm ci
cd ..
scripts/e2e.sh 32

18
.github/workflows/publish-docker.yml vendored
View file

@ -7,8 +7,9 @@ name: Publish server Docker image
on:
push:
tags:
- "*"
branches: ["master"]
pull_request:
branches: ["master"]
env:
# Use docker.io for Docker Hub if empty
@ -17,8 +18,9 @@ env:
IMAGE_NAME: ${{ github.repository }}
jobs:
build-docker:
runs-on: ubuntu-latest
publish-docker:
runs-on: self-hosted
permissions:
contents: read
packages: write
@ -33,7 +35,7 @@ jobs:
# Install the cosign tool except on PR
# https://github.com/sigstore/cosign-installer
- name: Install cosign
if: github.event_name != 'pull_request'
if: ${{ github.ref_type == 'tag' }}
uses: sigstore/cosign-installer@59acb6260d9c0ba8f4a2f9d9b48431a222b68e20 #v3.5.0
with:
cosign-release: "v2.2.4"
@ -47,7 +49,7 @@ jobs:
# Login against a Docker registry except on PR
# https://github.com/docker/login-action
- name: Log into registry ${{ env.REGISTRY }}
if: github.event_name != 'pull_request'
if: ${{ github.ref_type == 'tag' }}
uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0
with:
registry: ${{ env.REGISTRY }}
@ -69,7 +71,7 @@ jobs:
uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v5.0.0
with:
context: backend
push: ${{ github.event_name != 'pull_request' }}
push: ${{ github.ref_type == 'tag' }}
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
cache-from: type=gha
@ -81,7 +83,7 @@ jobs:
# transparency data even for private images, pass --force to cosign below.
# https://github.com/sigstore/cosign
- name: Sign the published Docker image
if: ${{ github.event_name != 'pull_request' }}
if: ${{ github.ref_type == 'tag' }}
env:
# https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions#using-an-intermediate-environment-variable
TAGS: ${{ steps.meta.outputs.tags }}

16
.github/workflows/publish-plugin.yml vendored
View file

@ -2,29 +2,27 @@ name: Publish Obsidian plugin
on:
push:
tags:
- "*"
tags: ["*"]
env:
CARGO_TERM_COLOR: always
jobs:
build-plugin:
publish-plugin:
runs-on: self-hosted
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- name: Use Node.js
uses: actions/setup-node@v3
- name: Setup Node.js environment
uses: actions/setup-node@v4.2.0
with:
node-version: "18.x"
node-version: "22.x"
check-latest: true
- name: Build wasm
run: |
cd backend
rustup install nightly
rustup default nightly
cargo install wasm-pack
wasm-pack build --target web sync_lib

1
.gitignore vendored
View file

@ -10,6 +10,7 @@ backend/target
frontend/*/dist
backend/db.sqlite3*
backend/databases
backend/config.yml
*.log

79
README.md
View file

@ -1,65 +1,56 @@
## VaultLink self-hosted Obsidian sync plugin
# VaultLink self-hosted Obsidian plugin for file syncing
[![Check](https://github.com/schmelczer/vault-link/actions/workflows/check.yml/badge.svg)](https://github.com/schmelczer/vault-link/actions/workflows/check.yml)
[![E2E tests](https://github.com/schmelczer/vault-link/actions/workflows/e2e.yml/badge.svg)](https://github.com/schmelczer/vault-link/actions/workflows/e2e.yml)
[![Publish server Docker image](https://github.com/schmelczer/vault-link/actions/workflows/publish-docker.yml/badge.svg)](https://github.com/schmelczer/vault-link/actions/workflows/publish-docker.yml)
[![Publish Obsidian plugin](https://github.com/schmelczer/vault-link/actions/workflows/publish-plugin.yml/badge.svg)](https://github.com/schmelczer/vault-link/actions/workflows/publish-plugin.yml)
## Install [nvm](https://github.com/nvm-sh/nvm)
## Develop
### Install [nvm](https://github.com/nvm-sh/nvm)
- `curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh | bash`
- `nvm install 20`
- `nvm use 20`
- Optionally set the system-wide default: `nvm alias default 20`
- `nvm install 22`
- `nvm use 22`
- Optionally set the system-wide default: `nvm alias default 22`
## Set up Rust
### Set up Rust
- Install [`rustup`](https://rustup.rs): `curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh`
- `sudo apt install llvm -y`
- `rustup self update`
- `rustup update`
- `rustup install nightly`
- `rustup default nightly`
- `rustup component add llvm-tools-preview`
- `cargo install cargo-generate cargo-fuzz cargo-insta rustfilt cargo-binutils`
- Install [`wasm-pack`](https://rustwasm.github.io/wasm-pack/installer): `curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh`
- `cargo install cargo-insta sqlx-cli cargo-edit`
## Publish new version
### Install Obsidian on Linux
```sh
./bump-version.sh patch
```
## Update HTTP API TS bindings
```sh
npm install -g openapi-typescript
openapi-typescript http://localhost:3030/api.json --output frontend/sync-client/src/services/types.ts
```
```
todo: enable
[workspace.lints.clippy]
single_call_fn = { level = "allow", priority = 1 }
absolute_paths = { level = "allow", priority = 1 }
arithmetic_side_effects = { level = "allow", priority = 1 }
similar_names = { level = "allow", priority = 1 }
self_named_module_files = { level = "allow", priority = 1 }
single_char_lifetime_names = { level = "allow", priority = 1 }
missing_docs_in_private_items = { level = "allow", priority = 1 }
question_mark_used = { level = "allow", priority = 1 }
implicit_return = { level = "allow", priority = 1 }
pedantic = { level = "warn", priority = 0 }
cargo = { level = "warn", priority = 0 }
```
apt install flatpak
flatpak remote-add --if-not-exists flathub https://dl.flathub.org/repo/flathub.flatpakrepo
flatpak install flathub md.obsidian.Obsidian
flatpak run md.obsidian.Obsidian
```
### Scripts
#### Update HTTP API TS bindings
```sh
scripts/update-api-types.sh
```
#### Publish new version
```sh
scripts/bump-version.sh patch
```
#### Run E2E tests
```sh
scripts/e2e.sh
```
And to clean up the logs & database files, run `scripts/clean-up.sh`
```

35
backend/Cargo.lock generated
View file

@ -105,12 +105,6 @@ dependencies = [
"backtrace",
]
[[package]]
name = "arbitrary"
version = "1.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dde20b3d026af13f561bdd0f15edf01fc734f0dafcedbaf42bba506a9517f223"
[[package]]
name = "async-trait"
version = "0.1.85"
@ -381,8 +375,6 @@ version = "1.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f34d93e62b03caf570cccc334cbc6c2fceca82f39211051345108adcba3eebdc"
dependencies = [
"jobserver",
"libc",
"shlex",
]
@ -1228,15 +1220,6 @@ version = "1.0.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674"
[[package]]
name = "jobserver"
version = "0.1.32"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0"
dependencies = [
"libc",
]
[[package]]
name = "js-sys"
version = "0.3.76"
@ -1290,16 +1273,6 @@ version = "0.2.167"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "09d6582e104315a817dff97f75133544b2e094ee22447d2acf4a74e189ba06fc"
[[package]]
name = "libfuzzer-sys"
version = "0.4.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9b9569d2f74e257076d8c6bfa73fb505b46b851e51ddaecc825944aa3bed17fa"
dependencies = [
"arbitrary",
"cc",
]
[[package]]
name = "libm"
version = "0.2.11"
@ -1781,14 +1754,6 @@ dependencies = [
"test-case",
]
[[package]]
name = "reconcile-fuzz"
version = "0.0.30"
dependencies = [
"libfuzzer-sys",
"reconcile",
]
[[package]]
name = "redox_syscall"
version = "0.5.7"

17
backend/Cargo.toml
View file

@ -2,7 +2,6 @@
resolver = "2"
members = [
"reconcile",
"fuzz",
"sync_server",
"sync_lib"
]
@ -57,3 +56,19 @@ uninlined_format_args = "warn"
unnested_or_patterns = "warn"
unused_self = "warn"
verbose_file_reads = "warn"
cast_possible_truncation = { level = "allow", priority = 1 }
doc_link_with_quotes = { level = "allow", priority = 1 }
cast_sign_loss = { level = "allow", priority = 1 }
cast_possible_wrap = { level = "allow", priority = 1 }
struct_field_names = { level = "allow", priority = 1 }
single_call_fn = { level = "allow", priority = 1 }
absolute_paths = { level = "allow", priority = 1 }
arithmetic_side_effects = { level = "allow", priority = 1 }
similar_names = { level = "allow", priority = 1 }
self_named_module_files = { level = "allow", priority = 1 }
single_char_lifetime_names = { level = "allow", priority = 1 }
missing_docs_in_private_items = { level = "allow", priority = 1 }
question_mark_used = { level = "allow", priority = 1 }
implicit_return = { level = "allow", priority = 1 }
pedantic = { level = "warn", priority = 0 }

4
backend/Dockerfile
View file

@ -3,8 +3,6 @@ FROM rust:1.83 AS builder
WORKDIR /usr/src/backend
RUN apt update && apt install -y musl-tools
RUN rustup install nightly && rustup default nightly
RUN rustup target add x86_64-unknown-linux-musl
RUN cargo install sqlx-cli
COPY . .
@ -23,7 +21,7 @@ RUN apk add --no-cache curl
COPY --from=builder /usr/src/backend/target/x86_64-unknown-linux-musl/release/sync_server /app/sync_server
VOLUME /data
VOLUME /data/databases
EXPOSE 3000/tcp
WORKDIR /data

4
backend/fuzz/.gitignore vendored
View file

@ -1,4 +0,0 @@
target
corpus
artifacts
coverage

25
backend/fuzz/Cargo.toml
View file

@ -1,25 +0,0 @@
[package]
name = "reconcile-fuzz"
version.workspace = true
edition.workspace = true
authors.workspace = true
license.workspace = true
repository.workspace = true
publish = false
[package.metadata]
cargo-fuzz = true
[dependencies]
libfuzzer-sys = "0.4"
reconcile = { path = "../reconcile" }
[[bin]]
name = "reconcile"
path = "fuzz_targets/reconcile.rs"
test = false
doc = false
bench = false
[lints]
workspace = true

8
backend/fuzz/fuzz_targets/reconcile.rs
View file

@ -1,8 +0,0 @@
#![no_main]
use libfuzzer_sys::fuzz_target;
fuzz_target!(|texts: (String, String, String)| {
let (original, left, right) = texts;
let _ = reconcile::reconcile(&original, &left, &right);
});

7
backend/reconcile/src/diffs/myers.rs
View file

@ -38,7 +38,7 @@ use crate::{
/// execution time permitted before it bails and falls back to an approximation.
pub fn diff<T>(old: &[Token<T>], new: &[Token<T>]) -> Vec<RawOperation<T>>
where
T: PartialEq + Clone,
T: PartialEq + Clone + std::fmt::Debug,
{
let max_d = (old.len() + new.len()).div_ceil(2) + 1;
let mut vb = V::new(max_d);
@ -99,7 +99,6 @@ impl IndexMut<isize> for V {
}
}
#[inline(always)]
fn split_at(range: Range<usize>, at: usize) -> (Range<usize>, Range<usize>) {
(range.start..at, at..range.end)
}
@ -124,7 +123,7 @@ fn find_middle_snake<T>(
vb: &mut V,
) -> Option<(usize, usize)>
where
T: PartialEq + Clone,
T: PartialEq + Clone + std::fmt::Debug,
{
let n = old_range.len();
let m = new_range.len();
@ -230,7 +229,7 @@ fn conquer<T>(
vb: &mut V,
result: &mut Vec<RawOperation<T>>,
) where
T: PartialEq + Clone,
T: PartialEq + Clone + std::fmt::Debug,
{
// Check for common prefix
let common_prefix_len = common_prefix_len(old, old_range.clone(), new, new_range.clone());

10
backend/reconcile/src/diffs/raw_operation.rs
View file

@ -3,7 +3,7 @@ use crate::tokenizer::token::Token;
#[derive(Debug, Clone, PartialEq)]
pub enum RawOperation<T>
where
T: PartialEq + Clone,
T: PartialEq + Clone + std::fmt::Debug,
{
Insert(Vec<Token<T>>),
Delete(Vec<Token<T>>),
@ -12,13 +12,13 @@ where
impl<T> RawOperation<T>
where
T: PartialEq + Clone,
T: PartialEq + Clone + std::fmt::Debug,
{
pub fn tokens(&self) -> &Vec<Token<T>> {
match self {
RawOperation::Insert(tokens) => tokens,
RawOperation::Delete(tokens) => tokens,
RawOperation::Equal(tokens) => tokens,
RawOperation::Insert(tokens)
| RawOperation::Delete(tokens)
| RawOperation::Equal(tokens) => tokens,
}
}

36
backend/reconcile/src/operation_transformation.rs
View file

@ -37,7 +37,7 @@ pub fn reconcile_with_tokenizer<F, T>(
tokenizer: &Tokenizer<T>,
) -> String
where
T: PartialEq + Clone,
T: PartialEq + Clone + std::fmt::Debug,
{
let left_operations = EditedText::from_strings_with_tokenizer(original, left, tokenizer);
let right_operations = EditedText::from_strings_with_tokenizer(original, right, tokenizer);
@ -73,7 +73,8 @@ mod test {
"original_1 edit_1 original_3",
);
// One deleted a large range, the other deleted subranges and inserted as well
// One deleted a large range, the other deleted subranges and inserted as
// well
test_merge_both_ways(
"original_1 original_2 original_3 original_4 original_5",
"original_1 original_5",
@ -120,9 +121,6 @@ mod test {
"hi, my friend!",
);
// test_merge_both_ways("hello world", "world !", "hi hello world", "hi world
// !");
test_merge_both_ways(
"both delete the same word",
"both the same word",
@ -147,7 +145,33 @@ mod test {
);
}
#[ignore = "it's too slow"]
#[test]
fn test_reconcile_idempotent_inserts() {
// Both inserted the same prefix; this should get deduped
test_merge_both_ways(
"hi ",
"hi there ",
"hi there my friend ",
"hi there my friend ",
);
// The prefix of the 2nd appears on the 1st so it shouldn't get duplicated
test_merge_both_ways(
"hi ",
"hi there you ",
"hi there my friend ",
"hi there my friend you ",
);
test_merge_both_ways("a", "a b c", "a b c d", "a b c d");
test_merge_both_ways(
" |7ca2b36d-6ee7-49eb-8eb1-d77e4cc1a001| ",
" |7ca2b36d-6ee7-49eb-8eb1-d77e4cc1a001| |cd9195cc-103a-4f13-90c8-4fba0ba421ee| |d39156cc-cfd6-42a8-b70a-75020896069d| |fbad794c-9c47-41f2-a343-490284ecb5a0| |dup| ",
" |7ca2b36d-6ee7-49eb-8eb1-d77e4cc1a001| |cd9195cc-103a-4f13-90c8-4fba0ba421ee| |dup| ",
" |7ca2b36d-6ee7-49eb-8eb1-d77e4cc1a001| |cd9195cc-103a-4f13-90c8-4fba0ba421ee| |d39156cc-cfd6-42a8-b70a-75020896069d| |fbad794c-9c47-41f2-a343-490284ecb5a0| |dup| |dup| ");
}
#[test_matrix( [
"pride_and_prejudice.txt",
"romeo_and_juliet.txt",

24
backend/reconcile/src/operation_transformation/edited_text.rs
View file

@ -25,7 +25,7 @@ use crate::{
#[derive(Debug, Clone, PartialEq, Default)]
pub struct EditedText<'a, T>
where
T: PartialEq + Clone,
T: PartialEq + Clone + std::fmt::Debug,
{
text: &'a str,
operations: Vec<OrderedOperation<T>>,
@ -46,7 +46,7 @@ impl<'a> EditedText<'a, String> {
impl<'a, T> EditedText<'a, T>
where
T: PartialEq + Clone,
T: PartialEq + Clone + std::fmt::Debug,
{
/// Create an `EditedText` from the given original (old) and updated (new)
/// strings. The returned `EditedText` represents the changes from the
@ -65,7 +65,6 @@ where
Self::new(
original,
// Self::cook_operations(diff),
Self::cook_operations(Self::elongate_operations(diff)).collect(),
)
}
@ -191,7 +190,7 @@ where
pub fn merge(self, other: Self) -> Self {
debug_assert_eq!(
self.text, other.text,
"EditedText-s must be derived from the same text to be mergable"
"`EditedText`-s must be derived from the same text to be mergable"
);
let mut left_merge_context = MergeContext::default();
@ -207,9 +206,21 @@ where
|(operation, _)| {
(
operation.order,
// Operations on left and right must come in the same order so that
// Operations on the left and right must come in the same order so that
// inserts can be merged with other inserts and deletes with deletes.
usize::from(matches!(operation.operation, Operation::Delete { .. })),
// Make sure that the ordering is deterministic regardless which text
// is left or right.
match &operation.operation {
Operation::Insert { text, .. } => text
.iter()
.map(super::super::tokenizer::token::Token::original)
.collect::<String>(),
Operation::Delete {
deleted_character_count,
..
} => deleted_character_count.to_string(),
},
)
},
)
@ -232,6 +243,7 @@ where
}
/// Apply the operations to the text and return the resulting text.
#[must_use]
pub fn apply(&self) -> String {
let mut builder: StringBuilder<'_> = StringBuilder::new(self.text);
@ -282,7 +294,7 @@ mod tests {
let original = "hello world! ...";
let left = "Hello world! I'm Andras.";
let right = "Hello world! How are you?";
let expected = "Hello world! I'm Andras.How are you?";
let expected = "Hello world! How are you? I'm Andras.";
let operations_1 = EditedText::from_strings(original, left);
let operations_2 = EditedText::from_strings(original, right);

8
backend/reconcile/src/operation_transformation/merge_context.rs
View file

@ -5,7 +5,7 @@ use crate::operation_transformation::Operation;
#[derive(Clone)]
pub struct MergeContext<T>
where
T: PartialEq + Clone,
T: PartialEq + Clone + std::fmt::Debug,
{
last_operation: Option<Operation<T>>,
pub shift: i64,
@ -13,7 +13,7 @@ where
impl<T> Default for MergeContext<T>
where
T: PartialEq + Clone,
T: PartialEq + Clone + std::fmt::Debug,
{
fn default() -> Self {
MergeContext {
@ -25,7 +25,7 @@ where
impl<T> Debug for MergeContext<T>
where
T: PartialEq + Clone,
T: PartialEq + Clone + std::fmt::Debug,
{
fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
f.debug_struct("MergeContext")
@ -37,7 +37,7 @@ where
impl<T> MergeContext<T>
where
T: PartialEq + Clone,
T: PartialEq + Clone + std::fmt::Debug,
{
pub fn last_operation(&self) -> Option<&Operation<T>> { self.last_operation.as_ref() }

53
backend/reconcile/src/operation_transformation/operation.rs
View file

@ -1,7 +1,5 @@
use core::{
fmt::{Debug, Display},
ops::Range,
};
use core::fmt::{Debug, Display};
use std::ops::Range;
#[cfg(feature = "serde")]
use serde::{Deserialize, Serialize};
@ -9,7 +7,10 @@ use serde::{Deserialize, Serialize};
use super::merge_context::MergeContext;
use crate::{
Token,
utils::{find_common_overlap::find_common_overlap, string_builder::StringBuilder},
utils::{
find_longest_prefix_contained_within::find_longest_prefix_contained_within,
string_builder::StringBuilder,
},
};
/// Represents a change that can be applied to a text document.
@ -19,7 +20,7 @@ use crate::{
#[derive(Clone, PartialEq)]
pub enum Operation<T>
where
T: PartialEq + Clone,
T: PartialEq + Clone + std::fmt::Debug,
{
Insert {
index: usize,
@ -37,7 +38,7 @@ where
impl<T> Operation<T>
where
T: PartialEq + Clone,
T: PartialEq + Clone + std::fmt::Debug,
{
/// Creates an insert operation with the given index and text.
/// If the text is empty (meaning that the operation would be a no-op),
@ -81,15 +82,8 @@ where
})
}
/// Tries to apply the operation to the given `ropey::Rope` text, returning
/// the modified text.
///
/// # Errors
///
/// Returns a `SyncLibError::OperationApplicationError` if the operation
/// cannot be applied.
///
/// # Panics
/// Applies the operation to the given `StringBuilder`, returning the
/// modified `StringBuilder`.
///
/// When compiled in debug mode, panics if a delete operation is attempted
/// on a range of text that does not match the text to be deleted.
@ -114,7 +108,7 @@ where
builder.delete(self.range());
}
};
}
builder
}
@ -122,8 +116,7 @@ where
/// Returns the index of the first character that the operation affects.
pub fn start_index(&self) -> usize {
match self {
Operation::Insert { index, .. } => *index,
Operation::Delete { index, .. } => *index,
Operation::Insert { index, .. } | Operation::Delete { index, .. } => *index,
}
}
@ -137,6 +130,7 @@ where
}
/// Returns the range of indices of characters that the operation affects.
#[allow(clippy::range_plus_one)]
pub fn range(&self) -> Range<usize> { self.start_index()..self.end_index() + 1 }
/// Returns the number of affected characters. It is always greater than 0
@ -212,17 +206,20 @@ where
..
}),
) => {
let offset_in_tokens = find_common_overlap(previous_inserted_text, &text);
let trimmed_length_in_tokens = previous_inserted_text.len() - offset_in_tokens;
let trimmed_length = previous_inserted_text
// In case the current insert's prefix appears in the previously inserted text,
// we can trim the current insert to only include the non-overlapping part.
// This way, we don't end up duplicating text.
let offset_in_tokens =
find_longest_prefix_contained_within(previous_inserted_text, &text);
let offset_in_length = text
.iter()
.skip(offset_in_tokens)
.take(offset_in_tokens)
.map(Token::get_original_length)
.sum::<usize>();
let trimmed_operation =
Operation::create_insert(index, text[trimmed_length_in_tokens..].to_vec());
Operation::create_insert(index, text[offset_in_tokens..].to_vec());
affecting_context.shift -= trimmed_length as i64;
affecting_context.shift -= offset_in_length as i64;
produced_context.shift += trimmed_operation
.as_ref()
.map(Operation::len)
@ -297,7 +294,7 @@ where
impl<T> Display for Operation<T>
where
T: PartialEq + Clone,
T: PartialEq + Clone + std::fmt::Debug,
{
fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
match self {
@ -341,7 +338,7 @@ where
impl<T> Debug for Operation<T>
where
T: PartialEq + Clone,
T: PartialEq + Clone + std::fmt::Debug,
{
fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { write!(f, "{self}") }
}
@ -353,7 +350,7 @@ mod tests {
use super::*;
#[test]
#[should_panic]
#[should_panic(expected = "Shifted index must be non-negative")]
fn test_shifting_error() {
insta::assert_debug_snapshot!(
Operation::create_insert(1, vec!["hi".into()])

12
backend/reconcile/src/operation_transformation/snapshots/reconcile__operation_transformation__edited_text__tests__calculate_operations.snap
View file

@ -8,19 +8,19 @@ EditedText {
operations: [
OrderedOperation {
order: 0,
operation: <insert 'Hello, my friend! ' from index 0>,
operation: <insert 'Hello, my friend!' from index 0>,
},
OrderedOperation {
order: 0,
operation: <delete 'hello world! ' from index 18>,
operation: <delete 'hello world!' from index 17>,
},
OrderedOperation {
order: 21,
operation: <insert 'you doing? Albert' from index 26>,
order: 20,
operation: <insert ' you doing? Albert' from index 25>,
},
OrderedOperation {
order: 21,
operation: <delete 'you? Adam' from index 43>,
order: 20,
operation: <delete ' you? Adam' from index 43>,
},
],
}

6
backend/reconcile/src/tokenizer/snapshots/reconcile__tokenizer__word_tokenizer__tests__with_snapshots-2.snap Normal file
View file

@ -0,0 +1,6 @@
---
source: reconcile/src/tokenizer/word_tokenizer.rs
expression: "word_tokenizer(\"\")"
snapshot_kind: text
---
[]

15
backend/reconcile/src/tokenizer/snapshots/reconcile__tokenizer__word_tokenizer__tests__with_snapshots-3.snap Normal file
View file

@ -0,0 +1,15 @@
---
source: reconcile/src/tokenizer/word_tokenizer.rs
expression: "word_tokenizer(\" what? \")"
snapshot_kind: text
---
[
Token {
normalised: "what?",
original: " what?",
},
Token {
normalised: "",
original: " ",
},
]

23
backend/reconcile/src/tokenizer/snapshots/reconcile__tokenizer__word_tokenizer__tests__with_snapshots-4.snap Normal file
View file

@ -0,0 +1,23 @@
---
source: reconcile/src/tokenizer/word_tokenizer.rs
expression: "word_tokenizer(\" hello, \\nwhere are you?\")"
snapshot_kind: text
---
[
Token {
normalised: "hello,",
original: " hello,",
},
Token {
normalised: "where",
original: " \nwhere",
},
Token {
normalised: "are",
original: " are",
},
Token {
normalised: "you?",
original: " you?",
},
]

15
backend/reconcile/src/tokenizer/snapshots/reconcile__tokenizer__word_tokenizer__tests__with_snapshots.snap Normal file
View file

@ -0,0 +1,15 @@
---
source: reconcile/src/tokenizer/word_tokenizer.rs
expression: "word_tokenizer(\"Hi there!\")"
snapshot_kind: text
---
[
Token {
normalised: "Hi",
original: "Hi",
},
Token {
normalised: "there!",
original: " there!",
},
]

13
backend/reconcile/src/tokenizer/token.rs
View file

@ -8,24 +8,19 @@ use serde::{Deserialize, Serialize};
#[derive(Debug, Clone)]
pub struct Token<T>
where
T: PartialEq + Clone,
T: PartialEq + Clone + std::fmt::Debug,
{
normalised: T,
original: String,
}
impl From<&str> for Token<String> {
fn from(s: &str) -> Self {
Token {
normalised: s.to_owned(),
original: s.to_owned(),
}
}
fn from(s: &str) -> Self { Token::new(s.trim().to_owned(), s.to_owned()) }
}
impl<T> Token<T>
where
T: PartialEq + Clone,
T: PartialEq + Clone + std::fmt::Debug,
{
pub fn new(normalised: T, original: String) -> Self {
Token {
@ -43,7 +38,7 @@ where
impl<T> PartialEq for Token<T>
where
T: PartialEq + Clone,
T: PartialEq + Clone + std::fmt::Debug,
{
fn eq(&self, other: &Self) -> bool { self.normalised == other.normalised }
}

47
backend/reconcile/src/tokenizer/word_tokenizer.rs
View file

@ -1,7 +1,48 @@
use super::token::Token;
/// Splits on whitespace keeping the leading whitespace.
///
///
/// ## Example
///
/// "Hi there!" -> ["Hi", " there!"]
pub fn word_tokenizer(text: &str) -> Vec<Token<String>> {
text.split_inclusive(char::is_whitespace)
.map(|s| Token::new(s.to_owned(), s.to_owned()))
.collect()
let mut result: Vec<Token<String>> = Vec::new();
let mut last_whitespace = 0;
let mut previous_char_is_whitespace = true;
for (i, c) in text.char_indices() {
let is_current_char_whitespace = c.is_whitespace();
if !previous_char_is_whitespace && is_current_char_whitespace {
result.push(text[last_whitespace..i].into());
last_whitespace = i;
}
previous_char_is_whitespace = is_current_char_whitespace;
}
if last_whitespace < text.len() {
result.push(text[last_whitespace..].into());
}
result
}
#[cfg(test)]
mod tests {
use insta::assert_debug_snapshot;
use super::*;
#[test]
fn test_with_snapshots() {
assert_debug_snapshot!(word_tokenizer("Hi there!"));
assert_debug_snapshot!(word_tokenizer(""));
assert_debug_snapshot!(word_tokenizer(" what? "));
assert_debug_snapshot!(word_tokenizer(" hello, \nwhere are you?"));
}
}

2
backend/reconcile/src/utils.rs
View file

@ -1,6 +1,6 @@
pub mod common_prefix_len;
pub mod common_suffix_len;
pub mod find_common_overlap;
pub mod find_longest_prefix_contained_within;
pub mod merge_iters;
pub mod ordered_operation;
pub mod side;

71
backend/reconcile/src/utils/find_common_overlap.rs
View file

@ -1,71 +0,0 @@
use crate::Token;
/// Given two lists of tokens, returns the offset in the first (old) list from
/// which the two lists have the same tokens until the end of the first list.
/// Thus, the suffix of the old list from the offset to the end is equal to a
/// prefix of the new list.
///
/// If there is no overlap, the function returns the maxmium offset, the length
/// of the old list.
///
/// ## Example
///
/// ```not_rust
/// old: [0, 1, 9, 0, 2, 5]
/// new: [9, 0, 2, 5, 1]
/// ```
/// > results in an offset of 2
pub fn find_common_overlap<T>(old: &[Token<T>], new: &[Token<T>]) -> usize
where
T: PartialEq + Clone,
{
let minimum_offset = old.len().saturating_sub(new.len());
for offset in minimum_offset..old.len() {
if old.iter().skip(offset).zip(new.iter()).all(|(a, b)| a == b) {
return offset;
}
}
old.len()
}
#[cfg(test)]
mod tests {
use pretty_assertions::assert_eq;
use super::*;
#[test]
fn test_common_overlap() {
assert_eq!(find_common_overlap(&["".into()], &["".into()]), 0);
assert_eq!(
find_common_overlap(
&["a".into(), "b".into(), "c".into()],
&["b".into(), "c".into(), "a".into()]
),
1
);
assert_eq!(
find_common_overlap(
&["a".into(), "a".into(), "a".into()],
&["a".into(), "b".into(), "c".into()]
),
2
);
assert_eq!(
find_common_overlap(
&["a".into(), "b".into(), "c".into()],
&["d".into(), "e".into(), "a".into()]
),
3
);
assert_eq!(
find_common_overlap(&["a".into(), "a".into()], &["a".into()]),
1
);
}
}

103
backend/reconcile/src/utils/find_longest_prefix_contained_within.rs Normal file
View file

@ -0,0 +1,103 @@
use crate::Token;
/// Given two lists of tokens, returns `length` where `old` list somewhere
/// within contains the `length` prefix of the `new` list.
///
/// ## Example
///
/// ```not_rust
/// old: [0, 1, 9, 0, 2, 5]
/// new: [9, 0, 2, 5, 1]
/// ```
/// > results in an length of 4
///
///
/// ```not_rust
/// old: [0, 1, 9, 0, 2, 5]
/// new: [0, 2]
/// ```
/// > results in an length of 2
///
/// ```not_rust
/// old: [0, 1, 9, 0, 2, 5]
/// new: [0, 4]
/// ```
/// > results in an length of 1
pub fn find_longest_prefix_contained_within<T>(old: &[Token<T>], new: &[Token<T>]) -> usize
where
T: PartialEq + Clone + std::fmt::Debug,
{
let max_possible = new.len().min(old.len());
for len in (1..=max_possible).rev() {
let prefix = &new[..len];
if old.windows(len).any(|window| window == prefix) {
return len;
}
}
0
}
#[cfg(test)]
mod tests {
use pretty_assertions::assert_eq;
use super::*;
#[test]
fn test_common_overlap() {
assert_eq!(
find_longest_prefix_contained_within(&["".into()], &["".into()]),
1
);
assert_eq!(
find_longest_prefix_contained_within(
&["a".into(), "b".into(), "c".into()],
&["b".into(), "c".into(), "a".into()]
),
2
);
assert_eq!(
find_longest_prefix_contained_within(
&["a".into(), "b".into(), "c".into()],
&["b".into(), "c".into()]
),
2
);
assert_eq!(
find_longest_prefix_contained_within(
&["a".into(), "b".into(), "c".into()],
&["b".into()]
),
1
);
assert_eq!(
find_longest_prefix_contained_within(
&["a".into(), "b".into(), "c".into(), "b".into(), "a".into()],
&["b".into(), "a".into()]
),
2
);
assert_eq!(
find_longest_prefix_contained_within(
&["a".into(), "a".into(), "a".into()],
&["a".into(), "b".into(), "c".into()]
),
1
);
assert_eq!(
find_longest_prefix_contained_within(
&["a".into(), "b".into(), "c".into()],
&["d".into(), "e".into(), "a".into()]
),
0
);
}
}

3
backend/reconcile/src/utils/merge_iters.rs
View file

@ -46,8 +46,7 @@ where
};
match order {
Some(Ordering::Less) | None => self.left.next(),
Some(Ordering::Equal) => self.left.next(),
Some(Ordering::Less | Ordering::Equal) | None => self.left.next(),
Some(Ordering::Greater) => self.right.next(),
}
}

2
backend/reconcile/src/utils/ordered_operation.rs
View file

@ -7,7 +7,7 @@ use crate::operation_transformation::Operation;
#[derive(Debug, Clone, PartialEq)]
pub struct OrderedOperation<T>
where
T: PartialEq + Clone,
T: PartialEq + Clone + std::fmt::Debug,
{
pub order: usize,
pub operation: Operation<T>,

4
backend/rust-toolchain.toml Normal file
View file

@ -0,0 +1,4 @@
[toolchain]
channel = "nightly-2025-03-14"
targets = [ "x86_64-unknown-linux-gnu", "x86_64-unknown-linux-musl" ]
profile = "default"

46
backend/sync_lib/src/lib.rs
View file

@ -18,7 +18,20 @@ pub mod errors;
/// Encode binary data for easy transport over HTTP. Inverse of
/// `base64_to_bytes`.
///
/// # Arguments
///
/// - `input`: The binary data to encode.
///
/// # Returns
///
/// The base64-encoded string.
///
/// # Panics
///
/// If the input is not valid UTF-8.
#[wasm_bindgen(js_name = bytesToBase64)]
#[must_use]
pub fn bytes_to_base64(input: &[u8]) -> String {
set_panic_hook();
@ -26,6 +39,19 @@ pub fn bytes_to_base64(input: &[u8]) -> String {
}
/// Inverse of `bytes_to_base64`.
/// Decode base64-encoded data into binary data.
///
/// # Arguments
///
/// - `input`: The base64-encoded string.
///
/// # Returns
///
/// The decoded binary data.
///
/// # Errors
///
/// If the input is not valid base64.
#[wasm_bindgen(js_name = base64ToBytes)]
pub fn base64_to_bytes(input: &str) -> Result<Vec<u8>, SyncLibError> {
set_panic_hook();
@ -36,7 +62,22 @@ pub fn base64_to_bytes(input: &str) -> Result<Vec<u8>, SyncLibError> {
/// Merge two documents with a common parent. Relies on `reconcile::reconcile`
/// for texts and returns the right document as-is if either of the updated
/// documents is binary.
///
/// # Arguments
///
/// - `parent`: The common parent document.
/// - `left`: The left document updated by one user.
/// - `right`: The right document updated by another user.
///
/// # Returns
///
/// The merged document.
///
/// # Panics
///
/// If any of the input documents are not valid UTF-8 strings.
#[wasm_bindgen]
#[must_use]
pub fn merge(parent: &[u8], left: &[u8], right: &[u8]) -> Vec<u8> {
set_panic_hook();
@ -54,6 +95,7 @@ pub fn merge(parent: &[u8], left: &[u8], right: &[u8]) -> Vec<u8> {
/// WASM wrapper around `reconcile::reconcile` for text merging.
#[wasm_bindgen(js_name = mergeText)]
#[must_use]
pub fn merge_text(parent: &str, left: &str, right: &str) -> String {
set_panic_hook();
@ -63,10 +105,11 @@ pub fn merge_text(parent: &str, left: &str, right: &str) -> String {
/// Heuristically determine if the given data is a binary or a text file's
/// content.
#[wasm_bindgen(js_name = isBinary)]
#[must_use]
pub fn is_binary(data: &[u8]) -> bool {
set_panic_hook();
if data.iter().any(|&b| b == 0) {
if data.contains(&0) {
// Even though the NUL character is valid in UTF-8, it's highly suspicious in
// human-readable text.
return true;
@ -77,6 +120,7 @@ pub fn is_binary(data: &[u8]) -> bool {
/// We don't want to support merging structured data like JSON, YAML, etc.
#[wasm_bindgen(js_name = isFileTypeMergable)]
#[must_use]
pub fn is_file_type_mergable(path_or_file_name: &str) -> bool {
set_panic_hook();

3
backend/sync_lib/tests/web.rs
View file

@ -26,7 +26,8 @@ fn test_base64_to_bytes_error() {
fn merge_text() {
let left = b"hello ";
let right = b"world";
assert_eq!(merge(b"", left, right), b"hello world".to_vec());
let result = merge(b"", left, right);
assert_eq!(result, b"hello world");
}
#[wasm_bindgen_test(unsupported = test)]

9
backend/sync_server/src/config.rs
View file

@ -42,9 +42,12 @@ impl Config {
}
pub async fn load_from_file(path: &Path) -> Result<Self> {
let contents = fs::read_to_string(path)
.await
.with_context(|| format!("Cannot load configuration from disk from ({path:?})"))?;
let contents = fs::read_to_string(path).await.with_context(|| {
format!(
"Cannot load configuration from disk from {}",
path.display()
)
})?;
let config = serde_yaml::from_str(&contents).context("Failed to parse configuration")?;

18
backend/sync_server/src/config/database_config.rs
View file

@ -1,31 +1,33 @@
use std::path::PathBuf;
use log::debug;
use serde::{Deserialize, Serialize};
use crate::consts::{DEFAULT_MAX_CONNECTIONS, DEFAULT_SQLITE_URL};
use crate::consts::{DEFAULT_DATABASES_DIRECTORY_PATH, DEFAULT_MAX_CONNECTIONS};
#[derive(Debug, Deserialize, Serialize, Clone)]
pub struct DatabaseConfig {
#[serde(default = "default_sqlite_url")]
pub sqlite_url: String,
#[serde(default = "default_databases_directory_path")]
pub databases_directory_path: PathBuf,
#[serde(default = "default_max_connections")]
pub max_connections: u32,
}
fn default_sqlite_url() -> String {
debug!("Using default sqlite url: {}", DEFAULT_SQLITE_URL);
DEFAULT_SQLITE_URL.to_owned()
fn default_databases_directory_path() -> PathBuf {
debug!("Using default databases directory path: {DEFAULT_DATABASES_DIRECTORY_PATH:?}");
PathBuf::from(DEFAULT_DATABASES_DIRECTORY_PATH)
}
fn default_max_connections() -> u32 {
debug!("Using default max connections: {}", DEFAULT_MAX_CONNECTIONS);
debug!("Using default max connections: {DEFAULT_MAX_CONNECTIONS}");
DEFAULT_MAX_CONNECTIONS
}
impl Default for DatabaseConfig {
fn default() -> Self {
Self {
sqlite_url: default_sqlite_url(),
databases_directory_path: default_databases_directory_path(),
max_connections: default_max_connections(),
}
}

9
backend/sync_server/src/config/server_config.rs
View file

@ -15,20 +15,17 @@ pub struct ServerConfig {
}
fn default_host() -> String {
debug!("Using default server host: {}", DEFAULT_HOST);
debug!("Using default server host: {DEFAULT_HOST}");
DEFAULT_HOST.to_owned()
}
fn default_port() -> u16 {
debug!("Using default server port: {}", DEFAULT_PORT);
debug!("Using default server port: {DEFAULT_PORT}");
DEFAULT_PORT
}
fn default_max_body_size_mb() -> usize {
debug!(
"Using default max body size (MB): {}",
DEFAULT_MAX_BODY_SIZE_MB
);
debug!("Using default max body size (MB): {DEFAULT_MAX_BODY_SIZE_MB}");
DEFAULT_MAX_BODY_SIZE_MB
}

2
backend/sync_server/src/consts.rs
View file

@ -1,5 +1,5 @@
pub const CONFIG_PATH: &str = "config.yml";
pub const DEFAULT_SQLITE_URL: &str = "db.sqlite3";
pub const DEFAULT_DATABASES_DIRECTORY_PATH: &str = "databases";
pub const DEFAULT_HOST: &str = "127.0.0.1";
pub const DEFAULT_PORT: u16 = 3000;
pub const DEFAULT_MAX_CONNECTIONS: u32 = 12;

181
backend/sync_server/src/database.rs
View file

@ -1,4 +1,5 @@
use core::{str::FromStr as _, time::Duration};
use core::time::Duration;
use std::{collections::HashMap, sync::Arc};
use anyhow::{Context as _, Result};
use models::{
@ -7,19 +8,66 @@ use models::{
use sqlx::{sqlite::SqliteConnectOptions, types::chrono::Utc};
pub mod models;
use sqlx::{Pool, Sqlite, sqlite::SqlitePoolOptions};
use tokio::sync::Mutex;
use uuid::fmt::Hyphenated;
use crate::config::database_config::DatabaseConfig;
#[derive(Clone, Debug)]
pub struct Database {
connection_pool: Pool<Sqlite>,
config: DatabaseConfig,
connection_pools: Arc<Mutex<HashMap<VaultId, Pool<Sqlite>>>>,
}
pub type Transaction<'a> = sqlx::Transaction<'a, Sqlite>;
impl Database {
pub async fn try_new(config: &DatabaseConfig) -> Result<Self> {
let connection_options = SqliteConnectOptions::from_str(&config.sqlite_url)?
tokio::fs::create_dir_all(&config.databases_directory_path)
.await
.with_context(|| {
format!(
"Failed to create databases directory: {}",
config.databases_directory_path.to_string_lossy()
)
})?;
let mut connection_pools = std::collections::HashMap::new();
let mut entries = tokio::fs::read_dir(&config.databases_directory_path).await?;
while let Some(entry) = entries.next_entry().await? {
if !entry.file_name().to_string_lossy().ends_with(".sqlite") {
continue;
}
let vault: VaultId = entry
.file_name()
.to_string_lossy()
.trim_end_matches(".sqlite")
.to_owned();
connection_pools.insert(
vault.clone(),
Self::create_vault_database(config, &vault).await?,
);
}
Ok(Self {
config: config.clone(),
connection_pools: Arc::new(Mutex::new(connection_pools)),
})
}
async fn create_vault_database(
config: &DatabaseConfig,
vault: &VaultId,
) -> Result<Pool<Sqlite>> {
let file_name = config
.databases_directory_path
.join(format!("{vault}.sqlite"));
let connection_options = SqliteConnectOptions::new()
.filename(file_name.clone())
.create_if_missing(true)
.busy_timeout(Duration::from_secs(3600))
.journal_mode(sqlx::sqlite::SqliteJournalMode::Wal);
@ -29,18 +77,11 @@ impl Database {
.test_before_acquire(true)
.connect_with(connection_options)
.await
.with_context(|| {
format!(
"Cannot connect to database with url: {}",
&config.sqlite_url
)
})?;
.with_context(|| format!("Cannot open database at {}", file_name.display()))?;
Self::run_migrations(&pool).await?;
Ok(Self {
connection_pool: pool,
})
Ok(pool)
}
async fn run_migrations(pool: &Pool<Sqlite>) -> Result<()> {
@ -50,17 +91,38 @@ impl Database {
.context("Cannot check for pending migrations")
}
async fn get_connection_pool(&mut self, vault: &VaultId) -> Result<Pool<Sqlite>> {
let mut pools = self.connection_pools.lock().await;
if !pools.contains_key(vault) {
let pool = Self::create_vault_database(&self.config, vault).await?;
pools.insert(vault.clone(), pool);
}
let pool = pools
.get(vault)
.expect("Pool was just inserted or already exists");
Ok(pool.clone())
}
/// Attempting to write from this transaction might result in a
/// database locked error. Use this transaction for read-only operations.
pub async fn create_readonly_transaction(&self) -> Result<Transaction<'_>> {
self.connection_pool
pub async fn create_readonly_transaction(
&mut self,
vault: &VaultId,
) -> Result<Transaction<'static>> {
self.get_connection_pool(vault)
.await?
.begin()
.await
.context("Cannot create transaction")
}
pub async fn create_write_transaction(&self) -> Result<Transaction<'_>> {
let mut transaction = self.create_readonly_transaction().await?;
pub async fn create_write_transaction(
&mut self,
vault: &VaultId,
) -> Result<Transaction<'static>> {
let mut transaction = self.create_readonly_transaction(vault).await?;
// sqlx doesn't support immediate transactions for sqlite: https://github.com/launchbadge/sqlx/issues/481
sqlx::query!("END; BEGIN IMMEDIATE;")
@ -72,7 +134,7 @@ impl Database {
/// Return the latest state of all documents in the vault
pub async fn get_latest_documents(
&self,
&mut self,
vault: &VaultId,
transaction: Option<&mut Transaction<'_>>,
) -> Result<Vec<DocumentVersionWithoutContent>> {
@ -80,24 +142,22 @@ impl Database {
DocumentVersionWithoutContent,
r#"
select
vault_id,
vault_update_id,
document_id as "document_id: uuid::Uuid",
document_id as "document_id: Hyphenated",
relative_path,
created_date as "created_date: chrono::DateTime<Utc>",
updated_date as "updated_date: chrono::DateTime<Utc>",
is_deleted
from latest_document_versions
where vault_id = ?
order by vault_update_id desc
"#,
vault,
);
if let Some(transaction) = transaction {
query.fetch_all(&mut **transaction).await
} else {
query.fetch_all(&self.connection_pool).await
query
.fetch_all(&self.get_connection_pool(vault).await?)
.await
}
.context("Cannot fetch latest documents")
}
@ -105,7 +165,7 @@ impl Database {
/// Return the latest state of all documents (including deleted) in the
/// vault which have changed since the given update id
pub async fn get_latest_documents_since(
&self,
&mut self,
vault: &VaultId,
vault_update_id: VaultUpdateId,
transaction: Option<&mut Transaction<'_>>,
@ -114,25 +174,24 @@ impl Database {
DocumentVersionWithoutContent,
r#"
select
vault_id,
vault_update_id,
document_id as "document_id: uuid::Uuid",
document_id as "document_id: Hyphenated",
relative_path,
created_date as "created_date: chrono::DateTime<Utc>",
updated_date as "updated_date: chrono::DateTime<Utc>",
is_deleted
from latest_document_versions
where vault_id = ? and vault_update_id > ?
where vault_update_id > ?
order by vault_update_id desc
"#,
vault,
vault_update_id
);
if let Some(transaction) = transaction {
query.fetch_all(&mut **transaction).await
} else {
query.fetch_all(&self.connection_pool).await
query
.fetch_all(&self.get_connection_pool(vault).await?)
.await
}
.with_context(|| {
format!("Cannot fetch latest documents since vault_update_id {vault_update_id}")
@ -140,7 +199,7 @@ impl Database {
}
pub async fn get_max_update_id_in_vault(
&self,
&mut self,
vault: &VaultId,
transaction: Option<&mut Transaction<'_>>,
) -> Result<i64> {
@ -148,22 +207,22 @@ impl Database {
r#"
select coalesce(max(vault_update_id), 0) as max_vault_update_id
from documents
where vault_id = ?
"#,
vault
);
if let Some(transaction) = transaction {
query.fetch_one(&mut **transaction).await
} else {
query.fetch_one(&self.connection_pool).await
query
.fetch_one(&self.get_connection_pool(vault).await?)
.await
}
.map(|row| row.max_vault_update_id)
.context("Cannot fetch max update id in vault")
}
pub async fn get_latest_document_by_path(
&self,
&mut self,
vault: &VaultId,
relative_path: &str,
transaction: Option<&mut Transaction<'_>>,
@ -172,68 +231,67 @@ impl Database {
StoredDocumentVersion,
r#"
select
vault_id,
vault_update_id,
document_id as "document_id: uuid::Uuid",
document_id as "document_id: Hyphenated",
relative_path,
created_date as "created_date: chrono::DateTime<Utc>",
updated_date as "updated_date: chrono::DateTime<Utc>",
content,
is_deleted
from latest_document_versions
where vault_id = ? and relative_path = ?
where relative_path = ?
order by vault_update_id desc -- `latest_document_versions` only contains a single latest version of each document, however,
-- multiple documents can have the same `relative_path`, if they have been deleted. That's
-- why we only care about the latest version of the document with the given relative path.
limit 1
"#,
vault,
relative_path
);
if let Some(transaction) = transaction {
query.fetch_optional(&mut **transaction).await
} else {
query.fetch_optional(&self.connection_pool).await
query
.fetch_optional(&self.get_connection_pool(vault).await?)
.await
}
.context("Cannot fetch latest document version")
}
pub async fn get_latest_document(
&self,
&mut self,
vault: &VaultId,
document_id: &DocumentId,
transaction: Option<&mut Transaction<'_>>,
) -> Result<Option<StoredDocumentVersion>> {
let document_id = document_id.as_hyphenated();
let query = sqlx::query_as!(
StoredDocumentVersion,
r#"
select
vault_id,
vault_update_id,
document_id as "document_id: uuid::Uuid",
document_id as "document_id: Hyphenated",
relative_path,
created_date as "created_date: chrono::DateTime<Utc>",
updated_date as "updated_date: chrono::DateTime<Utc>",
content,
is_deleted
from latest_document_versions
where vault_id = ? and document_id = ?
where document_id = ?
"#,
vault,
document_id
);
if let Some(transaction) = transaction {
query.fetch_optional(&mut **transaction).await
} else {
query.fetch_optional(&self.connection_pool).await
query
.fetch_optional(&self.get_connection_pool(vault).await?)
.await
}
.context("Cannot fetch latest document version")
}
pub async fn get_document_version(
&self,
&mut self,
vault: &VaultId,
vault_update_id: VaultUpdateId,
transaction: Option<&mut Transaction<'_>>,
@ -242,52 +300,49 @@ impl Database {
StoredDocumentVersion,
r#"
select
vault_id,
vault_update_id,
document_id as "document_id: uuid::Uuid",
document_id as "document_id: Hyphenated",
relative_path,
created_date as "created_date: chrono::DateTime<Utc>",
updated_date as "updated_date: chrono::DateTime<Utc>",
content,
is_deleted
from documents
where vault_id = ? and vault_update_id = ?"#,
vault,
where vault_update_id = ?"#,
vault_update_id
);
if let Some(transaction) = transaction {
query.fetch_optional(&mut **transaction).await
} else {
query.fetch_optional(&self.connection_pool).await
query
.fetch_optional(&self.get_connection_pool(vault).await?)
.await
}
.context("Cannot fetch document version")
}
pub async fn insert_document_version(
&self,
&mut self,
vault: &VaultId,
version: &StoredDocumentVersion,
transaction: Option<&mut Transaction<'_>>,
) -> Result<()> {
let document_id = version.document_id.as_hyphenated();
let query = sqlx::query!(
r#"
insert into documents (
vault_id,
vault_update_id,
document_id,
relative_path,
created_date,
updated_date,
content,
is_deleted
)
values (?, ?, ?, ?, ?, ?, ?, ?)
values (?, ?, ?, ?, ?, ?)
"#,
version.vault_id,
version.vault_update_id,
version.document_id,
document_id,
version.relative_path,
version.created_date,
version.updated_date,
version.content,
version.is_deleted
@ -296,7 +351,7 @@ impl Database {
if let Some(transaction) = transaction {
query.execute(&mut **transaction).await
} else {
query.execute(&self.connection_pool).await
query.execute(&self.get_connection_pool(vault).await?).await
}
.context("Cannot insert document version")?;

16
backend/sync_server/src/database/migrations/20241207143519_bootstrap.sql
View file

@ -1,25 +1,21 @@
CREATE TABLE IF NOT EXISTS documents (
vault_id TEXT NOT NULL,
vault_update_id INTEGER NOT NULL,
vault_update_id INTEGER NOT NULL PRIMARY KEY,
document_id TEXT NOT NULL,
relative_path TEXT NOT NULL,
created_date TIMESTAMP NOT NULL,
updated_date TIMESTAMP NOT NULL,
content BLOB NOT NULL,
is_deleted BOOLEAN NOT NULL,
PRIMARY KEY (vault_id, vault_update_id)
is_deleted BOOLEAN NOT NULL
);
CREATE VIEW IF NOT EXISTS latest_document_versions AS
SELECT d.*
FROM documents d
INNER JOIN (
SELECT vault_id, MAX(vault_update_id) AS max_version_id
SELECT MAX(vault_update_id) AS max_version_id
FROM documents
GROUP BY vault_id, document_id
GROUP BY document_id
) max_versions
ON d.vault_id = max_versions.vault_id
AND d.vault_update_id = max_versions.max_version_id;
ON d.vault_update_id = max_versions.max_version_id;
CREATE INDEX IF NOT EXISTS idx_documents_vault_id_relative_path
ON documents (vault_id, relative_path);
ON documents (relative_path);

14
backend/sync_server/src/database/models.rs
View file

@ -9,30 +9,24 @@ pub type DocumentId = uuid::Uuid;
#[derive(Debug, Clone)]
pub struct StoredDocumentVersion {
pub vault_id: VaultId,
pub vault_update_id: VaultUpdateId,
pub document_id: DocumentId,
pub relative_path: String,
pub created_date: DateTime<Utc>,
pub updated_date: DateTime<Utc>,
pub content: Vec<u8>,
pub is_deleted: bool,
}
impl PartialEq<Self> for StoredDocumentVersion {
fn eq(&self, other: &Self) -> bool {
self.vault_id == other.vault_id && self.vault_update_id == other.vault_update_id
}
fn eq(&self, other: &Self) -> bool { self.vault_update_id == other.vault_update_id }
}
#[derive(Debug, Clone, Serialize, JsonSchema)]
#[serde(rename_all = "camelCase")]
pub struct DocumentVersionWithoutContent {
pub vault_id: VaultId,
pub vault_update_id: VaultUpdateId,
pub document_id: DocumentId,
pub relative_path: String,
pub created_date: DateTime<Utc>,
pub updated_date: DateTime<Utc>,
pub is_deleted: bool,
}
@ -40,11 +34,9 @@ pub struct DocumentVersionWithoutContent {
impl From<StoredDocumentVersion> for DocumentVersionWithoutContent {
fn from(value: StoredDocumentVersion) -> Self {
Self {
vault_id: value.vault_id,
vault_update_id: value.vault_update_id,
document_id: value.document_id,
relative_path: value.relative_path,
created_date: value.created_date,
updated_date: value.updated_date,
is_deleted: value.is_deleted,
}
@ -54,11 +46,9 @@ impl From<StoredDocumentVersion> for DocumentVersionWithoutContent {
#[derive(Debug, Clone, Serialize, JsonSchema)]
#[serde(rename_all = "camelCase")]
pub struct DocumentVersion {
pub vault_id: VaultId,
pub vault_update_id: VaultUpdateId,
pub document_id: DocumentId,
pub relative_path: String,
pub created_date: DateTime<Utc>,
pub updated_date: DateTime<Utc>,
pub content_base64: String,
pub is_deleted: bool,
@ -67,11 +57,9 @@ pub struct DocumentVersion {
impl From<StoredDocumentVersion> for DocumentVersion {
fn from(value: StoredDocumentVersion) -> Self {
Self {
vault_id: value.vault_id,
vault_update_id: value.vault_update_id,
document_id: value.document_id,
relative_path: value.relative_path,
created_date: value.created_date,
updated_date: value.updated_date,
content_base64: bytes_to_base64(&value.content),
is_deleted: value.is_deleted,

17
backend/sync_server/src/errors.rs
View file

@ -33,12 +33,12 @@ pub enum SyncServerError {
impl SyncServerError {
pub fn serialize(&self) -> SerializedError {
match self {
Self::InitError(error) => error.into(),
Self::ClientError(error) => error.into(),
Self::ServerError(error) => error.into(),
Self::NotFound(error) => error.into(),
Self::Unauthorized(error) => error.into(),
Self::PermissionDeniedError(error) => error.into(),
Self::InitError(error)
| Self::ClientError(error)
| Self::ServerError(error)
| Self::NotFound(error)
| Self::Unauthorized(error)
| Self::PermissionDeniedError(error) => error.into(),
}
}
}
@ -48,9 +48,10 @@ impl IntoResponse for SyncServerError {
let body = Json(self.serialize());
match self {
Self::InitError(_) => (StatusCode::INTERNAL_SERVER_ERROR, body).into_response(),
Self::InitError(_) | Self::ServerError(_) => {
(StatusCode::INTERNAL_SERVER_ERROR, body).into_response()
}
Self::ClientError(_) => (StatusCode::BAD_REQUEST, body).into_response(),
Self::ServerError(_) => (StatusCode::INTERNAL_SERVER_ERROR, body).into_response(),
Self::NotFound(_) => (StatusCode::NOT_FOUND, body).into_response(),
Self::Unauthorized(_) => (StatusCode::UNAUTHORIZED, body).into_response(),
Self::PermissionDeniedError(_) => (StatusCode::FORBIDDEN, body).into_response(),

73
backend/sync_server/src/server.rs
View file

@ -16,6 +16,7 @@ use axum::{
extract::{DefaultBodyLimit, Request},
http::{self, HeaderValue, Method},
response::IntoResponse,
routing::IntoMakeService,
};
use log::{error, info};
use tokio::signal;
@ -30,7 +31,10 @@ use tower_http::{
};
use tracing::{Level, info_span};
use crate::errors::{SerializedError, not_found_error};
use crate::{
config::server_config::ServerConfig,
errors::{SerializedError, not_found_error},
};
mod app_state;
mod auth;
mod create_document;
@ -52,24 +56,9 @@ pub async fn create_server() -> Result<()> {
.await
.context("Failed to initialise app state")?;
let address = format!(
"{}:{}",
&app_state.config.server.host, &app_state.config.server.port
);
let mut api = OpenApi {
info: Info {
title: "VaultLink sync server".to_owned(),
summary: Some(
"Simple API for syncing documents between concurrent clients.".to_owned(),
),
description: Some(include_str!("../README.md").to_owned()),
version: env!("CARGO_PKG_VERSION").to_owned(),
..Info::default()
},
..OpenApi::default()
};
let server_config = app_state.config.server.clone();
let mut api = create_open_api();
let app = ApiRouter::new()
.api_route("/ping", get(ping::ping))
.api_route(
@ -140,11 +129,42 @@ pub async fn create_server() -> Result<()> {
.allow_methods([Method::GET, Method::POST, Method::PUT, Method::DELETE]),
)
.with_state(app_state)
.finish_api_with(&mut api, api_docs)
.finish_api_with(&mut api, add_api_docs_error_example)
.layer(Extension(Arc::new(api))) // https://github.com/tamasfe/aide/blob/507f4a8822bc0c13cbda0f589da1e0f4cbcdb812/examples/example-axum/src/main.rs#L39
.fallback(handler_404)
.into_make_service();
start_server(app, &server_config).await
}
async fn serve_api(Extension(api): Extension<Arc<OpenApi>>) -> impl IntoResponse { Json(api) }
fn create_open_api() -> OpenApi {
OpenApi {
info: Info {
title: "VaultLink sync server".to_owned(),
summary: Some(
"Simple API for syncing documents between concurrent clients.".to_owned(),
),
description: Some(include_str!("../README.md").to_owned()),
version: env!("CARGO_PKG_VERSION").to_owned(),
..Info::default()
},
..OpenApi::default()
}
}
fn add_api_docs_error_example(api: TransformOpenApi<'_>) -> TransformOpenApi<'_> {
api.default_response_with::<Json<SerializedError>, _>(|res| {
res.example(SerializedError {
message: "An error has occurred".to_owned(),
causes: vec![],
})
})
}
async fn start_server(app: IntoMakeService<axum::Router>, config: &ServerConfig) -> Result<()> {
let address = format!("{}:{}", config.host, config.port);
let listener = tokio::net::TcpListener::bind(address.clone())
.await
.with_context(|| format!("Failed to bind to address: {address}"))?;
@ -163,17 +183,6 @@ pub async fn create_server() -> Result<()> {
.context("Failed to start server")
}
async fn serve_api(Extension(api): Extension<Arc<OpenApi>>) -> impl IntoResponse { Json(api) }
fn api_docs(api: TransformOpenApi<'_>) -> TransformOpenApi<'_> {
api.default_response_with::<Json<SerializedError>, _>(|res| {
res.example(SerializedError {
message: "An error has occurred".to_owned(),
causes: vec![],
})
})
}
async fn shutdown_signal() {
let ctrl_c = async {
signal::ctrl_c()
@ -193,8 +202,8 @@ async fn shutdown_signal() {
let terminate = std::future::pending::<()>();
tokio::select! {
_ = ctrl_c => {},
_ = terminate => {},
() = ctrl_c => {},
() = terminate => {},
}
}

38
backend/sync_server/src/server/create_document.rs
View file

@ -6,7 +6,6 @@ use axum_extra::{
headers::{Authorization, authorization::Bearer},
};
use axum_jsonschema::Json;
use chrono::{DateTime, Utc};
use schemars::JsonSchema;
use serde::Deserialize;
use sync_lib::base64_to_bytes;
@ -17,7 +16,7 @@ use super::{
requests::{CreateDocumentVersion, CreateDocumentVersionMultipart},
};
use crate::{
database::models::{DocumentVersionWithoutContent, StoredDocumentVersion, VaultId},
database::models::{DocumentId, DocumentVersionWithoutContent, StoredDocumentVersion, VaultId},
errors::{SyncServerError, client_error, server_error},
utils::sanitize_path,
};
@ -44,8 +43,8 @@ pub async fn create_document_multipart(
auth_header,
state,
vault_id,
request.document_id,
request.relative_path,
request.created_date,
request.content.contents.to_vec(),
)
.await
@ -69,8 +68,8 @@ pub async fn create_document_json(
auth_header,
state,
vault_id,
request.document_id,
request.relative_path,
request.created_date,
content_bytes,
)
.await
@ -78,20 +77,39 @@ pub async fn create_document_json(
async fn internal_create_document(
auth_header: Authorization<Bearer>,
state: AppState,
mut state: AppState,
vault_id: VaultId,
document_id: Option<DocumentId>,
relative_path: String,
created_date: DateTime<Utc>,
content: Vec<u8>,
) -> Result<Json<DocumentVersionWithoutContent>, SyncServerError> {
auth(&state, auth_header.token())?;
let mut transaction = state
.database
.create_write_transaction()
.create_write_transaction(&vault_id)
.await
.map_err(server_error)?;
let document_id = match document_id {
Some(document_id) => {
let existing_version = state
.database
.get_latest_document(&vault_id, &document_id, Some(&mut transaction))
.await
.map_err(server_error)?;
if existing_version.is_some() {
return Err(client_error(anyhow::anyhow!(
"Document with the same ID already exists"
)));
}
document_id
}
None => uuid::Uuid::new_v4(),
};
let last_update_id = state
.database
.get_max_update_id_in_vault(&vault_id, Some(&mut transaction))
@ -101,19 +119,17 @@ async fn internal_create_document(
let sanitized_relative_path = sanitize_path(&relative_path);
let new_version = StoredDocumentVersion {
vault_id,
vault_update_id: last_update_id + 1,
document_id: uuid::Uuid::new_v4(),
document_id,
relative_path: sanitized_relative_path,
content,
created_date,
updated_date: chrono::Utc::now(),
is_deleted: false,
};
state
.database
.insert_document_version(&new_version, Some(&mut transaction))
.insert_document_version(&vault_id, &new_version, Some(&mut transaction))
.await
.map_err(server_error)?;

14
backend/sync_server/src/server/delete_document.rs
View file

@ -10,7 +10,7 @@ use serde::Deserialize;
use super::{app_state::AppState, auth::auth, requests::DeleteDocumentVersion};
use crate::{
database::models::{DocumentId, StoredDocumentVersion, VaultId},
database::models::{DocumentId, DocumentVersionWithoutContent, StoredDocumentVersion, VaultId},
errors::{SyncServerError, server_error},
utils::sanitize_path,
};
@ -29,14 +29,14 @@ pub async fn delete_document(
vault_id,
document_id,
}): Path<PathParams>,
State(state): State<AppState>,
State(mut state): State<AppState>,
Json(request): Json<DeleteDocumentVersion>,
) -> Result<(), SyncServerError> {
) -> Result<Json<DocumentVersionWithoutContent>, SyncServerError> {
auth(&state, auth_header.token())?;
let mut transaction = state
.database
.create_write_transaction()
.create_write_transaction(&vault_id)
.await
.map_err(server_error)?;
@ -47,19 +47,17 @@ pub async fn delete_document(
.map_err(server_error)?;
let new_version = StoredDocumentVersion {
vault_id,
vault_update_id: last_update_id + 1,
document_id,
relative_path: sanitize_path(&request.relative_path),
content: vec![],
created_date: request.created_date,
updated_date: chrono::Utc::now(),
is_deleted: true,
};
state
.database
.insert_document_version(&new_version, Some(&mut transaction))
.insert_document_version(&vault_id, &new_version, Some(&mut transaction))
.await
.map_err(server_error)?;
@ -69,5 +67,5 @@ pub async fn delete_document(
.context("Failed to commit successful transaction")
.map_err(server_error)?;
Ok(())
Ok(Json(new_version.into()))
}

10
backend/sync_server/src/server/fetch_document_version.rs
View file

@ -30,7 +30,7 @@ pub async fn fetch_document_version(
document_id,
vault_update_id,
}): Path<PathParams>,
State(state): State<AppState>,
State(mut state): State<AppState>,
) -> Result<Json<DocumentVersion>, SyncServerError> {
auth(&state, auth_header.token())?;
@ -39,12 +39,14 @@ pub async fn fetch_document_version(
.get_document_version(&vault_id, vault_update_id, None)
.await
.map_err(server_error)?
.map(Ok)
.unwrap_or_else(|| {
.map_or_else(
|| {
Err(not_found_error(anyhow!(
"Document with vault update id `{vault_update_id}` not found",
)))
})?;
},
Ok,
)?;
if result.document_id != document_id {
return Err(not_found_error(anyhow!(

10
backend/sync_server/src/server/fetch_document_version_content.rs
View file

@ -32,7 +32,7 @@ pub async fn fetch_document_version_content(
document_id,
vault_update_id,
}): Path<PathParams>,
State(state): State<AppState>,
State(mut state): State<AppState>,
) -> Result<Bytes, SyncServerError> {
auth(&state, auth_header.token())?;
@ -41,12 +41,14 @@ pub async fn fetch_document_version_content(
.get_document_version(&vault_id, vault_update_id, None)
.await
.map_err(server_error)?
.map(Ok)
.unwrap_or_else(|| {
.map_or_else(
|| {
Err(not_found_error(anyhow!(
"Document with vault update id `{vault_update_id}` not found",
)))
})?;
},
Ok,
)?;
if result.document_id != document_id {
return Err(not_found_error(anyhow!(

10
backend/sync_server/src/server/fetch_latest_document_version.rs
View file

@ -28,7 +28,7 @@ pub async fn fetch_latest_document_version(
vault_id,
document_id,
}): Path<PathParams>,
State(state): State<AppState>,
State(mut state): State<AppState>,
) -> Result<Json<DocumentVersion>, SyncServerError> {
auth(&state, auth_header.token())?;
@ -37,12 +37,14 @@ pub async fn fetch_latest_document_version(
.get_latest_document(&vault_id, &document_id, None)
.await
.map_err(server_error)?
.map(Ok)
.unwrap_or_else(|| {
.map_or_else(
|| {
Err(not_found_error(anyhow!(
"Document with id `{document_id}` not found",
)))
})?;
},
Ok,
)?;
Ok(Json(latest_version.into()))
}

2
backend/sync_server/src/server/fetch_latest_documents.rs
View file

@ -30,7 +30,7 @@ pub async fn fetch_latest_documents(
TypedHeader(auth_header): TypedHeader<Authorization<Bearer>>,
Path(PathParams { vault_id }): Path<PathParams>,
Query(QueryParams { since_update_id }): Query<QueryParams>,
State(state): State<AppState>,
State(mut state): State<AppState>,
) -> Result<Json<FetchLatestDocumentsResponse>, SyncServerError> {
auth(&state, auth_header.token())?;

14
backend/sync_server/src/server/requests.rs
View file

@ -1,24 +1,27 @@
use aide_axum_typed_multipart::FieldData;
use axum::body::Bytes;
use axum_typed_multipart::TryFromMultipart;
use chrono::{DateTime, Utc};
use schemars::JsonSchema;
use serde::{self, Deserialize};
use crate::database::models::VaultUpdateId;
use crate::database::models::{DocumentId, VaultUpdateId};
#[derive(Debug, Deserialize, JsonSchema)]
#[serde(rename_all = "camelCase")]
pub struct CreateDocumentVersion {
/// The client can decide the document id (if it wishes to) in order
/// to help with syncing. If the client does not provide a document id,
/// the server will generate one. If the client provides a document id
/// it must not already exist in the database.
pub document_id: Option<DocumentId>,
pub relative_path: String,
pub created_date: DateTime<Utc>,
pub content_base64: String,
}
#[derive(Debug, TryFromMultipart, JsonSchema)]
pub struct CreateDocumentVersionMultipart {
pub document_id: Option<DocumentId>,
pub relative_path: String,
pub created_date: DateTime<Utc>,
#[form_data(limit = "unlimited")]
pub content: FieldData<Bytes>,
}
@ -28,7 +31,6 @@ pub struct CreateDocumentVersionMultipart {
pub struct UpdateDocumentVersion {
pub parent_version_id: VaultUpdateId,
pub relative_path: String,
pub created_date: DateTime<Utc>,
pub content_base64: String,
}
@ -37,7 +39,6 @@ pub struct UpdateDocumentVersion {
pub struct UpdateDocumentVersionMultipart {
pub parent_version_id: VaultUpdateId,
pub relative_path: String,
pub created_date: DateTime<Utc>,
#[form_data(limit = "unlimited")]
pub content: FieldData<Bytes>,
}
@ -46,5 +47,4 @@ pub struct UpdateDocumentVersionMultipart {
#[serde(rename_all = "camelCase")]
pub struct DeleteDocumentVersion {
pub relative_path: String,
pub created_date: DateTime<Utc>,
}

30
backend/sync_server/src/server/update_document.rs
View file

@ -6,7 +6,6 @@ use axum_extra::{
headers::{Authorization, authorization::Bearer},
};
use axum_jsonschema::Json;
use chrono::{DateTime, Utc};
use log::info;
use schemars::JsonSchema;
use serde::Deserialize;
@ -50,7 +49,6 @@ pub async fn update_document_multipart(
document_id,
request.parent_version_id,
request.relative_path,
request.created_date,
request.content.contents.to_vec(),
)
.await
@ -77,21 +75,19 @@ pub async fn update_document_json(
document_id,
request.parent_version_id,
request.relative_path,
request.created_date,
content_bytes,
)
.await
}
#[allow(clippy::too_many_arguments)]
#[allow(clippy::too_many_arguments, clippy::too_many_lines)]
async fn internal_update_document(
auth_header: Authorization<Bearer>,
state: AppState,
mut state: AppState,
vault_id: VaultId,
document_id: DocumentId,
parent_version_id: VaultUpdateId,
relative_path: String,
created_date: DateTime<Utc>,
content: Vec<u8>,
) -> Result<Json<DocumentUpdateResponse>, SyncServerError> {
auth(&state, auth_header.token())?;
@ -114,7 +110,7 @@ async fn internal_update_document(
let mut transaction = state
.database
.create_write_transaction()
.create_write_transaction(&vault_id)
.await
.map_err(server_error)?;
@ -138,6 +134,18 @@ async fn internal_update_document(
Ok,
)?;
if latest_version.is_deleted {
transaction
.rollback()
.await
.context("Failed to roll back transaction")
.map_err(server_error)?;
return Ok(Json(DocumentUpdateResponse::FastForwardUpdate(
latest_version.into(),
)));
}
let sanitized_relative_path = sanitize_path(&relative_path);
// Return the latest version if the content and path are the same as the latest
@ -168,7 +176,7 @@ async fn internal_update_document(
let new_relative_path = if parent_document.relative_path == latest_version.relative_path
&& latest_version.relative_path != sanitized_relative_path
{
let mut new_relative_path = Default::default();
let mut new_relative_path = String::default();
for candidate in deduped_file_paths(&sanitized_relative_path) {
if state
.database
@ -188,19 +196,17 @@ async fn internal_update_document(
};
let new_version = StoredDocumentVersion {
vault_id,
document_id,
vault_update_id: last_update_id + 1,
relative_path: new_relative_path,
content: merged_content,
created_date,
updated_date: chrono::Utc::now(),
is_deleted: latest_version.is_deleted,
is_deleted: false,
};
state
.database
.insert_document_version(&new_version, Some(&mut transaction))
.insert_document_version(&vault_id, &new_version, Some(&mut transaction))
.await
.map_err(server_error)?;

8
frontend/obsidian-plugin/package.json
View file

@ -14,7 +14,7 @@
"license": "MIT",
"devDependencies": {
"@types/jest": "^29.5.14",
"@types/node": "^22.13.5",
"@types/node": "^22.13.10",
"css-loader": "^7.1.2",
"date-fns": "^4.1.0",
"file-loader": "^6.2.0",
@ -23,14 +23,14 @@
"mini-css-extract-plugin": "^2.9.2",
"obsidian": "1.8.7",
"resolve-url-loader": "^5.0.0",
"sass": "^1.85.0",
"sass": "^1.85.1",
"sass-loader": "^16.0.5",
"sync-client": "file:../sync-client",
"terser-webpack-plugin": "^5.3.11",
"terser-webpack-plugin": "^5.3.14",
"ts-jest": "^29.2.6",
"ts-loader": "^9.5.2",
"tslib": "2.8.1",
"typescript": "5.7.3",
"typescript": "5.8.2",
"url": "^0.11.4",
"virtual-scroller": "^1.13.1",
"webpack": "^5.98.0",

11
frontend/obsidian-plugin/src/obisidan-event-handler.ts
View file

@ -9,10 +9,7 @@ export class ObsidianFileEventHandler {
if (file instanceof TFile) {
this.client.logger.info(`File created: ${file.path}`);
await this.client.syncer.syncLocallyCreatedFile(
file.path,
new Date(file.stat.ctime)
);
await this.client.syncer.syncLocallyCreatedFile(file.path);
} else {
this.client.logger.debug(`Folder created: ${file.path}, ignored`);
}
@ -34,8 +31,7 @@ export class ObsidianFileEventHandler {
await this.client.syncer.syncLocallyUpdatedFile({
oldPath,
relativePath: file.path,
updateTime: new Date(file.stat.ctime)
relativePath: file.path
});
} else {
this.client.logger.debug(
@ -53,8 +49,7 @@ export class ObsidianFileEventHandler {
this.client.logger.info(`File modified: ${file.path}`);
await this.client.syncer.syncLocallyUpdatedFile({
relativePath: file.path,
updateTime: new Date(file.stat.ctime)
relativePath: file.path
});
} else {
this.client.logger.debug(`Folder modified: ${file.path}, ignored`);

1
frontend/obsidian-plugin/src/views/history-view.ts
View file

@ -60,7 +60,6 @@ export class HistoryView extends ItemView {
}
element.createEl("span", {
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
text: entry.relativePath
});

2
frontend/obsidian-plugin/src/views/logs-view.ts
View file

@ -1,6 +1,6 @@
import type { WorkspaceLeaf } from "obsidian";
import { ItemView } from "obsidian";
import type VaultLinkPlugin from "src/vault-link-plugin";
import type VaultLinkPlugin from "../vault-link-plugin";
import type { SyncClient } from "sync-client";
export class LogsView extends ItemView {

2
frontend/obsidian-plugin/src/views/settings-tab.ts
View file

@ -1,7 +1,7 @@
import type { App } from "obsidian";
import { Notice, PluginSettingTab, Setting } from "obsidian";
import type VaultLinkPlugin from "src/vault-link-plugin";
import type VaultLinkPlugin from "../vault-link-plugin";
import type { StatusDescription } from "./status-description";
import { LogsView } from "./logs-view";
import { HistoryView } from "./history-view";

2
frontend/obsidian-plugin/src/views/status-bar.ts
View file

@ -1,5 +1,5 @@
import type { HistoryStats, SyncClient } from "sync-client";
import type VaultLinkPlugin from "src/vault-link-plugin";
import type VaultLinkPlugin from "../vault-link-plugin";
export class StatusBar {
private readonly statusBarItem: HTMLElement;

3
frontend/obsidian-plugin/webpack.config.js
View file

@ -94,9 +94,6 @@ module.exports = (env, argv) => ({
alias: {
root: __dirname,
src: path.resolve(__dirname, "src")
},
fallback: {
url: require.resolve("url")
}
},
output: {

272
frontend/package-lock.json generated
View file

@ -12,11 +12,11 @@
],
"devDependencies": {
"concurrently": "^9.1.2",
"eslint": "9.21.0",
"eslint": "9.22.0",
"eslint-plugin-unused-imports": "^4.1.4",
"npm-check-updates": "^17.1.14",
"prettier": "^3.5.2",
"typescript-eslint": "8.24.1"
"npm-check-updates": "^17.1.15",
"prettier": "^3.5.3",
"typescript-eslint": "8.26.1"
}
},
"../backend/sync_lib/pkg": {
@ -39,7 +39,6 @@
},
"node_modules/@babel/code-frame": {
"version": "7.26.2",
"dev": true,
"license": "MIT",
"dependencies": {
"@babel/helper-validator-identifier": "^7.25.9",
@ -179,7 +178,6 @@
},
"node_modules/@babel/helper-validator-identifier": {
"version": "7.25.9",
"dev": true,
"license": "MIT",
"engines": {
"node": ">=6.9.0"
@ -194,12 +192,14 @@
}
},
"node_modules/@babel/helpers": {
"version": "7.26.9",
"version": "7.26.10",
"resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.26.10.tgz",
"integrity": "sha512-UPYc3SauzZ3JGgj87GgZ89JVdC5dj0AoetR5Bw6wj4niittNyFh6+eOGonYvJ1ao6B8lEa3Q3klS7ADZ53bc5g==",
"dev": true,
"license": "MIT",
"dependencies": {
"@babel/template": "^7.26.9",
"@babel/types": "^7.26.9"
"@babel/types": "^7.26.10"
},
"engines": {
"node": ">=6.9.0"
@ -463,7 +463,9 @@
}
},
"node_modules/@babel/types": {
"version": "7.26.9",
"version": "7.26.10",
"resolved": "https://registry.npmjs.org/@babel/types/-/types-7.26.10.tgz",
"integrity": "sha512-emqcG3vHrpxUKTrxcblR36dcrcoRDvKmnL/dCL6ZsHaShW80qxCAcNhzQZrpeM765VzEos+xOi4s+r4IXzTwdQ==",
"dev": true,
"license": "MIT",
"dependencies": {
@ -580,6 +582,16 @@
"node": "^18.18.0 || ^20.9.0 || >=21.1.0"
}
},
"node_modules/@eslint/config-helpers": {
"version": "0.1.0",
"resolved": "https://registry.npmjs.org/@eslint/config-helpers/-/config-helpers-0.1.0.tgz",
"integrity": "sha512-kLrdPDJE1ckPo94kmPPf9Hfd0DU0Jw6oKYrhe+pwSC0iTUInmTa+w6fw8sGgcfkFJGNdWOUeOaDM4quW4a7OkA==",
"dev": true,
"license": "Apache-2.0",
"engines": {
"node": "^18.18.0 || ^20.9.0 || >=21.1.0"
}
},
"node_modules/@eslint/core": {
"version": "0.12.0",
"resolved": "https://registry.npmjs.org/@eslint/core/-/core-0.12.0.tgz",
@ -618,9 +630,9 @@
}
},
"node_modules/@eslint/js": {
"version": "9.21.0",
"resolved": "https://registry.npmjs.org/@eslint/js/-/js-9.21.0.tgz",
"integrity": "sha512-BqStZ3HX8Yz6LvsF5ByXYrtigrV5AXADWLAGc7PH/1SxOb7/FIYYMszZZWiUou/GB9P2lXWk2SV4d+Z8h0nknw==",
"version": "9.22.0",
"resolved": "https://registry.npmjs.org/@eslint/js/-/js-9.22.0.tgz",
"integrity": "sha512-vLFajx9o8d1/oL2ZkpMYbkLv8nDB6yaIwFNt7nI4+I80U/z03SxmfOMsLbvWr3p7C+Wnoh//aOu2pQW8cS0HCQ==",
"dev": true,
"license": "MIT",
"engines": {
@ -1130,6 +1142,8 @@
},
"node_modules/@nodelib/fs.scandir": {
"version": "2.1.5",
"resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz",
"integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==",
"dev": true,
"license": "MIT",
"dependencies": {
@ -1142,6 +1156,8 @@
},
"node_modules/@nodelib/fs.stat": {
"version": "2.0.5",
"resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz",
"integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==",
"dev": true,
"license": "MIT",
"engines": {
@ -1150,6 +1166,8 @@
},
"node_modules/@nodelib/fs.walk": {
"version": "1.2.8",
"resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz",
"integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==",
"dev": true,
"license": "MIT",
"dependencies": {
@ -1235,7 +1253,6 @@
},
"node_modules/@redocly/ajv": {
"version": "8.11.2",
"dev": true,
"license": "MIT",
"dependencies": {
"fast-deep-equal": "^3.1.1",
@ -1250,17 +1267,14 @@
},
"node_modules/@redocly/ajv/node_modules/json-schema-traverse": {
"version": "1.0.0",
"dev": true,
"license": "MIT"
},
"node_modules/@redocly/config": {
"version": "0.20.3",
"dev": true,
"license": "MIT"
},
"node_modules/@redocly/openapi-core": {
"version": "1.29.0",
"dev": true,
"license": "MIT",
"dependencies": {
"@redocly/ajv": "^8.11.2",
@ -1280,7 +1294,6 @@
},
"node_modules/@redocly/openapi-core/node_modules/brace-expansion": {
"version": "2.0.1",
"dev": true,
"license": "MIT",
"dependencies": {
"balanced-match": "^1.0.0"
@ -1288,7 +1301,6 @@
},
"node_modules/@redocly/openapi-core/node_modules/minimatch": {
"version": "5.1.6",
"dev": true,
"license": "ISC",
"dependencies": {
"brace-expansion": "^2.0.1"
@ -1458,9 +1470,9 @@
"license": "MIT"
},
"node_modules/@types/node": {
"version": "22.13.5",
"resolved": "https://registry.npmjs.org/@types/node/-/node-22.13.5.tgz",
"integrity": "sha512-+lTU0PxZXn0Dr1NBtC7Y8cR21AJr87dLLU953CWA6pMxxv/UDc7jYAY90upcrie1nRcD6XNG5HOYEDtgW5TxAg==",
"version": "22.13.10",
"resolved": "https://registry.npmjs.org/@types/node/-/node-22.13.10.tgz",
"integrity": "sha512-I6LPUvlRH+O6VRUqYOcMudhaIdUVWfsjnZavnsraHvpBwaEyMN29ry+0UVJhImYL16xsscu0aske3yA+uPOWfw==",
"dev": true,
"license": "MIT",
"dependencies": {
@ -1494,15 +1506,17 @@
"license": "MIT"
},
"node_modules/@typescript-eslint/eslint-plugin": {
"version": "8.24.1",
"version": "8.26.1",
"resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.26.1.tgz",
"integrity": "sha512-2X3mwqsj9Bd3Ciz508ZUtoQQYpOhU/kWoUqIf49H8Z0+Vbh6UF/y0OEYp0Q0axOGzaBGs7QxRwq0knSQ8khQNA==",
"dev": true,
"license": "MIT",
"dependencies": {
"@eslint-community/regexpp": "^4.10.0",
"@typescript-eslint/scope-manager": "8.24.1",
"@typescript-eslint/type-utils": "8.24.1",
"@typescript-eslint/utils": "8.24.1",
"@typescript-eslint/visitor-keys": "8.24.1",
"@typescript-eslint/scope-manager": "8.26.1",
"@typescript-eslint/type-utils": "8.26.1",
"@typescript-eslint/utils": "8.26.1",
"@typescript-eslint/visitor-keys": "8.26.1",
"graphemer": "^1.4.0",
"ignore": "^5.3.1",
"natural-compare": "^1.4.0",
@ -1518,18 +1532,20 @@
"peerDependencies": {
"@typescript-eslint/parser": "^8.0.0 || ^8.0.0-alpha.0",
"eslint": "^8.57.0 || ^9.0.0",
"typescript": ">=4.8.4 <5.8.0"
"typescript": ">=4.8.4 <5.9.0"
}
},
"node_modules/@typescript-eslint/parser": {
"version": "8.24.1",
"version": "8.26.1",
"resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-8.26.1.tgz",
"integrity": "sha512-w6HZUV4NWxqd8BdeFf81t07d7/YV9s7TCWrQQbG5uhuvGUAW+fq1usZ1Hmz9UPNLniFnD8GLSsDpjP0hm1S4lQ==",
"dev": true,
"license": "MIT",
"dependencies": {
"@typescript-eslint/scope-manager": "8.24.1",
"@typescript-eslint/types": "8.24.1",
"@typescript-eslint/typescript-estree": "8.24.1",
"@typescript-eslint/visitor-keys": "8.24.1",
"@typescript-eslint/scope-manager": "8.26.1",
"@typescript-eslint/types": "8.26.1",
"@typescript-eslint/typescript-estree": "8.26.1",
"@typescript-eslint/visitor-keys": "8.26.1",
"debug": "^4.3.4"
},
"engines": {
@ -1541,16 +1557,18 @@
},
"peerDependencies": {
"eslint": "^8.57.0 || ^9.0.0",
"typescript": ">=4.8.4 <5.8.0"
"typescript": ">=4.8.4 <5.9.0"
}
},
"node_modules/@typescript-eslint/scope-manager": {
"version": "8.24.1",
"version": "8.26.1",
"resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-8.26.1.tgz",
"integrity": "sha512-6EIvbE5cNER8sqBu6V7+KeMZIC1664d2Yjt+B9EWUXrsyWpxx4lEZrmvxgSKRC6gX+efDL/UY9OpPZ267io3mg==",
"dev": true,
"license": "MIT",
"dependencies": {
"@typescript-eslint/types": "8.24.1",
"@typescript-eslint/visitor-keys": "8.24.1"
"@typescript-eslint/types": "8.26.1",
"@typescript-eslint/visitor-keys": "8.26.1"
},
"engines": {
"node": "^18.18.0 || ^20.9.0 || >=21.1.0"
@ -1561,12 +1579,14 @@
}
},
"node_modules/@typescript-eslint/type-utils": {
"version": "8.24.1",
"version": "8.26.1",
"resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-8.26.1.tgz",
"integrity": "sha512-Kcj/TagJLwoY/5w9JGEFV0dclQdyqw9+VMndxOJKtoFSjfZhLXhYjzsQEeyza03rwHx2vFEGvrJWJBXKleRvZg==",
"dev": true,
"license": "MIT",
"dependencies": {
"@typescript-eslint/typescript-estree": "8.24.1",
"@typescript-eslint/utils": "8.24.1",
"@typescript-eslint/typescript-estree": "8.26.1",
"@typescript-eslint/utils": "8.26.1",
"debug": "^4.3.4",
"ts-api-utils": "^2.0.1"
},
@ -1579,11 +1599,13 @@
},
"peerDependencies": {
"eslint": "^8.57.0 || ^9.0.0",
"typescript": ">=4.8.4 <5.8.0"
"typescript": ">=4.8.4 <5.9.0"
}
},
"node_modules/@typescript-eslint/types": {
"version": "8.24.1",
"version": "8.26.1",
"resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-8.26.1.tgz",
"integrity": "sha512-n4THUQW27VmQMx+3P+B0Yptl7ydfceUj4ON/AQILAASwgYdZ/2dhfymRMh5egRUrvK5lSmaOm77Ry+lmXPOgBQ==",
"dev": true,
"license": "MIT",
"engines": {
@ -1595,12 +1617,14 @@
}
},
"node_modules/@typescript-eslint/typescript-estree": {
"version": "8.24.1",
"version": "8.26.1",
"resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-8.26.1.tgz",
"integrity": "sha512-yUwPpUHDgdrv1QJ7YQal3cMVBGWfnuCdKbXw1yyjArax3353rEJP1ZA+4F8nOlQ3RfS2hUN/wze3nlY+ZOhvoA==",
"dev": true,
"license": "MIT",
"dependencies": {
"@typescript-eslint/types": "8.24.1",
"@typescript-eslint/visitor-keys": "8.24.1",
"@typescript-eslint/types": "8.26.1",
"@typescript-eslint/visitor-keys": "8.26.1",
"debug": "^4.3.4",
"fast-glob": "^3.3.2",
"is-glob": "^4.0.3",
@ -1616,11 +1640,13 @@
"url": "https://opencollective.com/typescript-eslint"
},
"peerDependencies": {
"typescript": ">=4.8.4 <5.8.0"
"typescript": ">=4.8.4 <5.9.0"
}
},
"node_modules/@typescript-eslint/typescript-estree/node_modules/brace-expansion": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz",
"integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==",
"dev": true,
"license": "MIT",
"dependencies": {
@ -1629,6 +1655,8 @@
},
"node_modules/@typescript-eslint/typescript-estree/node_modules/minimatch": {
"version": "9.0.5",
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.5.tgz",
"integrity": "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==",
"dev": true,
"license": "ISC",
"dependencies": {
@ -1642,14 +1670,16 @@
}
},
"node_modules/@typescript-eslint/utils": {
"version": "8.24.1",
"version": "8.26.1",
"resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-8.26.1.tgz",
"integrity": "sha512-V4Urxa/XtSUroUrnI7q6yUTD3hDtfJ2jzVfeT3VK0ciizfK2q/zGC0iDh1lFMUZR8cImRrep6/q0xd/1ZGPQpg==",
"dev": true,
"license": "MIT",
"dependencies": {
"@eslint-community/eslint-utils": "^4.4.0",
"@typescript-eslint/scope-manager": "8.24.1",
"@typescript-eslint/types": "8.24.1",
"@typescript-eslint/typescript-estree": "8.24.1"
"@typescript-eslint/scope-manager": "8.26.1",
"@typescript-eslint/types": "8.26.1",
"@typescript-eslint/typescript-estree": "8.26.1"
},
"engines": {
"node": "^18.18.0 || ^20.9.0 || >=21.1.0"
@ -1660,15 +1690,17 @@
},
"peerDependencies": {
"eslint": "^8.57.0 || ^9.0.0",
"typescript": ">=4.8.4 <5.8.0"
"typescript": ">=4.8.4 <5.9.0"
}
},
"node_modules/@typescript-eslint/visitor-keys": {
"version": "8.24.1",
"version": "8.26.1",
"resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-8.26.1.tgz",
"integrity": "sha512-AjOC3zfnxd6S4Eiy3jwktJPclqhFHNyd8L6Gycf9WUPoKZpgM5PjkxY1X7uSy61xVpiJDhhk7XT2NVsN3ALTWg==",
"dev": true,
"license": "MIT",
"dependencies": {
"@typescript-eslint/types": "8.24.1",
"@typescript-eslint/types": "8.26.1",
"eslint-visitor-keys": "^4.2.0"
},
"engines": {
@ -1909,7 +1941,6 @@
},
"node_modules/agent-base": {
"version": "7.1.3",
"dev": true,
"license": "MIT",
"engines": {
"node": ">= 14"
@ -1976,7 +2007,6 @@
},
"node_modules/ansi-colors": {
"version": "4.1.3",
"dev": true,
"license": "MIT",
"engines": {
"node": ">=6"
@ -2039,7 +2069,6 @@
},
"node_modules/argparse": {
"version": "2.0.1",
"dev": true,
"license": "Python-2.0"
},
"node_modules/async": {
@ -2161,7 +2190,6 @@
},
"node_modules/balanced-match": {
"version": "1.0.2",
"dev": true,
"license": "MIT"
},
"node_modules/big.js": {
@ -2249,7 +2277,6 @@
},
"node_modules/byte-base64": {
"version": "1.1.0",
"dev": true,
"license": "MIT"
},
"node_modules/call-bind-apply-helpers": {
@ -2335,7 +2362,6 @@
},
"node_modules/change-case": {
"version": "5.4.4",
"dev": true,
"license": "MIT"
},
"node_modules/char-regex": {
@ -2445,7 +2471,6 @@
},
"node_modules/colorette": {
"version": "1.4.0",
"dev": true,
"license": "MIT"
},
"node_modules/commander": {
@ -2597,7 +2622,6 @@
},
"node_modules/debug": {
"version": "4.4.0",
"dev": true,
"license": "MIT",
"dependencies": {
"ms": "^2.1.3"
@ -2822,18 +2846,19 @@
}
},
"node_modules/eslint": {
"version": "9.21.0",
"resolved": "https://registry.npmjs.org/eslint/-/eslint-9.21.0.tgz",
"integrity": "sha512-KjeihdFqTPhOMXTt7StsDxriV4n66ueuF/jfPNC3j/lduHwr/ijDwJMsF+wyMJethgiKi5wniIE243vi07d3pg==",
"version": "9.22.0",
"resolved": "https://registry.npmjs.org/eslint/-/eslint-9.22.0.tgz",
"integrity": "sha512-9V/QURhsRN40xuHXWjV64yvrzMjcz7ZyNoF2jJFmy9j/SLk0u1OLSZgXi28MrXjymnjEGSR80WCdab3RGMDveQ==",
"dev": true,
"license": "MIT",
"dependencies": {
"@eslint-community/eslint-utils": "^4.2.0",
"@eslint-community/regexpp": "^4.12.1",
"@eslint/config-array": "^0.19.2",
"@eslint/config-helpers": "^0.1.0",
"@eslint/core": "^0.12.0",
"@eslint/eslintrc": "^3.3.0",
"@eslint/js": "9.21.0",
"@eslint/js": "9.22.0",
"@eslint/plugin-kit": "^0.2.7",
"@humanfs/node": "^0.16.6",
"@humanwhocodes/module-importer": "^1.0.1",
@ -2845,7 +2870,7 @@
"cross-spawn": "^7.0.6",
"debug": "^4.3.2",
"escape-string-regexp": "^4.0.0",
"eslint-scope": "^8.2.0",
"eslint-scope": "^8.3.0",
"eslint-visitor-keys": "^4.2.0",
"espree": "^10.3.0",
"esquery": "^1.5.0",
@ -2896,7 +2921,9 @@
}
},
"node_modules/eslint-scope": {
"version": "8.2.0",
"version": "8.3.0",
"resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-8.3.0.tgz",
"integrity": "sha512-pUNxi75F8MJ/GdeKtVLSbYg4ZI34J6C0C7sbL4YOp2exGwen7ZsuBqKzUhXd0qMQ362yET3z+uPwKeg/0C2XCQ==",
"dev": true,
"license": "BSD-2-Clause",
"dependencies": {
@ -2991,7 +3018,6 @@
},
"node_modules/eventemitter3": {
"version": "5.0.1",
"dev": true,
"license": "MIT"
},
"node_modules/events": {
@ -3048,11 +3074,12 @@
},
"node_modules/fast-deep-equal": {
"version": "3.1.3",
"dev": true,
"license": "MIT"
},
"node_modules/fast-glob": {
"version": "3.3.3",
"resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.3.tgz",
"integrity": "sha512-7MptL8U0cqcFdzIzwOTHoilX9x5BrNqye7Z/LuC7kCMRio1EMSyqRK3BEAUD7sXRq4iT4AzTVuZdhgQ2TCvYLg==",
"dev": true,
"license": "MIT",
"dependencies": {
@ -3068,6 +3095,8 @@
},
"node_modules/fast-glob/node_modules/glob-parent": {
"version": "5.1.2",
"resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz",
"integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==",
"dev": true,
"license": "ISC",
"dependencies": {
@ -3111,7 +3140,9 @@
}
},
"node_modules/fastq": {
"version": "1.19.0",
"version": "1.19.1",
"resolved": "https://registry.npmjs.org/fastq/-/fastq-1.19.1.tgz",
"integrity": "sha512-GwLTyxkCXjXbxqIhTsMI2Nui8huMPtnxg7krajPJAjnEG/iiOS7i+zCtWGZR9G0NBKbXKh6X9m9UIsYX/N6vvQ==",
"dev": true,
"license": "ISC",
"dependencies": {
@ -3128,7 +3159,6 @@
},
"node_modules/fetch-retry": {
"version": "6.0.0",
"dev": true,
"license": "MIT"
},
"node_modules/file-entry-cache": {
@ -3449,7 +3479,6 @@
},
"node_modules/https-proxy-agent": {
"version": "7.0.6",
"dev": true,
"license": "MIT",
"dependencies": {
"agent-base": "^7.1.2",
@ -3536,7 +3565,6 @@
},
"node_modules/index-to-position": {
"version": "0.1.2",
"dev": true,
"license": "MIT",
"engines": {
"node": ">=18"
@ -4271,7 +4299,6 @@
},
"node_modules/js-levenshtein": {
"version": "1.1.6",
"dev": true,
"license": "MIT",
"engines": {
"node": ">=0.10.0"
@ -4279,12 +4306,10 @@
},
"node_modules/js-tokens": {
"version": "4.0.0",
"dev": true,
"license": "MIT"
},
"node_modules/js-yaml": {
"version": "4.1.0",
"dev": true,
"license": "MIT",
"dependencies": {
"argparse": "^2.0.1"
@ -4497,6 +4522,8 @@
},
"node_modules/merge2": {
"version": "1.4.1",
"resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz",
"integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==",
"dev": true,
"license": "MIT",
"engines": {
@ -4631,7 +4658,6 @@
},
"node_modules/ms": {
"version": "2.1.3",
"dev": true,
"license": "MIT"
},
"node_modules/nanoid": {
@ -4686,7 +4712,9 @@
}
},
"node_modules/npm-check-updates": {
"version": "17.1.14",
"version": "17.1.15",
"resolved": "https://registry.npmjs.org/npm-check-updates/-/npm-check-updates-17.1.15.tgz",
"integrity": "sha512-miATvKu5rjec/1wxc5TGDjpsucgtCHwRVZorZpDkS6NzdWXfnUWlN4abZddWb7XSijAuBNzzYglIdTm9SbgMVg==",
"dev": true,
"license": "Apache-2.0",
"bin": {
@ -4758,8 +4786,9 @@
}
},
"node_modules/openapi-fetch": {
"version": "0.13.4",
"dev": true,
"version": "0.13.5",
"resolved": "https://registry.npmjs.org/openapi-fetch/-/openapi-fetch-0.13.5.tgz",
"integrity": "sha512-AQK8T9GSKFREFlN1DBXTYsLjs7YV2tZcJ7zUWxbjMoQmj8dDSFRrzhLCbHPZWA1TMV3vACqfCxLEZcwf2wxV6Q==",
"license": "MIT",
"dependencies": {
"openapi-typescript-helpers": "^0.0.15"
@ -4767,7 +4796,6 @@
},
"node_modules/openapi-typescript": {
"version": "7.6.1",
"dev": true,
"license": "MIT",
"dependencies": {
"@redocly/openapi-core": "^1.28.0",
@ -4786,12 +4814,10 @@
},
"node_modules/openapi-typescript-helpers": {
"version": "0.0.15",
"dev": true,
"license": "MIT"
},
"node_modules/openapi-typescript/node_modules/parse-json": {
"version": "8.1.0",
"dev": true,
"license": "MIT",
"dependencies": {
"@babel/code-frame": "^7.22.13",
@ -4807,7 +4833,6 @@
},
"node_modules/openapi-typescript/node_modules/supports-color": {
"version": "9.4.0",
"dev": true,
"license": "MIT",
"engines": {
"node": ">=12"
@ -4818,7 +4843,6 @@
},
"node_modules/openapi-typescript/node_modules/type-fest": {
"version": "4.35.0",
"dev": true,
"license": "(MIT OR CC0-1.0)",
"engines": {
"node": ">=16"
@ -4873,7 +4897,6 @@
},
"node_modules/p-queue": {
"version": "8.1.0",
"dev": true,
"license": "MIT",
"dependencies": {
"eventemitter3": "^5.0.1",
@ -4888,7 +4911,6 @@
},
"node_modules/p-timeout": {
"version": "6.1.4",
"dev": true,
"license": "MIT",
"engines": {
"node": ">=14.16"
@ -4966,7 +4988,6 @@
},
"node_modules/picocolors": {
"version": "1.1.1",
"dev": true,
"license": "ISC"
},
"node_modules/picomatch": {
@ -5049,7 +5070,6 @@
},
"node_modules/pluralize": {
"version": "8.0.0",
"dev": true,
"license": "MIT",
"engines": {
"node": ">=4"
@ -5163,9 +5183,9 @@
}
},
"node_modules/prettier": {
"version": "3.5.2",
"resolved": "https://registry.npmjs.org/prettier/-/prettier-3.5.2.tgz",
"integrity": "sha512-lc6npv5PH7hVqozBR7lkBNOGXV9vMwROAPlumdBkX0wTbbzPu/U1hk5yL8p2pt4Xoc+2mkT8t/sow2YrV/M5qg==",
"version": "3.5.3",
"resolved": "https://registry.npmjs.org/prettier/-/prettier-3.5.3.tgz",
"integrity": "sha512-QQtaxnoDJeAkDvDKWCLiwIXkTgRhwYDEQCghU9Z6q03iyek/rxRh/2lC3HB7P8sWT2xC/y5JDctPLBIGzHKbhw==",
"dev": true,
"license": "MIT",
"bin": {
@ -5255,6 +5275,8 @@
},
"node_modules/queue-microtask": {
"version": "1.2.3",
"resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz",
"integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==",
"dev": true,
"funding": [
{
@ -5328,7 +5350,6 @@
},
"node_modules/require-from-string": {
"version": "2.0.2",
"dev": true,
"license": "MIT",
"engines": {
"node": ">=0.10.0"
@ -5411,7 +5432,9 @@
}
},
"node_modules/reusify": {
"version": "1.0.4",
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/reusify/-/reusify-1.1.0.tgz",
"integrity": "sha512-g6QUff04oZpHs0eG5p83rFLhHeV00ug/Yf9nZM6fLeUrPguBTkTQOdpAWWspMh55TZfVQDPaN3NQJfbVRAxdIw==",
"dev": true,
"license": "MIT",
"engines": {
@ -5421,6 +5444,8 @@
},
"node_modules/run-parallel": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz",
"integrity": "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==",
"dev": true,
"funding": [
{
@ -5469,7 +5494,9 @@
"license": "MIT"
},
"node_modules/sass": {
"version": "1.85.0",
"version": "1.85.1",
"resolved": "https://registry.npmjs.org/sass/-/sass-1.85.1.tgz",
"integrity": "sha512-Uk8WpxM5v+0cMR0XjX9KfRIacmSG86RH4DCCZjLU2rFh5tyutt9siAXJ7G+YfxQ99Q6wrRMbMlVl6KqUms71ag==",
"dev": true,
"license": "MIT",
"dependencies": {
@ -5871,7 +5898,9 @@
}
},
"node_modules/terser-webpack-plugin": {
"version": "5.3.11",
"version": "5.3.14",
"resolved": "https://registry.npmjs.org/terser-webpack-plugin/-/terser-webpack-plugin-5.3.14.tgz",
"integrity": "sha512-vkZjpUjb6OMS7dhV+tILUW6BhpDR7P2L/aQSAv+Uwk+m8KATX9EccViHTJR2qDtACKPIYndLGCyl3FMo+r2LMw==",
"dev": true,
"license": "MIT",
"dependencies": {
@ -6196,8 +6225,9 @@
}
},
"node_modules/typescript": {
"version": "5.7.3",
"dev": true,
"version": "5.8.2",
"resolved": "https://registry.npmjs.org/typescript/-/typescript-5.8.2.tgz",
"integrity": "sha512-aJn6wq13/afZp/jT9QZmwEjDqqvSGp1VT5GVg+f/t6/oVyrgXM6BY1h9BRh/O5p3PlUPAe+WuiEZOmb/49RqoQ==",
"license": "Apache-2.0",
"bin": {
"tsc": "bin/tsc",
@ -6208,13 +6238,15 @@
}
},
"node_modules/typescript-eslint": {
"version": "8.24.1",
"version": "8.26.1",
"resolved": "https://registry.npmjs.org/typescript-eslint/-/typescript-eslint-8.26.1.tgz",
"integrity": "sha512-t/oIs9mYyrwZGRpDv3g+3K6nZ5uhKEMt2oNmAPwaY4/ye0+EH4nXIPYNtkYFS6QHm+1DFg34DbglYBz5P9Xysg==",
"dev": true,
"license": "MIT",
"dependencies": {
"@typescript-eslint/eslint-plugin": "8.24.1",
"@typescript-eslint/parser": "8.24.1",
"@typescript-eslint/utils": "8.24.1"
"@typescript-eslint/eslint-plugin": "8.26.1",
"@typescript-eslint/parser": "8.26.1",
"@typescript-eslint/utils": "8.26.1"
},
"engines": {
"node": "^18.18.0 || ^20.9.0 || >=21.1.0"
@ -6225,7 +6257,7 @@
},
"peerDependencies": {
"eslint": "^8.57.0 || ^9.0.0",
"typescript": ">=4.8.4 <5.8.0"
"typescript": ">=4.8.4 <5.9.0"
}
},
"node_modules/undici-types": {
@ -6280,7 +6312,6 @@
},
"node_modules/uri-js-replace": {
"version": "1.0.1",
"dev": true,
"license": "MIT"
},
"node_modules/url": {
@ -6311,7 +6342,6 @@
},
"node_modules/uuid": {
"version": "11.1.0",
"dev": true,
"funding": [
"https://github.com/sponsors/broofa",
"https://github.com/sponsors/ctavan"
@ -6480,6 +6510,8 @@
},
"node_modules/webpack-merge": {
"version": "6.0.1",
"resolved": "https://registry.npmjs.org/webpack-merge/-/webpack-merge-6.0.1.tgz",
"integrity": "sha512-hXXvrjtx2PLYx4qruKl+kyRSLc52V+cCvMxRjmKwoA+CBbbF5GfIBtR6kCvl0fYGqTUPKB+1ktVmTHqMOzgCBg==",
"dev": true,
"license": "MIT",
"dependencies": {
@ -6643,7 +6675,6 @@
},
"node_modules/yaml-ast-parser": {
"version": "0.0.43",
"dev": true,
"license": "Apache-2.0"
},
"node_modules/yargs": {
@ -6665,7 +6696,6 @@
},
"node_modules/yargs-parser": {
"version": "21.1.1",
"dev": true,
"license": "ISC",
"engines": {
"node": ">=12"
@ -6698,7 +6728,7 @@
"license": "MIT",
"devDependencies": {
"@types/jest": "^29.5.14",
"@types/node": "^22.13.5",
"@types/node": "^22.13.10",
"css-loader": "^7.1.2",
"date-fns": "^4.1.0",
"file-loader": "^6.2.0",
@ -6707,14 +6737,14 @@
"mini-css-extract-plugin": "^2.9.2",
"obsidian": "1.8.7",
"resolve-url-loader": "^5.0.0",
"sass": "^1.85.0",
"sass": "^1.85.1",
"sass-loader": "^16.0.5",
"sync-client": "file:../sync-client",
"terser-webpack-plugin": "^5.3.11",
"terser-webpack-plugin": "^5.3.14",
"ts-jest": "^29.2.6",
"ts-loader": "^9.5.2",
"tslib": "2.8.1",
"typescript": "5.7.3",
"typescript": "5.8.2",
"url": "^0.11.4",
"virtual-scroller": "^1.13.1",
"webpack": "^5.98.0",
@ -6723,22 +6753,26 @@
},
"sync-client": {
"version": "0.0.0",
"devDependencies": {
"@types/jest": "^29.5.14",
"@types/node": "^22.13.5",
"dependencies": {
"byte-base64": "^1.1.0",
"fetch-retry": "^6.0.0",
"jest": "^29.7.0",
"openapi-fetch": "0.13.4",
"openapi-fetch": "0.13.5",
"openapi-typescript": "7.6.1",
"p-queue": "^8.1.0",
"uuid": "^11.1.0"
},
"devDependencies": {
"@types/jest": "^29.5.14",
"@types/node": "^22.13.10",
"jest": "^29.7.0",
"sync_lib": "file:../../backend/sync_lib/pkg",
"ts-jest": "^29.2.6",
"ts-loader": "^9.5.2",
"tslib": "2.8.1",
"typescript": "5.7.3",
"typescript": "5.8.2",
"webpack": "^5.98.0",
"webpack-cli": "^6.0.1"
"webpack-cli": "^6.0.1",
"webpack-merge": "^6.0.1"
}
},
"test-client": {
@ -6747,11 +6781,11 @@
"test-client": "dist/cli.js"
},
"devDependencies": {
"@types/node": "^22.13.5",
"@types/node": "^22.13.10",
"sync-client": "file:../sync-client",
"ts-loader": "^9.5.2",
"tslib": "2.8.1",
"typescript": "5.7.3",
"typescript": "5.8.2",
"uuid": "^11.1.0",
"webpack": "^5.98.0",
"webpack-cli": "^6.0.1"

8
frontend/package.json
View file

@ -21,10 +21,10 @@
},
"devDependencies": {
"concurrently": "^9.1.2",
"eslint": "9.21.0",
"eslint": "9.22.0",
"eslint-plugin-unused-imports": "^4.1.4",
"npm-check-updates": "^17.1.14",
"prettier": "^3.5.2",
"typescript-eslint": "8.24.1"
"npm-check-updates": "^17.1.15",
"prettier": "^3.5.3",
"typescript-eslint": "8.26.1"
}
}

33
frontend/sync-client/package.json
View file

@ -1,29 +1,36 @@
{
"name": "sync-client",
"version": "0.0.0",
"private": true,
"main": "dist/index.js",
"version": "0.0.30",
"main": "dist/sync-client.node.js",
"browser": "dist/sync-client.web.js",
"types": "dist/types/index.d.ts",
"files": [
"dist/**/*"
],
"scripts": {
"dev": "webpack watch --mode development",
"build": "webpack --mode production",
"test": "NODE_OPTIONS=\"$NODE_OPTIONS --experimental-vm-modules\" jest"
},
"dependencies": {
"byte-base64": "^1.1.0",
"fetch-retry": "^6.0.0",
"openapi-fetch": "0.13.5",
"openapi-typescript": "7.6.1",
"p-queue": "^8.1.0",
"uuid": "^11.1.0"
},
"devDependencies": {
"tslib": "2.8.1",
"typescript": "5.7.3",
"sync_lib": "file:../../backend/sync_lib/pkg",
"@types/jest": "^29.5.14",
"@types/node": "^22.13.5",
"@types/node": "^22.13.10",
"jest": "^29.7.0",
"ts-jest": "^29.2.6",
"p-queue": "^8.1.0",
"fetch-retry": "^6.0.0",
"byte-base64": "^1.1.0",
"openapi-fetch": "0.13.4",
"openapi-typescript": "7.6.1",
"ts-loader": "^9.5.2",
"tslib": "2.8.1",
"typescript": "5.8.2",
"webpack": "^5.98.0",
"webpack-cli": "^6.0.1"
"webpack-cli": "^6.0.1",
"webpack-merge": "^6.0.1",
"sync_lib": "file:../../backend/sync_lib/pkg"
}
}

3
frontend/sync-client/src/file-operations/document-locks.ts
View file

@ -1,6 +1,9 @@
import type { Logger } from "../tracing/logger";
import type { RelativePath } from "../persistence/database";
// Manages locks on documents to prevent concurrent modifications
// allowing the client's FileOperations implementation to be simpler.
// Locks are granted in a first-in-first-out order.
export class DocumentLocks {
private readonly locked = new Set<RelativePath>();
private readonly waiters = new Map<RelativePath, (() => void)[]>();

23
frontend/sync-client/src/file-operations/file-operations.test.ts
View file

@ -1,16 +1,27 @@
import type { FileSystemOperations } from "sync-client";
import type { Database, RelativePath } from "../persistence/database";
import type {
Database,
DocumentRecord,
RelativePath
} from "../persistence/database";
import { FileOperations } from "./file-operations";
import { Logger } from "../tracing/logger";
import { assertSetContainsExactly } from "../utils/assert-set-contains-exactly";
import type { FileSystemOperations } from "./filesystem-operations";
describe("File operations", () => {
class MockDatabase {
public async updatePath(
class MockDatabase implements Partial<Database> {
public getLatestDocumentByRelativePath(
_find: RelativePath
): DocumentRecord | undefined {
// no-op
return undefined;
}
public move(
_oldRelativePath: RelativePath,
_newRelativePath: RelativePath
): Promise<void> {
// this is called but irrelevant for this mock
): void {
// no-op
}
}

73
frontend/sync-client/src/file-operations/file-operations.ts
View file

@ -1,10 +1,6 @@
import type { Logger } from "src/tracing/logger";
import type { Logger } from "../tracing/logger";
import type { FileSystemOperations } from "./filesystem-operations";
import type {
Database,
DocumentId,
RelativePath
} from "src/persistence/database";
import type { Database, RelativePath } from "../persistence/database";
import { isBinary, isFileTypeMergable, mergeText } from "sync_lib";
import { SafeFileSystemOperations } from "./safe-filesystem-operations";
@ -17,7 +13,7 @@ export class FileOperations {
private readonly database: Database,
fs: FileSystemOperations
) {
this.fs = new SafeFileSystemOperations(fs);
this.fs = new SafeFileSystemOperations(fs, logger);
}
public async listAllFiles(): Promise<RelativePath[]> {
@ -35,7 +31,7 @@ export class FileOperations {
const decoder = new TextDecoder("utf-8");
// Normalize line endings to LF on Windows
// Normalize line-endings to LF on Windows
let text = decoder.decode(content);
text = text.replace(/\r\n/g, "\n");
@ -46,10 +42,6 @@ export class FileOperations {
return this.fs.getFileSize(path);
}
public async getModificationTime(path: RelativePath): Promise<Date> {
return this.fs.getModificationTime(path);
}
public async exists(path: RelativePath): Promise<boolean> {
return this.fs.exists(path);
}
@ -60,18 +52,23 @@ export class FileOperations {
path: RelativePath,
newContent: Uint8Array
): Promise<void> {
this.logger.debug(`Creating file: ${path}`);
await this.fs.write(path, newContent);
}
public async ensureClearPath(path: RelativePath): Promise<void> {
if (await this.fs.exists(path)) {
const deconflictedPath = await this.deconflictPath(path);
this.logger.debug(
`Didn't expect ${path} to exist, deconflicting by moving it to '${deconflictedPath}'`
);
await this.database.updatePath(path, deconflictedPath);
this.database.move(path, deconflictedPath);
await this.fs.rename(path, deconflictedPath);
} else {
await this.createParentDirectories(path);
}
await this.fs.write(path, newContent);
}
// Update the file at the given path.
@ -126,40 +123,25 @@ export class FileOperations {
return new TextEncoder().encode(resultText);
}
public async remove(path: RelativePath): Promise<void> {
public async delete(path: RelativePath): Promise<void> {
if (await this.exists(path)) {
this.logger.debug(`Deleting file: ${path}`);
return this.fs.delete(path);
} else {
this.logger.debug(`No need to delete '${path}', it doesn't exist`);
}
}
public async move(
oldPath: RelativePath,
newPath: RelativePath,
documentId?: DocumentId
newPath: RelativePath
): Promise<void> {
if (oldPath === newPath) {
return;
}
await this.ensureClearPath(newPath);
if (await this.fs.exists(newPath)) {
const deconflictedPath = await this.deconflictPath(newPath);
this.logger.debug(
`Conflict when moving '${oldPath}' to '${newPath}', the latter already exists, deconflicting by moving it to '${deconflictedPath}'`
);
const existingMetadata = this.database.getDocument(newPath);
if (
existingMetadata === undefined ||
existingMetadata.documentId !== documentId
) {
await this.database.updatePath(newPath, deconflictedPath);
await this.fs.rename(newPath, deconflictedPath);
} else {
await this.database.deleteDocument(newPath);
}
} else {
await this.createParentDirectories(newPath);
}
this.database.move(oldPath, newPath);
await this.fs.rename(oldPath, newPath);
}
@ -201,17 +183,12 @@ export class FileOperations {
);
stem = stem.replace(FileOperations.PARENTHESES_REGEX, "");
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
while (true) {
const newName =
currentCount === 0
? `${directory}${stem}${extension}`
: `${directory}${stem} (${currentCount})${extension}`;
if (await this.fs.exists(newName)) {
let newName = path;
do {
currentCount++;
} else {
newName = `${directory}${stem} (${currentCount})${extension}`;
} while (await this.fs.exists(newName));
return newName;
}
}
}
}

5
frontend/sync-client/src/file-operations/filesystem-operations.ts
View file

@ -1,4 +1,4 @@
import type { RelativePath } from "src/persistence/database";
import type { RelativePath } from "../persistence/database";
export interface FileSystemOperations {
listAllFiles: () => Promise<RelativePath[]>;
@ -9,11 +9,8 @@ export interface FileSystemOperations {
updater: (currentContent: string) => string
) => Promise<string>;
getFileSize: (path: RelativePath) => Promise<number>;
getModificationTime: (path: RelativePath) => Promise<Date>;
exists: (path: RelativePath) => Promise<boolean>;
createDirectory: (path: RelativePath) => Promise<void>;
delete: (path: RelativePath) => Promise<void>;
// Must be able to handle renaming to a file that already exists
rename: (oldPath: RelativePath, newPath: RelativePath) => Promise<void>;
}

105
frontend/sync-client/src/file-operations/safe-filesystem-operations.ts
View file

@ -1,5 +1,7 @@
import type { RelativePath } from "src/persistence/database";
import type { RelativePath } from "../persistence/database";
import type { FileSystemOperations } from "./filesystem-operations";
import type { Logger } from "../tracing/logger";
import { DocumentLocks } from "./document-locks";
export class FileNotFoundError extends Error {
public constructor(message: string) {
@ -9,71 +11,134 @@ export class FileNotFoundError extends Error {
}
// Decorate FileSystemOperations replacing errors with FileNotFoundError
// if the accessed file doesn't exist.
// if the accessed file doesn't exist. It also ensures that there's only
// ever a single request in-flight for any one file through the use of
// DocumentLocks.
export class SafeFileSystemOperations implements FileSystemOperations {
public constructor(private readonly fs: FileSystemOperations) {}
private readonly locks: DocumentLocks;
public constructor(
private readonly fs: FileSystemOperations,
private readonly logger: Logger
) {
this.locks = new DocumentLocks(logger);
}
public async listAllFiles(): Promise<RelativePath[]> {
return this.fs.listAllFiles();
}
public async read(path: RelativePath): Promise<Uint8Array> {
return this.safeOperation(path, async () => this.fs.read(path));
this.logger.debug(`Reading file: ${path}`);
return this.safeOperation(
path,
this.decorateToHoldLock(path, async () => this.fs.read(path)),
"read"
);
}
public async write(path: RelativePath, content: Uint8Array): Promise<void> {
return this.fs.write(path, content);
this.logger.debug(`Writing file: ${path}`);
return this.decorateToHoldLock(path, async () =>
this.fs.write(path, content)
)();
}
public async atomicUpdateText(
path: RelativePath,
updater: (currentContent: string) => string
): Promise<string> {
return this.safeOperation(path, async () =>
this.logger.debug(`Atomic update of file: ${path}`);
return this.safeOperation(
path,
this.decorateToHoldLock(path, async () =>
this.fs.atomicUpdateText(path, updater)
),
"atomicUpdateText"
);
}
public async getFileSize(path: RelativePath): Promise<number> {
return this.safeOperation(path, async () => this.fs.getFileSize(path));
}
public async getModificationTime(path: RelativePath): Promise<Date> {
return this.safeOperation(path, async () =>
this.fs.getModificationTime(path)
this.logger.debug(`Getting file size: ${path}`);
return this.safeOperation(
path,
this.decorateToHoldLock(path, async () =>
this.fs.getFileSize(path)
),
"getFileSize"
);
}
public async exists(path: RelativePath): Promise<boolean> {
return this.fs.exists(path);
this.logger.debug(`Checking if file exists: ${path}`);
return this.decorateToHoldLock(path, async () =>
this.fs.exists(path)
)();
}
public async createDirectory(path: RelativePath): Promise<void> {
return this.fs.createDirectory(path);
this.logger.debug(`Creating directory: ${path}`);
return this.decorateToHoldLock(path, async () =>
this.fs.createDirectory(path)
)();
}
public async delete(path: RelativePath): Promise<void> {
return this.fs.delete(path);
this.logger.debug(`Deleting file: ${path}`);
return this.decorateToHoldLock(path, async () =>
this.fs.delete(path)
)();
}
public async rename(
oldPath: RelativePath,
newPath: RelativePath
): Promise<void> {
return this.safeOperation(oldPath, async () =>
this.logger.debug(`Renaming file: ${oldPath} to ${newPath}`);
return this.safeOperation(
oldPath,
this.decorateToHoldLock([oldPath, newPath], async () =>
this.fs.rename(oldPath, newPath)
),
"rename"
);
}
private decorateToHoldLock<T>(
pathOrPaths: RelativePath | RelativePath[],
operation: () => Promise<T>
): () => Promise<T> {
return async () => {
const paths = Array.isArray(pathOrPaths)
? pathOrPaths
: [pathOrPaths];
await Promise.all(
paths.map(async (path) => this.locks.waitForDocumentLock(path))
);
try {
return await operation();
} finally {
await Promise.all(
paths.map((path) => {
this.locks.unlockDocument(path);
})
);
}
};
}
private async safeOperation<T>(
path: RelativePath,
operation: () => Promise<T>
operation: () => Promise<T>,
operationName: string
): Promise<T> {
// Without locking the file, this isn't atomic, however, it's good enough practicaly.
// This will only break if the file exists, gets deleted and then immediately
// recreated while `operation` is running.
if (!(await this.fs.exists(path))) {
throw new FileNotFoundError(path);
throw new FileNotFoundError(
`File not found: ${path} before trying to ${operationName}`
);
}
try {
return await operation();
@ -81,7 +146,9 @@ export class SafeFileSystemOperations implements FileSystemOperations {
if (await this.fs.exists(path)) {
throw error;
} else {
throw new FileNotFoundError(path);
throw new FileNotFoundError(
`File not found: ${path} when trying to ${operationName}`
);
}
}
}

297
frontend/sync-client/src/persistence/database.ts
View file

@ -1,23 +1,43 @@
import type { Logger } from "../tracing/logger";
export type VaultUpdateId = number;
export type DocumentId = string;
export type RelativePath = string;
export interface DocumentMetadata {
parentVersionId: VaultUpdateId;
documentId: DocumentId;
hash: string;
}
import type { Logger } from "src/tracing/logger";
export interface StoredDocumentMetadata {
relativePath: RelativePath;
documentId: DocumentId;
parentVersionId: VaultUpdateId;
hash: string;
}
export interface StoredDatabase {
documents: Record<RelativePath, DocumentMetadata>;
documents: StoredDocumentMetadata[];
lastSeenUpdateId: VaultUpdateId | undefined;
}
export class Database {
private documents = new Map<RelativePath, DocumentMetadata>();
/**
* Represents a document in the database.
*
* It is mutable and its content should always represent the latest
* state of the document on disk based on the update events we have seen.
*/
export interface DocumentRecord {
relativePath: RelativePath;
documentId: DocumentId;
metadata: DocumentMetadata | undefined;
isDeleted: boolean;
updates: Promise<void>[];
parallelVersion: number;
}
export class Database {
private documents: DocumentRecord[];
private lastSeenUpdateId: VaultUpdateId | undefined;
public constructor(
@ -26,16 +46,21 @@ export class Database {
private readonly saveData: (data: StoredDatabase) => Promise<void>
) {
initialState ??= {};
if (initialState.documents) {
for (const [relativePath, metadata] of Object.entries(
initialState.documents
)) {
this.documents.set(relativePath, metadata);
}
}
this.ensureConsistency();
this.logger.debug(`Loaded ${this.documents.size} documents`);
this.documents =
initialState.documents?.map(
({ relativePath, documentId, ...metadata }) => ({
relativePath,
documentId,
metadata,
isDeleted: false,
updates: [],
parallelVersion: 0
})
) ?? [];
this.ensureConsistency();
this.logger.debug(`Loaded ${this.documents.length} documents`);
this.lastSeenUpdateId = initialState.lastSeenUpdateId;
this.logger.debug(
@ -43,109 +68,213 @@ export class Database {
);
}
public getDocuments(): Map<RelativePath, DocumentMetadata> {
return this.documents;
public get length(): number {
return this.documents.length;
}
public get resolvedDocuments(): DocumentRecord[] {
const paths = new Map<string, DocumentRecord[]>();
this.documents
.filter(({ metadata }) => metadata !== undefined)
.forEach((record) =>
paths.set(record.relativePath, [
record,
...(paths.get(record.relativePath) ?? [])
])
);
return Array.from(paths.values()).map((records) => {
records.sort(
(a, b) => b.parallelVersion - a.parallelVersion // descending
);
if (
records.length > 1 &&
records.some((current, i) =>
i === 0
? false
: records[i - 1].parallelVersion ===
current.parallelVersion
)
) {
throw new Error(
`Multiple documents with the same parallel version and path at ${records[0].relativePath}`
);
}
return records[0];
});
}
public getLastSeenUpdateId(): VaultUpdateId | undefined {
return this.lastSeenUpdateId;
}
public async setLastSeenUpdateId(
value: VaultUpdateId | undefined
): Promise<void> {
public setLastSeenUpdateId(value: VaultUpdateId | undefined): void {
this.lastSeenUpdateId = value;
await this.save();
this.save();
}
public async resetSyncState(): Promise<void> {
this.documents = new Map();
public resetSyncState(): void {
this.documents = [];
this.lastSeenUpdateId = 0;
await this.save();
this.save();
}
public updateDocumentMetadata(
metadata: {
parentVersionId: VaultUpdateId;
hash: string;
},
toUpdate: DocumentRecord
): void {
if (!this.documents.includes(toUpdate)) {
throw new Error("Document not found in database");
}
toUpdate.metadata = metadata;
this.save();
}
public removeDocumentPromise(promise: Promise<void>): void {
const entry = this.documents.find(({ updates }) =>
updates.includes(promise)
);
if (entry === undefined) {
throw new Error("Document not found by update promise");
}
entry.updates = entry.updates.filter((update) => update !== promise);
// No need to save as Promises don't get serialized
}
public getLatestDocumentByRelativePath(
find: RelativePath
): DocumentRecord | undefined {
const candidates = this.documents.filter(
({ relativePath }) => relativePath === find
);
candidates.sort((a, b) => b.parallelVersion - a.parallelVersion); // descending
return candidates[0];
}
public async getResolvedDocumentByRelativePath(
relativePath: RelativePath,
promise: Promise<void>
): Promise<DocumentRecord> {
const entry = this.getLatestDocumentByRelativePath(relativePath);
if (entry === undefined) {
throw new Error(
`Document not found by relative path: ${relativePath}, ${JSON.stringify(
this.documents,
null,
2
)}`
);
}
const currentPromises = entry.updates;
entry.updates = [...currentPromises, promise];
await Promise.all(currentPromises);
return entry;
}
public createNewPendingDocument(
documentId: DocumentId,
relativePath: RelativePath,
promise: Promise<void>
): DocumentRecord {
const previousEntry =
this.getLatestDocumentByRelativePath(relativePath);
const entry = {
relativePath,
documentId,
metadata: undefined,
isDeleted: false,
updates: [promise],
parallelVersion:
previousEntry?.parallelVersion === undefined
? 0
: previousEntry.parallelVersion + 1
};
this.documents.push(entry);
this.save();
return entry;
}
public getDocumentByDocumentId(
documentId: DocumentId
): [RelativePath, DocumentMetadata] | undefined {
return [...this.documents.entries()].find(
([_, metadata]) => metadata.documentId === documentId
);
find: DocumentId
): DocumentRecord | undefined {
return this.documents.find(({ documentId }) => documentId === find);
}
public async setDocument({
documentId,
relativePath,
parentVersionId,
hash
}: {
documentId: DocumentId;
relativePath: RelativePath;
parentVersionId: VaultUpdateId;
hash: string;
}): Promise<void> {
this.documents.set(relativePath, {
documentId,
parentVersionId,
hash
});
await this.save();
}
public async removeDocument(relativePath: RelativePath): Promise<void> {
this.documents.delete(relativePath);
await this.save();
}
public getDocument(
relativePath: RelativePath
): DocumentMetadata | undefined {
return this.documents.get(relativePath);
}
public async deleteDocument(relativePath: RelativePath): Promise<void> {
this.documents.delete(relativePath);
await this.save();
}
public async updatePath(
public move(
oldRelativePath: RelativePath,
newRelativePath: RelativePath
): Promise<void> {
const document = this.documents.get(oldRelativePath);
if (!document) {
): void {
const oldDocument =
this.getLatestDocumentByRelativePath(oldRelativePath);
if (oldDocument === undefined) {
return;
}
const newDocument =
this.getLatestDocumentByRelativePath(newRelativePath);
if (newDocument?.isDeleted === false) {
throw new Error(
`Cannot update physical path for document that does not exist: ${oldRelativePath}`
`Document already exists at new location: ${newRelativePath}`
);
}
if (this.documents.has(newRelativePath)) {
oldDocument.relativePath = newRelativePath;
// We're in a strange state where the target of the move has just got deleted,
// however, its metadata might already have a bunch of updates queued up for
// the document at the new location. We need to keep these updates.
oldDocument.parallelVersion =
newDocument !== undefined ? newDocument.parallelVersion + 1 : 0;
this.save();
}
public delete(relativePath: RelativePath): void {
const candidate = this.getLatestDocumentByRelativePath(relativePath);
if (candidate === undefined) {
throw new Error(
`Cannot update physical path to path that is already in use: ${newRelativePath}`
`Document not found by relative path: ${relativePath}`
);
}
this.documents.delete(oldRelativePath);
this.documents.set(newRelativePath, document);
await this.save();
candidate.isDeleted = true;
}
private async save(): Promise<void> {
private save(): void {
this.ensureConsistency();
await this.saveData({
documents: Object.fromEntries(this.documents.entries()),
void this.saveData({
documents: this.resolvedDocuments.map(
({ relativePath, documentId, metadata }) => ({
documentId,
relativePath,
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
...metadata! // resolvedDocuments only returns docs with metadata set
})
),
lastSeenUpdateId: this.lastSeenUpdateId
});
}
private ensureConsistency(): void {
const allMetadata = Array.from(this.documents.entries());
const idToPath = new Map<string, Array<string>>();
const idToPath = new Map<string, string[]>();
allMetadata.forEach(([name, metadata]) => {
idToPath.set(metadata.documentId, [
...(idToPath.get(metadata.documentId) ?? []),
name
this.resolvedDocuments.forEach(({ relativePath, documentId }) => {
idToPath.set(documentId, [
...(idToPath.get(documentId) ?? []),
relativePath
]);
});

4
frontend/sync-client/src/persistence/settings.ts
View file

@ -1,5 +1,5 @@
import type { Logger } from "src/tracing/logger";
import { LogLevel } from "src/tracing/logger";
import type { Logger } from "../tracing/logger";
import { LogLevel } from "../tracing/logger";
export interface SyncSettings {
remoteUri: string;

51
frontend/sync-client/src/services/connected-state.ts Normal file
View file

@ -0,0 +1,51 @@
import type { Settings } from "../persistence/settings";
import type { Logger } from "../tracing/logger";
import { createPromise } from "../utils/create-promise";
import { retriedFetchFactory } from "../utils/retried-fetch";
export class ConnectedState {
private resolveIsSyncEnabled: (() => void) | undefined;
private syncIsEnabled: Promise<void> | undefined;
public constructor(
settings: Settings,
private readonly logger: Logger
) {
settings.addOnSettingsChangeHandlers((newSettings, oldSettings) => {
if (!oldSettings.isSyncEnabled && newSettings.isSyncEnabled) {
this.handleComingOnline();
} else if (
oldSettings.isSyncEnabled &&
!newSettings.isSyncEnabled
) {
this.handleGoingOffline();
}
});
}
public getFetchImplementation(
fetch: typeof globalThis.fetch,
{ doRetries = true }: { doRetries: boolean } = { doRetries: true }
): typeof globalThis.fetch {
const retriedFetch = doRetries
? retriedFetchFactory(this.logger, fetch)
: fetch;
return async (input: RequestInfo | URL): Promise<Response> => {
if (this.syncIsEnabled !== undefined) {
await this.syncIsEnabled;
}
return retriedFetch(input);
};
}
private handleComingOnline(): void {
this.logger.debug("Sync is enabled");
this.resolveIsSyncEnabled?.();
}
private handleGoingOffline(): void {
this.logger.debug("Sync is disabled");
[this.syncIsEnabled, this.resolveIsSyncEnabled] = createPromise();
}
}

45
frontend/sync-client/src/services/sync-service.ts
View file

@ -6,20 +6,22 @@ import type {
RelativePath,
VaultUpdateId
} from "../persistence/database";
import type { Logger } from "src/tracing/logger";
import { retriedFetchFactory } from "src/utils/retried-fetch";
import type { Settings } from "src/persistence/settings";
import type { Logger } from "../tracing/logger";
import type { Settings } from "../persistence/settings";
import type { ConnectedState } from "./connected-state";
export interface CheckConnectionResult {
isSuccessful: boolean;
message: string;
}
export class SyncService {
private client!: Client<paths>;
private clientWithoutRetries!: Client<paths>;
private _fetchImplementation: typeof globalThis.fetch = globalThis.fetch;
public constructor(
private readonly connectedState: ConnectedState,
private readonly settings: Settings,
private readonly logger: Logger
) {
@ -52,17 +54,19 @@ export class SyncService {
}
public async create({
documentId,
relativePath,
contentBytes,
createdDate
contentBytes
}: {
documentId?: DocumentId;
relativePath: RelativePath;
contentBytes: Uint8Array;
createdDate: Date;
}): Promise<components["schemas"]["DocumentUpdateResponse"]> {
}): Promise<components["schemas"]["DocumentVersionWithoutContent"]> {
const formData = new FormData();
if (documentId !== undefined) {
formData.append("document_id", documentId);
}
formData.append("relative_path", relativePath);
formData.append("created_date", createdDate.toISOString());
formData.append("content", new Blob([contentBytes]));
const response = await this.client.POST(
@ -100,18 +104,18 @@ export class SyncService {
parentVersionId,
documentId,
relativePath,
contentBytes,
createdDate
contentBytes
}: {
parentVersionId: VaultUpdateId;
documentId: DocumentId;
relativePath: RelativePath;
contentBytes: Uint8Array;
createdDate: Date;
}): Promise<components["schemas"]["DocumentUpdateResponse"]> {
this.logger.debug(
`Updating document ${documentId} with parent version ${parentVersionId} and relative path ${relativePath}`
);
const formData = new FormData();
formData.append("parent_version_id", parentVersionId.toString());
formData.append("created_date", createdDate.toISOString());
formData.append("relative_path", relativePath);
formData.append("content", new Blob([contentBytes]));
@ -149,13 +153,11 @@ export class SyncService {
public async delete({
documentId,
relativePath,
createdDate
relativePath
}: {
documentId: DocumentId;
relativePath: RelativePath;
createdDate: Date;
}): Promise<void> {
}): Promise<components["schemas"]["DocumentVersionWithoutContent"]> {
const response = await this.client.DELETE(
"/vaults/{vault_id}/documents/{document_id}",
{
@ -169,7 +171,6 @@ export class SyncService {
}
},
body: {
createdDate: createdDate.toISOString(),
relativePath
}
}
@ -295,11 +296,17 @@ export class SyncService {
private createClient(remoteUri: string): void {
this.client = createClient<paths>({
baseUrl: remoteUri,
fetch: retriedFetchFactory(this.logger, this._fetchImplementation)
fetch: this.connectedState.getFetchImplementation(
this._fetchImplementation
)
});
this.clientWithoutRetries = createClient<paths>({
baseUrl: remoteUri
baseUrl: remoteUri,
fetch: this.connectedState.getFetchImplementation(
this._fetchImplementation,
{ doRetries: false }
)
});
}
}

36
frontend/sync-client/src/services/types.ts
View file

@ -274,12 +274,13 @@ export interface paths {
};
};
responses: {
/** @description no content */
200: {
headers: {
[name: string]: unknown;
};
content?: never;
content: {
"application/json": components["schemas"]["DocumentVersionWithoutContent"];
};
};
default: {
headers: {
@ -451,26 +452,25 @@ export interface components {
Array_of_uint8: number[];
CreateDocumentVersion: {
contentBase64: string;
/** Format: date-time */
createdDate: string;
/**
* Format: uuid
* @description The client can decide the document id (if it wishes to) in order to help with syncing. If the client does not provide a document id, the server will generate one. If the client provides a document id it must not already exist in the database.
*/
documentId?: string | null;
relativePath: string;
};
CreateDocumentVersionMultipart: {
content: components["schemas"]["Array_of_uint8"];
/** Format: date-time */
created_date: string;
/** Format: uuid */
document_id?: string | null;
relative_path: string;
};
DeleteDocumentVersion: {
/** Format: date-time */
createdDate: string;
relativePath: string;
};
/** @description Response to a update document request. */
/** @description Response to an update document request. */
DocumentUpdateResponse:
| {
/** Format: date-time */
createdDate: string;
/** Format: uuid */
documentId: string;
isDeleted: boolean;
@ -479,14 +479,11 @@ export interface components {
type: "FastForwardUpdate";
/** Format: date-time */
updatedDate: string;
vaultId: string;
/** Format: int64 */
vaultUpdateId: number;
}
| {
contentBase64: string;
/** Format: date-time */
createdDate: string;
/** Format: uuid */
documentId: string;
isDeleted: boolean;
@ -495,34 +492,27 @@ export interface components {
type: "MergingUpdate";
/** Format: date-time */
updatedDate: string;
vaultId: string;
/** Format: int64 */
vaultUpdateId: number;
};
DocumentVersion: {
contentBase64: string;
/** Format: date-time */
createdDate: string;
/** Format: uuid */
documentId: string;
isDeleted: boolean;
relativePath: string;
/** Format: date-time */
updatedDate: string;
vaultId: string;
/** Format: int64 */
vaultUpdateId: number;
};
DocumentVersionWithoutContent: {
/** Format: date-time */
createdDate: string;
/** Format: uuid */
documentId: string;
isDeleted: boolean;
relativePath: string;
/** Format: date-time */
updatedDate: string;
vaultId: string;
/** Format: int64 */
vaultUpdateId: number;
};
@ -587,16 +577,12 @@ export interface components {
};
UpdateDocumentVersion: {
contentBase64: string;
/** Format: date-time */
createdDate: string;
/** Format: int64 */
parentVersionId: number;
relativePath: string;
};
UpdateDocumentVersionMultipart: {
content: components["schemas"]["Array_of_uint8"];
/** Format: date-time */
createdDate: string;
/** Format: int64 */
parentVersionId: number;
relativePath: string;

22
frontend/sync-client/src/sync-client.ts
View file

@ -12,6 +12,7 @@ import { SyncService } from "./services/sync-service";
import { Syncer } from "./sync-operations/syncer";
import type { FileSystemOperations } from "./file-operations/filesystem-operations";
import { FileOperations } from "./file-operations/file-operations";
import { ConnectedState } from "./services/connected-state";
export class SyncClient {
private remoteListenerIntervalId: NodeJS.Timeout | null = null;
@ -42,7 +43,7 @@ export class SyncClient {
}
public get documentCount(): number {
return this._database.getDocuments().size;
return this._database.length;
}
public set fetchImplementation(fetch: typeof globalThis.fetch) {
@ -90,7 +91,9 @@ export class SyncClient {
}
);
const syncService = new SyncService(settings, logger);
const connectedState = new ConnectedState(settings, logger);
const syncService = new SyncService(connectedState, settings, logger);
const syncer = new Syncer(
logger,
@ -117,18 +120,13 @@ export class SyncClient {
);
settings.addOnSettingsChangeHandlers((newSettings, oldSettings) => {
if (
newSettings.fetchChangesUpdateIntervalMs !==
oldSettings.fetchChangesUpdateIntervalMs
) {
client.registerRemoteEventListener(
newSettings.fetchChangesUpdateIntervalMs
);
if (!oldSettings.isSyncEnabled && newSettings.isSyncEnabled) {
syncer
.scheduleSyncForOfflineChanges()
.catch((_error: unknown) => {
logger.error(
"Failed to schedule sync for offline changes"
);
});
}
});
@ -148,7 +146,7 @@ export class SyncClient {
this.stop();
await this._syncer.reset();
this._history.reset();
await this._database.resetSyncState();
this._database.resetSyncState();
this.logger.reset();
}

403
frontend/sync-client/src/sync-operations/syncer.ts
View file

@ -1,15 +1,17 @@
import type { Database, RelativePath } from "../persistence/database";
import type { SyncService } from "src/services/sync-service";
import type { Logger } from "src/tracing/logger";
import type { SyncHistory } from "src/tracing/sync-history";
import type { SyncService } from "../services/sync-service";
import type { Logger } from "../tracing/logger";
import type { SyncHistory } from "../tracing/sync-history";
import PQueue from "p-queue";
import { hash } from "src/utils/hash";
import type { components } from "src/services/types";
import type { Settings } from "src/persistence/settings";
import type { FileOperations } from "src/file-operations/file-operations";
import { findMatchingFileBasedOnHash } from "src/utils/find-matching-file-based-on-hash";
import { hash } from "../utils/hash";
import { v4 as uuidv4 } from "uuid";
import type { components } from "../services/types";
import type { Settings } from "../persistence/settings";
import type { FileOperations } from "../file-operations/file-operations";
import { findMatchingFile } from "../utils/find-matching-file";
import { UnrestrictedSyncer } from "./unrestricted-syncer";
import { FileNotFoundError } from "../file-operations/safe-filesystem-operations";
import { createPromise } from "../utils/create-promise";
export class Syncer {
private readonly remainingOperationsListeners: ((
@ -18,17 +20,15 @@ export class Syncer {
private readonly syncQueue: PQueue;
private runningScheduleSyncForOfflineChanges: Promise<void> | undefined =
undefined;
private runningApplyRemoteChangesLocally: Promise<void> | undefined =
undefined;
private runningScheduleSyncForOfflineChanges: Promise<void> | undefined;
private runningApplyRemoteChangesLocally: Promise<void> | undefined;
private readonly internalSyncer: UnrestrictedSyncer;
public constructor(
private readonly logger: Logger,
private readonly database: Database,
private readonly settings: Settings,
settings: Settings,
private readonly syncService: SyncService,
private readonly operations: FileOperations,
history: SyncHistory
@ -45,7 +45,9 @@ export class Syncer {
});
this.syncQueue.on("active", () => {
this.emitRemainingOperationsChange(this.syncQueue.size);
this.remainingOperationsListeners.forEach((listener) => {
listener(this.syncQueue.size);
});
});
this.internalSyncer = new UnrestrictedSyncer(
@ -65,48 +67,131 @@ export class Syncer {
}
public async syncLocallyCreatedFile(
relativePath: RelativePath,
updateTime: Date
relativePath: RelativePath
): Promise<void> {
await this.syncQueue.add(async () =>
this.internalSyncer.unrestrictedSyncLocallyCreatedFile(
if (
this.database.getLatestDocumentByRelativePath(relativePath)
?.isDeleted === false
) {
this.logger.debug(
`Document ${relativePath} already exists in the database, skipping`
);
return;
}
const [promise, resolve, reject] = createPromise();
const document = this.database.createNewPendingDocument(
uuidv4(),
relativePath,
updateTime
)
promise
);
}
public async syncLocallyUpdatedFile(args: {
oldPath?: RelativePath;
relativePath: RelativePath;
updateTime: Date;
}): Promise<void> {
try {
await this.syncQueue.add(async () =>
this.internalSyncer.unrestrictedSyncLocallyUpdatedFile(args)
this.internalSyncer.unrestrictedSyncLocallyCreatedFile(document)
);
}
public async waitForSyncQueue(): Promise<void> {
return this.syncQueue.onEmpty();
resolve();
} catch (e) {
reject(e);
} finally {
this.database.removeDocumentPromise(promise);
}
}
public async syncLocallyDeletedFile(
relativePath: RelativePath
): Promise<void> {
// We have to have a record of the delete in case there's an in-flight update for the same
// document which finishes after the delete has succeeded and would introduce a phantom metadata record.
this.database.delete(relativePath);
const [promise, resolve, reject] = createPromise();
const document = await this.database.getResolvedDocumentByRelativePath(
relativePath,
promise
);
try {
await this.syncQueue.add(async () =>
this.internalSyncer.unrestrictedSyncLocallyDeletedFile(relativePath)
this.internalSyncer.unrestrictedSyncLocallyDeletedFile(document)
);
resolve();
} catch (e) {
reject(e);
} finally {
this.database.removeDocumentPromise(promise);
}
}
public async syncLocallyUpdatedFile({
oldPath,
relativePath
}: {
oldPath?: RelativePath;
relativePath: RelativePath;
}): Promise<void> {
if (
oldPath !== undefined &&
(this.database.getLatestDocumentByRelativePath(relativePath) ===
undefined ||
this.database.getLatestDocumentByRelativePath(relativePath)
?.isDeleted === true)
) {
if (oldPath === relativePath) {
throw new Error(
`Old path and new path are the same: ${oldPath}`
);
}
public async scheduleSyncForOfflineChanges(): Promise<void> {
if (!this.settings.getSettings().isSyncEnabled) {
this.database.move(oldPath, relativePath);
}
let document =
this.database.getLatestDocumentByRelativePath(relativePath);
if (document === undefined) {
this.logger.debug(
`Syncing is disabled, not uploading local changes`
`Cannot find document ${relativePath} in the database, skipping`
);
return;
}
if (this.runningScheduleSyncForOfflineChanges != null) {
if (document.isDeleted) {
this.logger.debug(
`Document ${relativePath} has been deleted locally, skipping`
);
return;
}
const [promise, resolve, reject] = createPromise();
document = await this.database.getResolvedDocumentByRelativePath(
relativePath,
promise
);
try {
await this.syncQueue.add(async () =>
this.internalSyncer.unrestrictedSyncLocallyUpdatedFile({
oldPath,
document
})
);
resolve();
} catch (e) {
reject(e);
} finally {
this.database.removeDocumentPromise(promise);
}
}
public async scheduleSyncForOfflineChanges(): Promise<void> {
if (this.runningScheduleSyncForOfflineChanges !== undefined) {
this.logger.debug("Uploading local changes is already in progress");
return this.runningScheduleSyncForOfflineChanges;
}
@ -127,13 +212,6 @@ export class Syncer {
}
public async applyRemoteChangesLocally(): Promise<void> {
if (!this.settings.getSettings().isSyncEnabled) {
this.logger.debug(
`Syncing is disabled, not fetching remote changes`
);
return;
}
if (this.runningApplyRemoteChangesLocally != null) {
this.logger.debug(
"Applying remote changes locally is already in progress"
@ -154,6 +232,10 @@ export class Syncer {
}
}
public async waitForSyncQueue(): Promise<void> {
return this.syncQueue.onEmpty();
}
public async reset(): Promise<void> {
this.syncQueue.clear();
await this.syncQueue.onEmpty();
@ -163,115 +245,15 @@ export class Syncer {
this.internalSyncer.reset();
}
private async syncRemotelyUpdatedFile(
remoteVersion: components["schemas"]["DocumentVersionWithoutContent"]
): Promise<void> {
await this.syncQueue.add(async () =>
this.internalSyncer.unrestrictedSyncRemotelyUpdatedFile(
remoteVersion
)
);
}
private async internalScheduleSyncForOfflineChanges(): Promise<void> {
const allLocalFiles = await this.operations.listAllFiles();
// This includes renamed files for now
let locallyPossiblyDeletedFiles = [
...this.database.getDocuments().entries()
].filter(([path, _]) => !allLocalFiles.includes(path));
await Promise.all(
allLocalFiles.map(async (relativePath) =>
this.syncQueue.add(async () => {
const metadata = this.database.getDocument(relativePath);
if (metadata) {
this.logger.debug(
`Document ${relativePath} might have been updated locally, scheduling sync to validate and update it`
);
return this.internalSyncer.unrestrictedSyncLocallyUpdatedFile(
{
relativePath,
updateTime:
await this.operations.getModificationTime(
relativePath
)
}
);
}
// Perhaps the file has been moved. Let's check by looking at the deleted files
const contentBytes =
await this.operations.read(relativePath);
const contentHash = hash(contentBytes);
// todo: make this smarter so that offline files can be renamed & edited at the same time
const originalFile = findMatchingFileBasedOnHash(
contentHash,
locallyPossiblyDeletedFiles
);
if (originalFile !== undefined) {
// `originalFile` hasn't been deleted but it got moved instead
locallyPossiblyDeletedFiles =
locallyPossiblyDeletedFiles.filter(
(item) => item[0] !== originalFile[0]
);
this.logger.debug(
`Document '${originalFile[0]}' was not found under its current path in the database but was found under a different path (${relativePath}), scheduling sync to move it`
);
return this.internalSyncer.unrestrictedSyncLocallyUpdatedFile(
{
oldPath: originalFile[0],
relativePath: relativePath,
updateTime:
await this.operations.getModificationTime(
relativePath
),
optimisations: {
contentBytes,
contentHash
}
}
);
}
this.logger.debug(
`Document ${relativePath} not found in database, scheduling sync to create it`
);
return this.internalSyncer.unrestrictedSyncLocallyCreatedFile(
relativePath,
await this.operations.getModificationTime(relativePath)
);
})
)
);
await Promise.all(
locallyPossiblyDeletedFiles.map(async ([relativePath, _]) => {
this.logger.debug(
`Document ${relativePath} has been deleted locally, scheduling sync to delete it`
);
if (await this.operations.exists(relativePath)) {
this.logger.debug(
`Document ${relativePath} actually exists locally, skipping`
);
return Promise.resolve();
}
// We're outside of the pqueue, so we need to call the public wrapper
return this.syncLocallyDeletedFile(relativePath);
})
);
}
private async internalApplyRemoteChangesLocally(): Promise<void> {
const remote = await this.syncService.getAll(
this.database.getLastSeenUpdateId()
const remote = await this.syncQueue.add(async () =>
this.syncService.getAll(this.database.getLastSeenUpdateId())
);
if (!remote) {
throw new Error("Failed to fetch remote changes");
}
if (remote.latestDocuments.length === 0) {
this.logger.debug("No remote changes to apply");
return;
@ -280,9 +262,7 @@ export class Syncer {
this.logger.info("Applying remote changes locally");
await Promise.all(
remote.latestDocuments.map(async (remoteDocument) =>
this.syncRemotelyUpdatedFile(remoteDocument)
)
remote.latestDocuments.map(this.syncRemotelyUpdatedFile.bind(this))
);
const lastSeenUpdateId = this.database.getLastSeenUpdateId();
@ -290,13 +270,124 @@ export class Syncer {
lastSeenUpdateId === undefined ||
remote.lastUpdateId > lastSeenUpdateId
) {
await this.database.setLastSeenUpdateId(remote.lastUpdateId);
this.database.setLastSeenUpdateId(remote.lastUpdateId);
}
}
private emitRemainingOperationsChange(remainingOperations: number): void {
this.remainingOperationsListeners.forEach((listener) => {
listener(remainingOperations);
private async syncRemotelyUpdatedFile(
remoteVersion: components["schemas"]["DocumentVersionWithoutContent"]
): Promise<void> {
let document = this.database.getDocumentByDocumentId(
remoteVersion.documentId
);
const [promise, resolve, reject] = createPromise();
if (document === undefined) {
await this.syncQueue.add(async () =>
this.internalSyncer.unrestrictedSyncRemotelyUpdatedFile(
remoteVersion
)
);
} else {
document = await this.database.getResolvedDocumentByRelativePath(
document.relativePath,
promise
);
try {
await this.syncQueue.add(async () =>
this.internalSyncer.unrestrictedSyncRemotelyUpdatedFile(
remoteVersion,
document
)
);
resolve();
} catch (e) {
reject(e);
} finally {
this.database.removeDocumentPromise(promise);
}
}
}
private async internalScheduleSyncForOfflineChanges(): Promise<void> {
const allLocalFiles = await this.operations.listAllFiles();
let locallyPossiblyDeletedFiles = [
...this.database.resolvedDocuments
].filter(({ relativePath }) => !allLocalFiles.includes(relativePath));
const updates = Promise.all(
allLocalFiles.map(async (relativePath) => {
if (
this.database.getLatestDocumentByRelativePath(relativePath)
?.metadata !== undefined
) {
this.logger.debug(
`Document ${relativePath} might have been updated locally, scheduling sync to validate and update it`
);
return this.syncLocallyUpdatedFile({
relativePath
});
}
// Perhaps the file has been moved; let's check by looking at the deleted files
const contentHash = await this.syncQueue.add(async () => {
const contentBytes =
await this.operations.read(relativePath); // this can throw FileNotFoundError
return hash(contentBytes);
});
if (contentHash == undefined) {
// The file was deleted before we had a chance to read it, no need to sync it here
return;
}
const originalFile = findMatchingFile(
contentHash,
locallyPossiblyDeletedFiles
);
if (originalFile !== undefined) {
// `originalFile` hasn't been deleted but it got moved instead
locallyPossiblyDeletedFiles =
locallyPossiblyDeletedFiles.filter(
(item) =>
item.relativePath !== originalFile.relativePath
);
this.logger.debug(
`Document '${originalFile.relativePath}' was not found under its current path in the database but was found under a different path (${relativePath}), scheduling sync to move it`
);
// We're outside of the pqueue, so we need to call the public wrapper
return this.syncLocallyUpdatedFile({
oldPath: originalFile.relativePath,
relativePath
});
}
this.logger.debug(
`Document ${relativePath} not found in database, scheduling sync to create it`
);
// We're outside of the pqueue, so we need to call the public wrapper
return this.syncLocallyCreatedFile(relativePath);
})
);
const deletes = Promise.all(
locallyPossiblyDeletedFiles.map(async ({ relativePath }) => {
this.logger.debug(
`Document ${relativePath} has been deleted locally, scheduling sync to delete it`
);
// We're outside of the pqueue, so we need to call the public wrapper
return this.syncLocallyDeletedFile(relativePath);
})
);
await Promise.all([updates, deletes]);
}
}

635
frontend/sync-client/src/sync-operations/unrestricted-syncer.ts
View file

@ -1,19 +1,24 @@
import type { Database, RelativePath } from "../persistence/database";
import type {
Database,
DocumentRecord,
RelativePath
} from "../persistence/database";
import type { SyncService } from "src/services/sync-service";
import type { Logger } from "src/tracing/logger";
import type { SyncHistory } from "src/tracing/sync-history";
import { SyncSource, SyncStatus, SyncType } from "src/tracing/sync-history";
import { hash } from "src/utils/hash";
import type { components } from "src/services/types";
import { deserialize } from "src/utils/deserialize";
import type { Settings } from "src/persistence/settings";
import type { FileOperations } from "src/file-operations/file-operations";
import { FileNotFoundError } from "src/file-operations/safe-filesystem-operations";
import { DocumentLocks } from "./document-locks";
import type { SyncService } from "../services/sync-service";
import type { Logger } from "../tracing/logger";
import type { SyncHistory } from "../tracing/sync-history";
import { SyncSource, SyncStatus, SyncType } from "../tracing/sync-history";
import { EMPTY_HASH, hash } from "../utils/hash";
import type { components } from "../services/types";
import { deserialize } from "../utils/deserialize";
import type { Settings } from "../persistence/settings";
import type { FileOperations } from "../file-operations/file-operations";
import { FileNotFoundError } from "../file-operations/safe-filesystem-operations";
import { DocumentLocks } from "../file-operations/document-locks";
import { createPromise } from "../utils/create-promise";
export class UnrestrictedSyncer {
private readonly locks = new DocumentLocks();
private readonly locks: DocumentLocks;
public constructor(
private readonly logger: Logger,
@ -22,240 +27,211 @@ export class UnrestrictedSyncer {
private readonly syncService: SyncService,
private readonly operations: FileOperations,
private readonly history: SyncHistory
) {}
) {
this.locks = new DocumentLocks(logger);
}
public async unrestrictedSyncLocallyCreatedFile(
relativePath: RelativePath,
updateTime: Date,
optimisations?: {
contentBytes?: Uint8Array;
contentHash?: string;
}
document: DocumentRecord
): Promise<void> {
await this.executeWhileHoldingFileLock(
[relativePath],
return this.executeSync(
document.relativePath,
SyncType.CREATE,
SyncSource.PUSH,
async () => {
if (
(await this.operations.getFileSize(relativePath)) / // this can throw FileNotFoundError
1024 /
1024 >
this.settings.getSettings().maxFileSizeMB
) {
this.history.addHistoryEntry({
status: SyncStatus.ERROR,
relativePath,
message: `File size exceeds the maximum file size limit of ${
this.settings.getSettings().maxFileSizeMB
}MB`,
type: SyncType.CREATE
});
return;
}
const contentBytes =
optimisations?.contentBytes ??
(await this.operations.read(relativePath)); // this can throw FileNotFoundError
let contentHash =
optimisations?.contentHash ?? hash(contentBytes);
const localMetadata = this.database.getDocument(relativePath);
if (localMetadata) {
this.logger.debug(
`Document metadata already exists for ${relativePath}, it must have been downloaded from the server`
);
if (localMetadata.hash === contentHash) {
this.history.addHistoryEntry({
status: SyncStatus.NO_OP,
relativePath,
message: `File hash matches with last synced version, no need to sync`,
type: SyncType.UPDATE
});
return;
}
}
const contentBytes = await this.operations.read(
document.relativePath
); // this can throw FileNotFoundError
const contentHash = hash(contentBytes);
const response = await this.syncService.create({
relativePath,
contentBytes,
createdDate: updateTime
documentId: document.documentId,
relativePath: document.relativePath,
contentBytes
});
this.history.addHistoryEntry({
status: SyncStatus.SUCCESS,
source: SyncSource.PUSH,
relativePath,
relativePath: document.relativePath,
message: `Successfully uploaded locally created file`,
type: SyncType.CREATE
});
// The response can't have a different relative path than the one we sent
// because the relative path is the key when finding existing documents
// when a create request is sent.
if (response.type === "MergingUpdate") {
const responseBytes = deserialize(response.contentBase64);
contentHash = hash(responseBytes);
await this.operations.write(
relativePath,
contentBytes,
responseBytes
);
this.history.addHistoryEntry({
status: SyncStatus.SUCCESS,
source: SyncSource.PULL,
relativePath,
message: `The file we created locally has already existed remotely, so we have merged them`,
type: SyncType.UPDATE
});
}
await this.database.setDocument({
documentId: response.documentId,
relativePath: response.relativePath,
this.database.updateDocumentMetadata(
{
parentVersionId: response.vaultUpdateId,
hash: contentHash
},
document
);
this.tryIncrementVaultUpdateId(response.vaultUpdateId);
}
);
}
public async unrestrictedSyncLocallyDeletedFile(
document: DocumentRecord
): Promise<void> {
await this.executeSync(
document.relativePath,
SyncType.DELETE,
SyncSource.PUSH,
async () => {
const response = await this.syncService.delete({
documentId: document.documentId,
relativePath: document.relativePath
});
await this.tryIncrementVaultUpdateId(response.vaultUpdateId);
this.history.addHistoryEntry({
status: SyncStatus.SUCCESS,
source: SyncSource.PUSH,
relativePath: document.relativePath,
message: `Successfully deleted locally deleted file on the remote server`,
type: SyncType.DELETE
});
this.database.updateDocumentMetadata(
{
parentVersionId: response.vaultUpdateId,
hash: EMPTY_HASH
},
document
);
}
);
}
public async unrestrictedSyncLocallyUpdatedFile({
oldPath,
relativePath,
updateTime,
optimisations
document,
force = false
}: {
oldPath?: RelativePath;
relativePath: RelativePath;
updateTime: Date;
optimisations?: {
contentBytes?: Uint8Array;
contentHash?: string;
};
force?: boolean;
document: DocumentRecord;
}): Promise<void> {
await this.executeWhileHoldingFileLock(
[oldPath, relativePath].filter((path) => path !== undefined),
await this.executeSync(
document.relativePath,
SyncType.UPDATE,
SyncSource.PUSH,
async () => {
// Check the new path first in case the metadata has been already moved
let localMetadata = this.database.getDocument(relativePath);
let metadataPath = relativePath;
const originalRelativePath = document.relativePath;
if (localMetadata === undefined && oldPath !== undefined) {
localMetadata = this.database.getDocument(oldPath);
metadataPath = oldPath;
}
if (!localMetadata) {
// It's fine, a subsequent sync operation must have dealt with this
if (document.metadata === undefined || document.isDeleted) {
this.logger.debug(
`Document ${document.relativePath} has been already deleted, no need to update it`
);
return;
}
if (
(await this.operations.getFileSize(relativePath)) / // this can throw FileNotFoundError
1024 /
1024 >
this.settings.getSettings().maxFileSizeMB
) {
this.history.addHistoryEntry({
status: SyncStatus.ERROR,
relativePath,
message: `File size exceeds the maximum file size limit of ${
this.settings.getSettings().maxFileSizeMB
}MB`,
type: SyncType.CREATE
});
return;
}
const contentBytes =
optimisations?.contentBytes ??
(await this.operations.read(relativePath)); // this can throw FileNotFoundError
let contentHash =
optimisations?.contentHash ?? hash(contentBytes);
const contentBytes = await this.operations.read(
document.relativePath
); // this can throw FileNotFoundError
let contentHash = hash(contentBytes);
if (
localMetadata.hash === contentHash &&
oldPath === undefined
document.metadata.hash === contentHash &&
oldPath === undefined &&
!force
) {
this.history.addHistoryEntry({
status: SyncStatus.NO_OP,
relativePath,
message: `File hash matches with last synced version, no need to sync`,
type: SyncType.UPDATE
});
this.logger.debug(
`File hash of ${document.relativePath} matches with last synced version and the path hasn't changed; no need to sync`
);
return;
}
const response = await this.syncService.put({
documentId: localMetadata.documentId,
parentVersionId: localMetadata.parentVersionId,
relativePath,
contentBytes,
createdDate: updateTime
documentId: document.documentId,
parentVersionId: document.metadata.parentVersionId,
relativePath: document.relativePath,
contentBytes
});
// `document` is mutable and reflects the latest state in the local database
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
if (document.isDeleted) {
this.logger.info(
`Document ${document.relativePath} has been deleted before we could finish updating it`
);
return;
}
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
if (document.metadata === undefined) {
throw new Error(
`Document ${document.relativePath} no longer has metadata after updating it, this cannot happen`
);
}
if (
document.metadata.parentVersionId >= response.vaultUpdateId
) {
this.logger.debug(
`Document ${document.relativePath} is already more up to date than the fetched version`
);
return;
}
this.history.addHistoryEntry({
status: SyncStatus.SUCCESS,
source: SyncSource.PUSH,
relativePath,
relativePath: document.relativePath,
message: `Successfully uploaded locally updated file to the remote server`,
type: SyncType.UPDATE
});
if (response.isDeleted) {
await this.operations.remove(oldPath ?? relativePath);
await this.database.removeDocument(oldPath ?? relativePath);
await this.tryIncrementVaultUpdateId(
response.vaultUpdateId
);
this.history.addHistoryEntry({
status: SyncStatus.SUCCESS,
source: SyncSource.PULL,
relativePath,
relativePath: document.relativePath,
message:
"The file we tried to update had been deleted remotely, therefore, we have deleted it locally",
type: SyncType.DELETE
});
this.database.delete(document.relativePath);
this.database.updateDocumentMetadata(
{
parentVersionId: response.vaultUpdateId,
hash: EMPTY_HASH
},
document
);
await this.operations.delete(document.relativePath);
this.tryIncrementVaultUpdateId(response.vaultUpdateId);
return;
}
if (
response.relativePath != relativePath &&
response.relativePath != oldPath
) {
await this.locks.waitForDocumentLock(response.relativePath);
let actualPath = document.relativePath;
if (response.relativePath != originalRelativePath) {
actualPath = response.relativePath;
await this.operations.move(
document.relativePath,
response.relativePath
); // this can throw FileNotFoundError
}
try {
if (response.relativePath != relativePath) {
// TODO: this can fail, that's bad
await this.operations.move(
// this can throw FileNotFoundError
relativePath,
response.relativePath,
response.documentId
this.database.updateDocumentMetadata(
{
parentVersionId: response.vaultUpdateId,
hash: contentHash
},
document
);
}
if (response.type === "MergingUpdate") {
const responseBytes = deserialize(
response.contentBase64
);
const responseBytes = deserialize(response.contentBase64);
contentHash = hash(responseBytes);
await this.operations.write(
response.relativePath,
actualPath,
contentBytes,
responseBytes
);
@ -263,111 +239,49 @@ export class UnrestrictedSyncer {
this.history.addHistoryEntry({
status: SyncStatus.SUCCESS,
source: SyncSource.PULL,
relativePath,
relativePath: document.relativePath,
message: `The file we updated had been updated remotely, so we downloaded the merged version`,
type: SyncType.UPDATE
});
}
if (metadataPath !== response.relativePath) {
await this.database.updatePath(
metadataPath,
response.relativePath
);
}
await this.database.setDocument({
documentId: localMetadata.documentId,
relativePath: response.relativePath,
parentVersionId: response.vaultUpdateId,
hash: contentHash
});
await this.tryIncrementVaultUpdateId(
response.vaultUpdateId
);
} finally {
if (
response.relativePath != relativePath &&
response.relativePath != oldPath
) {
this.locks.unlockDocument(response.relativePath);
}
}
}
);
}
public async unrestrictedSyncLocallyDeletedFile(
relativePath: RelativePath
): Promise<void> {
await this.executeWhileHoldingFileLock(
[relativePath],
SyncType.DELETE,
SyncSource.PUSH,
async () => {
const localMetadata = this.database.getDocument(relativePath);
if (!localMetadata) {
this.history.addHistoryEntry({
status: SyncStatus.NO_OP,
relativePath,
message: `Locally deleted file hasn't been uploaded yet, so there's no need to delete it on the remote server`,
type: SyncType.DELETE
});
return;
}
await this.syncService.delete({
documentId: localMetadata.documentId,
relativePath,
createdDate: new Date() // We got the event now, so it must have been deleted just now
});
this.history.addHistoryEntry({
status: SyncStatus.SUCCESS,
source: SyncSource.PUSH,
relativePath,
message: `Successfully deleted locally deleted file on the remote server`,
type: SyncType.DELETE
});
await this.database.removeDocument(relativePath);
this.tryIncrementVaultUpdateId(response.vaultUpdateId);
}
);
}
public async unrestrictedSyncRemotelyUpdatedFile(
remoteVersion: components["schemas"]["DocumentVersionWithoutContent"]
remoteVersion: components["schemas"]["DocumentVersionWithoutContent"],
document?: DocumentRecord
): Promise<void> {
await this.executeWhileHoldingFileLock(
[remoteVersion.relativePath],
await this.executeSync(
remoteVersion.relativePath,
SyncType.UPDATE,
SyncSource.PULL,
async () => {
let localMetadata = this.database.getDocumentByDocumentId(
remoteVersion.documentId
);
if (document?.metadata !== undefined) {
// If the file exists locally, let's pretend the user has updated it
// and deal with remote update/deletion within `unrestrictedSyncLocallyUpdatedFile`
if (
localMetadata &&
localMetadata[0] !== remoteVersion.relativePath
document.metadata.parentVersionId >=
remoteVersion.vaultUpdateId
) {
await this.locks.waitForDocumentLock(localMetadata[0]);
}
// Waiting for the new lock might take a while so we need to fetch the database
// entry again in case it's changed.
localMetadata = this.database.getDocumentByDocumentId(
remoteVersion.documentId
this.logger.debug(
`Document ${remoteVersion.relativePath} is already more up to date than the fetched version`
);
return;
}
if (!localMetadata) {
if (remoteVersion.isDeleted) {
this.history.addHistoryEntry({
status: SyncStatus.NO_OP,
source: SyncSource.PULL,
relativePath: remoteVersion.relativePath,
message: `Remotely deleted file hasn't been synced yet, so there's no need to delete it locally`,
type: SyncType.DELETE
return this.unrestrictedSyncLocallyUpdatedFile({
document,
force: true
});
} else if (remoteVersion.isDeleted) {
// Either the doc hasn't made it to us before and therefore we don't need to delete it,
// or we already have it, in which case the preceeding if will deal with it
this.logger.debug(
`Document ${remoteVersion.relativePath} has been deleted remotely, no need to sync`
);
return;
}
@ -376,18 +290,55 @@ export class UnrestrictedSyncer {
documentId: remoteVersion.documentId
})
).contentBase64;
document = this.database.getDocumentByDocumentId(
remoteVersion.documentId
);
if (document?.isDeleted === true) {
this.logger.info(
`Document ${remoteVersion.relativePath} has been deleted locally before we could finish updating it`
);
return;
}
if (
(document?.metadata?.parentVersionId ?? -1) >=
remoteVersion.vaultUpdateId
) {
this.logger.debug(
`Document ${remoteVersion.relativePath} is already more up to date than the fetched version`
);
return;
}
const contentBytes = deserialize(content);
await this.operations.ensureClearPath(
remoteVersion.relativePath
);
const [promise, resolve] = createPromise();
this.database.updateDocumentMetadata(
{
parentVersionId: remoteVersion.vaultUpdateId,
hash: hash(contentBytes)
},
this.database.createNewPendingDocument(
remoteVersion.documentId,
remoteVersion.relativePath,
promise
)
);
await this.operations.create(
remoteVersion.relativePath,
contentBytes
);
await this.database.setDocument({
documentId: remoteVersion.documentId,
relativePath: remoteVersion.relativePath,
parentVersionId: remoteVersion.vaultUpdateId,
hash: hash(contentBytes)
});
resolve();
this.database.removeDocumentPromise(promise);
this.history.addHistoryEntry({
status: SyncStatus.SUCCESS,
source: SyncSource.PULL,
@ -395,134 +346,56 @@ export class UnrestrictedSyncer {
message: `Successfully downloaded remote file which hadn't existed locally`,
type: SyncType.CREATE
});
return;
}
const [relativePath, metadata] = localMetadata;
if (remoteVersion.vaultUpdateId <= metadata.parentVersionId) {
this.logger.debug(
`Document ${relativePath} is already up to date`
);
return;
}
try {
if (remoteVersion.isDeleted) {
await this.operations.remove(relativePath);
await this.database.removeDocument(relativePath);
this.history.addHistoryEntry({
status: SyncStatus.SUCCESS,
source: SyncSource.PULL,
relativePath: remoteVersion.relativePath,
message: `Successfully deleted remotely deleted file locally`,
type: SyncType.DELETE
});
} else {
// TODO: this can fail, that's bad
const currentContent =
await this.operations.read(relativePath); // this can throw FileNotFoundError
const currentHash = hash(currentContent);
if (currentHash !== metadata.hash) {
this.logger.info(
`Document ${relativePath} has been updated both remotely and locally, letting the local file update event handle it`
);
return;
}
const content = (
await this.syncService.get({
documentId: remoteVersion.documentId
})
).contentBase64;
const contentBytes = deserialize(content);
const contentHash = hash(contentBytes);
if (relativePath !== remoteVersion.relativePath) {
// TODO: this can fail, that's bad
await this.operations.move(
// this can throw FileNotFoundError
relativePath,
remoteVersion.relativePath,
remoteVersion.documentId
);
await this.database.updatePath(
relativePath,
remoteVersion.relativePath
);
}
await this.operations.write(
remoteVersion.relativePath,
currentContent,
contentBytes
);
await this.database.setDocument({
documentId: remoteVersion.documentId,
relativePath: remoteVersion.relativePath,
parentVersionId: remoteVersion.vaultUpdateId,
hash: contentHash
});
this.history.addHistoryEntry({
status: SyncStatus.SUCCESS,
source: SyncSource.PULL,
relativePath: remoteVersion.relativePath,
message: `Successfully updated remotely updated file locally`,
type: SyncType.UPDATE
});
}
} finally {
if (relativePath !== remoteVersion.relativePath) {
this.locks.unlockDocument(relativePath);
}
}
}
);
}
public async executeWhileHoldingFileLock(
lockedPaths: RelativePath[],
public async executeSync<T>(
relativePath: RelativePath,
syncType: SyncType,
syncSource: SyncSource,
fn: () => Promise<void>
): Promise<void> {
const relativePath = lockedPaths[lockedPaths.length - 1];
if (!this.settings.getSettings().isSyncEnabled) {
this.logger.info(
`Syncing is disabled, not syncing ${relativePath}`
);
return;
}
fn: () => Promise<T>
): Promise<T | undefined> {
if (!this.operations.isFileEligibleForSync(relativePath)) {
this.logger.info(
`File ${relativePath} is not eligible for syncing`
);
this.history.addHistoryEntry({
status: SyncStatus.ERROR,
relativePath,
message: `File ${relativePath} is not eligible for syncing`,
type: syncType
});
return;
}
this.logger.debug(
`Syncing ${relativePath} (${syncSource} - ${syncType})`
);
await Promise.all(
lockedPaths.map(this.locks.waitForDocumentLock.bind(this.locks))
);
try {
await fn();
if (
(await this.operations.exists(relativePath)) &&
(await this.operations.getFileSize(relativePath)) / // this can throw FileNotFoundError
1024 /
1024 >
this.settings.getSettings().maxFileSizeMB
) {
this.history.addHistoryEntry({
status: SyncStatus.ERROR,
relativePath,
message: `File size exceeds the maximum file size limit of ${
this.settings.getSettings().maxFileSizeMB
}MB`,
type: syncType
});
return;
}
return await fn();
} catch (e) {
if (e instanceof FileNotFoundError) {
// A subsequent sync operation must have been creating to deal with this
this.history.addHistoryEntry({
status: SyncStatus.NO_OP,
relativePath,
message: `Skip ${syncSource.toLocaleLowerCase()} file because it no longer exists when trying to ${syncType.toLocaleLowerCase()} it`,
type: syncType,
source: syncSource
});
this.logger.info(
`Skip ${syncSource.toLocaleLowerCase()} file because it no longer exists when trying to ${syncType.toLocaleLowerCase()} it`
);
} else {
this.history.addHistoryEntry({
status: SyncStatus.ERROR,
@ -533,8 +406,6 @@ export class UnrestrictedSyncer {
});
throw e;
}
} finally {
lockedPaths.forEach(this.locks.unlockDocument.bind(this.locks));
}
}
@ -542,11 +413,9 @@ export class UnrestrictedSyncer {
this.locks.reset();
}
private async tryIncrementVaultUpdateId(
responseVaultUpdateId: number
): Promise<void> {
private tryIncrementVaultUpdateId(responseVaultUpdateId: number): void {
if (this.database.getLastSeenUpdateId() === responseVaultUpdateId - 1) {
await this.database.setLastSeenUpdateId(responseVaultUpdateId);
this.database.setLastSeenUpdateId(responseVaultUpdateId);
}
}
}

2
frontend/sync-client/src/tracing/sync-history.ts
View file

@ -1,4 +1,4 @@
import type { RelativePath } from "src/persistence/database";
import type { RelativePath } from "../persistence/database";
import type { Logger } from "./logger";
export interface CommonHistoryEntry {

15
frontend/sync-client/src/utils/create-promise.ts Normal file
View file

@ -0,0 +1,15 @@
export function createPromise<T = void>(): [
Promise<T>,
(value: T) => void,
(error: unknown) => void
] {
let resolve: undefined | ((resolved: T) => void) = undefined;
let reject: undefined | ((error: unknown) => void) = undefined;
const creationPromise = new Promise<T>(
(resolve_, reject_) => ((resolve = resolve_), (reject = reject_))
);
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
return [creationPromise, resolve!, reject!];
}

13
frontend/sync-client/src/utils/find-matching-file-based-on-hash.ts
View file

@ -1,13 +0,0 @@
import type { DocumentMetadata, RelativePath } from "src/persistence/database";
import { EMPTY_HASH } from "./hash";
export function findMatchingFileBasedOnHash(
contentHash: string,
candidates: [RelativePath, DocumentMetadata][]
): [RelativePath, DocumentMetadata] | undefined {
if (contentHash === EMPTY_HASH) {
return undefined;
}
return candidates.find(([_, metadata]) => metadata.hash === contentHash);
}

14
frontend/sync-client/src/utils/find-matching-file.ts Normal file
View file

@ -0,0 +1,14 @@
import type { DocumentRecord } from "../persistence/database";
import { EMPTY_HASH } from "./hash";
// TODO: make this smarter so that offline files can be renamed & edited at the same time
export function findMatchingFile(
contentHash: string,
candidates: DocumentRecord[]
): DocumentRecord | undefined {
if (contentHash === EMPTY_HASH) {
return undefined;
}
return candidates.find(({ metadata }) => metadata?.hash === contentHash);
}

2
frontend/sync-client/src/utils/hash.ts
View file

@ -6,7 +6,7 @@ export function hash(content: Uint8Array): string {
result = (result << 5) - result + content[i];
result |= 0; // Convert to 32bit integer
}
return Math.abs(result).toString(16);
return Math.abs(result).toString(16).padStart(8, "0");
}
export const EMPTY_HASH = hash(new Uint8Array(0));

3
frontend/sync-client/src/utils/retried-fetch.ts
View file

@ -1,6 +1,6 @@
import * as fetchRetryFactory from "fetch-retry";
import type { RequestInitRetryParams } from "fetch-retry";
import type { Logger } from "src/tracing/logger";
import type { Logger } from "../tracing/logger";
function getUrlFromInput(input: RequestInfo | URL): string {
if (input instanceof URL) {
@ -31,7 +31,6 @@ export function retriedFetchFactory(
}
return false;
},
retries: 6,
retryDelay: (attempt) => Math.pow(1.5, attempt) * 500,
...init
});

9
frontend/sync-client/tsconfig.json
View file

@ -1,12 +1,15 @@
{
"compilerOptions": {
"baseUrl": ".",
"module": "ESNext",
"target": "ESNext",
"strict": true,
"moduleResolution": "bundler",
"allowSyntheticDefaultImports": true,
"lib": ["DOM", "ESNext"]
"moduleResolution": "bundler",
"lib": [
"DOM" // to get "fetch"
],
"declaration": true,
"declarationDir": "./dist/types"
},
"exclude": ["./dist"]
}

46
frontend/sync-client/webpack.config.js
View file

@ -1,25 +1,13 @@
const path = require("path");
const { merge } = require("webpack-merge");
module.exports = (_env, _argv) => ({
const common = {
entry: "./src/index.ts",
devtool: "source-map",
target: "node",
module: {
rules: [
{
test: /\.ts$/,
use: [
{
loader: "ts-loader",
options: {
compilerOptions: {
declaration: true,
declarationDir: "./dist/types"
},
transpileOnly: false
}
}
]
use: ["ts-loader"]
},
{
test: /\.wasm$/,
@ -28,22 +16,40 @@ module.exports = (_env, _argv) => ({
]
},
optimization: {
// the consuming project should take care of minification
minimize: false
},
resolve: {
extensions: [".ts", ".js"],
extensions: [".ts"],
alias: {
root: __dirname,
src: path.resolve(__dirname, "src")
}
},
performance: {
hints: false // it's a library, no need to warn about its size
}
};
module.exports = [
merge(common, {
target: "web",
output: {
clean: true,
filename: "index.js",
path: path.resolve(__dirname, "dist"),
filename: "sync-client.web.js",
library: {
name: "SyncClient",
type: "umd"
},
path: path.resolve(__dirname, "dist")
globalObject: "this"
}
});
}),
merge(common, {
target: "node",
output: {
path: path.resolve(__dirname, "dist"),
filename: "sync-client.node.js",
libraryTarget: "commonjs2"
}
})
];

6
frontend/test-client/package.json
View file

@ -1,6 +1,6 @@
{
"name": "test-client",
"version": "0.0.0",
"version": "0.0.30",
"private": true,
"bin": {
"test-client": "./dist/cli.js"
@ -11,11 +11,11 @@
"test": "jest --passWithNoTests"
},
"devDependencies": {
"@types/node": "^22.13.5",
"@types/node": "^22.13.10",
"sync-client": "file:../sync-client",
"ts-loader": "^9.5.2",
"tslib": "2.8.1",
"typescript": "5.7.3",
"typescript": "5.8.2",
"uuid": "^11.1.0",
"webpack": "^5.98.0",
"webpack-cli": "^6.0.1"

57
frontend/test-client/src/agent/mock-agent.ts
View file

@ -18,9 +18,10 @@ export class MockAgent extends MockClient {
initialSettings: Partial<SyncSettings>,
public readonly name: string,
private readonly doDeletes: boolean,
useSlowFileEvents: boolean,
private readonly jitterScaleInSeconds: number
) {
super(initialSettings);
super(initialSettings, useSlowFileEvents);
}
public async init(): Promise<void> {
@ -46,27 +47,33 @@ export class MockAgent extends MockClient {
? "(online) "
: "(offline)";
const formatted = `[${this.name} ${state}] ${logLine.timestamp.toISOString()} ${logLine.level} ${logLine.message}`;
// HACK: we have to ensure the file has been synced if we want to change it offline without data loss
const historyEntry = /.*History entry: (.*.md).*/.exec(
logLine.message
);
if (historyEntry) {
this.doNotTouchWhileOffline =
this.doNotTouchWhileOffline.filter(
(file) => file !== historyEntry[1]
);
}
switch (logLine.level) {
case LogLevel.ERROR:
console.error(formatted);
if (!this.useSlowFileEvents) {
// Let's not ignore errors
process.exit(1);
// eslint-disable-next-line @typescript-eslint/no-floating-promises
sleep(100).then(() => process.exit(1));
}
break;
case LogLevel.WARNING:
console.warn(formatted);
break;
case LogLevel.INFO:
// HACK: we have to ensure the file has been synced if we want to change it offline without data loss
const result = /.*History entry: (.*.md).*/.exec(
logLine.message
);
if (result) {
this.doNotTouchWhileOffline =
this.doNotTouchWhileOffline.filter(
(file) => file !== result[1]
);
}
console.info(formatted);
break;
case LogLevel.DEBUG:
@ -84,11 +91,10 @@ export class MockAgent extends MockClient {
this.changeFetchChangesUpdateIntervalMsAction.bind(this)
];
if (
this.client.settings.getSettings().isSyncEnabled &&
this.doNotTouchWhileOffline.length === 0
) {
if (this.client.settings.getSettings().isSyncEnabled) {
if (this.doNotTouchWhileOffline.length === 0) {
options.push(this.disableSyncAction.bind(this));
}
} else {
options.push(this.enableSyncAction.bind(this));
}
@ -186,6 +192,14 @@ export class MockAgent extends MockClient {
}
public assertAllContentIsPresentOnce(): void {
if (this.useSlowFileEvents) {
this.client.logger.info(
// We can't ensure that we have seen every single update
`Skipping content check for ${this.name} because slow file events are enabled`
);
return;
}
for (const content of this.writtenContents) {
const found = Array.from(this.localFiles.keys()).filter((key) => {
return new TextDecoder()
@ -215,7 +229,7 @@ export class MockAgent extends MockClient {
);
assert(
fileContent.split(content).length == 2,
`Content ${content} (of ${this.name}) found more than once in file ${file}. File content:\n${fileContent}`
`Content ${content} (of ${this.name}) found more than once in '${file}'. File content:\n${fileContent}`
);
}
}
@ -237,10 +251,7 @@ export class MockAgent extends MockClient {
`Decided to create file ${file} with content ${content}`
);
return this.create(
file,
new TextEncoder().encode(` |${content}| `)
);
return this.create(file, new TextEncoder().encode(` ${content} `));
}
private async changeFetchChangesUpdateIntervalMsAction(): Promise<void> {
@ -314,7 +325,7 @@ export class MockAgent extends MockClient {
`Decided to update file ${file} with ${content}`
);
this.doNotTouchWhileOffline.push(file);
await this.atomicUpdateText(file, (old) => old + ` |${content}| `);
await this.atomicUpdateText(file, (old) => old + ` ${content} `);
}
private async deleteFileAction(files: RelativePath[]): Promise<void> {

79
frontend/test-client/src/agent/mock-client.ts
View file

@ -1,9 +1,10 @@
import type {
RelativePath,
FileSystemOperations,
SyncSettings
import { assert } from "../utils/assert";
import {
type RelativePath,
type FileSystemOperations,
type SyncSettings,
SyncClient
} from "sync-client";
import { SyncClient } from "sync-client";
export class MockClient implements FileSystemOperations {
protected readonly localFiles = new Map<string, Uint8Array>();
@ -11,7 +12,8 @@ export class MockClient implements FileSystemOperations {
protected data: object | undefined = undefined;
public constructor(
private readonly initialSettings: Partial<SyncSettings>
private readonly initialSettings: Partial<SyncSettings>,
protected readonly useSlowFileEvents: boolean
) {}
public async init(): Promise<void> {
@ -22,9 +24,10 @@ export class MockClient implements FileSystemOperations {
await Promise.all(
Object.keys(this.initialSettings).map(async (key) => {
const settingKey = key as keyof SyncSettings; // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion
return this.client.settings.setSetting(
key as keyof SyncSettings, // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion
this.initialSettings[key as keyof SyncSettings] // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion
settingKey,
this.initialSettings[settingKey]! // eslint-disable-line @typescript-eslint/no-non-null-assertion
);
})
);
@ -46,13 +49,6 @@ export class MockClient implements FileSystemOperations {
return (await this.read(path)).length;
}
public async getModificationTime(path: RelativePath): Promise<Date> {
if (!this.localFiles.has(path)) {
throw new Error(`File ${path} does not exist`);
}
return new Date();
}
public async exists(path: RelativePath): Promise<boolean> {
return this.localFiles.has(path);
}
@ -68,7 +64,10 @@ export class MockClient implements FileSystemOperations {
`Creating file ${path} with content ${new TextDecoder().decode(newContent)}`
);
this.localFiles.set(path, newContent);
void this.client.syncer.syncLocallyCreatedFile(path, new Date());
this.runCallback(() => {
void this.client.syncer.syncLocallyCreatedFile(path);
});
}
public async createDirectory(_path: RelativePath): Promise<void> {
@ -88,28 +87,51 @@ export class MockClient implements FileSystemOperations {
const newContentUint8Array = new TextEncoder().encode(newContent);
this.localFiles.set(path, newContentUint8Array);
if (!this.useSlowFileEvents) {
const existingParts = currentContent
.split(" ")
.map((part) => part.trim());
const newParts = newContent.split(" ").map((part) => part.trim());
existingParts.forEach((part) =>
// all changes should be additive
{
assert(
newParts.includes(part),
`Part ${part} not found in new content`
);
}
);
}
this.client.logger.info(
`Updated file ${path} with:\n current content: ${currentContent}\n new content: ${newContent}`
);
this.runCallback(() => {
void this.client.syncer.syncLocallyUpdatedFile({
relativePath: path,
updateTime: new Date()
relativePath: path
});
});
return newContent;
}
public async write(path: RelativePath, content: Uint8Array): Promise<void> {
const hasExisted = this.localFiles.has(path);
this.localFiles.set(path, content);
this.client.logger.info(
`Updated file ${path} with:\n new content: ${new TextDecoder().decode(content)}`
);
this.runCallback(() => {
if (hasExisted) {
void this.client.syncer.syncLocallyUpdatedFile({
relativePath: path,
updateTime: new Date()
relativePath: path
});
} else {
void this.client.syncer.syncLocallyCreatedFile(path);
}
});
}
@ -118,7 +140,10 @@ export class MockClient implements FileSystemOperations {
`Deleting file: ${path} with:\n content ${new TextDecoder().decode(this.localFiles.get(path))}`
);
this.localFiles.delete(path);
this.runCallback(() => {
void this.client.syncer.syncLocallyDeletedFile(path);
});
}
public async rename(
@ -138,10 +163,20 @@ export class MockClient implements FileSystemOperations {
`Renamed file: ${oldPath} -> ${newPath} with:\n content ${new TextDecoder().decode(file)}`
);
this.runCallback(() => {
void this.client.syncer.syncLocallyUpdatedFile({
oldPath,
relativePath: newPath,
updateTime: new Date()
relativePath: newPath
});
});
}
private runCallback(callback: () => void): void {
if (this.useSlowFileEvents) {
// we aren't the best client and it takes some time to notice changes
setTimeout(callback, 100);
} else {
callback();
}
}
}

71
frontend/test-client/src/cli.ts
View file

@ -3,20 +3,26 @@ import { MockAgent } from "./agent/mock-agent";
import { sleep } from "./utils/sleep";
import { v4 as uuidv4 } from "uuid";
let slowFileEvents = false;
async function runTest({
agentCount,
concurrency,
iterations,
doDeletes,
useSlowFileEvents,
jitterScaleInSeconds
}: {
agentCount: number;
concurrency: number;
iterations: number;
doDeletes: boolean;
useSlowFileEvents: boolean;
jitterScaleInSeconds: number;
}): Promise<void> {
const settings = `with ${agentCount} agents, concurrency ${concurrency}, iterations ${iterations}, doDeletes ${doDeletes}, jitterScaleInSeconds ${jitterScaleInSeconds}`;
slowFileEvents = useSlowFileEvents;
const settings = `with ${agentCount} agents, concurrency ${concurrency}, iterations ${iterations}, doDeletes ${doDeletes}, jitterScaleInSeconds ${jitterScaleInSeconds}, useSlowFileEvents ${useSlowFileEvents}`;
console.info(`Running test ${settings}`);
const initialSettings: Partial<SyncSettings> = {
@ -34,6 +40,7 @@ async function runTest({
initialSettings,
`agent-${i}`,
doDeletes,
useSlowFileEvents,
jitterScaleInSeconds
)
);
@ -52,12 +59,24 @@ async function runTest({
// Each agent can have unpushed changes which might conflict with eachother so each has to resolve the conflicts & push, and
for (const client of clients) {
try {
await client.finish();
} catch (err) {
if (!slowFileEvents) {
throw err;
}
}
}
// then we need a second pass to ensure that all agents pull the same state.
for (const client of clients) {
try {
await client.finish();
} catch (err) {
if (!slowFileEvents) {
throw err;
}
}
}
console.info("Agents finished successfully");
@ -78,41 +97,49 @@ async function runTest({
console.info(`Content check for ${client.name} passed`);
});
console.info(`Test passed with ${settings}`);
console.info(`Test passed ${settings}`);
} catch (err) {
console.error(`Test failed with ${settings}`);
console.error(`Test failed ${settings}`);
throw err;
}
}
async function runTests(): Promise<void> {
const agentCounts = [2, 10];
const jitterScaleInSeconds = [0.5, 3, 0];
const concurrencies = [1, 16];
const iterations = [50, 300];
const doDeletes = [false, true];
for (const agentCount of agentCounts) {
for (const concurrency of concurrencies) {
for (const jitter of jitterScaleInSeconds) {
for (const iteration of iterations) {
for (const deleteFiles of doDeletes) {
while (true) {
for (const useSlowFileEvents of [false, true]) {
for (const concurrency of [
16,
1 // test with concurrency 1 to check for deadlocks
]) {
for (const doDeletes of [true, false]) {
await runTest({
agentCount,
agentCount: 3,
concurrency,
iterations: iteration,
doDeletes: deleteFiles,
jitterScaleInSeconds: jitter
iterations: 100,
doDeletes,
useSlowFileEvents,
jitterScaleInSeconds: 0.75
});
}
}
}
}
}
}
}
process.on("uncaughtException", (error) => {
if (slowFileEvents) {
return;
}
console.error("Uncaught Exception:", error);
process.exit(1);
});
process.on("unhandledRejection", (reason, _promise) => {
if (slowFileEvents) {
return;
}
console.error("Unhandled Rejection:", reason);
process.exit(1);
});
runTests()
.then(() => {
process.exit(0);

3
frontend/test-client/webpack.config.js
View file

@ -12,8 +12,7 @@ module.exports = {
rules: [
{
test: /\.ts$/,
use: "ts-loader",
exclude: /node_modules/
use: "ts-loader"
}
]
},

0
frontend/manifest.json → manifest.json
View file

10
bump-version.sh → scripts/bump-version.sh
View file

@ -27,20 +27,20 @@ cd backend
cargo set-version --bump patch
echo "Bumping frontend versions"
cd ../plugin
npm version patch
cd ../frontend
npm version patch --workspaces
echo "Updating frontend dependencies to match the new backend versions"
cd ../backend/sync_lib
wasm-pack build --target web --features console_error_panic_hook
cd ../../plugin
cd ../../frontend
npm install
cd ..
cp plugin/manifest.json manifest.json # for BRAT, otherwise it wouldn't update
cp frontend/obsidian-plugin/manifest.json manifest.json # for BRAT, otherwise it wouldn't update
# Commit and tag
Commit and tag
git add .
TAG=$(node -p "require('./plugin/package.json').version")
git commit -m "Bump versions to $TAG"

4
scripts/clean-up.sh Executable file
View file

@ -0,0 +1,4 @@
#!/bin/bash
rm -rf backend/databases
rm -rf logs

79
scripts/e2e.sh Executable file
View file

@ -0,0 +1,79 @@
#!/bin/bash
set -e
set -o pipefail
# Check if the argument is provided
if [ $# -eq 0 ]; then
echo "Usage: $0 <number_of_processes>"
exit 1
fi
# Get the number of processes from the first argument
process_count=$1
mkdir -p logs
cd frontend
npm run build
pids=()
for i in $(seq 1 $process_count); do
node test-client/dist/cli.js > "../logs/log_${i}.log" 2>&1 &
pids+=($!)
done
cd -
print_failed_log() {
for i in $(seq 1 $process_count); do
if [ -n "${pids[$i-1]}" ] && ! kill -0 ${pids[$i-1]} 2>/dev/null; then
# Get the exit code of the process
wait ${pids[$i-1]}
exit_code=$?
# Only consider non-zero exit codes as failures
if [ $exit_code -ne 0 ]; then
echo "Process ${pids[$i-1]} failed with exit code $exit_code. Log file: $(pwd)/logs/log_${i}.log"
return 0
else
echo "Process ${pids[$i-1]} completed successfully with exit code 0"
# Mark this PID as processed by setting it to empty
pids[$i-1]=""
fi
fi
done
return 1
}
echo "Monitoring $process_count processes"
# Monitor processes
while true; do
if print_failed_log; then
# Kill remaining processes
for pid in "${pids[@]}"; do
if [ -n "$pid" ]; then
kill $pid 2>/dev/null || true
fi
done
exit 1
fi
# Check if all processes have completed
all_done=true
for pid in "${pids[@]}"; do
if [ -n "$pid" ] && kill -0 $pid 2>/dev/null; then
all_done=false
break
fi
done
if $all_done; then
echo "All processes completed successfully"
exit 0
fi
sleep 0.2
done

4
scripts/update-api-types.sh Executable file
View file

@ -0,0 +1,4 @@
#!/bin/bash
npm install -g openapi-typescript
openapi-typescript http://localhost:3030/api.json --output frontend/sync-client/src/services/types.ts