Fix syncing when network latency is present (#4)

* WIP

* Add debug

* Dedupe inserts

* Add deterministic ordering

* Fix whitespaces

* Update insta

* Add integration test script

* Rename

* Add test

* Working for non-deletes

* omg it mostly works for deletes

* Isdeleted fix

* remove created dates

* update api

* Take document id

* No max attempt

* works

* Use string uuids

* .

* working!!!! (hopefully)

* Improve bundling

* Add module

* lint

* .

* lint

* Fix CI

* use toolchain

* clean up

* Add useSlowFileEvents

* Delete fuzz

* Fix CI

* use docker

* fix script

* clean up

* Clean up

* change node version

* Build docker image on every commit

* fix ci

* 1 db per vault

* Add scritps folder

* Bump versions

* Lint

* .

* Fix tests for real

* Style

* .

* try

* Consistent ordering

* Fix tests

* hmm

* .

* Clean up diff

* Fixes

* .

* Fix version bump

* .

* .

* .

.github/workflows/check.yml

@@ -17,11 +17,14 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - name: Setup
+      - name: Setup Node.js environment
+        uses: actions/setup-node@v4.2.0
+        with:
+          node-version: "22.x"
+          check-latest: true
+
+      - name: Setup rust
         run: |
-          rustup install nightly
-          rustup default nightly
-          rustup component add clippy rustfmt
           cargo install sqlx-cli
           cd backend
           sqlx database create --database-url sqlite://db.sqlite3
@@ -44,7 +47,7 @@ jobs:
           cd backend
           cargo test --verbose
           cd sync_lib
-          # wasm-pack test --node # todo: fix this in CI
+          wasm-pack test --node
 
       - name: Lint frontend
         run: |

.github/workflows/e2e.yml

@@ -0,0 +1,45 @@
+name: E2E tests
+
+on:
+  push:
+    branches: ["master"]
+  pull_request:
+    branches: ["master"]
+
+env:
+  CARGO_TERM_COLOR: always
+  RUSTFLAGS: "-Dwarnings"
+
+jobs:
+  build:
+    runs-on: self-hosted
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup Node.js environment
+        uses: actions/setup-node@v4.2.0
+        with:
+          node-version: "22.x"
+          check-latest: true
+
+      - name: Setup rust
+        run: |
+          cargo install sqlx-cli wasm-pack
+          cd backend
+          sqlx database create --database-url sqlite://db.sqlite3
+          sqlx migrate run --source sync_server/src/database/migrations --database-url sqlite://db.sqlite3
+
+      - name: Build wasm
+        run: |
+          cd backend
+          wasm-pack build --target web sync_lib
+
+      - name: E2E tests
+        run: |
+          cd backend
+          RUST_BACKTRACE=1 cargo run -p sync_server &
+          cd ../frontend
+          npm ci
+          cd ..
+          scripts/e2e.sh 32

.github/workflows/publish-docker.yml

@@ -7,8 +7,9 @@ name: Publish server Docker image
 
 on:
   push:
-    tags:
-      - "*"
+    branches: ["master"]
+  pull_request:
+    branches: ["master"]
 
 env:
   # Use docker.io for Docker Hub if empty
@@ -17,8 +18,9 @@ env:
   IMAGE_NAME: ${{ github.repository }}
 
 jobs:
-  build-docker:
-    runs-on: ubuntu-latest
+  publish-docker:
+    runs-on: self-hosted
+
     permissions:
       contents: read
       packages: write
@@ -33,7 +35,7 @@ jobs:
       # Install the cosign tool except on PR
       # https://github.com/sigstore/cosign-installer
       - name: Install cosign
-        if: github.event_name != 'pull_request'
+        if: ${{ github.ref_type == 'tag' }}
         uses: sigstore/cosign-installer@59acb6260d9c0ba8f4a2f9d9b48431a222b68e20 #v3.5.0
         with:
           cosign-release: "v2.2.4"
@@ -47,7 +49,7 @@ jobs:
       # Login against a Docker registry except on PR
       # https://github.com/docker/login-action
       - name: Log into registry ${{ env.REGISTRY }}
-        if: github.event_name != 'pull_request'
+        if: ${{ github.ref_type == 'tag' }}
         uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0
         with:
           registry: ${{ env.REGISTRY }}
@@ -69,7 +71,7 @@ jobs:
         uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v5.0.0
         with:
           context: backend
-          push: ${{ github.event_name != 'pull_request' }}
+          push: ${{ github.ref_type == 'tag' }}
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
           cache-from: type=gha
@@ -81,7 +83,7 @@ jobs:
       # transparency data even for private images, pass --force to cosign below.
       # https://github.com/sigstore/cosign
       - name: Sign the published Docker image
-        if: ${{ github.event_name != 'pull_request' }}
+        if: ${{ github.ref_type == 'tag' }}
         env:
           # https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions#using-an-intermediate-environment-variable
           TAGS: ${{ steps.meta.outputs.tags }}

.github/workflows/publish-plugin.yml

@@ -2,29 +2,27 @@ name: Publish Obsidian plugin
 
 on:
   push:
-    tags:
-      - "*"
+    tags: ["*"]
 
 env:
   CARGO_TERM_COLOR: always
 
 jobs:
-  build-plugin:
+  publish-plugin:
     runs-on: self-hosted
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
-      - name: Use Node.js
-        uses: actions/setup-node@v3
+      - name: Setup Node.js environment
+        uses: actions/setup-node@v4.2.0
         with:
-          node-version: "18.x"
+          node-version: "22.x"
+          check-latest: true
 
       - name: Build wasm
         run: |
           cd backend
-          rustup install nightly
-          rustup default nightly
           cargo install wasm-pack
           wasm-pack build --target web sync_lib