Compare commits

...
Sign in to create a new pull request.

8 commits

Author SHA1 Message Date
45b86cffe4 split: sync-client services layer
Add build-vault-url helper. Rewrite fetch-controller and websocket-manager
(plus their tests). Update server-config and sync-service to consume the
new error types and the regenerated API types from previous chunks.
2026-05-08 21:36:41 +01:00
9d99a4ac23 split: sync-client utils and errors reorganization
Move error classes from services/ and file-operations/ into a new errors/
directory (authentication-error, server-version-mismatch-error,
sync-reset-error, file-not-found-error), plus add file-already-exists-error
and http-client-error. Update consts.ts and utils/* (await-all,
create-client-id, hash, rate-limit, find-matching-file). Replace
data-structures (locks, min-covered, event-listeners, fix-sized-cache) and
add debugging utilities (in-memory-file-system, log-to-console,
slow-web-socket-factory). Removes utils/create-promise.ts.
2026-05-08 21:36:29 +01:00
f7beb31d8f split: regenerated TS API type bindings
Auto-generated TS types regenerated from Rust ts-rs derives, mirrored into
frontend/sync-client/src/services/types/ and frontend/history-ui/src/lib/types/.
Adds ListVaultsResponse, VaultHistoryResponse, VaultInfo and updates several
existing types; removes DeleteDocumentVersion and UpdateDocumentVersion.
2026-05-08 21:36:13 +01:00
042233c4d7 split: server websocket + cursors
src/server/websocket.rs handshake/catch-up rewrite, app_state/cursors.rs,
app_state/websocket/{broadcasts,models,utils}.rs.
2026-05-08 21:35:52 +01:00
4ba439b874 split: server REST endpoints + rate limiting
server.rs router rewrite, auth.rs, device_id_header.rs, requests.rs,
responses.rs, plus per-endpoint changes: create/update/delete_document,
fetch_document_version{,_content,s}, fetch_latest_documents, index.rs.
Adds: fetch_vault_history, list_vaults, rate_limit (new files).
2026-05-08 21:35:41 +01:00
2d5edc6ec5 split: server database (app_state, migrations, models)
src/app_state.rs, src/app_state/database.rs (large schema/query rewrite),
two new migrations (add_idempotency_key, add_creation_vault_update_id),
and src/app_state/database/models.rs.
2026-05-08 21:35:30 +01:00
a9ce09b59d split: server foundation (Cargo, config, errors, utils, main)
Cargo.{toml,lock} bumps, build.rs, config-e2e.yml, rust-toolchain.toml,
src/config/* (database/logging/server/user configs), src/consts.rs,
src/errors.rs, src/main.rs, and src/utils/* (dedup_paths,
find_first_available_path, rotating_file_writer, sanitize_path).
2026-05-08 21:35:18 +01:00
70f97c4b16 split: CI workflows, scripts, root tooling, and docs
Some checks failed
Check / build (pull_request) Has been cancelled
E2E tests / build (pull_request) Has been cancelled
Publish CLI / publish-docker (pull_request) Has been cancelled
Publish server Docker image / publish-docker (pull_request) Has been cancelled
Forgejo workflows (new), GitHub workflow tweaks, .gitignore/.vscode, root
package-lock, rustfmt.toml, scripts/* updates, docs/ updates including
data-flow / authentication / server-setup, CLAUDE.md and README updates.
2026-05-08 21:35:07 +01:00
132 changed files with 7251 additions and 4281 deletions

View file

@ -0,0 +1,35 @@
name: Check
on:
push:
branches: ["main"]
pull_request:
branches: ["main"]
workflow_dispatch:
env:
CARGO_TERM_COLOR: always
RUSTFLAGS: "-Dwarnings"
jobs:
build:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Setup Node.js environment
uses: actions/setup-node@v4
with:
node-version: "25.x"
- name: Setup Rust toolchain
uses: dtolnay/rust-toolchain@stable
with:
toolchain: "1.92.0"
components: clippy, rustfmt
- name: Lint & test
run: scripts/check.sh

View file

@ -0,0 +1,38 @@
name: Deploy Documentation
on:
push:
branches:
- main
paths:
- "docs/**"
- ".forgejo/workflows/deploy-docs.yml"
workflow_dispatch:
concurrency:
group: pages
cancel-in-progress: false
jobs:
build:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Setup Node.js environment
uses: actions/setup-node@v4
with:
node-version: "25.x"
- name: Build docs
run: scripts/build-docs.sh
- name: Upload artifact
uses: actions/upload-artifact@v4
with:
name: docs
path: docs/.vitepress/dist

View file

@ -0,0 +1,71 @@
name: E2E tests
on:
push:
branches: ["main"]
pull_request:
branches: ["main"]
schedule:
- cron: "0 * * * *"
workflow_dispatch:
concurrency:
group: e2e-tests
cancel-in-progress: false
env:
RUSTFLAGS: "-Dwarnings"
jobs:
build:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Setup Node.js environment
uses: actions/setup-node@v4
with:
node-version: "25.x"
- name: Setup Rust toolchain
uses: dtolnay/rust-toolchain@stable
with:
toolchain: "1.92.0"
components: clippy, rustfmt
- name: Setup rust
run: |
which sqlx || cargo install sqlx-cli
cd sync-server
sqlx database create --database-url sqlite://db.sqlite3
sqlx migrate run --source src/app_state/database/migrations --database-url sqlite://db.sqlite3
- name: E2E tests
run: |
cd sync-server
cargo run config-e2e.yml --color never &
SERVER_PID=$!
cd ..
scripts/e2e.sh 8
EXIT_CODE=$?
kill $SERVER_PID 2>/dev/null || true
wait $SERVER_PID 2>/dev/null || true
exit $EXIT_CODE
- name: Upload e2e logs
if: always()
uses: actions/upload-artifact@v4
with:
name: e2e-logs
path: logs/
retention-days: 30
- name: Cleanup
if: always()
run: scripts/clean-up.sh

View file

@ -0,0 +1,51 @@
name: Publish CLI
on:
push:
branches: ["main"]
tags: ["*"]
pull_request:
branches: ["main"]
jobs:
publish-docker:
runs-on: ubuntu-docker
steps:
- name: Checkout repository
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Extract registry hostname
id: registry
run: echo "host=$(echo '${{ github.server_url }}' | sed 's|https\?://||')" >> "$GITHUB_OUTPUT"
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Log into container registry
uses: docker/login-action@v3
with:
registry: ${{ steps.registry.outputs.host }}
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Extract Docker metadata
id: meta
uses: docker/metadata-action@v5
with:
images: ${{ steps.registry.outputs.host }}/${{ github.repository }}-cli
- name: Build and push Docker image
id: build-and-push
uses: docker/build-push-action@v5
with:
context: frontend
file: frontend/local-client-cli/Dockerfile
platforms: linux/amd64
push: true
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
cache-from: type=registry,ref=${{ steps.registry.outputs.host }}/${{ github.repository }}-cli:buildcache
cache-to: type=registry,ref=${{ steps.registry.outputs.host }}/${{ github.repository }}-cli:buildcache,mode=max

View file

@ -0,0 +1,71 @@
name: Publish Obsidian plugin
on:
push:
tags: ["*"]
env:
CARGO_TERM_COLOR: always
jobs:
publish-plugin:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Setup Node.js environment
uses: actions/setup-node@v4
with:
node-version: "25.x"
- name: Build plugin
run: |
cd frontend
npm ci
npm run build
- name: Setup Rust toolchain
uses: dtolnay/rust-toolchain@stable
with:
toolchain: "1.92.0"
components: clippy, rustfmt
- name: Install cross-compilation tools
run: |
apt update
apt install -y gcc-aarch64-linux-gnu musl-tools gcc-mingw-w64-x86-64 jq
- name: Build Linux and Windows binaries
run: ./scripts/build-sync-server-binaries.sh
- name: Create release
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
SERVER_URL: ${{ github.server_url }}
REPO: ${{ github.repository }}
run: |
tag="${GITHUB_REF#refs/tags/}"
mkdir -p release
cp frontend/obsidian-plugin/dist/* release/
cp sync-server/artifacts/sync-server-* release/
# Create draft release via Forgejo API
RELEASE_ID=$(curl -s -X POST \
"${SERVER_URL}/api/v1/repos/${REPO}/releases" \
-H "Authorization: token ${GITHUB_TOKEN}" \
-H "Content-Type: application/json" \
-d "{\"tag_name\": \"${tag}\", \"name\": \"${tag}\", \"draft\": true}" \
| jq -r '.id')
# Upload release assets
for file in release/*; do
filename=$(basename "$file")
curl -s -X POST \
"${SERVER_URL}/api/v1/repos/${REPO}/releases/${RELEASE_ID}/assets?name=${filename}" \
-H "Authorization: token ${GITHUB_TOKEN}" \
-F "attachment=@${file}"
done

View file

@ -0,0 +1,51 @@
name: Publish server Docker image
on:
push:
branches: ["main"]
tags: ["*"]
pull_request:
branches: ["main"]
jobs:
publish-docker:
runs-on: ubuntu-docker
steps:
- name: Checkout repository
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Extract registry hostname
id: registry
run: echo "host=$(echo '${{ github.server_url }}' | sed 's|https\?://||')" >> "$GITHUB_OUTPUT"
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Log into container registry
if: github.ref_type == 'tag'
uses: docker/login-action@v3
with:
registry: ${{ steps.registry.outputs.host }}
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Extract Docker metadata
id: meta
uses: docker/metadata-action@v5
with:
images: ${{ steps.registry.outputs.host }}/${{ github.repository }}
- name: Build and push Docker image
id: build-and-push
uses: docker/build-push-action@v5
with:
context: sync-server
platforms: linux/amd64,linux/arm64
push: ${{ github.ref_type == 'tag' }}
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
cache-from: type=registry,ref=${{ steps.registry.outputs.host }}/${{ github.repository }}:buildcache
cache-to: type=registry,ref=${{ steps.registry.outputs.host }}/${{ github.repository }}:buildcache,mode=max

View file

@ -23,13 +23,13 @@ jobs:
- name: Setup Node.js environment - name: Setup Node.js environment
uses: actions/setup-node@v4.2.0 uses: actions/setup-node@v4.2.0
with: with:
node-version: "22.x" node-version: "25.x"
check-latest: true check-latest: true
- name: Setup Rust toolchain - name: Setup Rust toolchain
uses: dtolnay/rust-toolchain@stable uses: dtolnay/rust-toolchain@stable
with: with:
toolchain: "1.89.0" toolchain: "1.92.0"
components: clippy, rustfmt components: clippy, rustfmt
- name: Lint & test - name: Lint & test

View file

@ -5,8 +5,8 @@ on:
branches: branches:
- main - main
paths: paths:
- 'docs/**' - "docs/**"
- '.github/workflows/deploy-docs.yml' - ".github/workflows/deploy-docs.yml"
workflow_dispatch: workflow_dispatch:
permissions: permissions:
@ -28,12 +28,11 @@ jobs:
with: with:
fetch-depth: 0 fetch-depth: 0
- name: Setup Node - name: Setup Node.js environment
uses: actions/setup-node@v4 uses: actions/setup-node@v4.2.0
with: with:
node-version: 22 node-version: "25.x"
cache: npm check-latest: true
cache-dependency-path: docs/package-lock.json
- name: Setup Pages - name: Setup Pages
uses: actions/configure-pages@v4 uses: actions/configure-pages@v4

View file

@ -6,7 +6,7 @@ on:
pull_request: pull_request:
branches: ["main"] branches: ["main"]
schedule: schedule:
- cron: '0 * * * *' - cron: "0 * * * *"
workflow_dispatch: workflow_dispatch:
concurrency: concurrency:
@ -28,13 +28,13 @@ jobs:
- name: Setup Node.js environment - name: Setup Node.js environment
uses: actions/setup-node@v4.2.0 uses: actions/setup-node@v4.2.0
with: with:
node-version: "22.x" node-version: "25.x"
check-latest: true check-latest: true
- name: Setup Rust toolchain - name: Setup Rust toolchain
uses: dtolnay/rust-toolchain@stable uses: dtolnay/rust-toolchain@stable
with: with:
toolchain: "1.89.0" toolchain: "1.92.0"
components: clippy, rustfmt components: clippy, rustfmt
- name: Setup rust - name: Setup rust

View file

@ -19,7 +19,7 @@ jobs:
- name: Setup Node.js environment - name: Setup Node.js environment
uses: actions/setup-node@v4.2.0 uses: actions/setup-node@v4.2.0
with: with:
node-version: "22.x" node-version: "25.x"
check-latest: true check-latest: true
- name: Build plugin - name: Build plugin
@ -31,7 +31,7 @@ jobs:
- name: Setup Rust toolchain - name: Setup Rust toolchain
uses: dtolnay/rust-toolchain@stable uses: dtolnay/rust-toolchain@stable
with: with:
toolchain: "1.89.0" toolchain: "1.92.0"
components: clippy, rustfmt components: clippy, rustfmt
- name: Install cross-compilation tools - name: Install cross-compilation tools

9
.gitignore vendored
View file

@ -7,15 +7,18 @@ node_modules
# Frontend build folders # Frontend build folders
frontend/*/dist frontend/*/dist
sync-server/db.sqlite3*
sync-server/databases
# Rust build folders # Rust build folders
sync-server/target sync-server/target
sync-server/artifacts sync-server/artifacts
sync-server/bindings/*.ts sync-server/bindings/*.ts
# build folders
sync-server/db.sqlite3*
**/databases
*.log *.log
*.sqlx *.sqlx
target target
.task

View file

@ -5,6 +5,6 @@
"**/dist": true, "**/dist": true,
"**/node_modules": true, "**/node_modules": true,
"**/.sqlx": true, "**/.sqlx": true,
"**/target": true, "**/target": true
}, }
} }

195
CLAUDE.md
View file

@ -2,109 +2,154 @@
This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
## Project Overview ## Project shape
VaultLink is a self-hosted Obsidian plugin for real-time collaborative file syncing. The project consists of a Rust-based sync server and a TypeScript frontend with three main components: an Obsidian plugin, a sync client library, and a test client. VaultLink is a self-hosted Obsidian file-sync system. Two halves of one repo:
## Architecture - `sync-server/` — Rust (axum + sqlx/SQLite). Source of truth for vault state, broadcasts changes via WebSocket.
- `frontend/` — npm workspaces. The sync engine (`sync-client`) is consumed by an Obsidian plugin, a standalone CLI, a fuzz E2E harness, a scripted determinism harness, and a history UI.
### Core Components The HTTP/WS API types are generated from Rust (`ts-rs`) and mirrored into the TS workspaces. **Never hand-edit files in `frontend/sync-client/src/services/types/` or `frontend/history-ui/src/lib/types/`** — run `scripts/update-api-types.sh` after changing anything Serde-derived in the server.
- **sync-server/**: Rust-based WebSocket server with SQLite database for document versioning and real-time synchronization ### Frontend workspaces
- **frontend/sync-client/**: TypeScript library providing core sync functionality, WebSocket management, and file operations
- **frontend/obsidian-plugin/**: Obsidian plugin that integrates the sync client with Obsidian's API
- **frontend/test-client/**: CLI testing tool for the sync functionality
### Key Technologies - `sync-client` — the sync engine; published to consumers via `dist/`. All other TS workspaces depend on it via `file:../sync-client`.
- `obsidian-plugin` — Obsidian plugin built from `sync-client`.
- `local-client-cli` — same engine wrapped as a standalone CLI.
- `history-ui` — vault-history web UI.
- `test-client` — fuzz E2E harness (random ops across N processes).
- `deterministic-tests` — scripted multi-client tests with an in-memory FS, run against a real server.
- **Backend**: Rust with Axum framework, SQLite with SQLx, WebSockets for real-time sync ## Common commands
- **Frontend**: TypeScript, Webpack for bundling, Jest for testing
- **Sync Algorithm**: Uses reconcile-text library for operational transformation
## Development Commands Pre-push hygiene (formats, lints, runs tests, requires clean git state):
### Server Development ```sh
```bash scripts/check.sh --fix
cd sync-server
cargo run config-e2e.yml # Start development server
cargo test --verbose # Run Rust tests
cargo clippy --all-targets --all-features # Lint Rust code
cargo clippy --all-targets --all-features --fix --allow-dirty --allow-staged # Auto-fix clippy warnings
cargo fmt --all -- --check # Check Rust formatting
cargo fmt --all # Auto-format Rust code
cargo machete --with-metadata # Detect unused dependencies
``` ```
### Frontend Development Run the fuzz E2E (N parallel processes):
```bash
```sh
scripts/e2e.sh 12
# Logs land in logs/log_<i>.log. Clean with scripts/clean-up.sh
```
Run deterministic tests (require a release-built server in `sync-server/target/release/sync_server` — they spawn it themselves):
```sh
cd sync-server && cargo build --release && cd ..
cd frontend cd frontend
npm run dev # Start development mode (watches sync-client and obsidian-plugin) npm run build -w sync-client -w deterministic-tests
npm run build # Build all workspaces node deterministic-tests/dist/cli.js # all
npm run test # Run all tests node deterministic-tests/dist/cli.js --filter=rename # subset
npm run lint # Lint and format TypeScript code node deterministic-tests/dist/cli.js --filter=… -j 4 # cap parallelism
``` ```
### Database Setup (Development) Run a single sync-client unit test by file:
```bash
```sh
cd frontend/sync-client && npx tsx --test 'src/**/sync-event-queue.test.ts'
```
Server: dev runs from `sync-server/` against `config-e2e.yml`:
```sh
cd sync-server
cargo run config-e2e.yml # dev
cargo build --release # used by both e2e harnesses
cargo test # unit + ts-rs binding export tests
```
Frontend dev (sync-client + obsidian-plugin watch in parallel):
```sh
cd frontend && npm install && npm run dev
```
Regenerate TS bindings from Rust types (touches `frontend/{sync-client,history-ui}/src/.../types/`):
```sh
scripts/update-api-types.sh
```
## SQLite / sqlx
The server uses `sqlx::query!` macros that need a prepared `.sqlx` cache to compile offline. Touching any SQL means regenerating it:
```sh
cd sync-server cd sync-server
sqlx database create --database-url sqlite://db.sqlite3 sqlx database create --database-url sqlite://db.sqlite3
sqlx migrate run --source src/app_state/database/migrations --database-url sqlite://db.sqlite3 sqlx migrate run --source src/app_state/database/migrations --database-url sqlite://db.sqlite3
cargo sqlx prepare --workspace cargo sqlx prepare --workspace
``` ```
### Initial Setup New migrations: `sqlx migrate add --source src/app_state/database/migrations <name>`.
```bash
# Install required cargo tools ## Sync engine architecture
cargo install sqlx-cli cargo-machete cargo-edit
Read `frontend/sync-client/src/sync-operations/` to follow the sync engine; the rest of `sync-client` is plumbing (filesystem ops, persistence, services, telemetry).
The engine is **two independent loops with separate invariants**:
- **Wire loop** (`syncer.ts`) — drains the single-consumer FIFO queue. HTTP and WS handlers update record fields (`remoteRelativePath`, `parentVersionId`, `remoteHash`) and write content to the file at `record.localPath`. They never move files for path placement.
- **Path reconciler** (`reconciler.ts`) — runs after every drained event. Best-effort pass that moves files to make `localPath === remoteRelativePath`. The move graph is topologically sorted; cycles are resolved by reading every file in the cycle into memory and writing each back to its new slot (no tmp files). Records with pending local events are skipped on each pass — the reconciler operates only on settled records. Failures (slot occupied by an untracked file, etc.) are silent skips; the next pass retries.
**`SyncEventQueue`** (`sync-event-queue.ts`) holds:
- `byDocId: Map<DocumentId, DocumentRecord>` — primary record store.
- `byLocalPath: Map<RelativePath, DocumentRecord>` — derived index for path lookups, maintained at every mutation point.
- `events: SyncEvent[]` — pending wire ops in FIFO drain order.
```ts
DocumentRecord = {
documentId,
parentVersionId,
remoteHash?,
remoteRelativePath,
localPath: RelativePath | undefined
}
``` ```
### Scripts `localPath === undefined` means the doc has no local file yet — typically a remote create whose target slot was occupied at receive time; the reconciler will fetch and place when the slot frees (the bytes wait in `pendingPlacementContent`).
- `scripts/check.sh`: Full CI check (builds, lints, tests both server and frontend)
- `scripts/check.sh --fix`: Same as above but auto-fixes linting and formatting issues
- `scripts/e2e.sh`: End-to-end testing
- `scripts/clean-up.sh`: Clean logs and database files
- `scripts/bump-version.sh patch`: Publish new version
- `scripts/update-api-types.sh`: Update TypeScript bindings from Rust types
## Code Structure Local FS events from the watcher update `localPath` synchronously at enqueue time via `setLocalPath` / `upsertRecord`. The wire loop never updates it for path placement; only the reconciler does. A user rename onto a tracked slot enqueues a `LocalDelete` for the displaced doc (the OS rename clobbered its content) and clears that doc's `localPath`.
### Workspace Configuration **Pending creates** use a `Promise<DocumentId>` chain to serialize dependent ops (`LocalUpdate`, `LocalDelete`) behind the still-in-flight `LocalCreate`. `resolveCreate` resolves the promise once the server returns a docId, and `replacePendingDocumentId` swaps the resolved id across already-queued events. `findLatestCreateForPath` is the lookup the watcher uses to attach dependents; `updatePendingCreatePath` rewrites a pending create's `event.path` in place when the user renames the file before its create has acked.
The frontend uses npm workspaces with four packages:
- `sync-client`: Core synchronization logic
- `obsidian-plugin`: Obsidian-specific integration
- `test-client`: Testing utilities
- `local-client-cli`: Standalone CLI for VaultLink sync client
### Type Generation **Watermark.** `lastSeenUpdateId` uses a `MinCovered` (a contiguous-prefix tracker over a stream of integers): we only advance the published min when the next consecutive id has been processed, so out-of-order RemoteChange ids don't fool the WebSocket handshake into requesting a too-recent catch-up.
Rust structs generate TypeScript types via ts-rs crate, stored in `sync-server/bindings/` and used by frontend packages.
### Key Files **Server catch-up.** The server's WS handshake replays events newer than the client's `last_seen_vault_update_id` from the `latest_document_versions` view (one row per doc, the latest). On those replayed rows `is_new_file` means _new to this client_ (`creation_vault_update_id > last_seen_vault_update_id`), not "this row is the doc's first version" — necessary because the catch-up only carries the latest version; if a doc was created and updated past the watermark, the client never sees its create otherwise.
- `sync-server/src/`: Rust server implementation with WebSocket handlers
- `frontend/sync-client/src/sync-client.ts`: Main sync client entry point
- `frontend/obsidian-plugin/src/vault-link-plugin.ts`: Main Obsidian plugin class
- `frontend/sync-client/src/services/sync-service.ts`: Core synchronization logic
## Testing ## Edge-case patterns the sync engine has to survive
### Running Tests The two-loop split defuses most of the old race catalogue (slot-collision stashes, conflict-uuid divergence, `MoveOnConflict.NEW`/`EXISTING` policy choices) by separating wire transport from path placement. What's left:
- Server: `cargo test --verbose`
- Frontend: `npm run test` (runs Jest across all workspaces)
- E2E: `scripts/e2e.sh`
### Test Structure **Pending-create docId is a `Promise`, not a string, until the create acks.** Any `LocalUpdate` / `LocalDelete` queued behind a still-in-flight `LocalCreate` carries the create's `resolvers.promise` as its `documentId`. `replacePendingDocumentId` swaps the resolved id across queued events when the create resolves; `===` comparisons against the resolved string elsewhere will silently fail until that swap runs. Anything that walks `events[]` looking for a docId match must either run after the swap or be tolerant of `Promise`-typed ids.
- Rust: Unit tests alongside source files
- TypeScript: `.test.ts` files using Jest
- E2E: Uses test-client to simulate multiple concurrent users
## Code Style **`processCreate` reads `event.path` live, not `event.originalPath`.** The watcher rewrites `event.path` in place via `updatePendingCreatePath` when the user renames a pending-create file. `originalPath` was removed from `LocalCreate` events specifically because reading it would send the stale pre-rename path to the server.
### Rust **`record.localPath` mutates in place across awaits.** When the watcher renames a doc while a drain handler is awaiting an HTTP roundtrip, the queue mutates the in-flight event's record so subsequent reads see the new path. Snapshotting `record.localPath` into a local at function entry and using it after an `await` reads/writes a now-vacated slot. Read `record.localPath` live; only snapshot for the deliberate "did it change while I was awaiting" comparison.
- Uses extensive Clippy lints (see Cargo.toml)
- Follows pedantic linting rules
- Forbids unsafe code
- Uses cargo fmt with default settings
### TypeScript **Reconciler-defer is the wire-loop's contract with the reconciler.** The reconciler skips records where `hasPendingLocalEventsForDocumentId` returns true. Wire-loop handlers can therefore freely write `remoteRelativePath` to whatever the server returned — even if it disagrees with `localPath` — knowing the reconciler won't move the file out from under a queued user rename.
- Prettier configuration: 4-space tabs, trailing commas removed, LF line endings
- ESLint with unused imports plugin **Watermark advancement is load-bearing both ways.** Branches that _skip_ a remote event without advancing `lastSeenUpdateId` create permanent gaps that re-deliver forever. Branches that _advance_ without applying the content lose data: the server has no further event to re-deliver, the catch-up only carries the latest version, and any state in between is gone. Don't advance unless the event was actually applied (or deliberately discarded after weighing both halves).
- Consistent across all three frontend packages
**`isNewFile` semantics differ between catch-up and real-time.** On WS handshake replay it means _new to this client_ (`creation_vault_update_id > last_seen_vault_update_id`); on real-time broadcasts it means _this version is the create_ (`creation_vault_update_id == vault_update_id`). A handler that decides based on one interpretation will be wrong on the other channel; reasoning about fetch-and-treat-as-new vs. ignore needs to know which channel delivered the event.
**Pause / disable-sync mid-flight** is the one race the new model doesn't structurally fix. An HTTP that committed server-side but whose response was discarded leaves the server holding a doc the client has no record of. Resume → offline scan → server-side dedupe handles it (the server merges the duplicate create into the existing doc), but if the merge produces a deconflict, the client picks up an extra file. Out of scope for the two-loop split.
**Cycle reconciliation uses in-memory content swap.** When the move graph contains a cycle, the reconciler reads every file in the cycle into memory and writes each back to its new slot, with no tmp files. A write-ahead marker at `.vaultlink/swap-<uuid>.json` lists each leg; on startup the reconciler reads the marker, hashes each `from` to determine which legs ran, and replays the rest. The `.vaultlink/**` glob is hard-coded as an internal ignore pattern so swap markers don't get sync'd.
## Two complementary E2E harnesses
- **`test-client` (fuzz):** random ops across N parallel processes for many minutes. Used by `scripts/e2e.sh`. Catches bugs nobody thought to write a test for, but reproductions are noisy.
- **`deterministic-tests`:** scripted scenarios with an in-memory FS pinned to a real server. Used to _capture_ a fuzz-discovered bug as a minimal repro before fixing it. See `frontend/deterministic-tests/README.md` for the step grammar (`pause-server`, `pause-websocket`, `barrier`, `assert-consistent`, etc.).
When a fuzz failure surfaces, the workflow is: root-cause from logs → write a deterministic test that fails on the bug → fix → confirm both the deterministic test and `e2e.sh` pass.
## Style
- TS: 4-space indent, no tabs, LF, prettier (`trailingComma: "none"`). YAML/MD use 2-space indent.
- Rust: `rustfmt.toml` enforces 4-space spaces, LF.
- Lint: ESLint for TS, Clippy for Rust, `cargo machete` for unused deps. All wired into `scripts/check.sh`.

View file

@ -8,12 +8,12 @@
## Develop ## Develop
### Install [nvm](https://github.com/nvm-sh/nvm) ### Set up Node.JS 25 with [nvm](https://github.com/nvm-sh/nvm)
- `curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh | bash` - `curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh | bash`
- `nvm install 22` - `nvm install 25`
- `nvm use 22` - `nvm use 25`
- Optionally set the system-wide default: `nvm alias default 22` - Optionally, set the system-wide default: `nvm alias default 25`
### Set up Rust ### Set up Rust

View file

@ -2,12 +2,7 @@
"version": "0.2", "version": "0.2",
"language": "en-GB", "language": "en-GB",
"dictionaries": ["en-gb"], "dictionaries": ["en-gb"],
"ignorePaths": [ "ignorePaths": ["node_modules", ".vitepress/dist", ".vitepress/cache", "package-lock.json"],
"node_modules",
".vitepress/dist",
".vitepress/cache",
"package-lock.json"
],
"words": [ "words": [
"VaultLink", "VaultLink",
"Obsidian", "Obsidian",

View file

@ -361,11 +361,11 @@ VALUES (?, ?, ?);
```json ```json
{ {
"type": "upload_file", "type": "upload_file",
"path": "notes/example.md", "path": "notes/example.md",
"content": "File content here...", "content": "File content here...",
"base_version": 10, "base_version": 10,
"timestamp": "2024-01-01T12:00:00Z" "timestamp": "2024-01-01T12:00:00Z"
} }
``` ```
@ -373,8 +373,8 @@ VALUES (?, ?, ?);
```json ```json
{ {
"type": "download_file", "type": "download_file",
"path": "notes/example.md" "path": "notes/example.md"
} }
``` ```
@ -382,8 +382,8 @@ VALUES (?, ?, ?);
```json ```json
{ {
"type": "delete_file", "type": "delete_file",
"path": "notes/old.md" "path": "notes/old.md"
} }
``` ```
@ -391,8 +391,8 @@ VALUES (?, ?, ?);
```json ```json
{ {
"type": "list_files", "type": "list_files",
"since_version": 0 "since_version": 0
} }
``` ```
@ -402,11 +402,11 @@ VALUES (?, ?, ?);
```json ```json
{ {
"type": "file_updated", "type": "file_updated",
"path": "notes/example.md", "path": "notes/example.md",
"version": 11, "version": 11,
"size": 1024, "size": 1024,
"hash": "abc123..." "hash": "abc123..."
} }
``` ```
@ -414,10 +414,10 @@ VALUES (?, ?, ?);
```json ```json
{ {
"type": "file_content", "type": "file_content",
"path": "notes/example.md", "path": "notes/example.md",
"content": "Updated content...", "content": "Updated content...",
"version": 11 "version": 11
} }
``` ```
@ -425,9 +425,9 @@ VALUES (?, ?, ?);
```json ```json
{ {
"type": "file_deleted", "type": "file_deleted",
"path": "notes/old.md", "path": "notes/old.md",
"version": 12 "version": 12
} }
``` ```
@ -435,9 +435,9 @@ VALUES (?, ?, ?);
```json ```json
{ {
"type": "sync_complete", "type": "sync_complete",
"total_files": 150, "total_files": 150,
"current_version": 200 "current_version": 200
} }
``` ```
@ -445,9 +445,9 @@ VALUES (?, ?, ?);
```json ```json
{ {
"type": "error", "type": "error",
"message": "File too large", "message": "File too large",
"code": "FILE_TOO_LARGE" "code": "FILE_TOO_LARGE"
} }
``` ```

View file

@ -53,7 +53,7 @@ Central authority for synchronisation. Rust + Axum framework.
**Technology**: **Technology**:
- **Language**: Rust 1.89+ - **Language**: Rust 1.92+
- **Framework**: Axum (async web framework) - **Framework**: Axum (async web framework)
- **Database**: SQLite with SQLx - **Database**: SQLite with SQLx
- **Protocol**: WebSockets for real-time communication - **Protocol**: WebSockets for real-time communication

View file

@ -243,9 +243,9 @@ users:
2. Client sends authentication message: 2. Client sends authentication message:
```json ```json
{ {
"type": "auth", "type": "auth",
"token": "user-token", "token": "user-token",
"vault": "vault-name" "vault": "vault-name"
} }
``` ```
3. Server validates: 3. Server validates:

View file

@ -75,7 +75,7 @@ chmod +x sync_server-linux-x86_64
### Build from Source ### Build from Source
Requirements: Rust 1.89.0+, SQLite development headers, SQLx CLI Requirements: Rust 1.92.0+, SQLite development headers, SQLx CLI
```bash ```bash
# Clone the repository # Clone the repository

5960
docs/package-lock.json generated

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,8 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
import type { DocumentWithCursors } from "./DocumentWithCursors";
export type ClientCursors = {
userName: string;
deviceId: string;
documentsWithCursors: Array<DocumentWithCursors>;
};

View file

@ -1,7 +1,7 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
export interface UpdateDocumentVersion { export type CreateDocumentVersion = {
parent_version_id: bigint;
relative_path: string; relative_path: string;
content: number[]; last_seen_vault_update_id: number;
} content: Array<number>;
};

View file

@ -0,0 +1,6 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
import type { DocumentWithCursors } from "./DocumentWithCursors";
export type CursorPositionFromClient = {
documentsWithCursors: Array<DocumentWithCursors>;
};

View file

@ -0,0 +1,4 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
import type { ClientCursors } from "./ClientCursors";
export type CursorPositionFromServer = { clients: Array<ClientCursors> };

View file

@ -1,5 +1,3 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
export interface DeleteDocumentVersion { export type CursorSpan = { start: number; end: number };
relativePath: string;
}

View file

@ -0,0 +1,10 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
import type { DocumentVersion } from "./DocumentVersion";
import type { DocumentVersionWithoutContent } from "./DocumentVersionWithoutContent";
/**
* Response to a create/update document request.
*/
export type DocumentUpdateResponse =
| ({ type: "FastForwardUpdate" } & DocumentVersionWithoutContent)
| ({ type: "MergingUpdate" } & DocumentVersion);

View file

@ -0,0 +1,12 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
export type DocumentVersion = {
vaultUpdateId: number;
documentId: string;
relativePath: string;
updatedDate: string;
contentBase64: string;
isDeleted: boolean;
userId: string;
deviceId: string;
};

View file

@ -0,0 +1,16 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
export type DocumentVersionWithoutContent = {
vaultUpdateId: number;
documentId: string;
relativePath: string;
updatedDate: string;
isDeleted: boolean;
userId: string;
deviceId: string;
contentSize: number;
/**
* True iff this is the first version of the document
*/
isNewFile: boolean;
};

View file

@ -0,0 +1,9 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
import type { CursorSpan } from "./CursorSpan";
export type DocumentWithCursors = {
vaultUpdateId: number | null;
documentId: string;
relativePath: string;
cursors: Array<CursorSpan>;
};

View file

@ -0,0 +1,13 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
import type { DocumentVersionWithoutContent } from "./DocumentVersionWithoutContent";
/**
* Response to a fetch latest documents request.
*/
export type FetchLatestDocumentsResponse = {
latestDocuments: Array<DocumentVersionWithoutContent>;
/**
* The update ID of the latest document in the response.
*/
lastUpdateId: bigint;
};

View file

@ -0,0 +1,11 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
import type { VaultInfo } from "./VaultInfo";
/**
* Response to listing vaults accessible to the authenticated user.
*/
export type ListVaultsResponse = {
vaults: Array<VaultInfo>;
hasMore: boolean;
userName: string;
};

View file

@ -0,0 +1,25 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
/**
* Response to a ping request.
*/
export type PingResponse = {
/**
* Semantic version of the server.
*/
serverVersion: string;
/**
* Whether the client is authenticated based on the sent Authorization
* header.
*/
isAuthenticated: boolean;
/**
* List of file extensions that are allowed to be merged.
*/
mergeableFileExtensions: Array<string>;
/**
* API version ensuring backwards & forwards compatibility between the client
* and server.
*/
supportedApiVersion: number;
};

View file

@ -0,0 +1,7 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
export type SerializedError = {
errorType: string;
message: string;
causes: Array<string>;
};

View file

@ -0,0 +1,7 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
export type UpdateTextDocumentVersion = {
parentVersionId: number;
relativePath: string | null;
content: Array<number | string>;
};

View file

@ -0,0 +1,10 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
import type { DocumentVersionWithoutContent } from "./DocumentVersionWithoutContent";
/**
* Response to a vault history request (paginated).
*/
export type VaultHistoryResponse = {
versions: Array<DocumentVersionWithoutContent>;
hasMore: boolean;
};

View file

@ -0,0 +1,10 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
/**
* Summary of a single vault returned by the list-vaults endpoint.
*/
export type VaultInfo = {
name: string;
documentCount: number;
createdAt: string | null;
};

View file

@ -0,0 +1,7 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
import type { CursorPositionFromClient } from "./CursorPositionFromClient";
import type { WebSocketHandshake } from "./WebSocketHandshake";
export type WebSocketClientMessage =
| ({ type: "handshake" } & WebSocketHandshake)
| ({ type: "cursorPositions" } & CursorPositionFromClient);

View file

@ -0,0 +1,7 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
export type WebSocketHandshake = {
token: string;
deviceId: string;
lastSeenVaultUpdateId: number | null;
};

View file

@ -0,0 +1,7 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
import type { CursorPositionFromServer } from "./CursorPositionFromServer";
import type { WebSocketVaultUpdate } from "./WebSocketVaultUpdate";
export type WebSocketServerMessage =
| ({ type: "vaultUpdate" } & WebSocketVaultUpdate)
| ({ type: "cursorPositions" } & CursorPositionFromServer);

View file

@ -0,0 +1,4 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
import type { DocumentVersionWithoutContent } from "./DocumentVersionWithoutContent";
export type WebSocketVaultUpdate = { document: DocumentVersionWithoutContent };

View file

@ -1,6 +1,6 @@
export const TIMEOUT_FOR_MERGING_HISTORY_ENTRIES_IN_SECONDS = 60; export const TIMEOUT_FOR_MERGING_HISTORY_ENTRIES_IN_SECONDS = 60;
export const DIFF_CACHE_SIZE_MB = 2;
export const MAX_LOG_MESSAGE_COUNT = 100000; export const MAX_LOG_MESSAGE_COUNT = 100000;
export const MAX_HISTORY_ENTRY_COUNT = 5000; export const MAX_HISTORY_ENTRY_COUNT = 5000;
export const SUPPORTED_API_VERSION = 2; export const SUPPORTED_API_VERSION = 3;
export const WEBSOCKET_DISCONNECT_TIMEOUT_IN_S = 10; export const WEBSOCKET_DISCONNECT_TIMEOUT_IN_SECONDS = 10;
export const WEBSOCKET_CONNECTION_TIMEOUT_IN_SECONDS = 10;

View file

@ -0,0 +1,9 @@
export class FileAlreadyExistsError extends Error {
public constructor(
message: string,
public readonly filePath: string
) {
super(message);
this.name = "FileAlreadyExistsError";
}
}

View file

@ -0,0 +1,9 @@
export class HttpClientError extends Error {
public constructor(
public readonly statusCode: number,
message: string
) {
super(message);
this.name = "HttpClientError";
}
}

View file

@ -0,0 +1,8 @@
import type { Settings } from "../persistence/settings";
export function buildVaultUrl(settings: Settings, path: string): string {
const { vaultName, remoteUri } = settings.getSettings();
const remoteUriWithoutTrailingSlash = remoteUri.replace(/\/+$/, "");
const encodedVaultName = encodeURIComponent(vaultName.trim());
return `${remoteUriWithoutTrailingSlash}/vaults/${encodedVaultName}${path}`;
}

View file

@ -3,7 +3,7 @@ import { describe, it, mock, beforeEach, afterEach } from "node:test";
import assert from "node:assert"; import assert from "node:assert";
import { FetchController } from "./fetch-controller"; import { FetchController } from "./fetch-controller";
import { Logger } from "../tracing/logger"; import { Logger } from "../tracing/logger";
import { SyncResetError } from "./sync-reset-error"; import { SyncResetError } from "../errors/sync-reset-error";
import { sleep } from "../utils/sleep"; import { sleep } from "../utils/sleep";
describe("FetchController", () => { describe("FetchController", () => {

View file

@ -1,6 +1,5 @@
import type { Logger } from "../tracing/logger"; import type { Logger } from "../tracing/logger";
import { createPromise } from "../utils/create-promise"; import { SyncResetError } from "../errors/sync-reset-error";
import { SyncResetError } from "./sync-reset-error";
/** /**
* Offers a resettable fetch implementation that waits until syncing is enabled * Offers a resettable fetch implementation that waits until syncing is enabled
@ -13,37 +12,43 @@ export class FetchController {
// Promise resolves on the next state change: sync enabled/disabled or reset started/ended // Promise resolves on the next state change: sync enabled/disabled or reset started/ended
private until: Promise<symbol>; private until: Promise<symbol>;
private resolveUntil: (result: symbol) => unknown; private resolveUntil: (value: symbol | PromiseLike<symbol>) => void;
private rejectUntil: (reason: unknown) => unknown; private rejectUntil: (reason?: unknown) => void;
public constructor( public constructor(
private _canFetch: boolean, private _canFetch: boolean,
private readonly logger: Logger private readonly logger: Logger
) { ) {
[this.until, this.resolveUntil, this.rejectUntil] = ({
createPromise<symbol>(); promise: this.until,
resolve: this.resolveUntil,
reject: this.rejectUntil
} = Promise.withResolvers<symbol>());
} }
/** /**
* Whether the fetch implementation can immediately send requests once outside of a reset. * Whether the fetch implementation can immediately send requests once outside of a reset.
*/ */
public get canFetch(): boolean { public get canFetch(): boolean {
return this._canFetch; return this._canFetch;
} }
/** /**
* Allow or disallow fetching. The changes only take effect if not resetting. * Allow or disallow fetching. The changes only take effect if not resetting.
* When called during a reset, its effect is deferred until the reset is finished. * When called during a reset, its effect is deferred until the reset is finished.
* *
* @param canFetch Whether fetching is enabled * @param canFetch Whether fetching is enabled
*/ */
public set canFetch(canFetch: boolean) { public set canFetch(canFetch: boolean) {
this._canFetch = canFetch; this._canFetch = canFetch;
if (!this.isResetting) { if (!this.isResetting) {
const previousResolve = this.resolveUntil; const previousResolve = this.resolveUntil;
[this.until, this.resolveUntil, this.rejectUntil] = ({
createPromise<symbol>(); promise: this.until,
resolve: this.resolveUntil,
reject: this.rejectUntil
} = Promise.withResolvers<symbol>());
previousResolve(FetchController.UNTIL_RESOLUTION); previousResolve(FetchController.UNTIL_RESOLUTION);
} }
} }
@ -59,9 +64,9 @@ export class FetchController {
} }
/** /**
* Starts a reset, causing all ongoing and future fetches to be rejected * Starts a reset, causing all ongoing and future fetches to be rejected
* with a SyncResetError until finishReset is called. * with a SyncResetError until finishReset is called.
*/ */
public startReset(): void { public startReset(): void {
this.isResetting = true; this.isResetting = true;
this.rejectUntil(new SyncResetError()); this.rejectUntil(new SyncResetError());
@ -72,32 +77,36 @@ export class FetchController {
} }
/** /**
* Finishes a reset, allowing fetches to proceed or wait again depending on * Finishes a reset, allowing fetches to proceed or wait again depending on
* the current sync settings. * the current sync settings.
*/ */
public finishReset(): void { public finishReset(): void {
if (!this.isResetting) { if (!this.isResetting) {
return; return;
} }
this.isResetting = false; this.isResetting = false;
[this.until, this.resolveUntil, this.rejectUntil] = createPromise(); ({
promise: this.until,
resolve: this.resolveUntil,
reject: this.rejectUntil
} = Promise.withResolvers<symbol>());
} }
/** /**
* *
* |------------------|---------------|-----------------------------------------------------| * |------------------|---------------|-----------------------------------------------------|
* | | Sync enabled | Sync disabled | * | | Sync enabled | Sync disabled |
* |------------------|-------------- |-----------------------------------------------------| * |------------------|-------------- |-----------------------------------------------------|
* | During reset | Rejects with SyncResetError without sending request | * | During reset | Rejects with SyncResetError without sending request |
* |------------------|-------------- |-----------------------------------------------------| * |------------------|-------------- |-----------------------------------------------------|
* | Outside of reset | Same as fetch | Blocks until sync is enabled and then same as fetch | * | Outside of reset | Same as fetch | Blocks until sync is enabled and then same as fetch |
* |------------------|---------------|-----------------------------------------------------| * |------------------|---------------|-----------------------------------------------------|
* *
* @param logger for errors * @param logger for errors
* @param fetch to wrap * @param fetch to wrap
* @returns a wrapped fetch implementation affected by the FetchController state * @returns a wrapped fetch implementation affected by the FetchController state
*/ */
public getControlledFetchImplementation( public getControlledFetchImplementation(
logger: Logger, logger: Logger,
fetch: typeof globalThis.fetch = globalThis.fetch fetch: typeof globalThis.fetch = globalThis.fetch

View file

@ -1,6 +1,7 @@
import { SUPPORTED_API_VERSION } from "../consts"; import { SUPPORTED_API_VERSION } from "../consts";
import { AuthenticationError } from "./authentication-error"; import { AuthenticationError } from "../errors/authentication-error";
import { ServerVersionMismatchError } from "./server-version-mismatch-error"; import { ServerVersionMismatchError } from "../errors/server-version-mismatch-error";
import type { Settings } from "../persistence/settings";
import type { SyncService } from "./sync-service"; import type { SyncService } from "./sync-service";
import type { PingResponse } from "./types/PingResponse"; import type { PingResponse } from "./types/PingResponse";
@ -14,7 +15,20 @@ export class ServerConfig {
private response: Promise<PingResponse> | undefined; private response: Promise<PingResponse> | undefined;
private config: ServerConfigData | undefined; private config: ServerConfigData | undefined;
public constructor(private readonly syncService: SyncService) {} public constructor(
private readonly syncService: SyncService,
settings: Settings
) {
settings.onSettingsChanged.add((newSettings, oldSettings) => {
if (
newSettings.token !== oldSettings.token ||
newSettings.vaultName !== oldSettings.vaultName ||
newSettings.remoteUri !== oldSettings.remoteUri
) {
this.reset();
}
});
}
private static validateConfig(config: ServerConfigData): void { private static validateConfig(config: ServerConfigData): void {
if (config.supportedApiVersion !== SUPPORTED_API_VERSION) { if (config.supportedApiVersion !== SUPPORTED_API_VERSION) {
@ -34,11 +48,6 @@ export class ServerConfig {
} }
} }
// warm the cache
public async initialize(): Promise<void> {
await this.getConfig();
}
public async checkConnection(forceUpdate = false): Promise<{ public async checkConnection(forceUpdate = false): Promise<{
isSuccessful: boolean; isSuccessful: boolean;
message: string; message: string;
@ -46,7 +55,7 @@ export class ServerConfig {
try { try {
let { response } = this; let { response } = this;
if (!response || forceUpdate) { if (!response || forceUpdate) {
response = this.response = this.syncService.ping(); response = this.startPing();
} }
const result: PingResponse = await response; // it must be defined, otherwise we would have thrown above const result: PingResponse = await response; // it must be defined, otherwise we would have thrown above
@ -73,7 +82,7 @@ export class ServerConfig {
public async getConfig(): Promise<ServerConfigData> { public async getConfig(): Promise<ServerConfigData> {
if (!this.config) { if (!this.config) {
this.response ??= this.syncService.ping(); this.response ??= this.startPing();
this.config = await this.response; this.config = await this.response;
} }
@ -86,4 +95,15 @@ export class ServerConfig {
this.response = undefined; this.response = undefined;
this.config = undefined; this.config = undefined;
} }
private async startPing(): Promise<PingResponse> {
const pending = this.syncService.ping().catch((e: unknown) => {
if (this.response === pending) {
this.response = undefined;
}
throw e;
});
this.response = pending;
return pending;
}
} }

View file

@ -2,25 +2,27 @@ import type {
DocumentId, DocumentId,
RelativePath, RelativePath,
VaultUpdateId VaultUpdateId
} from "../persistence/database"; } from "../sync-operations/types";
import type { Logger } from "../tracing/logger"; import type { Logger } from "../tracing/logger";
import type { Settings } from "../persistence/settings"; import type { Settings } from "../persistence/settings";
import type { FetchController } from "./fetch-controller"; import type { FetchController } from "./fetch-controller";
import { sleep } from "../utils/sleep"; import { sleep } from "../utils/sleep";
import { SyncResetError } from "./sync-reset-error"; import { SyncResetError } from "../errors/sync-reset-error";
import { HttpClientError } from "../errors/http-client-error";
import type { SerializedError } from "./types/SerializedError"; import type { SerializedError } from "./types/SerializedError";
import type { DocumentVersionWithoutContent } from "./types/DocumentVersionWithoutContent"; import type { DocumentVersionWithoutContent } from "./types/DocumentVersionWithoutContent";
import type { DocumentUpdateResponse } from "./types/DocumentUpdateResponse"; import type { DocumentUpdateResponse } from "./types/DocumentUpdateResponse";
import type { DocumentVersion } from "./types/DocumentVersion"; import type { DocumentVersion } from "./types/DocumentVersion";
import type { FetchLatestDocumentsResponse } from "./types/FetchLatestDocumentsResponse"; import type { FetchLatestDocumentsResponse } from "./types/FetchLatestDocumentsResponse";
import type { PingResponse } from "./types/PingResponse"; import type { PingResponse } from "./types/PingResponse";
import type { DeleteDocumentVersion } from "./types/DeleteDocumentVersion";
import type { UpdateTextDocumentVersion } from "./types/UpdateTextDocumentVersion"; import type { UpdateTextDocumentVersion } from "./types/UpdateTextDocumentVersion";
import { buildVaultUrl } from "./build-vault-url";
export class SyncService { export class SyncService {
private readonly client: typeof globalThis.fetch; private readonly client: typeof globalThis.fetch;
private readonly pingClient: typeof globalThis.fetch; private readonly pingClient: typeof globalThis.fetch;
private isStopped = false;
public constructor( public constructor(
private readonly deviceId: string, private readonly deviceId: string,
@ -65,28 +67,68 @@ export class SyncService {
return result; return result;
} }
private static async throwIfNotOk(
response: Response,
operation: string
): Promise<void> {
if (response.ok) {
return;
}
const message = `Failed to ${operation}: ${await SyncService.errorFromResponse(response)}`;
// 429 is the only 4xx the server uses for *transient* contention
// (`WriteBusyError` → HTTP 429). Every other 4xx means the request
// is permanently rejected and shouldn't be retried.
if (response.status === 429) {
throw new Error(message);
}
if (response.status >= 400 && response.status < 500) {
throw new HttpClientError(response.status, message);
}
throw new Error(message);
}
/**
* Signal that the service is shutting down so any in-flight
* `retryForever` exits at its next iteration instead of looping
* indefinitely after the rest of the client has stopped. Idempotent.
*/
public stop(): void {
this.isStopped = true;
}
/**
* Re-enable the service after a `stop()`. Used when the client pauses
* and resumes syncing within the same lifecycle (e.g. user toggles
* sync off and on).
*/
public resume(): void {
this.isStopped = false;
}
public async create({ public async create({
documentId,
relativePath, relativePath,
lastSeenVaultUpdateId,
contentBytes contentBytes
}: { }: {
documentId?: DocumentId;
relativePath: RelativePath; relativePath: RelativePath;
lastSeenVaultUpdateId: VaultUpdateId;
contentBytes: Uint8Array; contentBytes: Uint8Array;
}): Promise<DocumentVersionWithoutContent> { }): Promise<DocumentUpdateResponse> {
return this.retryForever(async () => { return this.retryForever(async () => {
const formData = new FormData(); const formData = new FormData();
if (documentId !== undefined) {
formData.append("document_id", documentId);
}
formData.append("relative_path", relativePath); formData.append("relative_path", relativePath);
formData.append(
"last_seen_vault_update_id",
lastSeenVaultUpdateId.toString()
);
formData.append( formData.append(
"content", "content",
new Blob([new Uint8Array(contentBytes)]) new Blob([new Uint8Array(contentBytes)])
); );
this.logger.debug( this.logger.debug(
`Creating document with id ${documentId} and relative path ${relativePath}` `Creating document with relative path ${relativePath}`
); );
const response = await this.client(this.getUrl("/documents"), { const response = await this.client(this.getUrl("/documents"), {
@ -95,16 +137,10 @@ export class SyncService {
headers: this.getDefaultHeaders() headers: this.getDefaultHeaders()
}); });
if (!response.ok) { await SyncService.throwIfNotOk(response, "create document");
throw new Error(
`Failed to create document: ${await SyncService.errorFromResponse(
response
)}`
);
}
const result: DocumentVersionWithoutContent = const result: DocumentUpdateResponse =
(await response.json()) as DocumentVersionWithoutContent; // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion (await response.json()) as DocumentUpdateResponse; // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion
this.logger.debug(`Created document ${JSON.stringify(result)}`); this.logger.debug(`Created document ${JSON.stringify(result)}`);
@ -120,17 +156,17 @@ export class SyncService {
}: { }: {
parentVersionId: VaultUpdateId; parentVersionId: VaultUpdateId;
documentId: DocumentId; documentId: DocumentId;
relativePath: RelativePath; relativePath: RelativePath | undefined;
content: (number | string)[]; content: (number | string)[];
}): Promise<DocumentUpdateResponse> { }): Promise<DocumentUpdateResponse> {
return this.retryForever(async () => { return this.retryForever(async () => {
this.logger.debug( this.logger.debug(
`Updating text document ${documentId} with parent version ${parentVersionId} and relative path ${relativePath}, content [${content.join(", ")}]` `Updating text document ${documentId} with parent version ${parentVersionId} and relative path ${relativePath ?? "<unchanged>"}, content [${content.join(", ")}]`
); );
const request: UpdateTextDocumentVersion = { const request: UpdateTextDocumentVersion = {
parentVersionId, parentVersionId,
relativePath, relativePath: relativePath ?? null,
content content
}; };
@ -143,13 +179,7 @@ export class SyncService {
} }
); );
if (!response.ok) { await SyncService.throwIfNotOk(response, "update document");
throw new Error(
`Failed to update document: ${await SyncService.errorFromResponse(
response
)}`
);
}
const result: DocumentUpdateResponse = const result: DocumentUpdateResponse =
(await response.json()) as DocumentUpdateResponse; // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion (await response.json()) as DocumentUpdateResponse; // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion
@ -172,16 +202,18 @@ export class SyncService {
}: { }: {
parentVersionId: VaultUpdateId; parentVersionId: VaultUpdateId;
documentId: DocumentId; documentId: DocumentId;
relativePath: RelativePath; relativePath: RelativePath | undefined;
contentBytes: Uint8Array; contentBytes: Uint8Array;
}): Promise<DocumentUpdateResponse> { }): Promise<DocumentUpdateResponse> {
return this.retryForever(async () => { return this.retryForever(async () => {
this.logger.debug( this.logger.debug(
`Updating binary document ${documentId} with parent version ${parentVersionId} and relative path ${relativePath}` `Updating binary document ${documentId} with parent version ${parentVersionId} and relative path ${relativePath ?? "<unchanged>"}`
); );
const formData = new FormData(); const formData = new FormData();
formData.append("parent_version_id", parentVersionId.toString()); formData.append("parent_version_id", parentVersionId.toString());
formData.append("relative_path", relativePath); if (relativePath !== undefined) {
formData.append("relative_path", relativePath);
}
formData.append( formData.append(
"content", "content",
new Blob([new Uint8Array(contentBytes)]) new Blob([new Uint8Array(contentBytes)])
@ -196,13 +228,7 @@ export class SyncService {
} }
); );
if (!response.ok) { await SyncService.throwIfNotOk(response, "update document");
throw new Error(
`Failed to update document: ${await SyncService.errorFromResponse(
response
)}`
);
}
const result: DocumentUpdateResponse = const result: DocumentUpdateResponse =
(await response.json()) as DocumentUpdateResponse; // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion (await response.json()) as DocumentUpdateResponse; // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion
@ -218,44 +244,29 @@ export class SyncService {
} }
public async delete({ public async delete({
documentId, documentId
relativePath
}: { }: {
documentId: DocumentId; documentId: DocumentId;
relativePath: RelativePath;
}): Promise<DocumentVersionWithoutContent> { }): Promise<DocumentVersionWithoutContent> {
return this.retryForever(async () => { return this.retryForever(async () => {
const request: DeleteDocumentVersion = { this.logger.debug(`Delete document with id ${documentId}`);
relativePath
};
this.logger.debug(
`Delete document with id ${documentId} and relative path ${relativePath}`
);
// The server identifies the document by its URL path; no body
// is needed. Sending one was a leftover of an earlier shape.
const response = await this.client( const response = await this.client(
this.getUrl(`/documents/${documentId}`), this.getUrl(`/documents/${documentId}`),
{ {
method: "DELETE", method: "DELETE",
body: JSON.stringify(request), headers: this.getDefaultHeaders()
headers: this.getDefaultHeaders({ type: "json" })
} }
); );
if (!response.ok) { await SyncService.throwIfNotOk(response, "delete document");
throw new Error(
`Failed to delete document: ${await SyncService.errorFromResponse(
response
)}`
);
}
const result: DocumentVersionWithoutContent = const result: DocumentVersionWithoutContent =
(await response.json()) as DocumentVersionWithoutContent; // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion (await response.json()) as DocumentVersionWithoutContent; // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion
this.logger.debug( this.logger.debug(`Deleted document with id ${documentId}`);
`Deleted document ${relativePath} with id ${documentId}`
);
return result; return result;
}); });
@ -276,13 +287,7 @@ export class SyncService {
} }
); );
if (!response.ok) { await SyncService.throwIfNotOk(response, "get document");
throw new Error(
`Failed to get document: ${await SyncService.errorFromResponse(
response
)}`
);
}
const result: DocumentVersion = const result: DocumentVersion =
(await response.json()) as DocumentVersion; // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion (await response.json()) as DocumentVersion; // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion
@ -314,13 +319,10 @@ export class SyncService {
} }
); );
if (!response.ok) { await SyncService.throwIfNotOk(
throw new Error( response,
`Failed to get document: ${await SyncService.errorFromResponse( "get document version content"
response );
)}`
);
}
const result = await response.bytes(); const result = await response.bytes();
this.logger.debug( this.logger.debug(
@ -341,19 +343,13 @@ export class SyncService {
const url = new URL(this.getUrl("/documents")); const url = new URL(this.getUrl("/documents"));
if (since !== undefined) { if (since !== undefined) {
url.searchParams.append("since", since.toString()); url.searchParams.append("since_update_id", since.toString());
} }
const response = await this.client(url.toString(), { const response = await this.client(url.toString(), {
headers: this.getDefaultHeaders() headers: this.getDefaultHeaders()
}); });
if (!response.ok) { await SyncService.throwIfNotOk(response, "get documents");
throw new Error(
`Failed to get documents: ${await SyncService.errorFromResponse(
response
)}`
);
}
const result: FetchLatestDocumentsResponse = const result: FetchLatestDocumentsResponse =
(await response.json()) as FetchLatestDocumentsResponse; // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion (await response.json()) as FetchLatestDocumentsResponse; // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion
@ -390,10 +386,7 @@ export class SyncService {
} }
private getUrl(path: string): string { private getUrl(path: string): string {
const { vaultName, remoteUri } = this.settings.getSettings(); return buildVaultUrl(this.settings, path);
const remoteUriWithoutTrailingSlash = remoteUri.replace(/\/+$/, "");
const encodedVaultName = encodeURIComponent(vaultName.trim());
return `${remoteUriWithoutTrailingSlash}/vaults/${encodedVaultName}${path}`;
} }
private getDefaultHeaders( private getDefaultHeaders(
@ -414,13 +407,17 @@ export class SyncService {
private async retryForever<T>(fn: () => Promise<T>): Promise<T> { private async retryForever<T>(fn: () => Promise<T>): Promise<T> {
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
while (true) { while (true) {
this.throwIfStopped();
try { try {
return await fn(); return await fn();
} catch (e) { } catch (e) {
// We must not retry errors coming from reset if (
if (e instanceof SyncResetError) { e instanceof SyncResetError ||
e instanceof HttpClientError
) {
throw e; throw e;
} }
this.throwIfStopped();
const retryInterval = const retryInterval =
this.settings.getSettings().networkRetryIntervalMs; this.settings.getSettings().networkRetryIntervalMs;
@ -431,4 +428,10 @@ export class SyncService {
} }
} }
} }
private throwIfStopped(): void {
if (this.isStopped) {
throw new SyncResetError();
}
}
} }

View file

@ -1,13 +1,7 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
export interface CreateDocumentVersion { export interface CreateDocumentVersion {
/**
* The client can decide the document id (if it wishes to) in order
* to help with syncing. If the client does not provide a document id,
* the server will generate one. If the client provides a document id
* it must not already exist in the database.
*/
document_id: string | null;
relative_path: string; relative_path: string;
last_seen_vault_update_id: number;
content: number[]; content: number[];
} }

View file

@ -3,7 +3,7 @@ import type { DocumentVersion } from "./DocumentVersion";
import type { DocumentVersionWithoutContent } from "./DocumentVersionWithoutContent"; import type { DocumentVersionWithoutContent } from "./DocumentVersionWithoutContent";
/** /**
* Response to an update document request. * Response to a create/update document request.
*/ */
export type DocumentUpdateResponse = export type DocumentUpdateResponse =
| ({ type: "FastForwardUpdate" } & DocumentVersionWithoutContent) | ({ type: "FastForwardUpdate" } & DocumentVersionWithoutContent)

View file

@ -9,4 +9,8 @@ export interface DocumentVersionWithoutContent {
userId: string; userId: string;
deviceId: string; deviceId: string;
contentSize: number; contentSize: number;
/**
* True iff this is the first version of the document
*/
isNewFile: boolean;
} }

View file

@ -2,8 +2,8 @@
import type { CursorSpan } from "./CursorSpan"; import type { CursorSpan } from "./CursorSpan";
export interface DocumentWithCursors { export interface DocumentWithCursors {
vault_update_id: number | null; vaultUpdateId: number | null;
document_id: string; documentId: string;
relative_path: string; relativePath: string;
cursors: CursorSpan[]; cursors: CursorSpan[];
} }

View file

@ -7,7 +7,7 @@ import type { DocumentVersionWithoutContent } from "./DocumentVersionWithoutCont
export interface FetchLatestDocumentsResponse { export interface FetchLatestDocumentsResponse {
latestDocuments: DocumentVersionWithoutContent[]; latestDocuments: DocumentVersionWithoutContent[];
/** /**
* The update ID of the latest document in the response. * The update ID of the latest document in the response.
*/ */
lastUpdateId: bigint; lastUpdateId: bigint;
} }

View file

@ -0,0 +1,11 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
import type { VaultInfo } from "./VaultInfo";
/**
* Response to listing vaults accessible to the authenticated user.
*/
export interface ListVaultsResponse {
vaults: VaultInfo[];
hasMore: boolean;
userName: string;
}

View file

@ -5,21 +5,21 @@
*/ */
export interface PingResponse { export interface PingResponse {
/** /**
* Semantic version of the server. * Semantic version of the server.
*/ */
serverVersion: string; serverVersion: string;
/** /**
* Whether the client is authenticated based on the sent Authorization * Whether the client is authenticated based on the sent Authorization
* header. * header.
*/ */
isAuthenticated: boolean; isAuthenticated: boolean;
/** /**
* List of file extensions that are allowed to be merged. * List of file extensions that are allowed to be merged.
*/ */
mergeableFileExtensions: string[]; mergeableFileExtensions: string[];
/** /**
* API version ensuring backwards & forwards compatibility between the client * API version ensuring backwards & forwards compatibility between the client
* and server. * and server.
*/ */
supportedApiVersion: number; supportedApiVersion: number;
} }

View file

@ -2,6 +2,6 @@
export interface UpdateTextDocumentVersion { export interface UpdateTextDocumentVersion {
parentVersionId: number; parentVersionId: number;
relativePath: string; relativePath: string | null;
content: (number | string)[]; content: (number | string)[];
} }

View file

@ -0,0 +1,10 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
import type { DocumentVersionWithoutContent } from "./DocumentVersionWithoutContent";
/**
* Response to a vault history request (paginated).
*/
export interface VaultHistoryResponse {
versions: DocumentVersionWithoutContent[];
hasMore: boolean;
}

View file

@ -0,0 +1,10 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
/**
* Summary of a single vault returned by the list-vaults endpoint.
*/
export interface VaultInfo {
name: string;
documentCount: number;
createdAt: string | null;
}

View file

@ -2,6 +2,5 @@
import type { DocumentVersionWithoutContent } from "./DocumentVersionWithoutContent"; import type { DocumentVersionWithoutContent } from "./DocumentVersionWithoutContent";
export interface WebSocketVaultUpdate { export interface WebSocketVaultUpdate {
documents: DocumentVersionWithoutContent[]; document: DocumentVersionWithoutContent;
isInitialSync: boolean;
} }

View file

@ -4,8 +4,7 @@ import assert from "node:assert";
import { WebSocketManager } from "./websocket-manager"; import { WebSocketManager } from "./websocket-manager";
import type { Logger } from "../tracing/logger"; import type { Logger } from "../tracing/logger";
import type { Settings } from "../persistence/settings"; import type { Settings } from "../persistence/settings";
// eslint-disable-next-line @typescript-eslint/no-require-imports import { awaitAll } from "../utils/await-all";
const WebSocket = require("ws") as typeof globalThis.WebSocket;
class MockCloseEvent extends Event { class MockCloseEvent extends Event {
public code: number; public code: number;
@ -91,10 +90,8 @@ function createMockFn<T extends (...args: unknown[]) => unknown>(
describe("WebSocketManager", () => { describe("WebSocketManager", () => {
let mockLogger: Logger = undefined as unknown as Logger; let mockLogger: Logger = undefined as unknown as Logger;
let mockSettings: Settings = undefined as unknown as Settings; let mockSettings: Settings = undefined as unknown as Settings;
let deviceId = "test-device-123";
beforeEach(() => { beforeEach(() => {
deviceId = "test-device-123";
const noop = (): void => { const noop = (): void => {
// Intentionally empty for mock // Intentionally empty for mock
}; };
@ -116,7 +113,6 @@ describe("WebSocketManager", () => {
it("cleans up promises after message handling", async () => { it("cleans up promises after message handling", async () => {
const manager = new WebSocketManager( const manager = new WebSocketManager(
deviceId,
mockLogger, mockLogger,
mockSettings, mockSettings,
MockWebSocket as unknown as typeof WebSocket MockWebSocket as unknown as typeof WebSocket
@ -146,7 +142,6 @@ describe("WebSocketManager", () => {
it("cleans up cursor position promises", async () => { it("cleans up cursor position promises", async () => {
const manager = new WebSocketManager( const manager = new WebSocketManager(
deviceId,
mockLogger, mockLogger,
mockSettings, mockSettings,
MockWebSocket as unknown as typeof WebSocket MockWebSocket as unknown as typeof WebSocket
@ -176,7 +171,6 @@ describe("WebSocketManager", () => {
it("logs handshake send errors", async () => { it("logs handshake send errors", async () => {
const manager = new WebSocketManager( const manager = new WebSocketManager(
deviceId,
mockLogger, mockLogger,
mockSettings, mockSettings,
MockWebSocket as unknown as typeof WebSocket MockWebSocket as unknown as typeof WebSocket
@ -205,7 +199,6 @@ describe("WebSocketManager", () => {
it("completes stop with timeout protection", async () => { it("completes stop with timeout protection", async () => {
const manager = new WebSocketManager( const manager = new WebSocketManager(
deviceId,
mockLogger, mockLogger,
mockSettings, mockSettings,
MockWebSocket as unknown as typeof WebSocket MockWebSocket as unknown as typeof WebSocket
@ -220,7 +213,6 @@ describe("WebSocketManager", () => {
it("clears old handlers on reconnection", async () => { it("clears old handlers on reconnection", async () => {
const manager = new WebSocketManager( const manager = new WebSocketManager(
deviceId,
mockLogger, mockLogger,
mockSettings, mockSettings,
MockWebSocket as unknown as typeof WebSocket MockWebSocket as unknown as typeof WebSocket
@ -255,9 +247,68 @@ describe("WebSocketManager", () => {
await manager.stop(); await manager.stop();
}); });
it("handles concurrent stop() calls without stranding either caller", async () => {
// Real WebSocket.close() doesn't fire onclose synchronously, and the
// socket stays reachable across the close handshake. Model that
// here so the manager's `while (isWebSocketConnected)` loop is
// actually awaiting when the second stop() races in. Static OPEN
// is required because the manager compares readyState against
// `factory.OPEN`.
class AsyncCloseWebSocket extends MockWebSocket {
public static readonly OPEN = WebSocket.OPEN;
public override close(code?: number, reason?: string): void {
if (
this.readyState === WebSocket.CLOSED ||
(this as { _closing?: boolean })._closing === true
) {
return;
}
(this as { _closing?: boolean })._closing = true;
setTimeout(() => {
this.readyState = WebSocket.CLOSED;
this.onclose?.(
new MockCloseEvent("close", {
code: code ?? 1000,
reason: reason ?? ""
})
);
}, 5);
}
}
const manager = new WebSocketManager(
mockLogger,
mockSettings,
AsyncCloseWebSocket as unknown as typeof WebSocket
);
manager.start();
await new Promise((resolve) => setTimeout(resolve, 50));
const start = Date.now();
// Two concurrent stops mimic destroy() racing onSettingsChange.
await awaitAll([manager.stop(), manager.stop()]);
const elapsed = Date.now() - start;
// Both should resolve via the normal close path; if the second call
// had clobbered the first's resolver, the first would have been
// stranded until the 10s disconnect timeout.
assert.ok(
elapsed < 1000,
`concurrent stop() took ${elapsed}ms — expected fast resolution`
);
const errorCalls = (mockLogger.error as unknown as { calls: unknown[] })
.calls;
assert.strictEqual(
errorCalls.length,
0,
"no timeout-recovery error should be logged"
);
});
it("tracks message handling promises", async () => { it("tracks message handling promises", async () => {
const manager = new WebSocketManager( const manager = new WebSocketManager(
deviceId,
mockLogger, mockLogger,
mockSettings, mockSettings,
MockWebSocket as unknown as typeof WebSocket MockWebSocket as unknown as typeof WebSocket

View file

@ -4,12 +4,15 @@ import type { WebSocketServerMessage } from "./types/WebSocketServerMessage";
import type { WebSocketClientMessage } from "./types/WebSocketClientMessage"; import type { WebSocketClientMessage } from "./types/WebSocketClientMessage";
import type { CursorPositionFromClient } from "./types/CursorPositionFromClient"; import type { CursorPositionFromClient } from "./types/CursorPositionFromClient";
import type { ClientCursors } from "./types/ClientCursors"; import type { ClientCursors } from "./types/ClientCursors";
import { createPromise } from "../utils/create-promise";
import type { WebSocketVaultUpdate } from "./types/WebSocketVaultUpdate"; import type { WebSocketVaultUpdate } from "./types/WebSocketVaultUpdate";
import { WEBSOCKET_DISCONNECT_TIMEOUT_IN_S } from "../consts"; import {
WEBSOCKET_DISCONNECT_TIMEOUT_IN_SECONDS,
WEBSOCKET_CONNECTION_TIMEOUT_IN_SECONDS
} from "../consts";
import { removeFromArray } from "../utils/remove-from-array"; import { removeFromArray } from "../utils/remove-from-array";
import { EventListeners } from "../utils/data-structures/event-listeners"; import { EventListeners } from "../utils/data-structures/event-listeners";
import { awaitAll } from "../utils/await-all"; import { awaitAll } from "../utils/await-all";
import { buildVaultUrl } from "./build-vault-url";
export class WebSocketManager { export class WebSocketManager {
public readonly onWebSocketStatusChanged = new EventListeners< public readonly onWebSocketStatusChanged = new EventListeners<
@ -26,32 +29,22 @@ export class WebSocketManager {
private isStopped = true; private isStopped = true;
private resolveDisconnectingPromise: null | (() => unknown) = null; private resolveDisconnectingPromise: null | (() => unknown) = null;
private stopPromise: Promise<void> | null = null;
private reconnectTimeoutId: ReturnType<typeof setTimeout> | undefined; private reconnectTimeoutId: ReturnType<typeof setTimeout> | undefined;
private connectionTimeoutId: ReturnType<typeof setTimeout> | undefined;
private readonly outstandingPromises: Promise<unknown>[] = []; private readonly outstandingPromises: Promise<unknown>[] = [];
private webSocket: WebSocket | undefined; private webSocket: WebSocket | undefined;
private readonly webSocketFactoryImplementation: typeof globalThis.WebSocket;
public constructor( public constructor(
private readonly deviceId: string,
private readonly logger: Logger, private readonly logger: Logger,
private readonly settings: Settings, private readonly settings: Settings,
webSocketImplementation?: typeof globalThis.WebSocket private readonly webSocketFactoryImplementation: typeof globalThis.WebSocket = WebSocket
) { ) {}
if (webSocketImplementation) {
this.webSocketFactoryImplementation = webSocketImplementation; public get hasOutstandingWork(): boolean {
} else { return this.outstandingPromises.length > 0;
if (
typeof globalThis !== "undefined" &&
typeof globalThis.WebSocket === "undefined"
) {
// eslint-disable-next-line
this.webSocketFactoryImplementation = require("ws"); // polyfill for WebSocket in Node.js
} else {
this.webSocketFactoryImplementation = WebSocket;
}
}
} }
public get isWebSocketConnected(): boolean { public get isWebSocketConnected(): boolean {
@ -67,49 +60,14 @@ export class WebSocketManager {
} }
public async stop(): Promise<void> { public async stop(): Promise<void> {
const [promise, resolve] = createPromise(); // Concurrent callers (e.g. destroy() and onSettingsChange) must share
this.resolveDisconnectingPromise = resolve; // the same disconnect; otherwise the second call would overwrite
// resolveDisconnectingPromise and strand the first caller's await
this.isStopped = true; // until the timeout rejects.
this.stopPromise ??= this.performStop().finally(() => {
if (this.reconnectTimeoutId !== undefined) { this.stopPromise = null;
clearTimeout(this.reconnectTimeoutId);
this.reconnectTimeoutId = undefined;
}
this.webSocket?.close(1000, "WebSocketManager has been stopped");
// eslint-disable-next-line @typescript-eslint/init-declarations
let timeoutId: ReturnType<typeof setTimeout> | undefined;
const timeoutPromise = new Promise<void>((_, reject) => {
timeoutId = setTimeout(() => {
reject(
new Error(
`Timeout waiting for WebSocket to close after ${WEBSOCKET_DISCONNECT_TIMEOUT_IN_S} seconds`
)
);
}, WEBSOCKET_DISCONNECT_TIMEOUT_IN_S * 1000);
}); });
await this.stopPromise;
try {
while (this.isWebSocketConnected) {
await Promise.race([promise, timeoutPromise]);
}
} catch (error) {
this.logger.error(
`Error while waiting for WebSocket to close: ${String(error)}`
);
// Force cleanup even if close didn't work
this.resolveDisconnectingPromise();
this.resolveDisconnectingPromise = null;
} finally {
// Clear timeout to prevent unhandled rejection
if (timeoutId !== undefined) {
clearTimeout(timeoutId);
}
}
await this.waitUntilFinished();
} }
public async waitUntilFinished(): Promise<void> { public async waitUntilFinished(): Promise<void> {
@ -162,6 +120,59 @@ export class WebSocketManager {
} }
} }
private async performStop(): Promise<void> {
const { promise, resolve } = Promise.withResolvers<undefined>();
this.resolveDisconnectingPromise = (): void => {
resolve(undefined);
};
this.isStopped = true;
if (this.reconnectTimeoutId !== undefined) {
clearTimeout(this.reconnectTimeoutId);
this.reconnectTimeoutId = undefined;
}
if (this.connectionTimeoutId !== undefined) {
clearTimeout(this.connectionTimeoutId);
this.connectionTimeoutId = undefined;
}
this.webSocket?.close(1000, "WebSocketManager has been stopped");
// eslint-disable-next-line @typescript-eslint/init-declarations
let timeoutId: ReturnType<typeof setTimeout> | undefined;
const timeoutPromise = new Promise<void>((_, reject) => {
timeoutId = setTimeout(() => {
reject(
new Error(
`Timeout waiting for WebSocket to close after ${WEBSOCKET_DISCONNECT_TIMEOUT_IN_SECONDS} seconds`
)
);
}, WEBSOCKET_DISCONNECT_TIMEOUT_IN_SECONDS * 1000);
});
try {
while (this.isWebSocketConnected) {
await Promise.race([promise, timeoutPromise]);
}
} catch (error) {
this.logger.error(
`Error while waiting for WebSocket to close: ${String(error)}`
);
// Force cleanup even if close didn't work
this.resolveDisconnectingPromise();
this.resolveDisconnectingPromise = null;
} finally {
// Clear timeout to prevent unhandled rejection
if (timeoutId !== undefined) {
clearTimeout(timeoutId);
}
}
await this.waitUntilFinished();
}
private initializeWebSocket(): void { private initializeWebSocket(): void {
// Clean up old WebSocket handlers to prevent race conditions // Clean up old WebSocket handlers to prevent race conditions
if (this.webSocket) { if (this.webSocket) {
@ -171,26 +182,55 @@ export class WebSocketManager {
this.webSocket.onclose = null; this.webSocket.onclose = null;
this.webSocket.onmessage = null; this.webSocket.onmessage = null;
this.webSocket.onerror = null; this.webSocket.onerror = null;
this.webSocket.close(); this.webSocket.close(
1000,
"Closing previous WebSocket connection"
);
} catch (e) { } catch (e) {
this.logger.error( this.logger.error(
`Failed to close previous WebSocket connection: ${e}` `Failed to close previous WebSocket connection: ${e}`
); );
} }
// Abandon any outstanding handler promises from the previous
// connection. They'll still resolve in the background, but we
// no longer want `waitUntilFinished` / `stop` to block on
// post-reconnect state — and we definitely don't want their
// results applied against a now-stale socket.
this.outstandingPromises.length = 0;
} }
const wsUri = new URL(this.settings.getSettings().remoteUri); // Build the WS URL through the same vault-URL helper the HTTP client
wsUri.protocol = wsUri.protocol === "https" ? "wss" : "ws"; // uses so vault-name encoding, trailing-slash stripping, and any path
wsUri.pathname = `/vaults/${this.settings.getSettings().vaultName}/ws`; // prefix in `remoteUri` stay in sync between transports.
const wsUri = new URL(buildVaultUrl(this.settings, "/ws"));
wsUri.protocol = wsUri.protocol.startsWith("https") ? "wss" : "ws";
this.logger.info(`Connecting to WebSocket at ${wsUri.toString()}`); this.logger.info(`Connecting to WebSocket at ${wsUri.toString()}`);
this.webSocket = new this.webSocketFactoryImplementation(wsUri); const ws = new this.webSocketFactoryImplementation(wsUri);
this.webSocket = ws;
// Set connection timeout to handle cases where server is down and the WebSocket connection won't open.
// The callback closes the *captured* `ws` rather than `this.webSocket` so a delayed timeout cannot
// accidentally close a freshly-constructed replacement socket. (Closing the already-closed `ws` is a no-op.)
this.connectionTimeoutId = setTimeout(() => {
this.connectionTimeoutId = undefined;
this.logger.warn(
`WebSocket connection timeout after ${WEBSOCKET_CONNECTION_TIMEOUT_IN_SECONDS} seconds`
);
// Force close to trigger onclose handler which will schedule reconnection
ws.close(1000, "Connection timeout");
}, WEBSOCKET_CONNECTION_TIMEOUT_IN_SECONDS * 1000);
ws.onopen = (): void => {
if (this.connectionTimeoutId !== undefined) {
clearTimeout(this.connectionTimeoutId);
this.connectionTimeoutId = undefined;
}
this.webSocket.onopen = (): void => {
// Check if we've been stopped while connecting // Check if we've been stopped while connecting
if (this.isStopped) { if (this.isStopped) {
this.webSocket?.close( ws.close(
1000, 1000,
"WebSocketManager was stopped during connection" "WebSocketManager was stopped during connection"
); );
@ -200,7 +240,7 @@ export class WebSocketManager {
this.onWebSocketStatusChanged.trigger(true); this.onWebSocketStatusChanged.trigger(true);
}; };
this.webSocket.onmessage = (event): void => { ws.onmessage = (event): void => {
try { try {
// eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion
const message = JSON.parse( const message = JSON.parse(
@ -231,7 +271,18 @@ export class WebSocketManager {
} }
}; };
this.webSocket.onclose = (event): void => { ws.onerror = (error): void => {
this.logger.warn(
`WebSocket error occurred: ${error instanceof ErrorEvent ? error.message : "Unknown error"}`
);
};
ws.onclose = (event): void => {
if (this.connectionTimeoutId !== undefined) {
clearTimeout(this.connectionTimeoutId);
this.connectionTimeoutId = undefined;
}
this.logger.warn( this.logger.warn(
`WebSocket closed with code ${event.code} (${event.reason == "" ? "unknown reason" : event.reason})` `WebSocket closed with code ${event.code} (${event.reason == "" ? "unknown reason" : event.reason})`
); );
@ -241,10 +292,13 @@ export class WebSocketManager {
this.resolveDisconnectingPromise?.(); this.resolveDisconnectingPromise?.();
this.resolveDisconnectingPromise = null; this.resolveDisconnectingPromise = null;
} else { } else {
const delay =
this.settings.getSettings().webSocketRetryIntervalMs;
this.logger.info(`Reconnecting to WebSocket in ${delay}ms...`);
this.reconnectTimeoutId = setTimeout(() => { this.reconnectTimeoutId = setTimeout(() => {
this.reconnectTimeoutId = undefined; this.reconnectTimeoutId = undefined;
this.initializeWebSocket(); this.initializeWebSocket();
}, this.settings.getSettings().webSocketRetryIntervalMs); }, delay);
} }
}; };
} }
@ -252,22 +306,22 @@ export class WebSocketManager {
private async handleWebSocketMessage( private async handleWebSocketMessage(
message: WebSocketServerMessage message: WebSocketServerMessage
): Promise<void> { ): Promise<void> {
if (message.type === "vaultUpdate") { switch (message.type) {
await this.onRemoteVaultUpdateReceived.triggerAsync(message); case "vaultUpdate":
await this.onRemoteVaultUpdateReceived.triggerAsync(message);
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition return;
} else if (message.type === "cursorPositions") { case "cursorPositions":
this.logger.debug( this.logger.debug(
`Received cursor positions for ${JSON.stringify(message.clients)}` `Received cursor positions for ${JSON.stringify(message.clients)}`
); );
await this.onRemoteCursorsUpdateReceived.triggerAsync(
await this.onRemoteCursorsUpdateReceived.triggerAsync( message.clients
message.clients );
); return;
} else { default:
this.logger.warn( this.logger.warn(
`Received unknown message type: ${JSON.stringify(message)}` `Received unknown message type: ${JSON.stringify(message)}`
); );
} }
} }
} }

View file

@ -9,7 +9,7 @@ type ResolvedTuple<T extends readonly unknown[]> = {
export const awaitAll = async <T extends readonly unknown[]>( export const awaitAll = async <T extends readonly unknown[]>(
promises: PromiseTuple<T> promises: PromiseTuple<T>
): Promise<ResolvedTuple<T>> => { ): Promise<ResolvedTuple<T>> => {
// eslint-disable-next-line no-restricted-properties // eslint-disable-next-line no-restricted-properties, @typescript-eslint/await-thenable
const result = await Promise.allSettled(promises); const result = await Promise.allSettled(promises);
for (const res of result) { for (const res of result) {
if (res.status === "rejected") { if (res.status === "rejected") {

View file

@ -1,5 +1,3 @@
import { v4 as uuidv4 } from "uuid";
export function createClientId(): string { export function createClientId(): string {
// @ts-expect-error, injected by webpack // @ts-expect-error, injected by webpack
const packageVersion = __CURRENT_VERSION__; // eslint-disable-line const packageVersion = __CURRENT_VERSION__; // eslint-disable-line
@ -8,8 +6,8 @@ export function createClientId(): string {
typeof navigator !== "undefined" typeof navigator !== "undefined"
? navigator.platform // eslint-disable-line @typescript-eslint/no-deprecated ? navigator.platform // eslint-disable-line @typescript-eslint/no-deprecated
: typeof process !== "undefined" : typeof process !== "undefined"
? process.platform ? process.platform
: "unknown"; : "unknown";
return `vault-link/${packageVersion} (${uuidv4()}; ${platform})`; return `vault-link/${packageVersion} (${Math.round(Math.random() * 1e10)}; ${platform})`;
} }

View file

@ -1,25 +0,0 @@
type ResolveFunction<T> = undefined extends T
? (value?: T) => unknown
: (value: T) => unknown;
/**
* A type-safe utility function to create a Promise with resolve and reject functions.
* @returns A tuple containing a Promise, a resolve function, and a reject function.
*/
export function createPromise<T = unknown>(): [
Promise<T>,
ResolveFunction<T>,
(error: unknown) => unknown
] {
let resolve: undefined | ResolveFunction<T> = undefined;
let reject: undefined | ((error: unknown) => unknown) = undefined;
const creationPromise = new Promise<T>(
(resolve_, reject_) =>
// eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion
((resolve = resolve_ as ResolveFunction<T>), (reject = reject_))
);
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
return [creationPromise, resolve!, reject!];
}

View file

@ -13,56 +13,64 @@ export class EventListeners<TListener extends (...args: any[]) => any> {
} }
/** /**
* Adds a new listener to the collection. * Adds a new listener to the collection.
* *
* @param listener The listener callback to add * @param listener The listener callback to add
* @returns An unsubscribe function that removes this listener when called * @returns An unsubscribe function that removes this listener when called
*/ */
public add(listener: TListener): () => void { public add(listener: TListener): () => void {
this.listeners.push(listener); this.listeners.push(listener);
return () => this.remove(listener); return () => this.remove(listener);
} }
/** /**
* Removes a listener from the collection. * Removes a listener from the collection.
* *
* @param listener The listener callback to remove * @param listener The listener callback to remove
* @returns true if the listener was found and removed, false otherwise * @returns true if the listener was found and removed, false otherwise
*/ */
public remove(listener: TListener): boolean { public remove(listener: TListener): boolean {
return removeFromArray(this.listeners, listener); return removeFromArray(this.listeners, listener);
} }
/** /**
* Triggers all listeners synchronously with the provided arguments. * Triggers all listeners synchronously with the provided arguments.
* Any returned promises are ignored. Use triggerAsync() to await them. * Any returned promises are ignored. Use triggerAsync() to await them.
* *
* @param args The arguments to pass to each listener * @param args The arguments to pass to each listener
*/ */
public trigger(...args: Parameters<TListener>): void { public trigger(...args: Parameters<TListener>): void {
this.listeners.forEach((listener) => { const snapshot = this.listeners.slice();
for (const listener of snapshot) {
// allow removing listeners during the trigger loop
if (!this.listeners.includes(listener)) {
continue;
}
listener(...args); listener(...args);
}); }
} }
/** /**
* Triggers all listeners and awaits any promises they return. * Triggers all listeners and awaits any promises they return.
* Synchronous listeners are called immediately, and any async listeners * Synchronous listeners are called immediately, and any async listeners
* are awaited in parallel. * are awaited in parallel.
* *
* @param args The arguments to pass to each listener * @param args The arguments to pass to each listener
*/ */
public async triggerAsync(...args: Parameters<TListener>): Promise<void> { public async triggerAsync(...args: Parameters<TListener>): Promise<void> {
await awaitAll( const snapshot = this.listeners.slice();
this.listeners const promises: Promise<unknown>[] = [];
.map((listener) => { for (const listener of snapshot) {
// eslint-disable-next-line @typescript-eslint/no-unsafe-return if (!this.listeners.includes(listener)) {
return listener(...args); continue;
}) }
.filter((result): result is Promise<unknown> => { // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
return result instanceof Promise; const result = listener(...args);
}) if (result instanceof Promise) {
); promises.push(result);
}
}
await awaitAll(promises);
} }
public clear(): void { public clear(): void {

View file

@ -1,6 +1,6 @@
// Implements an in-memory fixed-size cache for document contents, // Implements an in-memory fixed-size cache for document contents,
import type { VaultUpdateId } from "../../persistence/database"; import type { VaultUpdateId } from "../../sync-operations/types";
// Doubly-linked list node for O(1) LRU operations // Doubly-linked list node for O(1) LRU operations
class LRUNode { class LRUNode {

View file

@ -1,22 +1,24 @@
import { describe, it, beforeEach } from "node:test"; import { describe, it, beforeEach } from "node:test";
import assert from "node:assert"; import assert from "node:assert";
import { Logger } from "../../tracing/logger"; import { Logger } from "../../tracing/logger";
import type { RelativePath } from "../../persistence/database"; import type { RelativePath } from "../../sync-operations/types";
import { Locks } from "./locks"; import { Locks } from "./locks";
import { awaitAll } from "../await-all"; import { awaitAll } from "../await-all";
import { sleep } from "../sleep"; import { sleep } from "../sleep";
import { SyncResetError } from "../../services/sync-reset-error"; import { SyncResetError } from "../../errors/sync-reset-error";
describe("withLock", () => { describe("withLock", () => {
const testPath: RelativePath = "test/document/path"; const testPath: RelativePath = "test/document/path";
const testPath2: RelativePath = "test/document/path2"; const testPath2: RelativePath = "test/document/path2";
const testPath3: RelativePath = "test/document/path3";
const logger = new Logger(); const logger = new Logger();
// eslint-disable-next-line @typescript-eslint/init-declarations // eslint-disable-next-line @typescript-eslint/init-declarations
let locks: Locks<RelativePath>; let locks: Locks<RelativePath>;
beforeEach(() => { beforeEach(() => {
locks = new Locks<RelativePath>(logger); locks = new Locks<RelativePath>("locks-test", logger);
}); });
it("should execute function with single key lock", async () => { it("should execute function with single key lock", async () => {
@ -56,22 +58,32 @@ describe("withLock", () => {
it("should sort multiple keys to prevent deadlocks", async () => { it("should sort multiple keys to prevent deadlocks", async () => {
const executionOrder: string[] = []; const executionOrder: string[] = [];
// Start two concurrent operations with keys in different orders await locks.waitForLock(testPath);
const promise1 = locks.withLock([testPath2, testPath], async () => {
executionOrder.push("operation1-start");
await sleep(50);
executionOrder.push("operation1-end");
return "result1";
});
const promise2 = locks.withLock([testPath, testPath2], async () => { const promise = awaitAll([
executionOrder.push("operation2-start"); locks.withLock([testPath2, testPath3, testPath], async () => {
await sleep(50); executionOrder.push("operation1-start");
executionOrder.push("operation2-end"); executionOrder.push("operation1-end");
return "result2"; return "result1";
}); }),
const [result1, result2] = await awaitAll([promise1, promise2]); locks.withLock([testPath3, testPath, testPath2], async () => {
executionOrder.push("operation2-start");
executionOrder.push("operation2-end");
return "result2";
})
]);
locks.unlock(testPath);
const [result1, result2] = await Promise.race([
promise,
new Promise<never>((_, reject) => {
setTimeout(() => {
reject(new Error("Deadlock detected"));
}, 1000);
})
]);
assert.strictEqual(result1, "result1"); assert.strictEqual(result1, "result1");
assert.strictEqual(result2, "result2"); assert.strictEqual(result2, "result2");
@ -234,13 +246,14 @@ describe("withLock", () => {
describe("reset", () => { describe("reset", () => {
const testPath: RelativePath = "test/document/path"; const testPath: RelativePath = "test/document/path";
const testPath2: RelativePath = "test/document/path2";
const logger = new Logger(); const logger = new Logger();
// eslint-disable-next-line @typescript-eslint/init-declarations // eslint-disable-next-line @typescript-eslint/init-declarations
let locks: Locks<RelativePath>; let locks: Locks<RelativePath>;
beforeEach(() => { beforeEach(() => {
locks = new Locks<RelativePath>(logger); locks = new Locks<RelativePath>("locks-test", logger);
}); });
it("should reject pending waiters with SyncResetError while running operation completes", async () => { it("should reject pending waiters with SyncResetError while running operation completes", async () => {
@ -289,4 +302,38 @@ describe("reset", () => {
const result = await locks.withLock(testPath, () => "success"); const result = await locks.withLock(testPath, () => "success");
assert.strictEqual(result, "success"); assert.strictEqual(result, "success");
}); });
it("should release partially acquired locks when reset interrupts multi-key acquisition", async () => {
// Hold testPath2 so multi-key acquisition will block on it
await locks.waitForLock(testPath2);
// Start multi-key lock that will acquire testPath first, then block on testPath2
const multiKeyPromise = locks.withLock(
[testPath, testPath2],
async () => "multi"
);
void multiKeyPromise.catch(() => {}); // eslint-disable-line @typescript-eslint/no-empty-function
// Wait for the multi-key operation to acquire testPath and start waiting on testPath2
await sleep(10);
// Reset should reject the waiting operation
locks.reset();
await assert.rejects(multiKeyPromise, (err: Error) => {
assert.ok(err instanceof SyncResetError);
return true;
});
// The key that was already acquired (testPath) should now be released
// This would hang/timeout if the lock was leaked
const result = await Promise.race([
locks.withLock(testPath, () => "success"),
sleep(100).then(() => {
throw new Error("Lock was not released - deadlock detected");
})
]);
assert.strictEqual(result, "success");
});
}); });

View file

@ -1,6 +1,5 @@
import { SyncResetError } from "../../services/sync-reset-error"; import { SyncResetError } from "../../errors/sync-reset-error";
import type { Logger } from "../../tracing/logger"; import type { Logger } from "../../tracing/logger";
import { awaitAll } from "../await-all";
/** /**
* Manages exclusive locks on items to prevent concurrent modifications. * Manages exclusive locks on items to prevent concurrent modifications.
@ -8,47 +7,53 @@ import { awaitAll } from "../await-all";
* *
* @template T The type of the key used for locking * @template T The type of the key used for locking
*/ */
/** Waiter entry with callbacks */
interface WaiterEntry {
resolve: () => unknown;
reject: (err: unknown) => unknown;
}
export class Locks<T> { export class Locks<T> {
/** Currently locked keys */ /** Currently locked keys */
private readonly locked = new Set<T>(); private readonly locked = new Set<T>();
/** Queue of resolve functions waiting for each key */ /** Queue of waiters for each key */
private readonly waiters = new Map< private readonly waiters = new Map<T, WaiterEntry[]>();
T,
[() => unknown, (err: unknown) => unknown][]
>();
public constructor(private readonly logger?: Logger) {} public constructor(
private readonly name: string,
private readonly logger?: Logger
) {}
/** /**
* Executes a function while holding exclusive locks on one or more keys. * Executes a function while holding exclusive locks on one or more keys.
* *
* This method ensures that the provided function runs with exclusive access to the * This method ensures that the provided function runs with exclusive access to the
* specified key(s). Multiple keys are sorted to prevent deadlocks when different * specified key(s). Multiple keys are sorted to prevent deadlocks when different
* operations request the same keys in different orders. * operations request the same keys in different orders.
* *
* @template R The return type of the function to execute * @template R The return type of the function to execute
* @param keyOrKeys A single key or array of keys to lock during function execution * @param keyOrKeys A single key or array of keys to lock during function execution
* @param fn The function to execute while holding the lock(s). Can be sync or async. * @param fn The function to execute while holding the lock(s). Can be sync or async.
* @returns A Promise that resolves to the return value of the executed function * @returns A Promise that resolves to the return value of the executed function
* *
* @example * @example
* ```typescript * ```typescript
* // Lock a single key * // Lock a single key
* const result = await locks.withLock('file1', () => { * const result = await locks.withLock('file1', () => {
* // Critical section - only one operation can access 'file1' at a time * // Critical section - only one operation can access 'file1' at a time
* return processFile('file1'); * return processFile('file1');
* }); * });
* *
* // Lock multiple keys (prevents deadlocks through consistent ordering) * // Lock multiple keys (prevents deadlocks through consistent ordering)
* await locks.withLock(['file1', 'file2'], async () => { * await locks.withLock(['file1', 'file2'], async () => {
* // Critical section - exclusive access to both files * // Critical section - exclusive access to both files
* await moveFile('file1', 'file2'); * await moveFile('file1', 'file2');
* }); * });
* ``` * ```
* *
* @throws Any error thrown by the provided function will be propagated after locks are released * @throws Any error thrown by the provided function will be propagated after locks are released
*/ */
public async withLock<R>( public async withLock<R>(
keyOrKeys: T | T[], keyOrKeys: T | T[],
fn: () => R | Promise<R> fn: () => R | Promise<R>
@ -59,12 +64,17 @@ export class Locks<T> {
const uniqueKeys = Array.from(new Set(keys)); const uniqueKeys = Array.from(new Set(keys));
uniqueKeys.sort((a, b) => String(a).localeCompare(String(b))); // Ensure consistent order to prevent deadlocks uniqueKeys.sort((a, b) => String(a).localeCompare(String(b))); // Ensure consistent order to prevent deadlocks
await awaitAll(uniqueKeys.map(async (key) => this.waitForLock(key))); const lockedKeys = [];
try { try {
for (const key of uniqueKeys) {
// Must acquire locks in-order (not concurrently) to prevent deadlocks
await this.waitForLock(key);
lockedKeys.push(key);
}
return await fn(); return await fn();
} finally { } finally {
uniqueKeys.forEach((key) => { lockedKeys.forEach((key) => {
this.unlock(key); this.unlock(key);
}); });
} }
@ -74,7 +84,7 @@ export class Locks<T> {
// Resolve all waiting promises before clearing to prevent deadlock // Resolve all waiting promises before clearing to prevent deadlock
// Any operation waiting for a lock will be granted access immediately // Any operation waiting for a lock will be granted access immediately
for (const waiting of this.waiters.values()) { for (const waiting of this.waiters.values()) {
for (const [_, reject] of waiting) { for (const { reject } of waiting) {
reject(new SyncResetError()); reject(new SyncResetError());
} }
} }
@ -82,13 +92,17 @@ export class Locks<T> {
this.waiters.clear(); this.waiters.clear();
} }
public isLocked(key: T): boolean {
return this.locked.has(key);
}
/** /**
* Attempts to acquire a lock immediately without waiting. * Attempts to acquire a lock immediately without waiting.
* Must call `unlock()` if successful. * Must call `unlock()` if successful.
* *
* @param key The key to lock * @param key The key to lock
* @returns `true` if lock acquired, `false` if already locked * @returns `true` if lock acquired, `false` if already locked
*/ */
public tryLock(key: T): boolean { public tryLock(key: T): boolean {
if (this.locked.has(key)) { if (this.locked.has(key)) {
return false; return false;
@ -100,18 +114,18 @@ export class Locks<T> {
} }
/** /**
* Waits to acquire a lock, blocking until available. * Waits to acquire a lock, blocking until available.
* Operations are queued in FIFO order. Must call `unlock()` when done. * Operations are queued in FIFO order. Must call `unlock()` when done.
* *
* @param key The key to wait for and lock * @param key The key to wait for and lock
* @returns Promise that resolves when lock is acquired * @returns Promise that resolves when lock is acquired
*/ */
public async waitForLock(key: T): Promise<void> { public async waitForLock(key: T): Promise<void> {
if (this.tryLock(key)) { if (this.tryLock(key)) {
return Promise.resolve(); return Promise.resolve();
} }
this.logger?.debug(`Waiting for lock on ${key}`); this.logger?.debug(`Waiting for lock '${this.name}' on '${key}'`);
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
// DefaultDict behavior // DefaultDict behavior
@ -121,28 +135,36 @@ export class Locks<T> {
this.waiters.set(key, waiting); this.waiters.set(key, waiting);
} }
waiting.push([resolve, reject]); waiting.push({
resolve,
reject
});
}); });
} }
/** /**
* Releases a lock and grants access to the next waiting operation in FIFO order. * Releases a lock and grants access to the next waiting operation in FIFO order.
* Removes the key from locked set if no waiters. * Removes the key from locked set if no waiters.
* *
* @param key The key to unlock * @param key The key to unlock
* @throws {Error} If key is not currently locked * @throws {Error} If key is not currently locked
*/ */
public unlock(key: T): void { public unlock(key: T): void {
if (!this.locked.has(key)) { if (!this.locked.has(key)) {
this.logger?.debug(
`Attempted to unlock '${this.name}' on '${key}' which is not locked`
);
return; return;
} }
// Remove first waiter to ensure FIFO order this.logger?.debug(`Releasing lock '${this.name}' on '${key}'`);
const [resolveNextWaiting, _] = this.waiters.get(key)?.shift() ?? [];
if (resolveNextWaiting) { // Remove first waiter to ensure FIFO order
this.logger?.debug(`Granted lock on ${key}`); const nextWaiter = this.waiters.get(key)?.shift();
resolveNextWaiting();
if (nextWaiter) {
this.logger?.debug(`Granted lock '${this.name}' on '${key}'`);
nextWaiter.resolve();
} else { } else {
this.locked.delete(key); this.locked.delete(key);
} }
@ -152,8 +174,8 @@ export class Locks<T> {
export class Lock { export class Lock {
private readonly locks: Locks<boolean>; private readonly locks: Locks<boolean>;
public constructor(logger?: Logger) { public constructor(name: string, logger?: Logger) {
this.locks = new Locks(logger); this.locks = new Locks(name, logger);
} }
public async withLock<R>(fn: () => R | Promise<R>): Promise<R> { public async withLock<R>(fn: () => R | Promise<R>): Promise<R> {

View file

@ -1,15 +1,15 @@
import { describe, it } from "node:test"; import { describe, it } from "node:test";
import assert from "node:assert"; import assert from "node:assert";
import { CoveredValues } from "./min-covered"; import { MinCovered } from "./min-covered";
describe("CoveredValues", () => { describe("MinCovered", () => {
it("should initialize with the given min value", () => { it("should initialize with the given min value", () => {
const covered = new CoveredValues(5); const covered = new MinCovered(5);
assert.strictEqual(covered.min, 5); assert.strictEqual(covered.min, 5);
}); });
it("should add values greater than min", () => { it("should add values greater than min", () => {
const covered = new CoveredValues(0); const covered = new MinCovered(0);
covered.add(3); covered.add(3);
assert.strictEqual(covered.min, 0); assert.strictEqual(covered.min, 0);
covered.add(1); covered.add(1);
@ -21,7 +21,7 @@ describe("CoveredValues", () => {
}); });
it("should ignore duplicate values", () => { it("should ignore duplicate values", () => {
const covered = new CoveredValues(0); const covered = new MinCovered(0);
covered.add(3); covered.add(3);
covered.add(3); covered.add(3);
covered.add(3); covered.add(3);
@ -32,7 +32,7 @@ describe("CoveredValues", () => {
}); });
it("should handle multiple consecutive values", () => { it("should handle multiple consecutive values", () => {
const covered = new CoveredValues(132); const covered = new MinCovered(132);
for (let i = 250; i > 132; i--) { for (let i = 250; i > 132; i--) {
assert.strictEqual(covered.min, 132); assert.strictEqual(covered.min, 132);
covered.add(i); covered.add(i);
@ -41,36 +41,32 @@ describe("CoveredValues", () => {
}); });
it("should handle adding values lower than current min", () => { it("should handle adding values lower than current min", () => {
const covered = new CoveredValues(5); const covered = new MinCovered(5);
covered.add(3); covered.add(3);
assert.strictEqual(covered.min, 5); assert.strictEqual(covered.min, 5);
covered.add(6); covered.add(6);
assert.strictEqual(covered.min, 6); assert.strictEqual(covered.min, 6);
}); });
it("should auto-advance when setting min value", () => { it("should auto-advance when adding the value that fills the next gap", () => {
const covered = new CoveredValues(5); const covered = new MinCovered(5);
covered.add(7); covered.add(7);
covered.add(8); covered.add(8);
covered.add(9); covered.add(9);
assert.strictEqual(covered.min, 5); assert.strictEqual(covered.min, 5);
// Setting min to 6 should auto-advance through 7, 8, 9 // Adding 6 fills the gap and auto-advances through 7, 8, 9
covered.min = 6; covered.add(6);
assert.strictEqual(covered.min, 9); assert.strictEqual(covered.min, 9);
covered.add(10); covered.add(10);
assert.strictEqual(covered.min, 10); assert.strictEqual(covered.min, 10);
}); });
it("should handle setting min value with no consecutive values", () => { it("should rewind when reset is called explicitly", () => {
const covered = new CoveredValues(5); const covered = new MinCovered(5);
covered.add(10); covered.add(7);
covered.add(15); covered.reset(3);
assert.strictEqual(covered.min, 5); assert.strictEqual(covered.min, 3);
// Setting min to 8 should not auto-advance (no consecutive values) covered.add(4);
covered.min = 8; assert.strictEqual(covered.min, 4);
assert.strictEqual(covered.min, 8);
// Add 9 to trigger auto-advance to 10
covered.add(9);
assert.strictEqual(covered.min, 10);
}); });
}); });

View file

@ -7,13 +7,13 @@
* *
* @example * @example
* ```typescript * ```typescript
* const covered = new CoveredValues(0); * const covered = new MinCovered(0);
* covered.add(2); // seenValues = [2], min = 0 * covered.add(2); // seenValues = [2], min = 0
* covered.add(1); // seenValues = [], min = 2 * covered.add(1); // seenValues = [], min = 2
* covered.min; // returns 2 * covered.min; // returns 2
* ``` * ```
*/ */
export class CoveredValues { export class MinCovered {
private seenValues: number[] = []; private seenValues: number[] = [];
public constructor(private minValue: number) {} public constructor(private minValue: number) {}
@ -22,12 +22,6 @@ export class CoveredValues {
return this.minValue; return this.minValue;
} }
public set min(value: number) {
this.minValue = Math.max(value, this.minValue);
this.seenValues = this.seenValues.filter((v) => v > this.minValue);
this.advanceMinWhilePossible();
}
public add(value: number | undefined): void { public add(value: number | undefined): void {
if (value === undefined || value < this.minValue) { if (value === undefined || value < this.minValue) {
return; return;
@ -49,6 +43,11 @@ export class CoveredValues {
this.advanceMinWhilePossible(); this.advanceMinWhilePossible();
} }
public reset(minValue?: number): void {
this.minValue = minValue ?? 0;
this.seenValues = [];
}
private advanceMinWhilePossible(): void { private advanceMinWhilePossible(): void {
while ( while (
this.seenValues.length > 0 && this.seenValues.length > 0 &&

View file

@ -0,0 +1,69 @@
import type { RelativePath } from "../../sync-operations/types";
import type { TextWithCursors } from "reconcile-text";
import type { FileSystemOperations } from "../../file-operations/filesystem-operations";
export class InMemoryFileSystem implements FileSystemOperations {
protected readonly files = new Map<string, Uint8Array>();
public async listFilesRecursively(
_root: RelativePath | undefined = undefined // we don't use multi-level paths during tests
): Promise<RelativePath[]> {
return Array.from(this.files.keys());
}
public async read(path: RelativePath): Promise<Uint8Array> {
const file = this.files.get(path);
if (!file) {
throw new Error(`File ${path} does not exist`);
}
return file;
}
public async write(path: RelativePath, content: Uint8Array): Promise<void> {
this.files.set(path, content);
}
public async atomicUpdateText(
path: RelativePath,
updater: (current: TextWithCursors) => TextWithCursors
): Promise<string> {
const file = this.files.get(path);
if (!file) {
throw new Error(`File ${path} does not exist`);
}
const currentContent = new TextDecoder().decode(file);
const newContent = updater({ text: currentContent, cursors: [] }).text;
this.files.set(path, new TextEncoder().encode(newContent));
return newContent;
}
public async getFileSize(path: RelativePath): Promise<number> {
return (await this.read(path)).length;
}
public async exists(path: RelativePath): Promise<boolean> {
return this.files.has(path);
}
public async createDirectory(_path: RelativePath): Promise<void> {
// This doesn't mean anything in our virtual FS representation
}
public async delete(path: RelativePath): Promise<void> {
this.files.delete(path);
}
public async rename(
oldPath: RelativePath,
newPath: RelativePath
): Promise<void> {
const file = this.files.get(oldPath);
if (!file) {
throw new Error(`File ${oldPath} does not exist`);
}
this.files.set(newPath, file);
if (oldPath !== newPath) {
this.files.delete(oldPath);
}
}
}

View file

@ -1,10 +1,44 @@
import type { SyncClient } from "../../sync-client"; /* eslint-disable no-console */
import type { LogLine } from "../../tracing/logger"; import type { Logger, LogLine } from "../../tracing/logger";
import { LogLevel } from "../../tracing/logger"; import { LogLevel } from "../../tracing/logger";
export function logToConsole(client: SyncClient): void { const COLORS = {
client.logger.onLogEmitted.add((logLine: LogLine) => { reset: "\x1b[0m",
const formatted = `${logLine.timestamp.toISOString()} ${logLine.level} ${logLine.message}`; red: "\x1b[31m",
yellow: "\x1b[33m",
blue: "\x1b[34m",
gray: "\x1b[90m"
};
export function logToConsole(
logger: Logger,
{ useColors = true }: { useColors?: boolean } = {}
): void {
logger.onLogEmitted.add((logLine: LogLine) => {
const timestamp = logLine.timestamp.toISOString();
const { message } = logLine;
let color = "";
let reset = "";
if (useColors) {
({ reset } = COLORS);
switch (logLine.level) {
case LogLevel.ERROR:
color = COLORS.red;
break;
case LogLevel.WARNING:
color = COLORS.yellow;
break;
case LogLevel.INFO:
color = COLORS.blue;
break;
case LogLevel.DEBUG:
color = COLORS.gray;
break;
}
}
const formatted = `${timestamp} ${color}${logLine.level}${reset} ${message}`;
switch (logLine.level) { switch (logLine.level) {
case LogLevel.ERROR: case LogLevel.ERROR:

View file

@ -11,7 +11,7 @@ export function slowWebSocketFactory(
private static readonly RECEIVE_KEY = "websocket-receive"; private static readonly RECEIVE_KEY = "websocket-receive";
private static readonly SEND_KEY = "websocket-send"; private static readonly SEND_KEY = "websocket-send";
private readonly locks = new Locks(logger); private readonly locks = new Locks(FlakyWebSocket.name, logger);
public set onopen(callback: ((event: Event) => void) | null) { public set onopen(callback: ((event: Event) => void) | null) {
super.onopen = async (event: Event): Promise<void> => { super.onopen = async (event: Event): Promise<void> => {

View file

@ -1,14 +1,17 @@
import type { DocumentRecord } from "../persistence/database"; import type { DocumentRecord } from "../sync-operations/types";
import { EMPTY_HASH } from "./hash"; import { EMPTY_HASH } from "./hash";
// TODO: make this smarter so that offline files can be renamed & edited at the same time // TODO: make this smarter so that offline files can be renamed & edited at the same time
export function findMatchingFile( export async function findMatchingFile(
contentHash: string, contentHash: string,
candidates: DocumentRecord[] candidates: DocumentRecord[]
): DocumentRecord | undefined { ): Promise<DocumentRecord | undefined> {
if (contentHash === EMPTY_HASH) { if (contentHash === (await EMPTY_HASH)) {
return undefined; return undefined;
} }
return candidates.find(({ metadata }) => metadata?.hash === contentHash); return candidates.find(
(record) =>
record.remoteHash !== undefined && record.remoteHash === contentHash
);
} }

View file

@ -1,12 +1,14 @@
// https://stackoverflow.com/questions/7616461/generate-a-hash-from-string-in-javascript export async function hash(content: Uint8Array): Promise<string> {
export function hash(content: Uint8Array): string { // Re-wrap into a fresh Uint8Array<ArrayBuffer> so SubtleCrypto's
let result = 0; // BufferSource overload accepts it without an unsafe type assertion.
// eslint-disable-next-line @typescript-eslint/prefer-for-of // The lib types require an ArrayBuffer-backed view; the source may
for (let i = 0; i < content.length; i++) { // be backed by SharedArrayBuffer in some runtimes.
result = (result << 5) - result + content[i]; const buffer = new ArrayBuffer(content.byteLength);
result |= 0; // Convert to 32bit integer new Uint8Array(buffer).set(content);
} const digest = await crypto.subtle.digest("SHA-256", buffer);
return Math.abs(result).toString(16).padStart(8, "0"); const bytes = new Uint8Array(digest);
return Array.from(bytes, (b) => b.toString(16).padStart(2, "0")).join("");
} }
export const EMPTY_HASH = hash(new Uint8Array(0)); // SHA-256 of empty content, computed once at import time
export const EMPTY_HASH: Promise<string> = hash(new Uint8Array());

View file

@ -1,4 +1,4 @@
import { createPromise } from "./create-promise"; import { awaitAll } from "./await-all";
import { sleep } from "./sleep"; import { sleep } from "./sleep";
/** /**
@ -45,18 +45,16 @@ export function rateLimit<
newArgs = undefined; newArgs = undefined;
} }
const [promise, resolve] = createPromise(); // `running` must signal both "minimum interval has elapsed" *and*
running = promise; // "fn() has finished" — otherwise an `fn` that takes longer than
sleep( // the interval would let a queued waiter fire a concurrent `fn`
const interval =
typeof minIntervalMs === "function" typeof minIntervalMs === "function"
? minIntervalMs() ? minIntervalMs()
: minIntervalMs : minIntervalMs;
) const fnPromise = fn(...args);
.then(resolve) running = awaitAll([fnPromise.catch(() => undefined), sleep(interval)]);
.catch(() => { return fnPromise;
// sleep cannot fail
});
return fn(...args);
}; };
return decoratedFn; return decoratedFn;

6
package-lock.json generated Normal file
View file

@ -0,0 +1,6 @@
{
"name": "vault-link",
"lockfileVersion": 3,
"requires": true,
"packages": {}
}

11
rustfmt.toml Normal file
View file

@ -0,0 +1,11 @@
# Rustfmt configuration
# This should match the .editorconfig settings
# Use spaces for indentation (matches .editorconfig indent_style = space)
hard_tabs = false
# Use 4 spaces for indentation (matches .editorconfig indent_size = 4)
tab_spaces = 4
# Use Unix line endings (matches .editorconfig end_of_line = lf)
newline_style = "Unix"

View file

@ -35,7 +35,8 @@ cd ..
cp frontend/obsidian-plugin/manifest.json manifest.json # for BRAT, otherwise it wouldn't update cp frontend/obsidian-plugin/manifest.json manifest.json # for BRAT, otherwise it wouldn't update
git ls-files | xargs npx eclint fix # Format all files across the project (frontend and backend)
npx -C frontend prettier --write "**/*.{ts,js,json,md,yml,yaml}"
# Commit and tag # Commit and tag
git add . git add .

View file

@ -30,8 +30,11 @@ fi
which cargo-machete || cargo install cargo-machete which cargo-machete || cargo install cargo-machete
cargo machete --with-metadata cargo machete --with-metadata
cd ..
scripts/update-api-types.sh # this will dirty up the git state if not up-to-date
echo "Running checks in frontend" echo "Running checks in frontend"
cd ../frontend cd frontend
if [[ "$FIX_MODE" == true ]]; then if [[ "$FIX_MODE" == true ]]; then
npm install npm install
@ -45,10 +48,11 @@ cd frontend
npm run build npm run build
npm run test npm run test
npm run lint npm run lint
cd ..
# Use git ls-files to only check tracked files, respecting .gitignore # Format all files across the project (frontend and backend)
# We always run in fix mode and then check with git status # Prettier respects .gitignore by default
git ls-files | xargs npx eclint fix npx -C frontend prettier --write "**/*.{ts,js,json,md,yml,yaml}"
if [[ "$FIX_MODE" == false ]] && [[ $(git status --porcelain) ]]; then if [[ "$FIX_MODE" == false ]] && [[ $(git status --porcelain) ]]; then
git status --porcelain git status --porcelain
@ -56,6 +60,4 @@ if [[ "$FIX_MODE" == false ]] && [[ $(git status --porcelain) ]]; then
exit 1 exit 1
fi fi
cd ..
echo "Success" echo "Success"

View file

@ -1,4 +1,4 @@
#!/bin/bash #!/bin/bash
rm -rf sync-server/databases rm -rf /host/tmp/vaultlink-e2e-databases
rm -rf logs rm -rf logs

View file

@ -19,35 +19,51 @@ process_count=$1
mkdir -p logs mkdir -p logs
# Build and restart the server
echo "Building server..."
cd sync-server
cargo build --release
# Kill any existing server process
echo "Stopping existing server..."
pkill -f "sync_server" 2>/dev/null || true
sleep 1
# Clean databases (uses tmpfs via /dev/shm for zero disk I/O)
echo "Cleaning databases..."
rm -rf /host/tmp/vaultlink-e2e-databases
# Start the server in the background
echo "Starting server..."
./target/release/sync_server config-e2e.yml &
server_pid=$!
echo "Server started with PID: $server_pid"
# Ensure server is killed on script exit
cleanup_server() {
if [ -n "$server_pid" ]; then
echo "Stopping server (PID: $server_pid)..."
kill $server_pid 2>/dev/null || true
wait $server_pid 2>/dev/null || true
server_pid=""
fi
}
trap cleanup_server EXIT
cd ..
cd frontend cd frontend
npm ci npm ci
npm run build npm run build
../scripts/utils/wait-for-server.sh ../scripts/utils/wait-for-server.sh
cd ..
scripts/update-api-types.sh
if [[ $(git status --porcelain) ]]; then
git status --porcelain
echo "Failing CI because the working directory is not clean after generating api types"
exit 1
fi
cd frontend
pids=() pids=()
for i in $(seq 1 $process_count); do for i in $(seq 1 $process_count); do
# Create a named pipe for this process node test-client/dist/cli.js > "../logs/log_${i}.log" 2>&1 &
pipe="/tmp/vaultlink_pipe_$$_$i"
mkfifo "$pipe"
# Start the node process writing to the pipe
node test-client/dist/cli.js > "$pipe" 2>&1 &
pid=$! pid=$!
pids+=($pid) pids+=($pid)
echo "Started process $i with PID: $pid" echo "Started process $i with PID: $pid (log: logs/log_${i}.log)"
# Read from pipe, prefix with PID
(sed "s/^/[PID $pid] /" < "$pipe" > "../logs/log_${i}.log"; rm "$pipe") &
done done
cd .. cd ..
@ -75,10 +91,25 @@ print_failed_log() {
return 1 return 1
} }
echo "Monitoring $process_count processes" E2E_TIMEOUT=${2:-3600}
start_time=$(date +%s)
echo "Monitoring $process_count processes (timeout: ${E2E_TIMEOUT}s)"
# Monitor processes # Monitor processes
while true; do while true; do
# Script-level timeout to prevent indefinite hangs
current_time=$(date +%s)
elapsed=$((current_time - start_time))
if [ $elapsed -ge $E2E_TIMEOUT ]; then
echo "E2E timeout reached (${E2E_TIMEOUT}s). Killing remaining processes."
for pid in "${pids[@]}"; do
if [ -n "$pid" ]; then
kill $pid 2>/dev/null || true
fi
done
exit 1
fi
if print_failed_log; then if print_failed_log; then
# Kill remaining processes # Kill remaining processes
for pid in "${pids[@]}"; do for pid in "${pids[@]}"; do
@ -99,6 +130,7 @@ while true; do
done done
if $all_done; then if $all_done; then
cleanup_server
echo "All processes completed successfully" echo "All processes completed successfully"
exit 0 exit 0
fi fi

View file

@ -8,9 +8,15 @@ cd sync-server
cargo test export_bindings cargo test export_bindings
cd - cd -
# Both target directories contain only generated bindings — wipe and copy
rm -f frontend/sync-client/src/services/types/*.ts
rm -f frontend/history-ui/src/lib/types/*.ts
cp -r sync-server/bindings/* frontend/sync-client/src/services/types/ cp -r sync-server/bindings/* frontend/sync-client/src/services/types/
cp -r sync-server/bindings/* frontend/history-ui/src/lib/types/
cd frontend cd frontend
npm run lint npm run lint
git ls-files | xargs npx eclint fix cd ..
cd -
# Format all files across the project (frontend and backend)
npx -C frontend prettier --write "**/*.{ts,js,json,md,yml,yaml}"

View file

@ -2,8 +2,10 @@
set -e set -e
TARGET_NODE_VERSION=25
node_version=$(node -v | sed 's/^v\([0-9]*\).*/\1/') node_version=$(node -v | sed 's/^v\([0-9]*\).*/\1/')
if [ "$node_version" != "22" ]; then if [ "$node_version" != "$TARGET_NODE_VERSION" ]; then
echo "Error: This script requires Node.js version 22, found: $node_version" echo "Error: This script requires Node.js version $TARGET_NODE_VERSION, found: $node_version"
exit 1 exit 1
fi fi

View file

@ -2,14 +2,14 @@
set -e set -e
SERVER_URL="http://localhost:3000" SERVER_URL="http://localhost:3010"
MAX_RETRIES=30 MAX_RETRIES=30
RETRY_INTERVAL_IN_SECONDS=5 RETRY_INTERVAL_IN_SECONDS=5
echo "Waiting for $SERVER_URL to become available..." echo "Waiting for $SERVER_URL to become available..."
count=0 count=0
while [ $count -lt $MAX_RETRIES ]; do while [ $count -lt $MAX_RETRIES ]; do
if curl -s -f -o /dev/null $SERVER_URL; then if curl -s -o /dev/null $SERVER_URL; then
echo "$SERVER_URL is now available!" echo "$SERVER_URL is now available!"
break break
fi fi

193
sync-server/Cargo.lock generated
View file

@ -337,10 +337,11 @@ checksum = "325918d6fe32f23b19878fe4b34794ae41fc19ddbe53b10571a4874d44ffd39b"
[[package]] [[package]]
name = "cc" name = "cc"
version = "1.2.2" version = "1.2.57"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f34d93e62b03caf570cccc334cbc6c2fceca82f39211051345108adcba3eebdc" checksum = "7a0dd1ca384932ff3641c8718a02769f1698e7563dc6974ffd03346116310423"
dependencies = [ dependencies = [
"find-msvc-tools",
"shlex", "shlex",
] ]
@ -456,6 +457,15 @@ version = "2.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5"
[[package]]
name = "crossbeam-channel"
version = "0.5.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2"
dependencies = [
"crossbeam-utils",
]
[[package]] [[package]]
name = "crossbeam-queue" name = "crossbeam-queue"
version = "0.3.11" version = "0.3.11"
@ -533,6 +543,15 @@ dependencies = [
"zeroize", "zeroize",
] ]
[[package]]
name = "deranged"
version = "0.5.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c"
dependencies = [
"powerfmt",
]
[[package]] [[package]]
name = "digest" name = "digest"
version = "0.10.7" version = "0.10.7"
@ -624,6 +643,12 @@ version = "2.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "486f806e73c5707928240ddc295403b1b93c96a02038563881c4a2fd84b81ac4" checksum = "486f806e73c5707928240ddc295403b1b93c96a02038563881c4a2fd84b81ac4"
[[package]]
name = "find-msvc-tools"
version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582"
[[package]] [[package]]
name = "flume" name = "flume"
version = "0.11.1" version = "0.11.1"
@ -1272,6 +1297,16 @@ version = "0.3.17"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a"
[[package]]
name = "mime_guess"
version = "2.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e"
dependencies = [
"mime",
"unicase",
]
[[package]] [[package]]
name = "miniz_oxide" name = "miniz_oxide"
version = "0.8.0" version = "0.8.0"
@ -1335,6 +1370,12 @@ dependencies = [
"zeroize", "zeroize",
] ]
[[package]]
name = "num-conv"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c6673768db2d862beb9b39a78fdcb1a69439615d5794a1be50caa9bc92c81967"
[[package]] [[package]]
name = "num-integer" name = "num-integer"
version = "0.1.46" version = "0.1.46"
@ -1463,6 +1504,12 @@ version = "0.3.31"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2" checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2"
[[package]]
name = "powerfmt"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391"
[[package]] [[package]]
name = "ppv-lite86" name = "ppv-lite86"
version = "0.2.20" version = "0.2.20"
@ -1582,12 +1629,12 @@ dependencies = [
[[package]] [[package]]
name = "reconcile-text" name = "reconcile-text"
version = "0.8.0" version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "599cf9539996a2a19e501110404c59ba62f4974009f8fb864a8b7151c15ee5a5" checksum = "52e0cf361887ea64c479ca871c1170dda761f84e122f2616b5579906a38d7557"
dependencies = [ dependencies = [
"serde", "serde",
"thiserror 2.0.17", "thiserror 2.0.18",
] ]
[[package]] [[package]]
@ -1648,6 +1695,40 @@ dependencies = [
"zeroize", "zeroize",
] ]
[[package]]
name = "rust-embed"
version = "8.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "04113cb9355a377d83f06ef1f0a45b8ab8cd7d8b1288160717d66df5c7988d27"
dependencies = [
"rust-embed-impl",
"rust-embed-utils",
"walkdir",
]
[[package]]
name = "rust-embed-impl"
version = "8.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "da0902e4c7c8e997159ab384e6d0fc91c221375f6894346ae107f47dd0f3ccaa"
dependencies = [
"proc-macro2",
"quote",
"rust-embed-utils",
"syn 2.0.90",
"walkdir",
]
[[package]]
name = "rust-embed-utils"
version = "8.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5bcdef0be6fe7f6fa333b1073c949729274b05f123a0ad7efcb8efd878e5c3b1"
dependencies = [
"sha2",
"walkdir",
]
[[package]] [[package]]
name = "rustc-demangle" name = "rustc-demangle"
version = "0.1.24" version = "0.1.24"
@ -1679,6 +1760,15 @@ version = "1.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f"
[[package]]
name = "same-file"
version = "1.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502"
dependencies = [
"winapi-util",
]
[[package]] [[package]]
name = "sanitize-filename" name = "sanitize-filename"
version = "0.6.0" version = "0.6.0"
@ -1916,7 +2006,7 @@ dependencies = [
"serde_json", "serde_json",
"sha2", "sha2",
"smallvec", "smallvec",
"thiserror 2.0.17", "thiserror 2.0.18",
"tokio", "tokio",
"tokio-stream", "tokio-stream",
"tracing", "tracing",
@ -2000,7 +2090,7 @@ dependencies = [
"smallvec", "smallvec",
"sqlx-core", "sqlx-core",
"stringprep", "stringprep",
"thiserror 2.0.17", "thiserror 2.0.18",
"tracing", "tracing",
"uuid", "uuid",
"whoami", "whoami",
@ -2039,7 +2129,7 @@ dependencies = [
"smallvec", "smallvec",
"sqlx-core", "sqlx-core",
"stringprep", "stringprep",
"thiserror 2.0.17", "thiserror 2.0.18",
"tracing", "tracing",
"uuid", "uuid",
"whoami", "whoami",
@ -2065,7 +2155,7 @@ dependencies = [
"serde", "serde",
"serde_urlencoded", "serde_urlencoded",
"sqlx-core", "sqlx-core",
"thiserror 2.0.17", "thiserror 2.0.18",
"tracing", "tracing",
"url", "url",
"uuid", "uuid",
@ -2100,6 +2190,12 @@ version = "2.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
[[package]]
name = "symlink"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a7973cce6668464ea31f176d85b13c7ab3bba2cb3b77a2ed26abd7801688010a"
[[package]] [[package]]
name = "syn" name = "syn"
version = "1.0.109" version = "1.0.109"
@ -2136,18 +2232,22 @@ dependencies = [
"futures", "futures",
"humantime-serde", "humantime-serde",
"log", "log",
"mime_guess",
"rand 0.9.0", "rand 0.9.0",
"reconcile-text", "reconcile-text",
"regex", "regex",
"rust-embed",
"sanitize-filename", "sanitize-filename",
"serde", "serde",
"serde_json", "serde_json",
"serde_yaml", "serde_yaml",
"sqlx", "sqlx",
"thiserror 2.0.17", "subtle",
"thiserror 2.0.18",
"tokio", "tokio",
"tower-http", "tower-http",
"tracing", "tracing",
"tracing-appender",
"tracing-subscriber", "tracing-subscriber",
"ts-rs", "ts-rs",
"uuid", "uuid",
@ -2203,11 +2303,11 @@ dependencies = [
[[package]] [[package]]
name = "thiserror" name = "thiserror"
version = "2.0.17" version = "2.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f63587ca0f12b72a0600bcba1d40081f830876000bb46dd2337a3051618f4fc8" checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4"
dependencies = [ dependencies = [
"thiserror-impl 2.0.17", "thiserror-impl 2.0.18",
] ]
[[package]] [[package]]
@ -2223,9 +2323,9 @@ dependencies = [
[[package]] [[package]]
name = "thiserror-impl" name = "thiserror-impl"
version = "2.0.17" version = "2.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913" checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
@ -2242,6 +2342,37 @@ dependencies = [
"once_cell", "once_cell",
] ]
[[package]]
name = "time"
version = "0.3.47"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c"
dependencies = [
"deranged",
"itoa",
"num-conv",
"powerfmt",
"serde_core",
"time-core",
"time-macros",
]
[[package]]
name = "time-core"
version = "0.1.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca"
[[package]]
name = "time-macros"
version = "0.2.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215"
dependencies = [
"num-conv",
"time-core",
]
[[package]] [[package]]
name = "tinystr" name = "tinystr"
version = "0.7.6" version = "0.7.6"
@ -2276,7 +2407,6 @@ dependencies = [
"bytes", "bytes",
"libc", "libc",
"mio", "mio",
"parking_lot",
"pin-project-lite", "pin-project-lite",
"signal-hook-registry", "signal-hook-registry",
"socket2", "socket2",
@ -2376,6 +2506,19 @@ dependencies = [
"tracing-core", "tracing-core",
] ]
[[package]]
name = "tracing-appender"
version = "0.2.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "050686193eb999b4bb3bc2acfa891a13da00f79734704c4b8b4ef1a10b368a3c"
dependencies = [
"crossbeam-channel",
"symlink",
"thiserror 2.0.18",
"time",
"tracing-subscriber",
]
[[package]] [[package]]
name = "tracing-attributes" name = "tracing-attributes"
version = "0.1.28" version = "0.1.28"
@ -2434,7 +2577,7 @@ checksum = "e640d9b0964e9d39df633548591090ab92f7a4567bc31d3891af23471a3365c6"
dependencies = [ dependencies = [
"chrono", "chrono",
"lazy_static", "lazy_static",
"thiserror 2.0.17", "thiserror 2.0.18",
"ts-rs-macros", "ts-rs-macros",
"uuid", "uuid",
] ]
@ -2481,6 +2624,12 @@ version = "0.10.4"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f720def6ce1ee2fc44d40ac9ed6d3a59c361c80a75a7aa8e75bb9baed31cf2ea" checksum = "f720def6ce1ee2fc44d40ac9ed6d3a59c361c80a75a7aa8e75bb9baed31cf2ea"
[[package]]
name = "unicase"
version = "2.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142"
[[package]] [[package]]
name = "unicode-bidi" name = "unicode-bidi"
version = "0.3.17" version = "0.3.17"
@ -2577,6 +2726,16 @@ version = "0.9.5"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
[[package]]
name = "walkdir"
version = "2.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b"
dependencies = [
"same-file",
"winapi-util",
]
[[package]] [[package]]
name = "wasi" name = "wasi"
version = "0.11.0+wasi-snapshot-preview1" version = "0.11.0+wasi-snapshot-preview1"

View file

@ -1,6 +1,6 @@
[package] [package]
name = "sync_server" name = "sync_server"
rust-version = "1.89.0" rust-version = "1.94.0"
authors = ["Andras Schmelczer <andras@schmelczer.dev>"] authors = ["Andras Schmelczer <andras@schmelczer.dev>"]
edition = "2024" edition = "2024"
license = "MIT" license = "MIT"
@ -10,7 +10,7 @@ version = "0.14.0"
[dependencies] [dependencies]
serde = { version = "1.0.219", default-features = false, features = ["derive"] } serde = { version = "1.0.219", default-features = false, features = ["derive"] }
thiserror = { version = "2.0.12", default-features = false } thiserror = { version = "2.0.12", default-features = false }
tokio = { version = "1.48.0", features = ["full"]} tokio = { version = "1.48.0", features = ["macros", "rt-multi-thread", "sync", "time", "net", "fs", "signal"]}
uuid = { version = "1.16.0", features = ["v4", "serde"] } uuid = { version = "1.16.0", features = ["v4", "serde"] }
log = { version = "0.4.28" } log = { version = "0.4.28" }
anyhow = { version = "1.0.100", features = ["backtrace"] } anyhow = { version = "1.0.100", features = ["backtrace"] }
@ -20,6 +20,7 @@ axum_typed_multipart = "0.11.0"
tower-http = { version = "0.6.1", features = ["cors", "trace", "limit", "timeout"] } tower-http = { version = "0.6.1", features = ["cors", "trace", "limit", "timeout"] }
tracing = "0.1.41" tracing = "0.1.41"
tracing-subscriber = { version = "0.3.20", features = ["fmt", "env-filter"]} tracing-subscriber = { version = "0.3.20", features = ["fmt", "env-filter"]}
tracing-appender = "0.2.5"
humantime-serde = "1.1.1" humantime-serde = "1.1.1"
sqlx = { version = "0.8.6", features = ["sqlite", "runtime-tokio", "uuid", "chrono"] } sqlx = { version = "0.8.6", features = ["sqlite", "runtime-tokio", "uuid", "chrono"] }
chrono = { version = "0.4.41", features = ["serde"] } chrono = { version = "0.4.41", features = ["serde"] }
@ -33,7 +34,10 @@ serde_json = "1.0.140"
bimap = "0.6.3" bimap = "0.6.3"
ts-rs = { version = "10.1", features = ["uuid-impl", "chrono-impl"] } ts-rs = { version = "10.1", features = ["uuid-impl", "chrono-impl"] }
base64 = "0.22.1" base64 = "0.22.1"
reconcile-text = { version = "0.8.0", features = ["serde"] } reconcile-text = { version = "0.11.0", features = ["serde"] }
rust-embed = "8.5"
mime_guess = "2.0"
subtle = "2.6.1"
[profile.release] [profile.release]
codegen-units = 1 codegen-units = 1

View file

@ -1,5 +1,16 @@
// generated by `sqlx migrate build-script`
fn main() { fn main() {
// trigger recompilation when a new migration is added // trigger recompilation when a new migration is added
println!("cargo:rerun-if-changed=migrations"); println!("cargo:rerun-if-changed=migrations");
// Ensure the history-ui dist directory exists so rust-embed can compile
// even when the frontend hasn't been built yet.
let dist_path = std::path::Path::new("../frontend/history-ui/dist");
if !dist_path.exists() {
std::fs::create_dir_all(dist_path).expect("Failed to create history-ui dist directory");
std::fs::write(
dist_path.join("index.html"),
"<!DOCTYPE html><html><body><p>Run <code>npm run build -w history-ui</code> first.</p></body></html>",
)
.expect("Failed to write placeholder index.html");
}
} }

View file

@ -1,32 +1,34 @@
database: database:
databases_directory_path: databases databases_directory_path: /host/tmp/vaultlink-e2e-databases
max_connections_per_vault: 12 max_connections_per_vault: 8
cursor_timeout: 1m cursor_timeout: 1m
server: server:
host: 0.0.0.0 host: 0.0.0.0
port: 3000 port: 3010
max_body_size_mb: 512 max_body_size_mb: 512
max_clients_per_vault: 256 max_clients_per_vault: 256
max_pending_websocket_connections: 4096
broadcast_channel_capacity: 1024
response_timeout: 30m response_timeout: 30m
mergeable_file_extensions: mergeable_file_extensions:
- md - md
- txt - txt
users: users:
user_configs: user_configs:
- name: admin - name: admin
token: test-token-change-me token: test-token-change-me
vault_access: vault_access:
type: allow_access_to_all type: allow_access_to_all
- name: other-admin - name: other-admin
token: test-token-change-me2 token: test-token-change-me2
vault_access: vault_access:
type: allow_access_to_all type: allow_access_to_all
- name: test - name: test
token: other-test-token token: other-test-token
vault_access: vault_access:
type: allow_list type: allow_list
allowed: allowed:
- default - default
logging: logging:
log_directory: logs log_directory: logs
log_rotation: 7days log_rotation: 7days

View file

@ -1,5 +1,5 @@
[toolchain] [toolchain]
channel = "1.89.0" channel = "1.94.0"
targets = [ targets = [
"x86_64-unknown-linux-gnu", "x86_64-unknown-linux-gnu",
"x86_64-unknown-linux-musl", "x86_64-unknown-linux-musl",

View file

@ -2,6 +2,8 @@ pub mod cursors;
pub mod database; pub mod database;
pub mod websocket; pub mod websocket;
use std::sync::{Arc, atomic::AtomicUsize};
use anyhow::Result; use anyhow::Result;
use cursors::Cursors; use cursors::Cursors;
use database::Database; use database::Database;
@ -15,21 +17,42 @@ pub struct AppState {
pub database: Database, pub database: Database,
pub cursors: Cursors, pub cursors: Cursors,
pub broadcasts: Broadcasts, pub broadcasts: Broadcasts,
/// Tracks WebSocket connections that have upgraded but not yet completed
/// the authentication handshake
pub pending_ws_connections: Arc<AtomicUsize>,
/// Send on this channel to stop background tasks (cursor cleanup,
/// idle-pool cleanup)
shutdown_tx: Arc<tokio::sync::watch::Sender<()>>,
} }
impl AppState { impl AppState {
pub async fn try_new(config: Config) -> Result<Self> { pub async fn try_new(config: Config) -> Result<Self> {
let (shutdown_tx, shutdown_rx) = tokio::sync::watch::channel(());
let broadcasts = Broadcasts::new(&config.server); let broadcasts = Broadcasts::new(&config.server);
let database = Database::try_new(&config.database, &broadcasts).await?; let database =
Database::try_new(&config.database, &broadcasts, shutdown_rx.clone()).await?;
let cursors: Cursors = Cursors::new(&config.database, &broadcasts); let cursors: Cursors = Cursors::new(&config.database, &broadcasts);
Cursors::start_background_task(cursors.clone()); Cursors::start_background_task(cursors.clone(), shutdown_rx);
Ok(Self { Ok(Self {
config, config,
database, database,
cursors, cursors,
broadcasts, broadcasts,
pending_ws_connections: Arc::new(AtomicUsize::new(0)),
shutdown_tx: Arc::new(shutdown_tx),
}) })
} }
/// Signal all background tasks (idle pool cleanup, cursor cleanup) to stop
pub fn shutdown(&self) {
let _ = self.shutdown_tx.send(());
}
/// Get a receiver to be notified when shutdown is triggered
pub fn subscribe_shutdown(&self) -> tokio::sync::watch::Receiver<()> {
self.shutdown_tx.subscribe()
}
} }

View file

@ -42,7 +42,9 @@ impl Cursors {
) { ) {
let mut vault_to_cursors = self.vault_to_cursors.lock().await; let mut vault_to_cursors = self.vault_to_cursors.lock().await;
let all_device_cursors = vault_to_cursors.entry(vault_id).or_insert_with(Vec::new); let all_device_cursors = vault_to_cursors
.entry(vault_id.clone())
.or_insert_with(Vec::new);
all_device_cursors.retain(|c| &c.client_cursors.device_id != device_id); all_device_cursors.retain(|c| &c.client_cursors.device_id != device_id);
all_device_cursors.push(ClientCursorsWithTimeToLive::new(ClientCursors { all_device_cursors.push(ClientCursorsWithTimeToLive::new(ClientCursors {
@ -52,7 +54,7 @@ impl Cursors {
})); }));
drop(vault_to_cursors); // Explicitly drop the lock before broadcasting to avoid deadlock drop(vault_to_cursors); // Explicitly drop the lock before broadcasting to avoid deadlock
self.broadcast_cursors().await; self.broadcast_cursors_for_vault(&vault_id).await;
} }
pub async fn get_cursors(&self, vault_id: &VaultId) -> Vec<ClientCursors> { pub async fn get_cursors(&self, vault_id: &VaultId) -> Vec<ClientCursors> {
@ -69,45 +71,81 @@ impl Cursors {
.unwrap_or_default() .unwrap_or_default()
} }
pub fn start_background_task(self) { pub fn start_background_task(self, mut shutdown: tokio::sync::watch::Receiver<()>) {
tokio::spawn(async move { tokio::spawn(async move {
loop { loop {
self.remove_expired_cursors().await; tokio::select! {
tokio::time::sleep(Duration::from_secs(1)).await; () = tokio::time::sleep(Duration::from_secs(1)) => {
self.remove_expired_cursors().await;
}
Ok(()) = shutdown.changed() => break,
}
} }
}); });
} }
async fn remove_expired_cursors(&self) { async fn remove_expired_cursors(&self) {
let mut vault_to_cursors = self.vault_to_cursors.lock().await; let changed_vaults: Vec<VaultId> = {
let mut vault_to_cursors = self.vault_to_cursors.lock().await;
for (_vault_id, cursors) in vault_to_cursors.iter_mut() { let mut changed = Vec::new();
cursors.retain(|cursor| !cursor.is_expired(self.config.cursor_timeout)); for (vault_id, cursors) in vault_to_cursors.iter_mut() {
let before = cursors.len();
cursors.retain(|cursor| !cursor.is_expired(self.config.cursor_timeout));
if cursors.len() != before {
changed.push(vault_id.clone());
}
}
// Remove empty vault entries to prevent unbounded growth
vault_to_cursors.retain(|_, cursors| !cursors.is_empty());
changed
};
for vault_id in &changed_vaults {
self.broadcast_cursors_for_vault(vault_id).await;
} }
} }
async fn broadcast_cursors(&self) { async fn broadcast_cursors_for_vault(&self, vault_id: &VaultId) {
let vault_to_cursors = self.vault_to_cursors.lock().await; let client_cursors: Vec<ClientCursors> = {
let vault_to_cursors = self.vault_to_cursors.lock().await;
vault_to_cursors
.get(vault_id)
.map(|cursors| cursors.iter().map(|c| c.client_cursors.clone()).collect())
.unwrap_or_default()
};
for (vault_id, cursors) in vault_to_cursors.iter() { self.broadcasts.send_document_update(
self.broadcasts vault_id.clone(),
.send_document_update( WebSocketServerMessageWithOrigin::new(WebSocketServerMessage::CursorPositions(
vault_id.clone(), CursorPositionFromServer {
WebSocketServerMessageWithOrigin::new(WebSocketServerMessage::CursorPositions( clients: client_cursors,
CursorPositionFromServer { },
clients: cursors.iter().map(|c| c.client_cursors.clone()).collect(), )),
}, );
)),
)
.await;
}
} }
pub async fn remove_cursors_of_device(&self, vault_id: &str, device_id: &str) { pub async fn remove_cursors_of_device(&self, vault_id: &VaultId, device_id: &DeviceId) {
let mut vault_to_cursors = self.vault_to_cursors.lock().await; let changed = {
let mut vault_to_cursors = self.vault_to_cursors.lock().await;
if let Some(cursors) = vault_to_cursors.get_mut(vault_id) { if let Some(cursors) = vault_to_cursors.get_mut(vault_id) {
cursors.retain(|c| c.client_cursors.device_id != device_id); let before = cursors.len();
cursors.retain(|c| c.client_cursors.device_id != *device_id);
let changed = cursors.len() != before;
if cursors.is_empty() {
vault_to_cursors.remove(vault_id);
}
changed
} else {
false
}
};
if changed {
self.broadcast_cursors_for_vault(vault_id).await;
} }
} }
} }

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,2 @@
CREATE INDEX IF NOT EXISTS idx_documents_document_id
ON documents (document_id, vault_update_id);

View file

@ -0,0 +1,20 @@
ALTER TABLE documents ADD COLUMN creation_vault_update_id INTEGER NOT NULL DEFAULT 0;
UPDATE documents
SET creation_vault_update_id = (
SELECT MIN(d2.vault_update_id)
FROM documents d2
WHERE d2.document_id = documents.document_id
);
DROP VIEW latest_document_versions;
CREATE VIEW IF NOT EXISTS latest_document_versions AS --recreate view as it now includes one more field
SELECT d.*
FROM documents d
INNER JOIN (
SELECT MAX(vault_update_id) AS max_version_id
FROM documents
GROUP BY document_id
) max_versions
ON d.vault_update_id = max_versions.max_version_id;

View file

@ -13,6 +13,7 @@ pub type DeviceId = String;
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
pub struct StoredDocumentVersion { pub struct StoredDocumentVersion {
pub vault_update_id: VaultUpdateId, pub vault_update_id: VaultUpdateId,
pub creation_vault_update_id: VaultUpdateId,
pub document_id: DocumentId, pub document_id: DocumentId,
pub relative_path: String, pub relative_path: String,
pub updated_date: DateTime<Utc>, pub updated_date: DateTime<Utc>,
@ -33,7 +34,7 @@ impl PartialEq<Self> for StoredDocumentVersion {
#[derive(TS, Debug, Clone, Serialize)] #[derive(TS, Debug, Clone, Serialize)]
#[serde(rename_all = "camelCase")] #[serde(rename_all = "camelCase")]
pub struct DocumentVersionWithoutContent { pub struct DocumentVersionWithoutContent {
#[ts(as = "i32")] #[ts(type = "number")]
pub vault_update_id: VaultUpdateId, pub vault_update_id: VaultUpdateId,
pub document_id: DocumentId, pub document_id: DocumentId,
@ -43,12 +44,16 @@ pub struct DocumentVersionWithoutContent {
pub user_id: UserId, pub user_id: UserId,
pub device_id: DeviceId, pub device_id: DeviceId,
#[ts(as = "i32")] #[ts(type = "number")]
pub content_size: u64, pub content_size: u64,
/// True iff this is the first version of the document
pub is_new_file: bool,
} }
impl From<StoredDocumentVersion> for DocumentVersionWithoutContent { impl From<StoredDocumentVersion> for DocumentVersionWithoutContent {
fn from(value: StoredDocumentVersion) -> Self { fn from(value: StoredDocumentVersion) -> Self {
let is_new_file = value.creation_vault_update_id == value.vault_update_id;
Self { Self {
vault_update_id: value.vault_update_id, vault_update_id: value.vault_update_id,
document_id: value.document_id, document_id: value.document_id,
@ -58,6 +63,7 @@ impl From<StoredDocumentVersion> for DocumentVersionWithoutContent {
user_id: value.user_id, user_id: value.user_id,
device_id: value.device_id, device_id: value.device_id,
content_size: value.content.len() as u64, content_size: value.content.len() as u64,
is_new_file,
} }
} }
} }
@ -65,7 +71,7 @@ impl From<StoredDocumentVersion> for DocumentVersionWithoutContent {
#[derive(TS, Debug, Clone, Serialize)] #[derive(TS, Debug, Clone, Serialize)]
#[serde(rename_all = "camelCase")] #[serde(rename_all = "camelCase")]
pub struct DocumentVersion { pub struct DocumentVersion {
#[ts(as = "i32")] #[ts(type = "number")]
pub vault_update_id: VaultUpdateId, pub vault_update_id: VaultUpdateId,
pub document_id: DocumentId, pub document_id: DocumentId,
@ -77,6 +83,25 @@ pub struct DocumentVersion {
pub device_id: DeviceId, pub device_id: DeviceId,
} }
/// Row struct for vault history queries (used by `sqlx::query_as!`)
#[derive(Debug)]
pub struct VaultHistoryRow {
pub vault_update_id: VaultUpdateId,
pub creation_vault_update_id: VaultUpdateId,
pub document_id: DocumentId,
pub relative_path: String,
pub updated_date: DateTime<Utc>,
pub is_deleted: bool,
pub user_id: String,
pub device_id: String,
pub content_size: Option<u64>,
}
pub struct VaultStats {
pub created_at: Option<DateTime<Utc>>,
pub document_count: u32,
}
impl From<StoredDocumentVersion> for DocumentVersion { impl From<StoredDocumentVersion> for DocumentVersion {
fn from(value: StoredDocumentVersion) -> Self { fn from(value: StoredDocumentVersion) -> Self {
Self { Self {

Some files were not shown because too many files have changed in this diff Show more