Compare commits

...
Sign in to create a new pull request.

6 commits

Author SHA1 Message Date
f7beb31d8f split: regenerated TS API type bindings
Auto-generated TS types regenerated from Rust ts-rs derives, mirrored into
frontend/sync-client/src/services/types/ and frontend/history-ui/src/lib/types/.
Adds ListVaultsResponse, VaultHistoryResponse, VaultInfo and updates several
existing types; removes DeleteDocumentVersion and UpdateDocumentVersion.
2026-05-08 21:36:13 +01:00
042233c4d7 split: server websocket + cursors
src/server/websocket.rs handshake/catch-up rewrite, app_state/cursors.rs,
app_state/websocket/{broadcasts,models,utils}.rs.
2026-05-08 21:35:52 +01:00
4ba439b874 split: server REST endpoints + rate limiting
server.rs router rewrite, auth.rs, device_id_header.rs, requests.rs,
responses.rs, plus per-endpoint changes: create/update/delete_document,
fetch_document_version{,_content,s}, fetch_latest_documents, index.rs.
Adds: fetch_vault_history, list_vaults, rate_limit (new files).
2026-05-08 21:35:41 +01:00
2d5edc6ec5 split: server database (app_state, migrations, models)
src/app_state.rs, src/app_state/database.rs (large schema/query rewrite),
two new migrations (add_idempotency_key, add_creation_vault_update_id),
and src/app_state/database/models.rs.
2026-05-08 21:35:30 +01:00
a9ce09b59d split: server foundation (Cargo, config, errors, utils, main)
Cargo.{toml,lock} bumps, build.rs, config-e2e.yml, rust-toolchain.toml,
src/config/* (database/logging/server/user configs), src/consts.rs,
src/errors.rs, src/main.rs, and src/utils/* (dedup_paths,
find_first_available_path, rotating_file_writer, sanitize_path).
2026-05-08 21:35:18 +01:00
70f97c4b16 split: CI workflows, scripts, root tooling, and docs
Some checks failed
Check / build (pull_request) Has been cancelled
E2E tests / build (pull_request) Has been cancelled
Publish CLI / publish-docker (pull_request) Has been cancelled
Publish server Docker image / publish-docker (pull_request) Has been cancelled
Forgejo workflows (new), GitHub workflow tweaks, .gitignore/.vscode, root
package-lock, rustfmt.toml, scripts/* updates, docs/ updates including
data-flow / authentication / server-setup, CLAUDE.md and README updates.
2026-05-08 21:35:07 +01:00
103 changed files with 6490 additions and 3834 deletions

View file

@ -0,0 +1,35 @@
name: Check
on:
push:
branches: ["main"]
pull_request:
branches: ["main"]
workflow_dispatch:
env:
CARGO_TERM_COLOR: always
RUSTFLAGS: "-Dwarnings"
jobs:
build:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Setup Node.js environment
uses: actions/setup-node@v4
with:
node-version: "25.x"
- name: Setup Rust toolchain
uses: dtolnay/rust-toolchain@stable
with:
toolchain: "1.92.0"
components: clippy, rustfmt
- name: Lint & test
run: scripts/check.sh

View file

@ -0,0 +1,38 @@
name: Deploy Documentation
on:
push:
branches:
- main
paths:
- "docs/**"
- ".forgejo/workflows/deploy-docs.yml"
workflow_dispatch:
concurrency:
group: pages
cancel-in-progress: false
jobs:
build:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Setup Node.js environment
uses: actions/setup-node@v4
with:
node-version: "25.x"
- name: Build docs
run: scripts/build-docs.sh
- name: Upload artifact
uses: actions/upload-artifact@v4
with:
name: docs
path: docs/.vitepress/dist

View file

@ -0,0 +1,71 @@
name: E2E tests
on:
push:
branches: ["main"]
pull_request:
branches: ["main"]
schedule:
- cron: "0 * * * *"
workflow_dispatch:
concurrency:
group: e2e-tests
cancel-in-progress: false
env:
RUSTFLAGS: "-Dwarnings"
jobs:
build:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Setup Node.js environment
uses: actions/setup-node@v4
with:
node-version: "25.x"
- name: Setup Rust toolchain
uses: dtolnay/rust-toolchain@stable
with:
toolchain: "1.92.0"
components: clippy, rustfmt
- name: Setup rust
run: |
which sqlx || cargo install sqlx-cli
cd sync-server
sqlx database create --database-url sqlite://db.sqlite3
sqlx migrate run --source src/app_state/database/migrations --database-url sqlite://db.sqlite3
- name: E2E tests
run: |
cd sync-server
cargo run config-e2e.yml --color never &
SERVER_PID=$!
cd ..
scripts/e2e.sh 8
EXIT_CODE=$?
kill $SERVER_PID 2>/dev/null || true
wait $SERVER_PID 2>/dev/null || true
exit $EXIT_CODE
- name: Upload e2e logs
if: always()
uses: actions/upload-artifact@v4
with:
name: e2e-logs
path: logs/
retention-days: 30
- name: Cleanup
if: always()
run: scripts/clean-up.sh

View file

@ -0,0 +1,51 @@
name: Publish CLI
on:
push:
branches: ["main"]
tags: ["*"]
pull_request:
branches: ["main"]
jobs:
publish-docker:
runs-on: ubuntu-docker
steps:
- name: Checkout repository
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Extract registry hostname
id: registry
run: echo "host=$(echo '${{ github.server_url }}' | sed 's|https\?://||')" >> "$GITHUB_OUTPUT"
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Log into container registry
uses: docker/login-action@v3
with:
registry: ${{ steps.registry.outputs.host }}
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Extract Docker metadata
id: meta
uses: docker/metadata-action@v5
with:
images: ${{ steps.registry.outputs.host }}/${{ github.repository }}-cli
- name: Build and push Docker image
id: build-and-push
uses: docker/build-push-action@v5
with:
context: frontend
file: frontend/local-client-cli/Dockerfile
platforms: linux/amd64
push: true
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
cache-from: type=registry,ref=${{ steps.registry.outputs.host }}/${{ github.repository }}-cli:buildcache
cache-to: type=registry,ref=${{ steps.registry.outputs.host }}/${{ github.repository }}-cli:buildcache,mode=max

View file

@ -0,0 +1,71 @@
name: Publish Obsidian plugin
on:
push:
tags: ["*"]
env:
CARGO_TERM_COLOR: always
jobs:
publish-plugin:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Setup Node.js environment
uses: actions/setup-node@v4
with:
node-version: "25.x"
- name: Build plugin
run: |
cd frontend
npm ci
npm run build
- name: Setup Rust toolchain
uses: dtolnay/rust-toolchain@stable
with:
toolchain: "1.92.0"
components: clippy, rustfmt
- name: Install cross-compilation tools
run: |
apt update
apt install -y gcc-aarch64-linux-gnu musl-tools gcc-mingw-w64-x86-64 jq
- name: Build Linux and Windows binaries
run: ./scripts/build-sync-server-binaries.sh
- name: Create release
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
SERVER_URL: ${{ github.server_url }}
REPO: ${{ github.repository }}
run: |
tag="${GITHUB_REF#refs/tags/}"
mkdir -p release
cp frontend/obsidian-plugin/dist/* release/
cp sync-server/artifacts/sync-server-* release/
# Create draft release via Forgejo API
RELEASE_ID=$(curl -s -X POST \
"${SERVER_URL}/api/v1/repos/${REPO}/releases" \
-H "Authorization: token ${GITHUB_TOKEN}" \
-H "Content-Type: application/json" \
-d "{\"tag_name\": \"${tag}\", \"name\": \"${tag}\", \"draft\": true}" \
| jq -r '.id')
# Upload release assets
for file in release/*; do
filename=$(basename "$file")
curl -s -X POST \
"${SERVER_URL}/api/v1/repos/${REPO}/releases/${RELEASE_ID}/assets?name=${filename}" \
-H "Authorization: token ${GITHUB_TOKEN}" \
-F "attachment=@${file}"
done

View file

@ -0,0 +1,51 @@
name: Publish server Docker image
on:
push:
branches: ["main"]
tags: ["*"]
pull_request:
branches: ["main"]
jobs:
publish-docker:
runs-on: ubuntu-docker
steps:
- name: Checkout repository
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Extract registry hostname
id: registry
run: echo "host=$(echo '${{ github.server_url }}' | sed 's|https\?://||')" >> "$GITHUB_OUTPUT"
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Log into container registry
if: github.ref_type == 'tag'
uses: docker/login-action@v3
with:
registry: ${{ steps.registry.outputs.host }}
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Extract Docker metadata
id: meta
uses: docker/metadata-action@v5
with:
images: ${{ steps.registry.outputs.host }}/${{ github.repository }}
- name: Build and push Docker image
id: build-and-push
uses: docker/build-push-action@v5
with:
context: sync-server
platforms: linux/amd64,linux/arm64
push: ${{ github.ref_type == 'tag' }}
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
cache-from: type=registry,ref=${{ steps.registry.outputs.host }}/${{ github.repository }}:buildcache
cache-to: type=registry,ref=${{ steps.registry.outputs.host }}/${{ github.repository }}:buildcache,mode=max

View file

@ -23,13 +23,13 @@ jobs:
- name: Setup Node.js environment
uses: actions/setup-node@v4.2.0
with:
node-version: "22.x"
node-version: "25.x"
check-latest: true
- name: Setup Rust toolchain
uses: dtolnay/rust-toolchain@stable
with:
toolchain: "1.89.0"
toolchain: "1.92.0"
components: clippy, rustfmt
- name: Lint & test

View file

@ -5,8 +5,8 @@ on:
branches:
- main
paths:
- 'docs/**'
- '.github/workflows/deploy-docs.yml'
- "docs/**"
- ".github/workflows/deploy-docs.yml"
workflow_dispatch:
permissions:
@ -28,12 +28,11 @@ jobs:
with:
fetch-depth: 0
- name: Setup Node
uses: actions/setup-node@v4
- name: Setup Node.js environment
uses: actions/setup-node@v4.2.0
with:
node-version: 22
cache: npm
cache-dependency-path: docs/package-lock.json
node-version: "25.x"
check-latest: true
- name: Setup Pages
uses: actions/configure-pages@v4

View file

@ -6,7 +6,7 @@ on:
pull_request:
branches: ["main"]
schedule:
- cron: '0 * * * *'
- cron: "0 * * * *"
workflow_dispatch:
concurrency:
@ -28,13 +28,13 @@ jobs:
- name: Setup Node.js environment
uses: actions/setup-node@v4.2.0
with:
node-version: "22.x"
node-version: "25.x"
check-latest: true
- name: Setup Rust toolchain
uses: dtolnay/rust-toolchain@stable
with:
toolchain: "1.89.0"
toolchain: "1.92.0"
components: clippy, rustfmt
- name: Setup rust

View file

@ -19,7 +19,7 @@ jobs:
- name: Setup Node.js environment
uses: actions/setup-node@v4.2.0
with:
node-version: "22.x"
node-version: "25.x"
check-latest: true
- name: Build plugin
@ -31,7 +31,7 @@ jobs:
- name: Setup Rust toolchain
uses: dtolnay/rust-toolchain@stable
with:
toolchain: "1.89.0"
toolchain: "1.92.0"
components: clippy, rustfmt
- name: Install cross-compilation tools

9
.gitignore vendored
View file

@ -7,15 +7,18 @@ node_modules
# Frontend build folders
frontend/*/dist
sync-server/db.sqlite3*
sync-server/databases
# Rust build folders
sync-server/target
sync-server/artifacts
sync-server/bindings/*.ts
# build folders
sync-server/db.sqlite3*
**/databases
*.log
*.sqlx
target
.task

View file

@ -5,6 +5,6 @@
"**/dist": true,
"**/node_modules": true,
"**/.sqlx": true,
"**/target": true,
},
"**/target": true
}
}

195
CLAUDE.md
View file

@ -2,109 +2,154 @@
This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
## Project Overview
## Project shape
VaultLink is a self-hosted Obsidian plugin for real-time collaborative file syncing. The project consists of a Rust-based sync server and a TypeScript frontend with three main components: an Obsidian plugin, a sync client library, and a test client.
VaultLink is a self-hosted Obsidian file-sync system. Two halves of one repo:
## Architecture
- `sync-server/` — Rust (axum + sqlx/SQLite). Source of truth for vault state, broadcasts changes via WebSocket.
- `frontend/` — npm workspaces. The sync engine (`sync-client`) is consumed by an Obsidian plugin, a standalone CLI, a fuzz E2E harness, a scripted determinism harness, and a history UI.
### Core Components
The HTTP/WS API types are generated from Rust (`ts-rs`) and mirrored into the TS workspaces. **Never hand-edit files in `frontend/sync-client/src/services/types/` or `frontend/history-ui/src/lib/types/`** — run `scripts/update-api-types.sh` after changing anything Serde-derived in the server.
- **sync-server/**: Rust-based WebSocket server with SQLite database for document versioning and real-time synchronization
- **frontend/sync-client/**: TypeScript library providing core sync functionality, WebSocket management, and file operations
- **frontend/obsidian-plugin/**: Obsidian plugin that integrates the sync client with Obsidian's API
- **frontend/test-client/**: CLI testing tool for the sync functionality
### Frontend workspaces
### Key Technologies
- `sync-client` — the sync engine; published to consumers via `dist/`. All other TS workspaces depend on it via `file:../sync-client`.
- `obsidian-plugin` — Obsidian plugin built from `sync-client`.
- `local-client-cli` — same engine wrapped as a standalone CLI.
- `history-ui` — vault-history web UI.
- `test-client` — fuzz E2E harness (random ops across N processes).
- `deterministic-tests` — scripted multi-client tests with an in-memory FS, run against a real server.
- **Backend**: Rust with Axum framework, SQLite with SQLx, WebSockets for real-time sync
- **Frontend**: TypeScript, Webpack for bundling, Jest for testing
- **Sync Algorithm**: Uses reconcile-text library for operational transformation
## Common commands
## Development Commands
Pre-push hygiene (formats, lints, runs tests, requires clean git state):
### Server Development
```bash
cd sync-server
cargo run config-e2e.yml # Start development server
cargo test --verbose # Run Rust tests
cargo clippy --all-targets --all-features # Lint Rust code
cargo clippy --all-targets --all-features --fix --allow-dirty --allow-staged # Auto-fix clippy warnings
cargo fmt --all -- --check # Check Rust formatting
cargo fmt --all # Auto-format Rust code
cargo machete --with-metadata # Detect unused dependencies
```sh
scripts/check.sh --fix
```
### Frontend Development
```bash
Run the fuzz E2E (N parallel processes):
```sh
scripts/e2e.sh 12
# Logs land in logs/log_<i>.log. Clean with scripts/clean-up.sh
```
Run deterministic tests (require a release-built server in `sync-server/target/release/sync_server` — they spawn it themselves):
```sh
cd sync-server && cargo build --release && cd ..
cd frontend
npm run dev # Start development mode (watches sync-client and obsidian-plugin)
npm run build # Build all workspaces
npm run test # Run all tests
npm run lint # Lint and format TypeScript code
npm run build -w sync-client -w deterministic-tests
node deterministic-tests/dist/cli.js # all
node deterministic-tests/dist/cli.js --filter=rename # subset
node deterministic-tests/dist/cli.js --filter=… -j 4 # cap parallelism
```
### Database Setup (Development)
```bash
Run a single sync-client unit test by file:
```sh
cd frontend/sync-client && npx tsx --test 'src/**/sync-event-queue.test.ts'
```
Server: dev runs from `sync-server/` against `config-e2e.yml`:
```sh
cd sync-server
cargo run config-e2e.yml # dev
cargo build --release # used by both e2e harnesses
cargo test # unit + ts-rs binding export tests
```
Frontend dev (sync-client + obsidian-plugin watch in parallel):
```sh
cd frontend && npm install && npm run dev
```
Regenerate TS bindings from Rust types (touches `frontend/{sync-client,history-ui}/src/.../types/`):
```sh
scripts/update-api-types.sh
```
## SQLite / sqlx
The server uses `sqlx::query!` macros that need a prepared `.sqlx` cache to compile offline. Touching any SQL means regenerating it:
```sh
cd sync-server
sqlx database create --database-url sqlite://db.sqlite3
sqlx migrate run --source src/app_state/database/migrations --database-url sqlite://db.sqlite3
cargo sqlx prepare --workspace
```
### Initial Setup
```bash
# Install required cargo tools
cargo install sqlx-cli cargo-machete cargo-edit
New migrations: `sqlx migrate add --source src/app_state/database/migrations <name>`.
## Sync engine architecture
Read `frontend/sync-client/src/sync-operations/` to follow the sync engine; the rest of `sync-client` is plumbing (filesystem ops, persistence, services, telemetry).
The engine is **two independent loops with separate invariants**:
- **Wire loop** (`syncer.ts`) — drains the single-consumer FIFO queue. HTTP and WS handlers update record fields (`remoteRelativePath`, `parentVersionId`, `remoteHash`) and write content to the file at `record.localPath`. They never move files for path placement.
- **Path reconciler** (`reconciler.ts`) — runs after every drained event. Best-effort pass that moves files to make `localPath === remoteRelativePath`. The move graph is topologically sorted; cycles are resolved by reading every file in the cycle into memory and writing each back to its new slot (no tmp files). Records with pending local events are skipped on each pass — the reconciler operates only on settled records. Failures (slot occupied by an untracked file, etc.) are silent skips; the next pass retries.
**`SyncEventQueue`** (`sync-event-queue.ts`) holds:
- `byDocId: Map<DocumentId, DocumentRecord>` — primary record store.
- `byLocalPath: Map<RelativePath, DocumentRecord>` — derived index for path lookups, maintained at every mutation point.
- `events: SyncEvent[]` — pending wire ops in FIFO drain order.
```ts
DocumentRecord = {
documentId,
parentVersionId,
remoteHash?,
remoteRelativePath,
localPath: RelativePath | undefined
}
```
### Scripts
- `scripts/check.sh`: Full CI check (builds, lints, tests both server and frontend)
- `scripts/check.sh --fix`: Same as above but auto-fixes linting and formatting issues
- `scripts/e2e.sh`: End-to-end testing
- `scripts/clean-up.sh`: Clean logs and database files
- `scripts/bump-version.sh patch`: Publish new version
- `scripts/update-api-types.sh`: Update TypeScript bindings from Rust types
`localPath === undefined` means the doc has no local file yet — typically a remote create whose target slot was occupied at receive time; the reconciler will fetch and place when the slot frees (the bytes wait in `pendingPlacementContent`).
## Code Structure
Local FS events from the watcher update `localPath` synchronously at enqueue time via `setLocalPath` / `upsertRecord`. The wire loop never updates it for path placement; only the reconciler does. A user rename onto a tracked slot enqueues a `LocalDelete` for the displaced doc (the OS rename clobbered its content) and clears that doc's `localPath`.
### Workspace Configuration
The frontend uses npm workspaces with four packages:
- `sync-client`: Core synchronization logic
- `obsidian-plugin`: Obsidian-specific integration
- `test-client`: Testing utilities
- `local-client-cli`: Standalone CLI for VaultLink sync client
**Pending creates** use a `Promise<DocumentId>` chain to serialize dependent ops (`LocalUpdate`, `LocalDelete`) behind the still-in-flight `LocalCreate`. `resolveCreate` resolves the promise once the server returns a docId, and `replacePendingDocumentId` swaps the resolved id across already-queued events. `findLatestCreateForPath` is the lookup the watcher uses to attach dependents; `updatePendingCreatePath` rewrites a pending create's `event.path` in place when the user renames the file before its create has acked.
### Type Generation
Rust structs generate TypeScript types via ts-rs crate, stored in `sync-server/bindings/` and used by frontend packages.
**Watermark.** `lastSeenUpdateId` uses a `MinCovered` (a contiguous-prefix tracker over a stream of integers): we only advance the published min when the next consecutive id has been processed, so out-of-order RemoteChange ids don't fool the WebSocket handshake into requesting a too-recent catch-up.
### Key Files
- `sync-server/src/`: Rust server implementation with WebSocket handlers
- `frontend/sync-client/src/sync-client.ts`: Main sync client entry point
- `frontend/obsidian-plugin/src/vault-link-plugin.ts`: Main Obsidian plugin class
- `frontend/sync-client/src/services/sync-service.ts`: Core synchronization logic
**Server catch-up.** The server's WS handshake replays events newer than the client's `last_seen_vault_update_id` from the `latest_document_versions` view (one row per doc, the latest). On those replayed rows `is_new_file` means _new to this client_ (`creation_vault_update_id > last_seen_vault_update_id`), not "this row is the doc's first version" — necessary because the catch-up only carries the latest version; if a doc was created and updated past the watermark, the client never sees its create otherwise.
## Testing
## Edge-case patterns the sync engine has to survive
### Running Tests
- Server: `cargo test --verbose`
- Frontend: `npm run test` (runs Jest across all workspaces)
- E2E: `scripts/e2e.sh`
The two-loop split defuses most of the old race catalogue (slot-collision stashes, conflict-uuid divergence, `MoveOnConflict.NEW`/`EXISTING` policy choices) by separating wire transport from path placement. What's left:
### Test Structure
- Rust: Unit tests alongside source files
- TypeScript: `.test.ts` files using Jest
- E2E: Uses test-client to simulate multiple concurrent users
**Pending-create docId is a `Promise`, not a string, until the create acks.** Any `LocalUpdate` / `LocalDelete` queued behind a still-in-flight `LocalCreate` carries the create's `resolvers.promise` as its `documentId`. `replacePendingDocumentId` swaps the resolved id across queued events when the create resolves; `===` comparisons against the resolved string elsewhere will silently fail until that swap runs. Anything that walks `events[]` looking for a docId match must either run after the swap or be tolerant of `Promise`-typed ids.
## Code Style
**`processCreate` reads `event.path` live, not `event.originalPath`.** The watcher rewrites `event.path` in place via `updatePendingCreatePath` when the user renames a pending-create file. `originalPath` was removed from `LocalCreate` events specifically because reading it would send the stale pre-rename path to the server.
### Rust
- Uses extensive Clippy lints (see Cargo.toml)
- Follows pedantic linting rules
- Forbids unsafe code
- Uses cargo fmt with default settings
**`record.localPath` mutates in place across awaits.** When the watcher renames a doc while a drain handler is awaiting an HTTP roundtrip, the queue mutates the in-flight event's record so subsequent reads see the new path. Snapshotting `record.localPath` into a local at function entry and using it after an `await` reads/writes a now-vacated slot. Read `record.localPath` live; only snapshot for the deliberate "did it change while I was awaiting" comparison.
### TypeScript
- Prettier configuration: 4-space tabs, trailing commas removed, LF line endings
- ESLint with unused imports plugin
- Consistent across all three frontend packages
**Reconciler-defer is the wire-loop's contract with the reconciler.** The reconciler skips records where `hasPendingLocalEventsForDocumentId` returns true. Wire-loop handlers can therefore freely write `remoteRelativePath` to whatever the server returned — even if it disagrees with `localPath` — knowing the reconciler won't move the file out from under a queued user rename.
**Watermark advancement is load-bearing both ways.** Branches that _skip_ a remote event without advancing `lastSeenUpdateId` create permanent gaps that re-deliver forever. Branches that _advance_ without applying the content lose data: the server has no further event to re-deliver, the catch-up only carries the latest version, and any state in between is gone. Don't advance unless the event was actually applied (or deliberately discarded after weighing both halves).
**`isNewFile` semantics differ between catch-up and real-time.** On WS handshake replay it means _new to this client_ (`creation_vault_update_id > last_seen_vault_update_id`); on real-time broadcasts it means _this version is the create_ (`creation_vault_update_id == vault_update_id`). A handler that decides based on one interpretation will be wrong on the other channel; reasoning about fetch-and-treat-as-new vs. ignore needs to know which channel delivered the event.
**Pause / disable-sync mid-flight** is the one race the new model doesn't structurally fix. An HTTP that committed server-side but whose response was discarded leaves the server holding a doc the client has no record of. Resume → offline scan → server-side dedupe handles it (the server merges the duplicate create into the existing doc), but if the merge produces a deconflict, the client picks up an extra file. Out of scope for the two-loop split.
**Cycle reconciliation uses in-memory content swap.** When the move graph contains a cycle, the reconciler reads every file in the cycle into memory and writes each back to its new slot, with no tmp files. A write-ahead marker at `.vaultlink/swap-<uuid>.json` lists each leg; on startup the reconciler reads the marker, hashes each `from` to determine which legs ran, and replays the rest. The `.vaultlink/**` glob is hard-coded as an internal ignore pattern so swap markers don't get sync'd.
## Two complementary E2E harnesses
- **`test-client` (fuzz):** random ops across N parallel processes for many minutes. Used by `scripts/e2e.sh`. Catches bugs nobody thought to write a test for, but reproductions are noisy.
- **`deterministic-tests`:** scripted scenarios with an in-memory FS pinned to a real server. Used to _capture_ a fuzz-discovered bug as a minimal repro before fixing it. See `frontend/deterministic-tests/README.md` for the step grammar (`pause-server`, `pause-websocket`, `barrier`, `assert-consistent`, etc.).
When a fuzz failure surfaces, the workflow is: root-cause from logs → write a deterministic test that fails on the bug → fix → confirm both the deterministic test and `e2e.sh` pass.
## Style
- TS: 4-space indent, no tabs, LF, prettier (`trailingComma: "none"`). YAML/MD use 2-space indent.
- Rust: `rustfmt.toml` enforces 4-space spaces, LF.
- Lint: ESLint for TS, Clippy for Rust, `cargo machete` for unused deps. All wired into `scripts/check.sh`.

View file

@ -8,12 +8,12 @@
## Develop
### Install [nvm](https://github.com/nvm-sh/nvm)
### Set up Node.JS 25 with [nvm](https://github.com/nvm-sh/nvm)
- `curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh | bash`
- `nvm install 22`
- `nvm use 22`
- Optionally set the system-wide default: `nvm alias default 22`
- `nvm install 25`
- `nvm use 25`
- Optionally, set the system-wide default: `nvm alias default 25`
### Set up Rust

View file

@ -2,12 +2,7 @@
"version": "0.2",
"language": "en-GB",
"dictionaries": ["en-gb"],
"ignorePaths": [
"node_modules",
".vitepress/dist",
".vitepress/cache",
"package-lock.json"
],
"ignorePaths": ["node_modules", ".vitepress/dist", ".vitepress/cache", "package-lock.json"],
"words": [
"VaultLink",
"Obsidian",

View file

@ -361,11 +361,11 @@ VALUES (?, ?, ?);
```json
{
"type": "upload_file",
"path": "notes/example.md",
"content": "File content here...",
"base_version": 10,
"timestamp": "2024-01-01T12:00:00Z"
"type": "upload_file",
"path": "notes/example.md",
"content": "File content here...",
"base_version": 10,
"timestamp": "2024-01-01T12:00:00Z"
}
```
@ -373,8 +373,8 @@ VALUES (?, ?, ?);
```json
{
"type": "download_file",
"path": "notes/example.md"
"type": "download_file",
"path": "notes/example.md"
}
```
@ -382,8 +382,8 @@ VALUES (?, ?, ?);
```json
{
"type": "delete_file",
"path": "notes/old.md"
"type": "delete_file",
"path": "notes/old.md"
}
```
@ -391,8 +391,8 @@ VALUES (?, ?, ?);
```json
{
"type": "list_files",
"since_version": 0
"type": "list_files",
"since_version": 0
}
```
@ -402,11 +402,11 @@ VALUES (?, ?, ?);
```json
{
"type": "file_updated",
"path": "notes/example.md",
"version": 11,
"size": 1024,
"hash": "abc123..."
"type": "file_updated",
"path": "notes/example.md",
"version": 11,
"size": 1024,
"hash": "abc123..."
}
```
@ -414,10 +414,10 @@ VALUES (?, ?, ?);
```json
{
"type": "file_content",
"path": "notes/example.md",
"content": "Updated content...",
"version": 11
"type": "file_content",
"path": "notes/example.md",
"content": "Updated content...",
"version": 11
}
```
@ -425,9 +425,9 @@ VALUES (?, ?, ?);
```json
{
"type": "file_deleted",
"path": "notes/old.md",
"version": 12
"type": "file_deleted",
"path": "notes/old.md",
"version": 12
}
```
@ -435,9 +435,9 @@ VALUES (?, ?, ?);
```json
{
"type": "sync_complete",
"total_files": 150,
"current_version": 200
"type": "sync_complete",
"total_files": 150,
"current_version": 200
}
```
@ -445,9 +445,9 @@ VALUES (?, ?, ?);
```json
{
"type": "error",
"message": "File too large",
"code": "FILE_TOO_LARGE"
"type": "error",
"message": "File too large",
"code": "FILE_TOO_LARGE"
}
```

View file

@ -53,7 +53,7 @@ Central authority for synchronisation. Rust + Axum framework.
**Technology**:
- **Language**: Rust 1.89+
- **Language**: Rust 1.92+
- **Framework**: Axum (async web framework)
- **Database**: SQLite with SQLx
- **Protocol**: WebSockets for real-time communication

View file

@ -243,9 +243,9 @@ users:
2. Client sends authentication message:
```json
{
"type": "auth",
"token": "user-token",
"vault": "vault-name"
"type": "auth",
"token": "user-token",
"vault": "vault-name"
}
```
3. Server validates:

View file

@ -75,7 +75,7 @@ chmod +x sync_server-linux-x86_64
### Build from Source
Requirements: Rust 1.89.0+, SQLite development headers, SQLx CLI
Requirements: Rust 1.92.0+, SQLite development headers, SQLx CLI
```bash
# Clone the repository

5960
docs/package-lock.json generated

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,8 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
import type { DocumentWithCursors } from "./DocumentWithCursors";
export type ClientCursors = {
userName: string;
deviceId: string;
documentsWithCursors: Array<DocumentWithCursors>;
};

View file

@ -1,7 +1,7 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
export interface UpdateDocumentVersion {
parent_version_id: bigint;
export type CreateDocumentVersion = {
relative_path: string;
content: number[];
}
last_seen_vault_update_id: number;
content: Array<number>;
};

View file

@ -0,0 +1,6 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
import type { DocumentWithCursors } from "./DocumentWithCursors";
export type CursorPositionFromClient = {
documentsWithCursors: Array<DocumentWithCursors>;
};

View file

@ -0,0 +1,4 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
import type { ClientCursors } from "./ClientCursors";
export type CursorPositionFromServer = { clients: Array<ClientCursors> };

View file

@ -1,5 +1,3 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
export interface DeleteDocumentVersion {
relativePath: string;
}
export type CursorSpan = { start: number; end: number };

View file

@ -0,0 +1,10 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
import type { DocumentVersion } from "./DocumentVersion";
import type { DocumentVersionWithoutContent } from "./DocumentVersionWithoutContent";
/**
* Response to a create/update document request.
*/
export type DocumentUpdateResponse =
| ({ type: "FastForwardUpdate" } & DocumentVersionWithoutContent)
| ({ type: "MergingUpdate" } & DocumentVersion);

View file

@ -0,0 +1,12 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
export type DocumentVersion = {
vaultUpdateId: number;
documentId: string;
relativePath: string;
updatedDate: string;
contentBase64: string;
isDeleted: boolean;
userId: string;
deviceId: string;
};

View file

@ -0,0 +1,16 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
export type DocumentVersionWithoutContent = {
vaultUpdateId: number;
documentId: string;
relativePath: string;
updatedDate: string;
isDeleted: boolean;
userId: string;
deviceId: string;
contentSize: number;
/**
* True iff this is the first version of the document
*/
isNewFile: boolean;
};

View file

@ -0,0 +1,9 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
import type { CursorSpan } from "./CursorSpan";
export type DocumentWithCursors = {
vaultUpdateId: number | null;
documentId: string;
relativePath: string;
cursors: Array<CursorSpan>;
};

View file

@ -0,0 +1,13 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
import type { DocumentVersionWithoutContent } from "./DocumentVersionWithoutContent";
/**
* Response to a fetch latest documents request.
*/
export type FetchLatestDocumentsResponse = {
latestDocuments: Array<DocumentVersionWithoutContent>;
/**
* The update ID of the latest document in the response.
*/
lastUpdateId: bigint;
};

View file

@ -0,0 +1,11 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
import type { VaultInfo } from "./VaultInfo";
/**
* Response to listing vaults accessible to the authenticated user.
*/
export type ListVaultsResponse = {
vaults: Array<VaultInfo>;
hasMore: boolean;
userName: string;
};

View file

@ -0,0 +1,25 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
/**
* Response to a ping request.
*/
export type PingResponse = {
/**
* Semantic version of the server.
*/
serverVersion: string;
/**
* Whether the client is authenticated based on the sent Authorization
* header.
*/
isAuthenticated: boolean;
/**
* List of file extensions that are allowed to be merged.
*/
mergeableFileExtensions: Array<string>;
/**
* API version ensuring backwards & forwards compatibility between the client
* and server.
*/
supportedApiVersion: number;
};

View file

@ -0,0 +1,7 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
export type SerializedError = {
errorType: string;
message: string;
causes: Array<string>;
};

View file

@ -0,0 +1,7 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
export type UpdateTextDocumentVersion = {
parentVersionId: number;
relativePath: string | null;
content: Array<number | string>;
};

View file

@ -0,0 +1,10 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
import type { DocumentVersionWithoutContent } from "./DocumentVersionWithoutContent";
/**
* Response to a vault history request (paginated).
*/
export type VaultHistoryResponse = {
versions: Array<DocumentVersionWithoutContent>;
hasMore: boolean;
};

View file

@ -0,0 +1,10 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
/**
* Summary of a single vault returned by the list-vaults endpoint.
*/
export type VaultInfo = {
name: string;
documentCount: number;
createdAt: string | null;
};

View file

@ -0,0 +1,7 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
import type { CursorPositionFromClient } from "./CursorPositionFromClient";
import type { WebSocketHandshake } from "./WebSocketHandshake";
export type WebSocketClientMessage =
| ({ type: "handshake" } & WebSocketHandshake)
| ({ type: "cursorPositions" } & CursorPositionFromClient);

View file

@ -0,0 +1,7 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
export type WebSocketHandshake = {
token: string;
deviceId: string;
lastSeenVaultUpdateId: number | null;
};

View file

@ -0,0 +1,7 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
import type { CursorPositionFromServer } from "./CursorPositionFromServer";
import type { WebSocketVaultUpdate } from "./WebSocketVaultUpdate";
export type WebSocketServerMessage =
| ({ type: "vaultUpdate" } & WebSocketVaultUpdate)
| ({ type: "cursorPositions" } & CursorPositionFromServer);

View file

@ -0,0 +1,4 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
import type { DocumentVersionWithoutContent } from "./DocumentVersionWithoutContent";
export type WebSocketVaultUpdate = { document: DocumentVersionWithoutContent };

View file

@ -1,13 +1,7 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
export interface CreateDocumentVersion {
/**
* The client can decide the document id (if it wishes to) in order
* to help with syncing. If the client does not provide a document id,
* the server will generate one. If the client provides a document id
* it must not already exist in the database.
*/
document_id: string | null;
relative_path: string;
last_seen_vault_update_id: number;
content: number[];
}

View file

@ -3,7 +3,7 @@ import type { DocumentVersion } from "./DocumentVersion";
import type { DocumentVersionWithoutContent } from "./DocumentVersionWithoutContent";
/**
* Response to an update document request.
* Response to a create/update document request.
*/
export type DocumentUpdateResponse =
| ({ type: "FastForwardUpdate" } & DocumentVersionWithoutContent)

View file

@ -9,4 +9,8 @@ export interface DocumentVersionWithoutContent {
userId: string;
deviceId: string;
contentSize: number;
/**
* True iff this is the first version of the document
*/
isNewFile: boolean;
}

View file

@ -2,8 +2,8 @@
import type { CursorSpan } from "./CursorSpan";
export interface DocumentWithCursors {
vault_update_id: number | null;
document_id: string;
relative_path: string;
vaultUpdateId: number | null;
documentId: string;
relativePath: string;
cursors: CursorSpan[];
}

View file

@ -7,7 +7,7 @@ import type { DocumentVersionWithoutContent } from "./DocumentVersionWithoutCont
export interface FetchLatestDocumentsResponse {
latestDocuments: DocumentVersionWithoutContent[];
/**
* The update ID of the latest document in the response.
*/
* The update ID of the latest document in the response.
*/
lastUpdateId: bigint;
}

View file

@ -0,0 +1,11 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
import type { VaultInfo } from "./VaultInfo";
/**
* Response to listing vaults accessible to the authenticated user.
*/
export interface ListVaultsResponse {
vaults: VaultInfo[];
hasMore: boolean;
userName: string;
}

View file

@ -5,21 +5,21 @@
*/
export interface PingResponse {
/**
* Semantic version of the server.
*/
* Semantic version of the server.
*/
serverVersion: string;
/**
* Whether the client is authenticated based on the sent Authorization
* header.
*/
* Whether the client is authenticated based on the sent Authorization
* header.
*/
isAuthenticated: boolean;
/**
* List of file extensions that are allowed to be merged.
*/
* List of file extensions that are allowed to be merged.
*/
mergeableFileExtensions: string[];
/**
* API version ensuring backwards & forwards compatibility between the client
* and server.
*/
* API version ensuring backwards & forwards compatibility between the client
* and server.
*/
supportedApiVersion: number;
}

View file

@ -2,6 +2,6 @@
export interface UpdateTextDocumentVersion {
parentVersionId: number;
relativePath: string;
relativePath: string | null;
content: (number | string)[];
}

View file

@ -0,0 +1,10 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
import type { DocumentVersionWithoutContent } from "./DocumentVersionWithoutContent";
/**
* Response to a vault history request (paginated).
*/
export interface VaultHistoryResponse {
versions: DocumentVersionWithoutContent[];
hasMore: boolean;
}

View file

@ -0,0 +1,10 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
/**
* Summary of a single vault returned by the list-vaults endpoint.
*/
export interface VaultInfo {
name: string;
documentCount: number;
createdAt: string | null;
}

View file

@ -2,6 +2,5 @@
import type { DocumentVersionWithoutContent } from "./DocumentVersionWithoutContent";
export interface WebSocketVaultUpdate {
documents: DocumentVersionWithoutContent[];
isInitialSync: boolean;
document: DocumentVersionWithoutContent;
}

6
package-lock.json generated Normal file
View file

@ -0,0 +1,6 @@
{
"name": "vault-link",
"lockfileVersion": 3,
"requires": true,
"packages": {}
}

11
rustfmt.toml Normal file
View file

@ -0,0 +1,11 @@
# Rustfmt configuration
# This should match the .editorconfig settings
# Use spaces for indentation (matches .editorconfig indent_style = space)
hard_tabs = false
# Use 4 spaces for indentation (matches .editorconfig indent_size = 4)
tab_spaces = 4
# Use Unix line endings (matches .editorconfig end_of_line = lf)
newline_style = "Unix"

View file

@ -35,7 +35,8 @@ cd ..
cp frontend/obsidian-plugin/manifest.json manifest.json # for BRAT, otherwise it wouldn't update
git ls-files | xargs npx eclint fix
# Format all files across the project (frontend and backend)
npx -C frontend prettier --write "**/*.{ts,js,json,md,yml,yaml}"
# Commit and tag
git add .

View file

@ -30,8 +30,11 @@ fi
which cargo-machete || cargo install cargo-machete
cargo machete --with-metadata
cd ..
scripts/update-api-types.sh # this will dirty up the git state if not up-to-date
echo "Running checks in frontend"
cd ../frontend
cd frontend
if [[ "$FIX_MODE" == true ]]; then
npm install
@ -45,10 +48,11 @@ cd frontend
npm run build
npm run test
npm run lint
cd ..
# Use git ls-files to only check tracked files, respecting .gitignore
# We always run in fix mode and then check with git status
git ls-files | xargs npx eclint fix
# Format all files across the project (frontend and backend)
# Prettier respects .gitignore by default
npx -C frontend prettier --write "**/*.{ts,js,json,md,yml,yaml}"
if [[ "$FIX_MODE" == false ]] && [[ $(git status --porcelain) ]]; then
git status --porcelain
@ -56,6 +60,4 @@ if [[ "$FIX_MODE" == false ]] && [[ $(git status --porcelain) ]]; then
exit 1
fi
cd ..
echo "Success"

View file

@ -1,4 +1,4 @@
#!/bin/bash
rm -rf sync-server/databases
rm -rf /host/tmp/vaultlink-e2e-databases
rm -rf logs

View file

@ -19,35 +19,51 @@ process_count=$1
mkdir -p logs
# Build and restart the server
echo "Building server..."
cd sync-server
cargo build --release
# Kill any existing server process
echo "Stopping existing server..."
pkill -f "sync_server" 2>/dev/null || true
sleep 1
# Clean databases (uses tmpfs via /dev/shm for zero disk I/O)
echo "Cleaning databases..."
rm -rf /host/tmp/vaultlink-e2e-databases
# Start the server in the background
echo "Starting server..."
./target/release/sync_server config-e2e.yml &
server_pid=$!
echo "Server started with PID: $server_pid"
# Ensure server is killed on script exit
cleanup_server() {
if [ -n "$server_pid" ]; then
echo "Stopping server (PID: $server_pid)..."
kill $server_pid 2>/dev/null || true
wait $server_pid 2>/dev/null || true
server_pid=""
fi
}
trap cleanup_server EXIT
cd ..
cd frontend
npm ci
npm run build
../scripts/utils/wait-for-server.sh
cd ..
scripts/update-api-types.sh
if [[ $(git status --porcelain) ]]; then
git status --porcelain
echo "Failing CI because the working directory is not clean after generating api types"
exit 1
fi
cd frontend
pids=()
for i in $(seq 1 $process_count); do
# Create a named pipe for this process
pipe="/tmp/vaultlink_pipe_$$_$i"
mkfifo "$pipe"
# Start the node process writing to the pipe
node test-client/dist/cli.js > "$pipe" 2>&1 &
node test-client/dist/cli.js > "../logs/log_${i}.log" 2>&1 &
pid=$!
pids+=($pid)
echo "Started process $i with PID: $pid"
# Read from pipe, prefix with PID
(sed "s/^/[PID $pid] /" < "$pipe" > "../logs/log_${i}.log"; rm "$pipe") &
echo "Started process $i with PID: $pid (log: logs/log_${i}.log)"
done
cd ..
@ -75,10 +91,25 @@ print_failed_log() {
return 1
}
echo "Monitoring $process_count processes"
E2E_TIMEOUT=${2:-3600}
start_time=$(date +%s)
echo "Monitoring $process_count processes (timeout: ${E2E_TIMEOUT}s)"
# Monitor processes
while true; do
# Script-level timeout to prevent indefinite hangs
current_time=$(date +%s)
elapsed=$((current_time - start_time))
if [ $elapsed -ge $E2E_TIMEOUT ]; then
echo "E2E timeout reached (${E2E_TIMEOUT}s). Killing remaining processes."
for pid in "${pids[@]}"; do
if [ -n "$pid" ]; then
kill $pid 2>/dev/null || true
fi
done
exit 1
fi
if print_failed_log; then
# Kill remaining processes
for pid in "${pids[@]}"; do
@ -99,6 +130,7 @@ while true; do
done
if $all_done; then
cleanup_server
echo "All processes completed successfully"
exit 0
fi

View file

@ -8,9 +8,15 @@ cd sync-server
cargo test export_bindings
cd -
# Both target directories contain only generated bindings — wipe and copy
rm -f frontend/sync-client/src/services/types/*.ts
rm -f frontend/history-ui/src/lib/types/*.ts
cp -r sync-server/bindings/* frontend/sync-client/src/services/types/
cp -r sync-server/bindings/* frontend/history-ui/src/lib/types/
cd frontend
npm run lint
git ls-files | xargs npx eclint fix
cd -
cd ..
# Format all files across the project (frontend and backend)
npx -C frontend prettier --write "**/*.{ts,js,json,md,yml,yaml}"

View file

@ -2,8 +2,10 @@
set -e
TARGET_NODE_VERSION=25
node_version=$(node -v | sed 's/^v\([0-9]*\).*/\1/')
if [ "$node_version" != "22" ]; then
echo "Error: This script requires Node.js version 22, found: $node_version"
if [ "$node_version" != "$TARGET_NODE_VERSION" ]; then
echo "Error: This script requires Node.js version $TARGET_NODE_VERSION, found: $node_version"
exit 1
fi

View file

@ -2,14 +2,14 @@
set -e
SERVER_URL="http://localhost:3000"
SERVER_URL="http://localhost:3010"
MAX_RETRIES=30
RETRY_INTERVAL_IN_SECONDS=5
echo "Waiting for $SERVER_URL to become available..."
count=0
while [ $count -lt $MAX_RETRIES ]; do
if curl -s -f -o /dev/null $SERVER_URL; then
if curl -s -o /dev/null $SERVER_URL; then
echo "$SERVER_URL is now available!"
break
fi

193
sync-server/Cargo.lock generated
View file

@ -337,10 +337,11 @@ checksum = "325918d6fe32f23b19878fe4b34794ae41fc19ddbe53b10571a4874d44ffd39b"
[[package]]
name = "cc"
version = "1.2.2"
version = "1.2.57"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f34d93e62b03caf570cccc334cbc6c2fceca82f39211051345108adcba3eebdc"
checksum = "7a0dd1ca384932ff3641c8718a02769f1698e7563dc6974ffd03346116310423"
dependencies = [
"find-msvc-tools",
"shlex",
]
@ -456,6 +457,15 @@ version = "2.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5"
[[package]]
name = "crossbeam-channel"
version = "0.5.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2"
dependencies = [
"crossbeam-utils",
]
[[package]]
name = "crossbeam-queue"
version = "0.3.11"
@ -533,6 +543,15 @@ dependencies = [
"zeroize",
]
[[package]]
name = "deranged"
version = "0.5.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c"
dependencies = [
"powerfmt",
]
[[package]]
name = "digest"
version = "0.10.7"
@ -624,6 +643,12 @@ version = "2.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "486f806e73c5707928240ddc295403b1b93c96a02038563881c4a2fd84b81ac4"
[[package]]
name = "find-msvc-tools"
version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582"
[[package]]
name = "flume"
version = "0.11.1"
@ -1272,6 +1297,16 @@ version = "0.3.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a"
[[package]]
name = "mime_guess"
version = "2.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e"
dependencies = [
"mime",
"unicase",
]
[[package]]
name = "miniz_oxide"
version = "0.8.0"
@ -1335,6 +1370,12 @@ dependencies = [
"zeroize",
]
[[package]]
name = "num-conv"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c6673768db2d862beb9b39a78fdcb1a69439615d5794a1be50caa9bc92c81967"
[[package]]
name = "num-integer"
version = "0.1.46"
@ -1463,6 +1504,12 @@ version = "0.3.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2"
[[package]]
name = "powerfmt"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391"
[[package]]
name = "ppv-lite86"
version = "0.2.20"
@ -1582,12 +1629,12 @@ dependencies = [
[[package]]
name = "reconcile-text"
version = "0.8.0"
version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "599cf9539996a2a19e501110404c59ba62f4974009f8fb864a8b7151c15ee5a5"
checksum = "52e0cf361887ea64c479ca871c1170dda761f84e122f2616b5579906a38d7557"
dependencies = [
"serde",
"thiserror 2.0.17",
"thiserror 2.0.18",
]
[[package]]
@ -1648,6 +1695,40 @@ dependencies = [
"zeroize",
]
[[package]]
name = "rust-embed"
version = "8.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "04113cb9355a377d83f06ef1f0a45b8ab8cd7d8b1288160717d66df5c7988d27"
dependencies = [
"rust-embed-impl",
"rust-embed-utils",
"walkdir",
]
[[package]]
name = "rust-embed-impl"
version = "8.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "da0902e4c7c8e997159ab384e6d0fc91c221375f6894346ae107f47dd0f3ccaa"
dependencies = [
"proc-macro2",
"quote",
"rust-embed-utils",
"syn 2.0.90",
"walkdir",
]
[[package]]
name = "rust-embed-utils"
version = "8.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5bcdef0be6fe7f6fa333b1073c949729274b05f123a0ad7efcb8efd878e5c3b1"
dependencies = [
"sha2",
"walkdir",
]
[[package]]
name = "rustc-demangle"
version = "0.1.24"
@ -1679,6 +1760,15 @@ version = "1.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f"
[[package]]
name = "same-file"
version = "1.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502"
dependencies = [
"winapi-util",
]
[[package]]
name = "sanitize-filename"
version = "0.6.0"
@ -1916,7 +2006,7 @@ dependencies = [
"serde_json",
"sha2",
"smallvec",
"thiserror 2.0.17",
"thiserror 2.0.18",
"tokio",
"tokio-stream",
"tracing",
@ -2000,7 +2090,7 @@ dependencies = [
"smallvec",
"sqlx-core",
"stringprep",
"thiserror 2.0.17",
"thiserror 2.0.18",
"tracing",
"uuid",
"whoami",
@ -2039,7 +2129,7 @@ dependencies = [
"smallvec",
"sqlx-core",
"stringprep",
"thiserror 2.0.17",
"thiserror 2.0.18",
"tracing",
"uuid",
"whoami",
@ -2065,7 +2155,7 @@ dependencies = [
"serde",
"serde_urlencoded",
"sqlx-core",
"thiserror 2.0.17",
"thiserror 2.0.18",
"tracing",
"url",
"uuid",
@ -2100,6 +2190,12 @@ version = "2.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
[[package]]
name = "symlink"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a7973cce6668464ea31f176d85b13c7ab3bba2cb3b77a2ed26abd7801688010a"
[[package]]
name = "syn"
version = "1.0.109"
@ -2136,18 +2232,22 @@ dependencies = [
"futures",
"humantime-serde",
"log",
"mime_guess",
"rand 0.9.0",
"reconcile-text",
"regex",
"rust-embed",
"sanitize-filename",
"serde",
"serde_json",
"serde_yaml",
"sqlx",
"thiserror 2.0.17",
"subtle",
"thiserror 2.0.18",
"tokio",
"tower-http",
"tracing",
"tracing-appender",
"tracing-subscriber",
"ts-rs",
"uuid",
@ -2203,11 +2303,11 @@ dependencies = [
[[package]]
name = "thiserror"
version = "2.0.17"
version = "2.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f63587ca0f12b72a0600bcba1d40081f830876000bb46dd2337a3051618f4fc8"
checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4"
dependencies = [
"thiserror-impl 2.0.17",
"thiserror-impl 2.0.18",
]
[[package]]
@ -2223,9 +2323,9 @@ dependencies = [
[[package]]
name = "thiserror-impl"
version = "2.0.17"
version = "2.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913"
checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5"
dependencies = [
"proc-macro2",
"quote",
@ -2242,6 +2342,37 @@ dependencies = [
"once_cell",
]
[[package]]
name = "time"
version = "0.3.47"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c"
dependencies = [
"deranged",
"itoa",
"num-conv",
"powerfmt",
"serde_core",
"time-core",
"time-macros",
]
[[package]]
name = "time-core"
version = "0.1.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca"
[[package]]
name = "time-macros"
version = "0.2.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215"
dependencies = [
"num-conv",
"time-core",
]
[[package]]
name = "tinystr"
version = "0.7.6"
@ -2276,7 +2407,6 @@ dependencies = [
"bytes",
"libc",
"mio",
"parking_lot",
"pin-project-lite",
"signal-hook-registry",
"socket2",
@ -2376,6 +2506,19 @@ dependencies = [
"tracing-core",
]
[[package]]
name = "tracing-appender"
version = "0.2.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "050686193eb999b4bb3bc2acfa891a13da00f79734704c4b8b4ef1a10b368a3c"
dependencies = [
"crossbeam-channel",
"symlink",
"thiserror 2.0.18",
"time",
"tracing-subscriber",
]
[[package]]
name = "tracing-attributes"
version = "0.1.28"
@ -2434,7 +2577,7 @@ checksum = "e640d9b0964e9d39df633548591090ab92f7a4567bc31d3891af23471a3365c6"
dependencies = [
"chrono",
"lazy_static",
"thiserror 2.0.17",
"thiserror 2.0.18",
"ts-rs-macros",
"uuid",
]
@ -2481,6 +2624,12 @@ version = "0.10.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f720def6ce1ee2fc44d40ac9ed6d3a59c361c80a75a7aa8e75bb9baed31cf2ea"
[[package]]
name = "unicase"
version = "2.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142"
[[package]]
name = "unicode-bidi"
version = "0.3.17"
@ -2577,6 +2726,16 @@ version = "0.9.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
[[package]]
name = "walkdir"
version = "2.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b"
dependencies = [
"same-file",
"winapi-util",
]
[[package]]
name = "wasi"
version = "0.11.0+wasi-snapshot-preview1"

View file

@ -1,6 +1,6 @@
[package]
name = "sync_server"
rust-version = "1.89.0"
rust-version = "1.94.0"
authors = ["Andras Schmelczer <andras@schmelczer.dev>"]
edition = "2024"
license = "MIT"
@ -10,7 +10,7 @@ version = "0.14.0"
[dependencies]
serde = { version = "1.0.219", default-features = false, features = ["derive"] }
thiserror = { version = "2.0.12", default-features = false }
tokio = { version = "1.48.0", features = ["full"]}
tokio = { version = "1.48.0", features = ["macros", "rt-multi-thread", "sync", "time", "net", "fs", "signal"]}
uuid = { version = "1.16.0", features = ["v4", "serde"] }
log = { version = "0.4.28" }
anyhow = { version = "1.0.100", features = ["backtrace"] }
@ -20,6 +20,7 @@ axum_typed_multipart = "0.11.0"
tower-http = { version = "0.6.1", features = ["cors", "trace", "limit", "timeout"] }
tracing = "0.1.41"
tracing-subscriber = { version = "0.3.20", features = ["fmt", "env-filter"]}
tracing-appender = "0.2.5"
humantime-serde = "1.1.1"
sqlx = { version = "0.8.6", features = ["sqlite", "runtime-tokio", "uuid", "chrono"] }
chrono = { version = "0.4.41", features = ["serde"] }
@ -33,7 +34,10 @@ serde_json = "1.0.140"
bimap = "0.6.3"
ts-rs = { version = "10.1", features = ["uuid-impl", "chrono-impl"] }
base64 = "0.22.1"
reconcile-text = { version = "0.8.0", features = ["serde"] }
reconcile-text = { version = "0.11.0", features = ["serde"] }
rust-embed = "8.5"
mime_guess = "2.0"
subtle = "2.6.1"
[profile.release]
codegen-units = 1

View file

@ -1,5 +1,16 @@
// generated by `sqlx migrate build-script`
fn main() {
// trigger recompilation when a new migration is added
println!("cargo:rerun-if-changed=migrations");
// Ensure the history-ui dist directory exists so rust-embed can compile
// even when the frontend hasn't been built yet.
let dist_path = std::path::Path::new("../frontend/history-ui/dist");
if !dist_path.exists() {
std::fs::create_dir_all(dist_path).expect("Failed to create history-ui dist directory");
std::fs::write(
dist_path.join("index.html"),
"<!DOCTYPE html><html><body><p>Run <code>npm run build -w history-ui</code> first.</p></body></html>",
)
.expect("Failed to write placeholder index.html");
}
}

View file

@ -1,32 +1,34 @@
database:
databases_directory_path: databases
max_connections_per_vault: 12
databases_directory_path: /host/tmp/vaultlink-e2e-databases
max_connections_per_vault: 8
cursor_timeout: 1m
server:
host: 0.0.0.0
port: 3000
port: 3010
max_body_size_mb: 512
max_clients_per_vault: 256
max_pending_websocket_connections: 4096
broadcast_channel_capacity: 1024
response_timeout: 30m
mergeable_file_extensions:
- md
- txt
- md
- txt
users:
user_configs:
- name: admin
token: test-token-change-me
vault_access:
type: allow_access_to_all
- name: other-admin
token: test-token-change-me2
vault_access:
type: allow_access_to_all
- name: test
token: other-test-token
vault_access:
type: allow_list
allowed:
- default
- name: admin
token: test-token-change-me
vault_access:
type: allow_access_to_all
- name: other-admin
token: test-token-change-me2
vault_access:
type: allow_access_to_all
- name: test
token: other-test-token
vault_access:
type: allow_list
allowed:
- default
logging:
log_directory: logs
log_rotation: 7days

View file

@ -1,5 +1,5 @@
[toolchain]
channel = "1.89.0"
channel = "1.94.0"
targets = [
"x86_64-unknown-linux-gnu",
"x86_64-unknown-linux-musl",

View file

@ -2,6 +2,8 @@ pub mod cursors;
pub mod database;
pub mod websocket;
use std::sync::{Arc, atomic::AtomicUsize};
use anyhow::Result;
use cursors::Cursors;
use database::Database;
@ -15,21 +17,42 @@ pub struct AppState {
pub database: Database,
pub cursors: Cursors,
pub broadcasts: Broadcasts,
/// Tracks WebSocket connections that have upgraded but not yet completed
/// the authentication handshake
pub pending_ws_connections: Arc<AtomicUsize>,
/// Send on this channel to stop background tasks (cursor cleanup,
/// idle-pool cleanup)
shutdown_tx: Arc<tokio::sync::watch::Sender<()>>,
}
impl AppState {
pub async fn try_new(config: Config) -> Result<Self> {
let (shutdown_tx, shutdown_rx) = tokio::sync::watch::channel(());
let broadcasts = Broadcasts::new(&config.server);
let database = Database::try_new(&config.database, &broadcasts).await?;
let database =
Database::try_new(&config.database, &broadcasts, shutdown_rx.clone()).await?;
let cursors: Cursors = Cursors::new(&config.database, &broadcasts);
Cursors::start_background_task(cursors.clone());
Cursors::start_background_task(cursors.clone(), shutdown_rx);
Ok(Self {
config,
database,
cursors,
broadcasts,
pending_ws_connections: Arc::new(AtomicUsize::new(0)),
shutdown_tx: Arc::new(shutdown_tx),
})
}
/// Signal all background tasks (idle pool cleanup, cursor cleanup) to stop
pub fn shutdown(&self) {
let _ = self.shutdown_tx.send(());
}
/// Get a receiver to be notified when shutdown is triggered
pub fn subscribe_shutdown(&self) -> tokio::sync::watch::Receiver<()> {
self.shutdown_tx.subscribe()
}
}

View file

@ -42,7 +42,9 @@ impl Cursors {
) {
let mut vault_to_cursors = self.vault_to_cursors.lock().await;
let all_device_cursors = vault_to_cursors.entry(vault_id).or_insert_with(Vec::new);
let all_device_cursors = vault_to_cursors
.entry(vault_id.clone())
.or_insert_with(Vec::new);
all_device_cursors.retain(|c| &c.client_cursors.device_id != device_id);
all_device_cursors.push(ClientCursorsWithTimeToLive::new(ClientCursors {
@ -52,7 +54,7 @@ impl Cursors {
}));
drop(vault_to_cursors); // Explicitly drop the lock before broadcasting to avoid deadlock
self.broadcast_cursors().await;
self.broadcast_cursors_for_vault(&vault_id).await;
}
pub async fn get_cursors(&self, vault_id: &VaultId) -> Vec<ClientCursors> {
@ -69,45 +71,81 @@ impl Cursors {
.unwrap_or_default()
}
pub fn start_background_task(self) {
pub fn start_background_task(self, mut shutdown: tokio::sync::watch::Receiver<()>) {
tokio::spawn(async move {
loop {
self.remove_expired_cursors().await;
tokio::time::sleep(Duration::from_secs(1)).await;
tokio::select! {
() = tokio::time::sleep(Duration::from_secs(1)) => {
self.remove_expired_cursors().await;
}
Ok(()) = shutdown.changed() => break,
}
}
});
}
async fn remove_expired_cursors(&self) {
let mut vault_to_cursors = self.vault_to_cursors.lock().await;
let changed_vaults: Vec<VaultId> = {
let mut vault_to_cursors = self.vault_to_cursors.lock().await;
for (_vault_id, cursors) in vault_to_cursors.iter_mut() {
cursors.retain(|cursor| !cursor.is_expired(self.config.cursor_timeout));
let mut changed = Vec::new();
for (vault_id, cursors) in vault_to_cursors.iter_mut() {
let before = cursors.len();
cursors.retain(|cursor| !cursor.is_expired(self.config.cursor_timeout));
if cursors.len() != before {
changed.push(vault_id.clone());
}
}
// Remove empty vault entries to prevent unbounded growth
vault_to_cursors.retain(|_, cursors| !cursors.is_empty());
changed
};
for vault_id in &changed_vaults {
self.broadcast_cursors_for_vault(vault_id).await;
}
}
async fn broadcast_cursors(&self) {
let vault_to_cursors = self.vault_to_cursors.lock().await;
async fn broadcast_cursors_for_vault(&self, vault_id: &VaultId) {
let client_cursors: Vec<ClientCursors> = {
let vault_to_cursors = self.vault_to_cursors.lock().await;
vault_to_cursors
.get(vault_id)
.map(|cursors| cursors.iter().map(|c| c.client_cursors.clone()).collect())
.unwrap_or_default()
};
for (vault_id, cursors) in vault_to_cursors.iter() {
self.broadcasts
.send_document_update(
vault_id.clone(),
WebSocketServerMessageWithOrigin::new(WebSocketServerMessage::CursorPositions(
CursorPositionFromServer {
clients: cursors.iter().map(|c| c.client_cursors.clone()).collect(),
},
)),
)
.await;
}
self.broadcasts.send_document_update(
vault_id.clone(),
WebSocketServerMessageWithOrigin::new(WebSocketServerMessage::CursorPositions(
CursorPositionFromServer {
clients: client_cursors,
},
)),
);
}
pub async fn remove_cursors_of_device(&self, vault_id: &str, device_id: &str) {
let mut vault_to_cursors = self.vault_to_cursors.lock().await;
pub async fn remove_cursors_of_device(&self, vault_id: &VaultId, device_id: &DeviceId) {
let changed = {
let mut vault_to_cursors = self.vault_to_cursors.lock().await;
if let Some(cursors) = vault_to_cursors.get_mut(vault_id) {
cursors.retain(|c| c.client_cursors.device_id != device_id);
if let Some(cursors) = vault_to_cursors.get_mut(vault_id) {
let before = cursors.len();
cursors.retain(|c| c.client_cursors.device_id != *device_id);
let changed = cursors.len() != before;
if cursors.is_empty() {
vault_to_cursors.remove(vault_id);
}
changed
} else {
false
}
};
if changed {
self.broadcast_cursors_for_vault(vault_id).await;
}
}
}

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,2 @@
CREATE INDEX IF NOT EXISTS idx_documents_document_id
ON documents (document_id, vault_update_id);

View file

@ -0,0 +1,20 @@
ALTER TABLE documents ADD COLUMN creation_vault_update_id INTEGER NOT NULL DEFAULT 0;
UPDATE documents
SET creation_vault_update_id = (
SELECT MIN(d2.vault_update_id)
FROM documents d2
WHERE d2.document_id = documents.document_id
);
DROP VIEW latest_document_versions;
CREATE VIEW IF NOT EXISTS latest_document_versions AS --recreate view as it now includes one more field
SELECT d.*
FROM documents d
INNER JOIN (
SELECT MAX(vault_update_id) AS max_version_id
FROM documents
GROUP BY document_id
) max_versions
ON d.vault_update_id = max_versions.max_version_id;

View file

@ -13,6 +13,7 @@ pub type DeviceId = String;
#[derive(Debug, Clone)]
pub struct StoredDocumentVersion {
pub vault_update_id: VaultUpdateId,
pub creation_vault_update_id: VaultUpdateId,
pub document_id: DocumentId,
pub relative_path: String,
pub updated_date: DateTime<Utc>,
@ -33,7 +34,7 @@ impl PartialEq<Self> for StoredDocumentVersion {
#[derive(TS, Debug, Clone, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct DocumentVersionWithoutContent {
#[ts(as = "i32")]
#[ts(type = "number")]
pub vault_update_id: VaultUpdateId,
pub document_id: DocumentId,
@ -43,12 +44,16 @@ pub struct DocumentVersionWithoutContent {
pub user_id: UserId,
pub device_id: DeviceId,
#[ts(as = "i32")]
#[ts(type = "number")]
pub content_size: u64,
/// True iff this is the first version of the document
pub is_new_file: bool,
}
impl From<StoredDocumentVersion> for DocumentVersionWithoutContent {
fn from(value: StoredDocumentVersion) -> Self {
let is_new_file = value.creation_vault_update_id == value.vault_update_id;
Self {
vault_update_id: value.vault_update_id,
document_id: value.document_id,
@ -58,6 +63,7 @@ impl From<StoredDocumentVersion> for DocumentVersionWithoutContent {
user_id: value.user_id,
device_id: value.device_id,
content_size: value.content.len() as u64,
is_new_file,
}
}
}
@ -65,7 +71,7 @@ impl From<StoredDocumentVersion> for DocumentVersionWithoutContent {
#[derive(TS, Debug, Clone, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct DocumentVersion {
#[ts(as = "i32")]
#[ts(type = "number")]
pub vault_update_id: VaultUpdateId,
pub document_id: DocumentId,
@ -77,6 +83,25 @@ pub struct DocumentVersion {
pub device_id: DeviceId,
}
/// Row struct for vault history queries (used by `sqlx::query_as!`)
#[derive(Debug)]
pub struct VaultHistoryRow {
pub vault_update_id: VaultUpdateId,
pub creation_vault_update_id: VaultUpdateId,
pub document_id: DocumentId,
pub relative_path: String,
pub updated_date: DateTime<Utc>,
pub is_deleted: bool,
pub user_id: String,
pub device_id: String,
pub content_size: Option<u64>,
}
pub struct VaultStats {
pub created_at: Option<DateTime<Utc>>,
pub document_count: u32,
}
impl From<StoredDocumentVersion> for DocumentVersion {
fn from(value: StoredDocumentVersion) -> Self {
Self {

View file

@ -1,69 +1,147 @@
use std::{collections::HashMap, sync::Arc};
use std::{
collections::HashMap,
sync::{Arc, Mutex as StdMutex},
};
use anyhow::Context;
use log::{debug, warn};
use log::{debug, info, warn};
use tokio::sync::{Mutex, broadcast};
use super::models::WebSocketServerMessageWithOrigin;
use crate::{
app_state::database::models::VaultId, config::server_config::ServerConfig, errors::server_error,
};
use super::models::{WebSocketServerMessage, WebSocketServerMessageWithOrigin};
use crate::{app_state::database::models::VaultId, config::server_config::ServerConfig};
#[derive(Debug, Clone)]
pub struct Broadcasts {
max_clients_per_vault: usize,
tx: Arc<Mutex<HashMap<VaultId, broadcast::Sender<WebSocketServerMessageWithOrigin>>>>,
broadcast_channel_capacity: usize,
// `tx` uses a blocking std::sync::Mutex because the critical section is
// a HashMap lookup plus a synchronous `broadcast::Sender::send`. Making
// this non-async lets `send_document_update` run without an `.await`,
// so an axum handler that is cancelled between `transaction.commit()`
// and the broadcast can never drop the notification mid-flight.
tx: Arc<StdMutex<HashMap<VaultId, broadcast::Sender<WebSocketServerMessageWithOrigin>>>>,
send_locks: Arc<Mutex<HashMap<VaultId, Arc<tokio::sync::Mutex<()>>>>>,
}
type TxMap = HashMap<VaultId, broadcast::Sender<WebSocketServerMessageWithOrigin>>;
impl Broadcasts {
pub fn new(server_config: &ServerConfig) -> Self {
Self {
max_clients_per_vault: server_config.max_clients_per_vault,
tx: Arc::new(Mutex::new(HashMap::new())),
broadcast_channel_capacity: server_config.broadcast_channel_capacity,
tx: Arc::new(StdMutex::new(HashMap::new())),
send_locks: Arc::new(Mutex::new(HashMap::new())),
}
}
pub async fn get_receiver(
/// Acquire a per-vault lock that serializes broadcasts in commit order.
/// Must be acquired before the insert, held through commit and broadcast.
pub async fn acquire_send_lock(&self, vault: &VaultId) -> tokio::sync::OwnedMutexGuard<()> {
let lock = {
let mut locks = self.send_locks.lock().await;
locks
.entry(vault.clone())
.or_insert_with(|| Arc::new(tokio::sync::Mutex::new(())))
.clone()
};
lock.lock_owned().await
}
/// Remove senders for vaults with no active receivers
fn prune_inactive_vaults(tx_map: &mut TxMap) -> Vec<VaultId> {
let mut pruned = Vec::new();
tx_map.retain(|vault, sender| {
let alive = sender.receiver_count() > 0;
if !alive {
pruned.push(vault.clone());
}
alive
});
pruned
}
pub fn get_receiver(
&self,
vault: VaultId,
) -> broadcast::Receiver<WebSocketServerMessageWithOrigin> {
let tx = self.get_or_create(vault).await;
max_clients: usize,
) -> Result<broadcast::Receiver<WebSocketServerMessageWithOrigin>, crate::errors::SyncServerError>
{
let mut tx_map = self
.tx
.lock()
.expect("broadcasts.tx mutex poisoned — a previous holder panicked");
tx.subscribe()
let count_before_prune = tx_map
.get(&vault)
.map_or(0, tokio::sync::broadcast::Sender::receiver_count);
let pruned = Self::prune_inactive_vaults(&mut tx_map);
let pruned_self = pruned.contains(&vault);
let sender = tx_map
.entry(vault.clone())
.or_insert_with(|| broadcast::channel(self.broadcast_channel_capacity).0);
// Hold the lock across the count check *and* the subscribe so the
// `max_clients` cap is atomic: two concurrent callers can't both
// observe `receiver_count() < max_clients` and both subscribe.
if sender.receiver_count() >= max_clients {
return Err(crate::errors::client_error(anyhow::anyhow!(
"Vault has reached the maximum number of clients ({max_clients})"
)));
}
let receiver = sender.subscribe();
let count_after = sender.receiver_count();
info!(
"[BCAST] get_receiver vault={vault} count_before_prune={count_before_prune} pruned_self={pruned_self} pruned_total={} count_after_subscribe={count_after}",
pruned.len()
);
Ok(receiver)
}
/// Notify all clients (who are subscribed to the vault) about an update.
/// We only log failures and don't propagate them.
pub async fn send_document_update(
&self,
vault: VaultId,
document: WebSocketServerMessageWithOrigin,
) {
let tx = self.get_or_create(vault.clone()).await;
/// Synchronous: safe to invoke from a handler between `commit()` and
/// function return without worrying about task cancellation dropping
/// the broadcast mid-flight. Failures are logged, never propagated.
pub fn send_document_update(&self, vault: VaultId, document: WebSocketServerMessageWithOrigin) {
let vault_update_id = match &document.message {
WebSocketServerMessage::VaultUpdate(u) => Some(u.document.vault_update_id),
WebSocketServerMessage::CursorPositions(_) => None,
};
let is_deleted = match &document.message {
WebSocketServerMessage::VaultUpdate(u) => Some(u.document.is_deleted),
WebSocketServerMessage::CursorPositions(_) => None,
};
let mut tx_map = self
.tx
.lock()
.expect("broadcasts.tx mutex poisoned — a previous holder panicked");
let count_before_prune = tx_map
.get(&vault)
.map_or(0, tokio::sync::broadcast::Sender::receiver_count);
let pruned = Self::prune_inactive_vaults(&mut tx_map);
let pruned_self = pruned.contains(&vault);
if tx.receiver_count() == 0 {
let sender = tx_map
.entry(vault.clone())
.or_insert_with(|| broadcast::channel(self.broadcast_channel_capacity).0);
let count_before_send = sender.receiver_count();
if count_before_send == 0 {
info!(
"[BCAST] send_document_update vault={vault} vuid={vault_update_id:?} is_deleted={is_deleted:?} count_before_prune={count_before_prune} pruned_self={pruned_self} count_before_send=0 SKIPPED"
);
debug!("Skipping broadcast, no clients connected for vault `{vault}`");
return;
}
let result = tx
.send(document)
.context("Cannot broadcast server message to websocket listeners")
.map_err(server_error);
if result.is_err() {
warn!("Failed to send message: {result:?}");
let send_result = sender.send(document);
match &send_result {
Ok(n) => info!(
"[BCAST] send_document_update vault={vault} vuid={vault_update_id:?} is_deleted={is_deleted:?} count_before_prune={count_before_prune} pruned_self={pruned_self} count_before_send={count_before_send} SENT delivered_to={n}"
),
Err(e) => warn!(
"[BCAST] send_document_update vault={vault} vuid={vault_update_id:?} is_deleted={is_deleted:?} count_before_prune={count_before_prune} pruned_self={pruned_self} count_before_send={count_before_send} FAILED err={e}"
),
}
}
async fn get_or_create(
&self,
vault: VaultId,
) -> broadcast::Sender<WebSocketServerMessageWithOrigin> {
let mut tx = self.tx.lock().await;
tx.entry(vault)
.or_insert_with(|| broadcast::channel(self.max_clients_per_vault).0.clone())
.clone()
}
}

View file

@ -11,7 +11,7 @@ pub struct WebSocketHandshake {
pub token: String,
pub device_id: DeviceId,
#[ts(as = "Option<i32>")]
#[ts(type = "number | null")]
pub last_seen_vault_update_id: Option<VaultUpdateId>,
}
@ -22,13 +22,14 @@ pub struct CursorPositionFromClient {
}
#[derive(TS, Serialize, Deserialize, Clone, Debug)]
#[serde(rename_all = "camelCase")]
pub struct DocumentWithCursors {
// It's None in case the document is dirty.
// We still want to sync the cursor to mark
// that it exists and can be client-side
// interpolated. However, the actual
// position is meaningless.
#[ts(as = "Option<u32>")]
#[ts(type = "number | null")]
pub vault_update_id: Option<VaultUpdateId>,
pub document_id: DocumentId,
@ -57,11 +58,19 @@ pub struct CursorPositionFromServer {
pub clients: Vec<ClientCursors>,
}
// One committed version. Non-delete updates are broadcast to every
// connected client *except* the device that authored them — that
// device already has the new state via its HTTP response. Deletes are
// broadcast to every client including the author: the author keeps
// the document in its sync queue until this receipt arrives so a late
// remote update can't sneak in between the HTTP response and the
// queue cleanup. The server also emits these one-at-a-time to catch
// up a freshly-connected client on versions committed while it was
// offline, in ascending `vault_update_id` order.
#[derive(TS, Serialize, Clone, Debug)]
#[serde(rename_all = "camelCase")]
pub struct WebSocketVaultUpdate {
pub documents: Vec<DocumentVersionWithoutContent>,
pub is_initial_sync: bool,
pub document: DocumentVersionWithoutContent,
}
#[derive(TS, Deserialize, Clone, Debug)]
@ -80,6 +89,10 @@ pub enum WebSocketServerMessage {
CursorPositions(CursorPositionFromServer),
}
/// Broadcast envelope carrying the message plus the device that produced
/// it. The per-recipient send task compares `origin_device_id` against
/// its own device id to fill in `originates_from_self` before the message
/// is serialized on the wire.
#[derive(Clone, Debug)]
pub struct WebSocketServerMessageWithOrigin {
pub origin_device_id: Option<DeviceId>,

View file

@ -9,7 +9,7 @@ use crate::{
database::models::{DocumentVersionWithoutContent, VaultId, VaultUpdateId},
},
config::user_config::User,
errors::{SyncServerError, server_error, unauthenticated_error},
errors::{SyncServerError, client_error, server_error, unauthenticated_error},
server::auth::auth,
};
@ -26,7 +26,7 @@ pub fn get_authenticated_handshake(
if let Some(Message::Text(message)) = message {
let message: WebSocketClientMessage = serde_json::from_str(&message)
.context("Failed to parse message")
.map_err(server_error)?;
.map_err(client_error)?;
match message {
WebSocketClientMessage::Handshake(handshake) => {
@ -44,21 +44,29 @@ pub fn get_authenticated_handshake(
}
}
/// Stream the documents the client missed while offline, bounded above
/// by `up_to_vault_update_id` so the catch-up is a stable snapshot at
/// exactly that cursor. The WebSocket handshake atomically subscribes
/// to the broadcast channel and snapshots this cursor under the per-
/// vault send lock; commits past the cursor are then delivered solely
/// through the broadcast channel (filtered by the same cursor on the
/// receive side), so every committed update is delivered exactly once.
pub async fn get_unseen_documents(
state: &AppState,
vault_id: &VaultId,
last_seen_vault_update_id: Option<VaultUpdateId>,
up_to_vault_update_id: VaultUpdateId,
) -> Result<Vec<DocumentVersionWithoutContent>, SyncServerError> {
if let Some(update_id) = last_seen_vault_update_id {
state
.database
.get_latest_documents_since(vault_id, update_id, None)
.get_latest_documents_since(vault_id, update_id, Some(up_to_vault_update_id), None)
.await
.map_err(server_error)
} else {
state
.database
.get_latest_documents(vault_id, None)
.get_latest_documents(vault_id, Some(up_to_vault_update_id), None)
.await
.map_err(server_error)
}

View file

@ -27,24 +27,34 @@ pub struct Config {
}
impl Config {
pub fn validate(&self) -> Result<()> {
self.server
.validate()
.context("Invalid server configuration")?;
self.logging
.validate()
.context("Invalid logging configuration")?;
self.database
.validate()
.context("Invalid database configuration")?;
Ok(())
}
pub async fn read_or_create(path: &Path) -> Result<Self> {
let config = if path.exists() {
info!(
"Loading configuration from `{}`",
path.canonicalize().unwrap().display()
);
Self::load_from_file(path).await?
let display_path = path.canonicalize().unwrap_or_else(|_| path.to_path_buf());
if path.exists() {
info!("Loading configuration from `{}`", display_path.display());
Self::load_from_file(path).await
} else {
Self::default()
};
config.write(path).await?;
info!(
"Updated configuration at `{}`",
path.canonicalize().unwrap().display()
);
Ok(config)
let config = Self::default();
config.write(path).await?;
info!(
"Created default configuration at `{}`",
display_path.display()
);
Ok(config)
}
}
pub async fn load_from_file(path: &Path) -> Result<Self> {

View file

@ -1,5 +1,6 @@
use std::{path::PathBuf, time::Duration};
use anyhow::{Result, ensure};
use log::debug;
use serde::{Deserialize, Serialize};
@ -34,6 +35,24 @@ fn default_cursor_timeout() -> Duration {
DEFAULT_CURSOR_TIMEOUT
}
impl DatabaseConfig {
pub fn validate(&self) -> Result<()> {
ensure!(
!self.databases_directory_path.as_os_str().is_empty(),
"databases_directory_path must not be empty"
);
ensure!(
self.max_connections_per_vault > 0,
"max_connections_per_vault must be greater than 0"
);
ensure!(
!self.cursor_timeout.is_zero(),
"cursor_timeout must be greater than 0"
);
Ok(())
}
}
impl Default for DatabaseConfig {
fn default() -> Self {
Self {

View file

@ -1,10 +1,13 @@
use std::time::Duration;
use anyhow::{Result, ensure};
use log::debug;
use serde::{Deserialize, Serialize};
use crate::{
consts::{DEFAULT_LOG_DIRECTORY, DEFAULT_LOG_LEVEL, DEFAULT_LOG_ROTATION_INTERVAL},
consts::{
DEFAULT_LOG_DIRECTORY, DEFAULT_LOG_LEVEL, DEFAULT_LOG_ROTATION_INTERVAL, DURATION_ZERO,
},
utils::log_level::LogLevel,
};
@ -20,6 +23,20 @@ pub struct LoggingConfig {
pub log_level: LogLevel,
}
impl LoggingConfig {
pub fn validate(&self) -> Result<()> {
ensure!(
!self.log_directory.is_empty(),
"log_directory must not be an empty string"
);
ensure!(
self.log_rotation > DURATION_ZERO,
"log_rotation must be greater than 0"
);
Ok(())
}
}
impl Default for LoggingConfig {
fn default() -> Self {
Self {

View file

@ -1,10 +1,13 @@
use anyhow::{Result, ensure};
use log::debug;
use serde::{Deserialize, Serialize};
use std::time::Duration;
use crate::consts::{
DEFAULT_HOST, DEFAULT_MAX_BODY_SIZE_MB, DEFAULT_MAX_CLIENTS_PER_VAULT,
DEFAULT_MERGEABLE_FILE_EXTENSIONS, DEFAULT_PORT, DEFAULT_RESPONSE_TIMEOUT_SECONDS,
DEFAULT_ALLOWED_ORIGINS, DEFAULT_BROADCAST_CHANNEL_CAPACITY, DEFAULT_HOST,
DEFAULT_MAX_BODY_SIZE_MB, DEFAULT_MAX_CLIENTS_PER_VAULT, DEFAULT_MAX_PENDING_WS_CONNECTIONS,
DEFAULT_MERGEABLE_FILE_EXTENSIONS, DEFAULT_PORT, DEFAULT_RATE_LIMIT_PER_USER_PER_SECOND,
DEFAULT_RESPONSE_TIMEOUT_SECONDS, DURATION_ZERO,
};
#[derive(Debug, Deserialize, Serialize, Clone, Default)]
@ -21,11 +24,56 @@ pub struct ServerConfig {
#[serde(default = "default_max_clients_per_vault")]
pub max_clients_per_vault: usize,
#[serde(default = "default_broadcast_channel_capacity")]
pub broadcast_channel_capacity: usize,
#[serde(default = "default_response_timeout", with = "humantime_serde")]
pub response_timeout: Duration,
#[serde(default = "default_mergeable_file_extensions")]
pub mergeable_file_extensions: Vec<String>,
/// Per-user maximum requests per second (keyed by bearer token).
/// `None` disables rate limiting.
#[serde(default = "default_rate_limit_per_user_per_second")]
pub rate_limit_per_user_per_second: Option<u64>,
/// Allowed CORS origins. Default: `["*"]` (allow all).
#[serde(default = "default_allowed_origins")]
pub allowed_origins: Vec<String>,
/// Maximum concurrent unauthenticated WebSocket connections waiting for
/// handshake. Limits resource consumption from clients that connect but
/// never authenticate.
#[serde(default = "default_max_pending_websocket_connections")]
pub max_pending_websocket_connections: usize,
}
impl ServerConfig {
pub fn validate(&self) -> Result<()> {
ensure!(
self.response_timeout > DURATION_ZERO,
"response_timeout must be greater than 0"
);
ensure!(
self.max_body_size_mb > 0,
"max_body_size_mb must be greater than 0"
);
ensure!(
self.max_clients_per_vault > 0,
"max_clients_per_vault must be greater than 0"
);
ensure!(
self.broadcast_channel_capacity > 0,
"broadcast_channel_capacity must be greater than 0"
);
ensure!(
self.max_pending_websocket_connections > 0,
"max_pending_websocket_connections must be greater than 0"
);
Ok(())
}
}
fn default_host() -> String {
@ -48,6 +96,11 @@ fn default_max_clients_per_vault() -> usize {
DEFAULT_MAX_CLIENTS_PER_VAULT
}
fn default_broadcast_channel_capacity() -> usize {
debug!("Using default broadcast channel capacity: {DEFAULT_BROADCAST_CHANNEL_CAPACITY}");
DEFAULT_BROADCAST_CHANNEL_CAPACITY
}
fn default_response_timeout() -> Duration {
debug!("Using default response timeout: {DEFAULT_RESPONSE_TIMEOUT_SECONDS:?}");
DEFAULT_RESPONSE_TIMEOUT_SECONDS
@ -60,3 +113,21 @@ fn default_mergeable_file_extensions() -> Vec<String> {
.map(|s| (*s).to_owned())
.collect()
}
fn default_rate_limit_per_user_per_second() -> Option<u64> {
debug!("Using default rate limit per second: {DEFAULT_RATE_LIMIT_PER_USER_PER_SECOND:?}");
DEFAULT_RATE_LIMIT_PER_USER_PER_SECOND
}
fn default_allowed_origins() -> Vec<String> {
debug!("Using default allowed origins: {DEFAULT_ALLOWED_ORIGINS:?}");
DEFAULT_ALLOWED_ORIGINS
.iter()
.map(|s| (*s).to_owned())
.collect()
}
fn default_max_pending_websocket_connections() -> usize {
debug!("Using default max pending WebSocket connections: {DEFAULT_MAX_PENDING_WS_CONNECTIONS}");
DEFAULT_MAX_PENDING_WS_CONNECTIONS
}

View file

@ -1,6 +1,7 @@
use bimap::BiHashMap;
use rand::{Rng, distr::Alphanumeric, rng};
use serde::{Deserialize, Deserializer, Serialize, de::Error};
use subtle::ConstantTimeEq;
use crate::app_state::database::models::VaultId;
@ -19,10 +20,19 @@ where
let mut user_token_map = BiHashMap::new();
for user in &users {
if let Some(existing_name) = user_token_map.get_by_right(&user.token) {
let redacted = if user.token.len() > 6 {
format!(
"{}...{}",
&user.token[..3],
&user.token[user.token.len() - 3..]
)
} else {
"***".to_owned()
};
return Err(D::Error::custom(format!(
"Duplicate user token found: `{}` for users `{}` and `{}`. User tokens must be \
unique.",
user.token, existing_name, user.name
"Duplicate user token found: `{redacted}` for users `{}` and `{}`. User tokens \
must be unique.",
existing_name, user.name
)));
}
@ -41,7 +51,9 @@ where
impl UserConfig {
pub fn get_user(&self, token: &str) -> Option<&User> {
self.user_configs.iter().find(|u| u.token == token)
self.user_configs
.iter()
.find(|u| u.token.as_bytes().ct_eq(token.as_bytes()).into())
}
}

View file

@ -2,22 +2,36 @@ use std::time::Duration;
use crate::utils::log_level::LogLevel;
pub const DURATION_ZERO: Duration = Duration::from_secs(0);
pub const DEFAULT_CONFIG_PATH: &str = "config.yml";
pub const DEFAULT_DATABASES_DIRECTORY_PATH: &str = "databases";
pub const DEFAULT_MAX_CONNECTIONS_PER_VAULT: u32 = 12;
pub const DEFAULT_MAX_CONNECTIONS_PER_VAULT: u32 = 6;
pub const DEFAULT_CURSOR_TIMEOUT: Duration = Duration::from_secs(60);
pub const DEFAULT_HOST: &str = "127.0.0.1";
pub const DEFAULT_PORT: u16 = 3000;
pub const DEFAULT_MAX_BODY_SIZE_MB: usize = 4096;
pub const DEFAULT_RESPONSE_TIMEOUT_SECONDS: Duration = Duration::from_secs(1800);
pub const DEFAULT_RESPONSE_TIMEOUT_SECONDS: Duration = Duration::from_mins(30);
pub const DEFAULT_MAX_CLIENTS_PER_VAULT: usize = 256;
pub const DEFAULT_BROADCAST_CHANNEL_CAPACITY: usize = 4096;
pub const DEFAULT_MAX_PENDING_WS_CONNECTIONS: usize = 128;
pub const DEFAULT_LOG_DIRECTORY: &str = "logs";
pub const DEFAULT_LOG_ROTATION_INTERVAL: Duration = Duration::from_secs(60 * 60 * 24); // 1 day
pub const DEFAULT_LOG_ROTATION_INTERVAL: Duration = Duration::from_hours(24);
pub const IDLE_POOL_TIMEOUT: Duration = Duration::from_mins(5);
pub const GRACEFUL_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(10);
pub const HANDSHAKE_TIMEOUT: Duration = Duration::from_secs(10);
pub const MAX_CURSOR_DOCUMENTS: usize = 1000;
pub const MAX_CURSORS_PER_DOCUMENT: usize = 100;
pub const MAX_RELATIVE_PATH_LEN: usize = 4096;
pub const DEFAULT_LOG_LEVEL: LogLevel = LogLevel::Info;
pub const DEFAULT_MERGEABLE_FILE_EXTENSIONS: &[&str] = &["md", "txt"];
pub const SUPPORTED_API_VERSION: u32 = 2;
pub const DEFAULT_RATE_LIMIT_PER_USER_PER_SECOND: Option<u64> = None;
pub const DEFAULT_ALLOWED_ORIGINS: &[&str] = &["*"];
pub const SUPPORTED_API_VERSION: u32 = 3;

View file

@ -5,7 +5,7 @@ use axum::{
http::StatusCode,
response::{IntoResponse, Response},
};
use log::{debug, error};
use log::{debug, error, warn};
use serde::Serialize;
use thiserror::Error;
use ts_rs::TS;
@ -29,6 +29,9 @@ pub enum SyncServerError {
#[error("Permission denied error: {0}")]
PermissionDeniedError(#[source] anyhow::Error),
#[error("Too many requests: {0}")]
TooManyRequests(#[source] anyhow::Error),
}
impl SyncServerError {
@ -39,7 +42,8 @@ impl SyncServerError {
| Self::ServerError(error)
| Self::NotFound(error)
| Self::Unauthenticated(error)
| Self::PermissionDeniedError(error) => error.into(),
| Self::PermissionDeniedError(error)
| Self::TooManyRequests(error) => error.into(),
}
}
}
@ -69,7 +73,22 @@ impl Display for SerializedError {
impl IntoResponse for SyncServerError {
fn into_response(self) -> Response {
let body = Json(self.serialize());
let serialized = self.serialize();
match &self {
Self::InitError(_) | Self::ServerError(_) => {
error!("{serialized}");
}
Self::ClientError(_) | Self::NotFound(_) => {
warn!("{serialized}");
}
Self::TooManyRequests(_) => {
warn!("{serialized}");
}
Self::Unauthenticated(_) | Self::PermissionDeniedError(_) => {}
}
let body = Json(serialized);
match self {
Self::InitError(_) | Self::ServerError(_) => {
@ -79,6 +98,7 @@ impl IntoResponse for SyncServerError {
Self::NotFound(_) => (StatusCode::NOT_FOUND, body).into_response(),
Self::Unauthenticated(_) => (StatusCode::UNAUTHORIZED, body).into_response(),
Self::PermissionDeniedError(_) => (StatusCode::FORBIDDEN, body).into_response(),
Self::TooManyRequests(_) => (StatusCode::TOO_MANY_REQUESTS, body).into_response(),
}
}
}
@ -102,6 +122,7 @@ impl From<&anyhow::Error> for SerializedError {
SyncServerError::NotFound(_) => "NotFound",
SyncServerError::Unauthenticated(_) => "Unauthenticated",
SyncServerError::PermissionDeniedError(_) => "PermissionDeniedError",
SyncServerError::TooManyRequests(_) => "TooManyRequests",
},
),
message: error.to_string(),
@ -139,3 +160,21 @@ pub fn permission_denied_error(error: anyhow::Error) -> SyncServerError {
debug!("Permission denied: {error:?}");
SyncServerError::PermissionDeniedError(error)
}
pub fn too_many_requests_error(error: anyhow::Error) -> SyncServerError {
debug!("Too many requests: {error:?}");
SyncServerError::TooManyRequests(error)
}
/// Maps a `create_write_transaction` error to 429 if the database is busy,
/// or 500 for all other failures.
pub fn write_transaction_error(error: anyhow::Error) -> SyncServerError {
if error
.downcast_ref::<crate::app_state::database::WriteBusyError>()
.is_some()
{
too_many_requests_error(error)
} else {
server_error(error)
}
}

View file

@ -16,6 +16,7 @@ use consts::DEFAULT_CONFIG_PATH;
use errors::{SyncServerError, init_error};
use log::info;
use server::create_server;
use tracing_appender::non_blocking::WorkerGuard;
use tracing_subscriber::{EnvFilter, fmt::format, layer::SubscriberExt, util::SubscriberInitExt};
use utils::rotating_file_writer::RotatingFileWriter;
@ -41,11 +42,14 @@ async fn main() -> ExitCode {
}
};
let mut result = set_up_logging(&args, &config.logging);
if result.is_ok() {
result = start_server(config).await;
let result = async {
config.validate().map_err(init_error)?;
// Hold the non-blocking writer guards until shutdown so the
// dedicated writer threads stay alive and flush queued log lines.
let _log_guards = set_up_logging(&args, &config.logging)?;
start_server(config).await
}
.await;
match result {
Ok(()) => ExitCode::SUCCESS,
@ -59,7 +63,7 @@ async fn main() -> ExitCode {
fn set_up_logging(
args: &Args,
logging_config: &config::logging_config::LoggingConfig,
) -> Result<(), SyncServerError> {
) -> Result<[WorkerGuard; 2], SyncServerError> {
let level_filter = logging_config.log_level.as_tracing_level();
let env_filter = EnvFilter::builder()
@ -80,6 +84,14 @@ fn set_up_logging(
.context("Failed to create rotating file writer")
.map_err(init_error)?;
// Decouple log emission from disk/stderr I/O. Without this, a tokio
// worker that holds the writer's std::sync::Mutex while a `write(2)`
// is throttled by the kernel (e.g. btrfs writeback) cascades the
// stall to every other worker that tries to log, freezing the whole
// runtime. The guards must outlive every emitter.
let (file_writer, file_guard) = tracing_appender::non_blocking(file_appender);
let (stderr_writer, stderr_guard) = tracing_appender::non_blocking(std::io::stderr());
let format = format()
.with_target(is_debug_mode)
.with_line_number(is_debug_mode)
@ -87,12 +99,12 @@ fn set_up_logging(
let stderr_layer = tracing_subscriber::fmt::layer()
.with_ansi(use_colors)
.with_writer(std::io::stderr)
.with_writer(stderr_writer)
.event_format(format.clone());
let file_layer = tracing_subscriber::fmt::layer()
.with_ansi(false)
.with_writer(file_appender)
.with_writer(file_writer)
.event_format(format);
tracing_subscriber::registry()
@ -103,7 +115,7 @@ fn set_up_logging(
.context("Failed to initialise tracing")
.map_err(init_error)?;
Ok(())
Ok([file_guard, stderr_guard])
}
async fn start_server(config: Config) -> Result<(), SyncServerError> {

View file

@ -4,27 +4,30 @@ mod delete_document;
mod device_id_header;
mod fetch_document_version;
mod fetch_document_version_content;
mod fetch_document_versions;
mod fetch_latest_document_version;
mod fetch_latest_documents;
mod fetch_vault_history;
mod index;
mod list_vaults;
mod ping;
mod rate_limit;
mod requests;
mod responses;
mod update_document;
mod websocket;
use anyhow::{Context as _, Result, anyhow};
use anyhow::{Context as _, Result};
use auth::auth_middleware;
use axum::{
Router,
extract::{DefaultBodyLimit, Request},
http::{self, HeaderValue, Method},
middleware,
response::IntoResponse,
routing::{IntoMakeService, delete, get, post, put},
};
use device_id_header::DEVICE_ID_HEADER_NAME;
use log::info;
use log::{info, warn};
use tokio::signal;
use tower_http::{
LatencyUnit,
@ -41,7 +44,7 @@ use tracing::{Level, info_span};
use crate::{
app_state::AppState,
config::{Config, server_config::ServerConfig},
errors::{client_error, not_found_error},
consts::GRACEFUL_SHUTDOWN_TIMEOUT,
};
pub async fn create_server(config: Config) -> Result<()> {
@ -51,26 +54,33 @@ pub async fn create_server(config: Config) -> Result<()> {
let server_config = app_state.config.server.clone();
let app = Router::new()
let mut app = Router::new()
.nest("/", get_authed_routes(app_state.clone()))
.route("/", get(index::index))
.route("/assets/*path", get(index::spa_assets))
.route("/vaults", get(list_vaults::list_vaults))
.route("/vaults/:vault_id/ping", get(ping::ping))
.route("/vaults/:vault_id/ws", get(websocket::websocket_handler))
.fallback(index::spa_fallback);
let cors_layer = build_cors_layer(&server_config).context("Invalid CORS configuration")?;
if let Some(rate_limit) = server_config.rate_limit_per_user_per_second {
info!("Rate limiting enabled: {rate_limit} requests/second per user");
let limiter = rate_limit::RateLimiter::new(rate_limit);
app = app.layer(middleware::from_fn_with_state(
limiter,
rate_limit::rate_limit_middleware,
));
}
let app = app
.layer(DefaultBodyLimit::disable())
.layer(RequestBodyLimitLayer::new(
app_state.config.server.max_body_size_mb * 1024 * 1024,
))
.layer(TimeoutLayer::new(server_config.response_timeout))
.layer(
CorsLayer::new()
.allow_origin("*".parse::<HeaderValue>().expect("Failed to parse origin"))
.allow_headers([
http::header::CONTENT_TYPE,
http::header::AUTHORIZATION,
DEVICE_ID_HEADER_NAME.clone(),
])
.allow_methods([Method::GET, Method::POST, Method::PUT, Method::DELETE]),
)
.layer(cors_layer)
.layer(
TraceLayer::new_for_http()
.make_span_with(|request: &Request<_>| {
@ -90,12 +100,39 @@ pub async fn create_server(config: Config) -> Result<()> {
.on_eos(DefaultOnEos::new())
.on_failure(DefaultOnFailure::new().level(Level::ERROR)),
)
.with_state(app_state)
.fallback(handle_404)
.fallback(handle_405)
.with_state(app_state.clone())
.into_make_service();
start_server(app, &server_config).await
start_server(app, &server_config, app_state).await
}
fn build_cors_layer(server_config: &ServerConfig) -> Result<CorsLayer> {
let origins = &server_config.allowed_origins;
let cors = if origins.len() == 1 && origins[0] == "*" {
info!("CORS: allowing all origins");
let header: HeaderValue = "*"
.parse()
.context("Failed to parse wildcard CORS origin")?;
CorsLayer::new().allow_origin(header)
} else {
let parsed: Vec<HeaderValue> = origins
.iter()
.map(|o| {
o.parse::<HeaderValue>()
.with_context(|| format!("Failed to parse CORS origin: `{o}`"))
})
.collect::<Result<Vec<_>>>()?;
CorsLayer::new().allow_origin(parsed)
};
Ok(cors
.allow_headers([
http::header::CONTENT_TYPE,
http::header::AUTHORIZATION,
DEVICE_ID_HEADER_NAME.clone(),
])
.allow_methods([Method::GET, Method::POST, Method::PUT, Method::DELETE]))
}
fn get_authed_routes(app_state: AppState) -> Router<AppState> {
@ -120,6 +157,10 @@ fn get_authed_routes(app_state: AppState) -> Router<AppState> {
"/vaults/:vault_id/documents/:document_id/text",
put(update_document::update_text),
)
.route(
"/vaults/:vault_id/documents/:document_id/versions",
get(fetch_document_versions::fetch_document_versions),
)
.route(
"/vaults/:vault_id/documents/:document_id/versions/:vault_update_id",
get(fetch_document_version::fetch_document_version),
@ -132,10 +173,18 @@ fn get_authed_routes(app_state: AppState) -> Router<AppState> {
"/vaults/:vault_id/documents/:document_id",
delete(delete_document::delete_document),
)
.route(
"/vaults/:vault_id/history",
get(fetch_vault_history::fetch_vault_history),
)
.layer(middleware::from_fn_with_state(app_state, auth_middleware))
}
async fn start_server(app: IntoMakeService<axum::Router>, config: &ServerConfig) -> Result<()> {
async fn start_server(
app: IntoMakeService<axum::Router>,
config: &ServerConfig,
app_state: AppState,
) -> Result<()> {
let address = format!("{}:{}", config.host, config.port);
let listener = tokio::net::TcpListener::bind(address.clone())
.await
@ -148,26 +197,46 @@ async fn start_server(app: IntoMakeService<axum::Router>, config: &ServerConfig)
.context("Failed to get local address")?
);
axum::serve(listener, app)
.with_graceful_shutdown(shutdown_signal())
.tcp_nodelay(true)
.await
.context("Failed to start server")
let mut shutdown_rx = app_state.subscribe_shutdown();
let server = axum::serve(listener, app)
.with_graceful_shutdown(async move {
shutdown_signal().await;
app_state.shutdown();
})
.tcp_nodelay(true);
tokio::select! {
result = server => result.context("Failed to start server"),
() = async {
let _ = shutdown_rx.changed().await;
info!(
"Shutdown signal received, waiting up to {}s for in-flight requests to complete...",
GRACEFUL_SHUTDOWN_TIMEOUT.as_secs()
);
tokio::time::sleep(GRACEFUL_SHUTDOWN_TIMEOUT).await;
warn!("Graceful shutdown timed out, forcing exit");
} => Ok(()),
}
}
async fn shutdown_signal() {
let ctrl_c = async {
signal::ctrl_c()
.await
.expect("failed to install Ctrl+C handler");
if let Err(e) = signal::ctrl_c().await {
log::error!("Failed to install Ctrl+C handler: {e}");
}
};
#[cfg(unix)]
let terminate = async {
signal::unix::signal(signal::unix::SignalKind::terminate())
.expect("failed to install signal handler")
.recv()
.await;
match signal::unix::signal(signal::unix::SignalKind::terminate()) {
Ok(mut signal) => {
signal.recv().await;
}
Err(e) => {
log::error!("Failed to install SIGTERM handler: {e}");
}
}
};
#[cfg(not(unix))]
@ -178,11 +247,3 @@ async fn shutdown_signal() {
() = terminate => {},
}
}
async fn handle_404() -> impl IntoResponse {
not_found_error(anyhow!("Page not found"))
}
async fn handle_405() -> impl IntoResponse {
client_error(anyhow!("Method not allowed"))
}

View file

@ -9,7 +9,7 @@ use axum_extra::{
TypedHeader,
headers::{Authorization, authorization::Bearer},
};
use log::info;
use log::{debug, info};
use crate::{
app_state::{AppState, database::models::VaultId},
@ -21,10 +21,12 @@ use crate::{
pub async fn auth_middleware(
State(state): State<AppState>,
Path(path_params): Path<HashMap<String, String>>,
TypedHeader(auth_header): TypedHeader<Authorization<Bearer>>,
auth_header: Option<TypedHeader<Authorization<Bearer>>>,
mut req: Request,
next: Next,
) -> Result<Response, SyncServerError> {
let auth_header = auth_header
.ok_or_else(|| unauthenticated_error(anyhow::anyhow!("Missing Authorization header")))?;
let token = auth_header.token().trim();
let vault_id = normalize_string(
path_params
@ -39,20 +41,24 @@ pub async fn auth_middleware(
Ok(next.run(req).await)
}
pub fn auth(state: &AppState, token: &str, vault_id: &VaultId) -> Result<User, SyncServerError> {
let user = state
pub fn authenticate(state: &AppState, token: &str) -> Result<User, SyncServerError> {
state
.config
.users
.get_user(token)
.cloned()
.ok_or_else(|| unauthenticated_error(anyhow::anyhow!("Invalid token")))?;
.ok_or_else(|| unauthenticated_error(anyhow::anyhow!("Invalid token")))
}
pub fn auth(state: &AppState, token: &str, vault_id: &VaultId) -> Result<User, SyncServerError> {
let user = authenticate(state, token)?;
if match user.vault_access {
VaultAccess::AllowAccessToAll => true,
VaultAccess::AllowList(AllowListedVaults { ref allowed }) => allowed.contains(vault_id),
} {
info!(
"User `{}` is authenticated and is authorised to access to vault `{vault_id}`",
debug!(
"User `{}` is authenticated and is authorised to access vault `{vault_id}`",
user.name
);

View file

@ -11,12 +11,14 @@ use super::{device_id_header::DeviceIdHeader, requests::CreateDocumentVersion};
use crate::{
app_state::{
AppState,
database::models::{DocumentVersionWithoutContent, StoredDocumentVersion, VaultId},
database::models::{StoredDocumentVersion, VaultId},
},
config::user_config::User,
errors::{SyncServerError, client_error, server_error},
errors::{SyncServerError, client_error, server_error, write_transaction_error},
server::{responses::DocumentUpdateResponse, update_document},
utils::{
find_first_available_path::find_first_available_path, normalize::normalize,
find_first_available_path::find_first_available_path, is_binary::is_binary,
is_file_type_mergable::is_file_type_mergable, normalize::normalize,
sanitize_path::sanitize_path,
},
};
@ -30,48 +32,137 @@ pub struct CreateDocumentPathParams {
/// Create a new document in case a document with the same doesn't exist
/// already. If a document with the same path exists, a new version is created
/// with their content merged.
///
/// Text content must be UTF-8 encoded. Clients are responsible for
/// transcoding other encodings (e.g. UTF-16) to UTF-8 before sending.
#[axum::debug_handler]
#[allow(clippy::too_many_lines)]
pub async fn create_document(
Path(CreateDocumentPathParams { vault_id }): Path<CreateDocumentPathParams>,
Extension(user): Extension<User>,
TypedHeader(device_id): TypedHeader<DeviceIdHeader>,
State(state): State<AppState>,
TypedMultipart(request): TypedMultipart<CreateDocumentVersion>,
) -> Result<Json<DocumentVersionWithoutContent>, SyncServerError> {
) -> Result<Json<DocumentUpdateResponse>, SyncServerError> {
debug!("Creating document in vault `{vault_id}`");
let mut transaction = state
.database
.create_write_transaction(&vault_id)
.await
.map_err(server_error)?;
.map_err(write_transaction_error)?;
let document_id = match request.document_id {
Some(document_id) => {
let existing_version = state
.database
.get_latest_document(&vault_id, &document_id, Some(&mut transaction))
.await
.map_err(server_error)?;
let sanitized_relative_path = sanitize_path(&request.relative_path).map_err(client_error)?;
let new_content = request.content.contents.to_vec();
if existing_version.is_some() {
return Err(client_error(anyhow::anyhow!(
"Document with the same ID `{document_id}` already exists"
)));
}
document_id
}
None => uuid::Uuid::new_v4(),
};
let last_update_id = state
let latest_version = state
.database
.get_max_update_id_in_vault(&vault_id, Some(&mut transaction))
.get_latest_non_deleted_document_by_path(
&vault_id,
&sanitized_relative_path,
Some(&mut *transaction),
)
.await
.map_err(server_error)?;
if let Some(latest_version) = latest_version {
// Only merge with an existing document the client couldn't have
// known about: its creation is newer than the client's last seen
// vault update to avoid creating cycles by merging two documents into one.
// This could happen if both clients know of document A at path P1,
// but client 2 moves it to P2 while client 1 creates a new document at P2,
// then client 1 would merge its new document with the moved version of A at P2
// that client 2 resulting in two files (P1 and P2) with the same doc id (A).
if latest_version.creation_vault_update_id > request.last_seen_vault_update_id
&& latest_version.creation_vault_update_id == latest_version.vault_update_id
// can't allow merging with a moved document as that could create a cycle
{
let is_mergeable_text = is_file_type_mergable(
&sanitized_relative_path,
&state.config.server.mergeable_file_extensions,
) && !is_binary(&latest_version.content)
&& !is_binary(&new_content);
if is_mergeable_text || new_content == latest_version.content {
return update_document::update_document(
&sanitized_relative_path,
Vec::new(),
vault_id,
latest_version.document_id,
Some(&request.relative_path),
new_content,
user,
device_id,
state,
transaction,
)
.await;
}
// For non-mergeable (binary) files with different content, don't
// merge, create a separate document at a deconflicted path so
// neither client's data is silently overwritten.
}
}
// Lost-create + local rename recovery. If this device has a doc
// the requesting client hasn't seen yet (its create succeeded
// server-side but the response was discarded — e.g. a sync
// reset mid-flight) and the new request carries the same content
// at a different path (the user renamed the file before the
// retry), bind the retry to that existing doc instead of
// creating a duplicate. The dedup is scoped tightly:
// - same `device_id` (only this client's own lost create),
// - `creation_vault_update_id > last_seen` (client never saw
// this doc, so it can't be deliberately creating another
// copy with matching content),
// - `creation == latest` (the doc has only its create version,
// nobody else has touched it; safe to relocate),
// - exact content match.
// Outside that window we fall through to the normal deconflict
// path, so legitimate "this device created a duplicate of an
// already-acknowledged file" flows still produce a new doc.
if let Some(lost_create) = state
.database
.find_unseen_lost_create_by_device_and_content(
&vault_id,
&device_id.0,
request.last_seen_vault_update_id,
&new_content,
Some(&mut *transaction),
)
.await
.map_err(server_error)?
{
info!(
"Lost-create recovery: binding retry at `{sanitized_relative_path}` to existing doc {} (was at `{}`) in vault `{vault_id}` for device `{}`",
lost_create.document_id,
lost_create.relative_path,
device_id.0
);
return update_document::update_document(
&sanitized_relative_path,
Vec::new(),
vault_id,
lost_create.document_id,
Some(&request.relative_path),
new_content,
user,
device_id,
state,
transaction,
)
.await;
}
let document_id = uuid::Uuid::new_v4();
let last_update_id = state
.database
.get_max_update_id_in_vault(&vault_id, Some(&mut *transaction))
.await
.map_err(server_error)?;
let sanitized_relative_path = sanitize_path(&request.relative_path);
let deduped_path = find_first_available_path(
&vault_id,
&sanitized_relative_path,
@ -87,11 +178,13 @@ pub async fn create_document(
);
}
let new_vault_update_id = last_update_id + 1;
let new_version = StoredDocumentVersion {
vault_update_id: last_update_id + 1,
vault_update_id: new_vault_update_id,
creation_vault_update_id: new_vault_update_id,
document_id,
relative_path: deduped_path,
content: request.content.contents.to_vec(),
content: new_content,
updated_date: chrono::Utc::now(),
is_deleted: false,
user_id: user.name,
@ -101,9 +194,11 @@ pub async fn create_document(
state
.database
.insert_document_version(&vault_id, &new_version, Some(transaction))
.insert_document_version(&vault_id, &new_version, transaction)
.await
.map_err(server_error)?;
Ok(Json(new_version.into()))
Ok(Json(DocumentUpdateResponse::FastForwardUpdate(
new_version.into(),
)))
}

View file

@ -1,4 +1,4 @@
use anyhow::Context;
use anyhow::{Context, anyhow};
use axum::{
Extension, Json,
extract::{Path, State},
@ -7,7 +7,7 @@ use axum_extra::TypedHeader;
use log::{debug, info};
use serde::Deserialize;
use super::{device_id_header::DeviceIdHeader, requests::DeleteDocumentVersion};
use super::device_id_header::DeviceIdHeader;
use crate::{
app_state::{
AppState,
@ -16,8 +16,8 @@ use crate::{
},
},
config::user_config::User,
errors::{SyncServerError, server_error},
utils::{normalize::normalize, sanitize_path::sanitize_path},
errors::{SyncServerError, not_found_error, server_error, write_transaction_error},
utils::normalize::normalize,
};
#[derive(Deserialize)]
@ -37,7 +37,6 @@ pub async fn delete_document(
Extension(user): Extension<User>,
TypedHeader(device_id): TypedHeader<DeviceIdHeader>,
State(state): State<AppState>,
Json(request): Json<DeleteDocumentVersion>,
) -> Result<Json<DocumentVersionWithoutContent>, SyncServerError> {
debug!("Deleting document `{document_id}` in vault `{vault_id}`");
@ -45,7 +44,7 @@ pub async fn delete_document(
.database
.create_write_transaction(&vault_id)
.await
.map_err(server_error)?;
.map_err(write_transaction_error)?;
let last_update_id = state
.database
@ -59,9 +58,18 @@ pub async fn delete_document(
.await
.map_err(server_error)?;
if let Some(latest_version) = &latest_version
&& latest_version.is_deleted
{
let Some(latest_version) = latest_version else {
transaction
.rollback()
.await
.context("Failed to roll back transaction")
.map_err(server_error)?;
return Err(not_found_error(anyhow!(
"Document `{document_id}` not found in vault `{vault_id}`"
)));
};
if latest_version.is_deleted {
transaction
.rollback()
.await
@ -69,15 +77,19 @@ pub async fn delete_document(
.map_err(server_error)?;
info!("Document `{document_id}` has already been deleted",);
return Ok(Json(latest_version.clone().into()));
return Ok(Json(latest_version.into()));
}
let latest_content = latest_version.map_or_else(Vec::new, |version| version.content); // in case the document has never existed before deleting it
let new_vault_update_id = last_update_id + 1;
let latest_relative_path = latest_version.relative_path;
let latest_content = latest_version.content;
let creation_vault_update_id = latest_version.creation_vault_update_id;
let new_version = StoredDocumentVersion {
vault_update_id: last_update_id + 1,
vault_update_id: new_vault_update_id,
creation_vault_update_id,
document_id,
relative_path: sanitize_path(&request.relative_path),
relative_path: latest_relative_path,
content: latest_content, // copy the content from the latest version
updated_date: chrono::Utc::now(),
is_deleted: true,
@ -88,7 +100,7 @@ pub async fn delete_document(
state
.database
.insert_document_version(&vault_id, &new_version, Some(transaction))
.insert_document_version(&vault_id, &new_version, transaction)
.await
.map_err(server_error)?;

View file

@ -16,20 +16,31 @@ impl Header for DeviceIdHeader {
{
let value = values.next().ok_or_else(headers::Error::invalid)?;
Ok(DeviceIdHeader(
value
.to_str()
.map_err(|_| headers::Error::invalid())?
.to_owned(),
))
let s = value.to_str().map_err(|_| headers::Error::invalid())?;
if s.is_empty() || s.len() > 256 {
return Err(headers::Error::invalid());
}
// Only allow safe characters to prevent log injection and similar attacks.
// Covers UUIDs, user-agent strings like "vault-link/1.0 (12345; linux)",
// and human-readable device names.
if !s
.chars()
.all(|c| c.is_ascii_alphanumeric() || "-_./ ();:@+,".contains(c))
{
return Err(headers::Error::invalid());
}
Ok(DeviceIdHeader(s.to_owned()))
}
fn encode<E>(&self, values: &mut E)
where
E: Extend<HeaderValue>,
{
let value = HeaderValue::from_static(Box::leak(self.0.clone().into_boxed_str()));
values.extend(std::iter::once(value));
if let Ok(value) = HeaderValue::from_str(&self.0) {
values.extend(std::iter::once(value));
}
}
}

View file

@ -11,7 +11,7 @@ use crate::{
AppState,
database::models::{DocumentId, DocumentVersion, VaultId, VaultUpdateId},
},
errors::{SyncServerError, not_found_error, server_error},
errors::{SyncServerError, client_error, not_found_error, server_error},
utils::normalize::normalize,
};
@ -52,7 +52,7 @@ pub async fn fetch_document_version(
)?;
if result.document_id != document_id {
return Err(not_found_error(anyhow!(
return Err(client_error(anyhow!(
"Document with document id `{document_id}` does not have a version with id \
`{vault_update_id}`",
)));

View file

@ -11,7 +11,7 @@ use crate::{
AppState,
database::models::{DocumentId, VaultId, VaultUpdateId},
},
errors::{SyncServerError, not_found_error, server_error},
errors::{SyncServerError, client_error, not_found_error, server_error},
utils::normalize::normalize,
};
@ -52,7 +52,7 @@ pub async fn fetch_document_version_content(
)?;
if result.document_id != document_id {
return Err(not_found_error(anyhow!(
return Err(client_error(anyhow!(
"Document with document id `{document_id}` does not have a version with id \
`{vault_update_id}`",
)));

View file

@ -0,0 +1,42 @@
use axum::{
Json,
extract::{Path, State},
};
use log::debug;
use serde::Deserialize;
use crate::{
app_state::{
AppState,
database::models::{DocumentId, DocumentVersionWithoutContent, VaultId},
},
errors::{SyncServerError, server_error},
utils::normalize::normalize,
};
#[derive(Deserialize)]
pub struct FetchDocumentVersionsPathParams {
#[serde(deserialize_with = "normalize")]
vault_id: VaultId,
document_id: DocumentId,
}
#[axum::debug_handler]
pub async fn fetch_document_versions(
Path(FetchDocumentVersionsPathParams {
vault_id,
document_id,
}): Path<FetchDocumentVersionsPathParams>,
State(state): State<AppState>,
) -> Result<Json<Vec<DocumentVersionWithoutContent>>, SyncServerError> {
debug!("Fetching all versions for document `{document_id}` in vault `{vault_id}`");
let versions = state
.database
.get_document_versions(&vault_id, &document_id, None)
.await
.map_err(server_error)?;
Ok(Json(versions))
}

View file

@ -37,13 +37,13 @@ pub async fn fetch_latest_documents(
let documents = if let Some(since_update_id) = since_update_id {
state
.database
.get_latest_documents_since(&vault_id, since_update_id, None)
.get_latest_documents_since(&vault_id, since_update_id, None, None)
.await
.map_err(server_error)
} else {
state
.database
.get_latest_documents(&vault_id, None)
.get_latest_documents(&vault_id, None, None)
.await
.map_err(server_error)
}?;

View file

@ -0,0 +1,70 @@
use axum::{
Json,
extract::{Path, Query, State},
};
use log::debug;
use serde::Deserialize;
use super::responses::VaultHistoryResponse;
use crate::{
app_state::{
AppState,
database::models::{VaultId, VaultUpdateId},
},
errors::{SyncServerError, client_error, server_error},
utils::normalize::normalize,
};
const DEFAULT_LIMIT: i64 = 50;
const MAX_LIMIT: i64 = 500;
#[derive(Deserialize)]
pub struct FetchVaultHistoryPathParams {
#[serde(deserialize_with = "normalize")]
vault_id: VaultId,
}
#[derive(Deserialize)]
pub struct QueryParams {
limit: Option<i64>,
before_update_id: Option<VaultUpdateId>,
}
#[axum::debug_handler]
pub async fn fetch_vault_history(
Path(FetchVaultHistoryPathParams { vault_id }): Path<FetchVaultHistoryPathParams>,
Query(QueryParams {
limit,
before_update_id,
}): Query<QueryParams>,
State(state): State<AppState>,
) -> Result<Json<VaultHistoryResponse>, SyncServerError> {
if let Some(id) = before_update_id
&& id <= 0
{
return Err(client_error(anyhow::anyhow!(
"before_update_id must be a positive integer"
)));
}
let limit = limit.unwrap_or(DEFAULT_LIMIT).clamp(1, MAX_LIMIT);
debug!(
"Fetching vault history for vault `{vault_id}` (limit={limit}, before={before_update_id:?})"
);
// Fetch one extra row to determine if there are more results
let mut versions = state
.database
.get_vault_history(&vault_id, limit + 1, before_update_id, None)
.await
.map_err(server_error)?;
#[allow(clippy::cast_sign_loss)] // limit is clamped to [1, 500] above
let has_more = versions.len() > limit as usize;
if has_more {
versions.pop();
}
Ok(Json(VaultHistoryResponse { versions, has_more }))
}

View file

@ -1,7 +1,77 @@
use axum::response::{Html, IntoResponse};
use axum::{
body::Body,
extract::{Path, State},
http::{StatusCode, header},
response::{Html, IntoResponse, Response},
};
use log::warn;
use rust_embed::Embed;
pub async fn index() -> impl IntoResponse {
const HTML_CONTENT: &str = include_str!("./assets/index.html");
let html_content = HTML_CONTENT;
Html(html_content)
use crate::app_state::AppState;
#[derive(Embed)]
#[folder = "../frontend/history-ui/dist/"]
struct HistoryUiAssets;
pub async fn index(State(_state): State<AppState>) -> impl IntoResponse {
if let Some(content) = HistoryUiAssets::get("index.html") {
Html(
std::str::from_utf8(content.data.as_ref())
.inspect_err(|e| warn!("Embedded index.html is not valid UTF-8: {e}"))
.unwrap_or("<h1>VaultLink</h1>")
.to_owned(),
)
.into_response()
} else {
warn!("No embedded index.html found — history UI may not have been built");
Html("<h1>VaultLink server</h1>".to_owned()).into_response()
}
}
pub async fn spa_assets(Path(path): Path<String>) -> impl IntoResponse {
// The route is /assets/*path so path is relative to assets/.
// The embedded files include the assets/ prefix from the dist directory.
let full_path = format!("assets/{path}");
if let Some(content) = HistoryUiAssets::get(&full_path) {
let mime = mime_guess::from_path(&full_path).first_or_octet_stream();
return Response::builder()
.status(StatusCode::OK)
.header(header::CONTENT_TYPE, mime.as_ref())
.body(Body::from(content.data.to_vec()))
.unwrap_or_else(|_| {
Response::builder()
.status(StatusCode::INTERNAL_SERVER_ERROR)
.body(Body::empty())
.unwrap_or_else(|_| Response::new(Body::empty()))
});
}
// Asset paths must match an embedded file — no SPA fallback.
// Serving index.html here would return 200 with text/html for missing
// .css/.js files, causing the browser to silently ignore the content.
Response::builder()
.status(StatusCode::NOT_FOUND)
.body(Body::from("Not found"))
.unwrap_or_else(|_| Response::new(Body::from("Not found")))
}
/// SPA fallback for production: serves index.html for client-side routes
/// (e.g. `/documents/123`).
pub async fn spa_fallback() -> impl IntoResponse {
match HistoryUiAssets::get("index.html") {
Some(content) => Response::builder()
.status(StatusCode::OK)
.header(header::CONTENT_TYPE, "text/html")
.body(Body::from(content.data.to_vec()))
.unwrap_or_else(|_| {
Response::builder()
.status(StatusCode::INTERNAL_SERVER_ERROR)
.body(Body::empty())
.unwrap_or_else(|_| Response::new(Body::empty()))
}),
None => Response::builder()
.status(StatusCode::NOT_FOUND)
.body(Body::from("Not found"))
.unwrap_or_else(|_| Response::new(Body::from("Not found"))),
}
}

View file

@ -0,0 +1,82 @@
use axum::{
Json,
extract::{Query, State},
};
use axum_extra::{
TypedHeader,
headers::{Authorization, authorization::Bearer},
};
use log::debug;
use serde::Deserialize;
use super::{
auth::authenticate,
responses::{ListVaultsResponse, VaultInfo},
};
use crate::{
app_state::AppState,
config::user_config::{AllowListedVaults, VaultAccess},
errors::{SyncServerError, server_error, unauthenticated_error},
};
const DEFAULT_LIMIT: usize = 50;
const MAX_LIMIT: usize = 200;
#[derive(Deserialize)]
pub struct QueryParams {
limit: Option<usize>,
after: Option<String>,
}
#[axum::debug_handler]
pub async fn list_vaults(
auth_header: Option<TypedHeader<Authorization<Bearer>>>,
Query(QueryParams { limit, after }): Query<QueryParams>,
State(state): State<AppState>,
) -> Result<Json<ListVaultsResponse>, SyncServerError> {
let auth_header = auth_header
.ok_or_else(|| unauthenticated_error(anyhow::anyhow!("Missing Authorization header")))?;
let user = authenticate(&state, auth_header.token().trim())?;
debug!("User `{}` listing accessible vaults", user.name);
let existing_vaults = state.database.list_vaults().await.map_err(server_error)?;
let mut accessible: Vec<String> = match user.vault_access {
VaultAccess::AllowAccessToAll => existing_vaults,
VaultAccess::AllowList(AllowListedVaults { ref allowed }) => existing_vaults
.into_iter()
.filter(|v| allowed.contains(v))
.collect(),
};
// Cursor-based pagination: skip vaults up to and including `after`
if let Some(ref cursor) = after {
accessible.retain(|v| v.as_str() > cursor.as_str());
}
let limit = limit.unwrap_or(DEFAULT_LIMIT).clamp(1, MAX_LIMIT);
let has_more = accessible.len() > limit;
accessible.truncate(limit);
let mut vaults = Vec::with_capacity(accessible.len());
for name in accessible {
let stats = state
.database
.get_vault_stats(&name)
.await
.map_err(server_error)?;
vaults.push(VaultInfo {
name,
document_count: stats.document_count,
created_at: stats.created_at,
});
}
Ok(Json(ListVaultsResponse {
vaults,
has_more,
user_name: user.name,
}))
}

View file

@ -0,0 +1,102 @@
use std::{
collections::HashMap,
sync::{Arc, Mutex},
time::Instant,
};
use axum::{extract::Request, http::StatusCode, middleware::Next, response::Response};
use axum_extra::{
TypedHeader,
headers::{Authorization, authorization::Bearer},
};
/// Per-user token-bucket rate limiter. Each bearer token gets its own bucket
/// that refills to `max_per_second` tokens every second.
#[derive(Clone, Debug)]
pub struct RateLimiter {
max_per_second: u64,
buckets: Arc<Mutex<HashMap<String, Arc<TokenBucket>>>>,
}
#[derive(Debug)]
struct TokenBucket {
state: Mutex<BucketState>,
max_tokens: u64,
}
#[derive(Debug)]
struct BucketState {
tokens: u64,
last_refill: Instant,
}
impl RateLimiter {
/// Create a new per-user rate limiter.
///
/// # Panics
///
/// Panics if `max_per_second` is 0.
pub fn new(max_per_second: u64) -> Self {
assert!(
max_per_second > 0,
"max_per_second must be > 0 (set rate_limit_per_user_per_second to null in config to disable)"
);
Self {
max_per_second,
buckets: Arc::new(Mutex::new(HashMap::new())),
}
}
fn get_or_create_bucket(&self, token: &str) -> Arc<TokenBucket> {
self.buckets
.lock()
.expect("rate limiter lock poisoned")
.entry(token.to_owned())
.or_insert_with(|| {
Arc::new(TokenBucket {
state: Mutex::new(BucketState {
tokens: self.max_per_second,
last_refill: Instant::now(),
}),
max_tokens: self.max_per_second,
})
})
.clone()
}
}
impl TokenBucket {
fn try_acquire(&self) -> bool {
let mut state = self.state.lock().expect("token bucket lock poisoned");
let now = Instant::now();
if now.duration_since(state.last_refill).as_secs() >= 1 {
state.tokens = self.max_tokens;
state.last_refill = now;
}
if state.tokens > 0 {
state.tokens -= 1;
true
} else {
false
}
}
}
pub async fn rate_limit_middleware(
axum::extract::State(limiter): axum::extract::State<RateLimiter>,
auth_header: Option<TypedHeader<Authorization<Bearer>>>,
req: Request,
next: Next,
) -> Result<Response, StatusCode> {
let Some(TypedHeader(auth)) = auth_header else {
return Ok(next.run(req).await);
};
let bucket = limiter.get_or_create_bucket(auth.token());
if bucket.try_acquire() {
Ok(next.run(req).await)
} else {
Err(StatusCode::TOO_MANY_REQUESTS)
}
}

View file

@ -4,18 +4,16 @@ use reconcile_text::NumberOrText;
use serde::{self, Deserialize};
use ts_rs::TS;
use crate::app_state::database::models::{DocumentId, VaultUpdateId};
use crate::app_state::database::models::VaultUpdateId;
#[derive(TS, Debug, TryFromMultipart)]
#[ts(export)]
pub struct CreateDocumentVersion {
/// The client can decide the document id (if it wishes to) in order
/// to help with syncing. If the client does not provide a document id,
/// the server will generate one. If the client provides a document id
/// it must not already exist in the database.
pub document_id: Option<DocumentId>,
pub relative_path: String,
#[ts(type = "number")]
pub last_seen_vault_update_id: VaultUpdateId,
#[ts(as = "Vec<u8>")]
#[form_data(limit = "unlimited")]
pub content: FieldData<Bytes>,
@ -24,7 +22,9 @@ pub struct CreateDocumentVersion {
#[derive(Debug, TryFromMultipart)]
pub struct UpdateBinaryDocumentVersion {
pub parent_version_id: VaultUpdateId,
pub relative_path: String,
// None on a content-only edit; Some on a user rename. When None,
// the server keeps the document at its current path.
pub relative_path: Option<String>,
#[form_data(limit = "unlimited")]
pub content: FieldData<Bytes>,
@ -34,18 +34,13 @@ pub struct UpdateBinaryDocumentVersion {
#[serde(rename_all = "camelCase")]
#[ts(export)]
pub struct UpdateTextDocumentVersion {
#[ts(as = "i32")]
#[ts(type = "number")]
pub parent_version_id: VaultUpdateId,
pub relative_path: String,
// None on a content-only edit; Some on a user rename. When None,
// the server keeps the document at its current path.
pub relative_path: Option<String>,
#[ts(type = "Array<number | string>")]
pub content: Vec<NumberOrText>,
}
#[derive(TS, Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
#[ts(export)]
pub struct DeleteDocumentVersion {
pub relative_path: String,
}

View file

@ -1,3 +1,4 @@
use chrono::{DateTime, Utc};
use serde::{self, Serialize};
use ts_rs::TS;
@ -36,7 +37,36 @@ pub struct FetchLatestDocumentsResponse {
pub last_update_id: VaultUpdateId,
}
/// Response to an update document request.
/// Response to a vault history request (paginated).
#[derive(TS, Debug, Clone, Serialize)]
#[serde(rename_all = "camelCase")]
#[ts(export)]
pub struct VaultHistoryResponse {
pub versions: Vec<DocumentVersionWithoutContent>,
pub has_more: bool,
}
/// Summary of a single vault returned by the list-vaults endpoint.
#[derive(TS, Debug, Clone, Serialize)]
#[serde(rename_all = "camelCase")]
#[ts(export)]
pub struct VaultInfo {
pub name: String,
pub document_count: u32,
pub created_at: Option<DateTime<Utc>>,
}
/// Response to listing vaults accessible to the authenticated user.
#[derive(TS, Debug, Clone, Serialize)]
#[serde(rename_all = "camelCase")]
#[ts(export)]
pub struct ListVaultsResponse {
pub vaults: Vec<VaultInfo>,
pub has_more: bool,
pub user_name: String,
}
/// Response to a create/update document request.
#[derive(TS, Debug, Clone, Serialize)]
#[serde(tag = "type")]
#[ts(export)]

View file

@ -16,10 +16,15 @@ use super::{
use crate::{
app_state::{
AppState,
database::models::{DocumentId, StoredDocumentVersion, VaultId, VaultUpdateId},
database::{
WriteTransaction,
models::{DocumentId, StoredDocumentVersion, VaultId, VaultUpdateId},
},
},
config::user_config::User,
errors::{SyncServerError, client_error, not_found_error, server_error},
errors::{
SyncServerError, client_error, not_found_error, server_error, write_transaction_error,
},
server::requests::UpdateBinaryDocumentVersion,
utils::{
find_first_available_path::find_first_available_path, is_binary::is_binary,
@ -46,18 +51,27 @@ pub async fn update_binary(
State(state): State<AppState>,
TypedMultipart(request): TypedMultipart<UpdateBinaryDocumentVersion>,
) -> Result<Json<DocumentUpdateResponse>, SyncServerError> {
let parent_document = get_parent_document(&state, &vault_id, request.parent_version_id).await?;
let parent_document =
get_parent_document(&state, &vault_id, &document_id, request.parent_version_id).await?;
let content = request.content.contents.to_vec();
let transaction = state
.database
.create_write_transaction(&vault_id)
.await
.map_err(write_transaction_error)?;
update_document(
parent_document,
&parent_document.relative_path,
parent_document.content,
vault_id,
document_id,
request.relative_path.as_deref(),
content,
user,
device_id,
state,
&request.relative_path,
content,
transaction,
)
.await
}
@ -74,28 +88,36 @@ pub async fn update_text(
State(state): State<AppState>,
Json(request): Json<UpdateTextDocumentVersion>,
) -> Result<Json<DocumentUpdateResponse>, SyncServerError> {
let parent_document = get_parent_document(&state, &vault_id, request.parent_version_id).await?;
let parent_document =
get_parent_document(&state, &vault_id, &document_id, request.parent_version_id).await?;
let edited_text = EditedText::from_diff(
str::from_utf8(&parent_document.content)
.expect("parent must be valid UTF-8 because it's a text document"),
request.content,
&*BuiltinTokenizer::Word,
)
.context("Failed to apply given diff to parent document")
.map_err(client_error)?;
let parent_text = str::from_utf8(&parent_document.content)
.context("Parent version contains binary content; use putBinary instead of putText")
.map_err(client_error)?;
let edited_text = EditedText::from_diff(parent_text, request.content, &*BuiltinTokenizer::Word)
.context("Failed to apply given diff to parent document")
.map_err(client_error)?;
let content = edited_text.apply().text().into_bytes();
let transaction = state
.database
.create_write_transaction(&vault_id)
.await
.map_err(write_transaction_error)?;
update_document(
parent_document,
&parent_document.relative_path,
parent_document.content,
vault_id,
document_id,
request.relative_path.as_deref(),
content,
user,
device_id,
state,
&request.relative_path,
content,
transaction,
)
.await
}
@ -103,9 +125,10 @@ pub async fn update_text(
async fn get_parent_document(
state: &AppState,
vault_id: &VaultId,
document_id: &DocumentId,
parent_version_id: VaultUpdateId,
) -> Result<StoredDocumentVersion, SyncServerError> {
state
let parent = state
.database
.get_document_version(vault_id, parent_version_id, None)
.await
@ -117,29 +140,36 @@ async fn get_parent_document(
)))
},
Ok,
)
)?;
if &parent.document_id != document_id {
return Err(client_error(anyhow!(
"Parent version `{parent_version_id}` does not belong to document `{document_id}`"
)));
}
Ok(parent)
}
#[allow(clippy::too_many_lines, clippy::too_many_arguments)]
async fn update_document(
parent_document: StoredDocumentVersion,
pub async fn update_document(
parent_relative_path: &str,
parent_content: Vec<u8>,
vault_id: VaultId,
document_id: DocumentId,
relative_path: Option<&str>,
content: Vec<u8>,
user: User,
device_id: DeviceIdHeader,
state: AppState,
relative_path: &str,
content: Vec<u8>,
mut transaction: WriteTransaction,
) -> Result<Json<DocumentUpdateResponse>, SyncServerError> {
debug!("Updating document `{document_id}` in vault `{vault_id}`");
let sanitized_relative_path = sanitize_path(relative_path);
let mut transaction = state
.database
.create_write_transaction(&vault_id)
.await
.map_err(server_error)?;
let sanitized_relative_path = relative_path
.map(sanitize_path)
.transpose()
.map_err(client_error)?;
let last_update_id = state
.database
@ -175,9 +205,12 @@ async fn update_document(
}
// Return the latest version if the content and path are the same as the latest
// version
if content == latest_version.content && sanitized_relative_path == latest_version.relative_path
{
// version. A missing relative_path means "keep current path", so the path
// is implicitly unchanged.
let path_unchanged = sanitized_relative_path
.as_deref()
.is_none_or(|p| p == latest_version.relative_path);
if content == latest_version.content && path_unchanged {
info!(
"Document content is the same as the latest version for `{document_id}`, skipping update"
);
@ -192,62 +225,89 @@ async fn update_document(
)));
}
// For mergability, use whichever path the new version will live at — the
// requested rename target if the client sent one, otherwise the existing
// server-side path.
let mergable_check_path = sanitized_relative_path
.as_deref()
.unwrap_or(&latest_version.relative_path);
let are_all_participants_mergable = is_file_type_mergable(
&sanitized_relative_path,
mergable_check_path,
&state.config.server.mergeable_file_extensions,
) && !is_binary(&parent_document.content)
) && !is_binary(&parent_content)
&& !is_binary(&latest_version.content)
&& !is_binary(&content);
let merged_content = if are_all_participants_mergable {
let (merged_content, is_different_from_request_content) = if are_all_participants_mergable {
info!("Merging changes for document `{document_id}` in vault `{vault_id}`");
reconcile(
str::from_utf8(&parent_document.content)
.expect("parent must be valid UTF-8 because it's not binary"),
&str::from_utf8(&latest_version.content)
.expect("latest_version must be valid UTF-8 because it's not binary")
.into(),
&str::from_utf8(&content)
.expect("content must be valid UTF-8 because it's not binary")
.into(),
&*BuiltinTokenizer::Word,
)
.apply()
.text()
.into_bytes()
let parent_text = str::from_utf8(&parent_content)
.context("Parent document content is not valid UTF-8")
.map_err(client_error)?;
let latest_text = str::from_utf8(&latest_version.content)
.context("Latest version content is not valid UTF-8")
.map_err(client_error)?;
let new_text = str::from_utf8(&content)
.context("New content is not valid UTF-8")
.map_err(client_error)?;
let parent_owned = parent_text.to_owned();
let latest_owned = latest_text.to_owned();
let new_owned = new_text.to_owned();
let content_clone = content.clone();
let (merged, is_different) = tokio::task::spawn_blocking(move || {
let merged = reconcile(
&parent_owned,
&latest_owned.into(),
&new_owned.into(),
&*BuiltinTokenizer::Word,
)
.apply()
.text()
.into_bytes();
let is_different = merged != content_clone;
(merged, is_different)
})
.await
.map_err(|e| server_error(anyhow::anyhow!("Reconcile task failed: {e}")))?;
(merged, is_different)
} else {
content.clone()
(content, false) // false means that the client doesn't need to refetch the file as we can ensure the remote and local versions are the same as LWW is the merging method for binary files
};
let is_different_from_request_content = merged_content != content;
// Rename resolution: only apply the client's rename if (a) the client
// requested one (`sanitized_relative_path` is `Some`) and (b) the
// document's path hasn't changed since this client's parent version.
// If the parent and latest paths differ, another client already renamed
// the document — keep the latest path (first rename wins). Content
// changes from both clients are still merged correctly via the 3-way
// reconcile above, independent of which rename wins. A missing
// relative_path means "keep current path" (content-only edit).
let new_relative_path = match sanitized_relative_path.as_deref() {
Some(requested)
if parent_relative_path == latest_version.relative_path
&& requested != latest_version.relative_path =>
{
let new_path =
find_first_available_path(&vault_id, requested, &state.database, &mut transaction)
.await
.map_err(server_error)?;
// We can only update the relative path if we're the first one to do so
let new_relative_path = if parent_document.relative_path == latest_version.relative_path
&& latest_version.relative_path != sanitized_relative_path
{
let new_path = find_first_available_path(
&vault_id,
&sanitized_relative_path,
&state.database,
&mut transaction,
)
.await
.map_err(server_error)?;
if new_path != requested {
info!(
"Document already exists at new location: `{requested}` when trying to update it in vault `{vault_id}`, deconflicting by creating at `{new_path}`"
);
}
if new_path != sanitized_relative_path {
info!(
"Document already exists at new location: `{sanitized_relative_path}` when trying to update it in vault `{vault_id}`, deconflicting by creating at `{new_path}`"
);
new_path
}
new_path
} else {
latest_version.relative_path.clone()
_ => latest_version.relative_path.clone(),
};
let new_version = StoredDocumentVersion {
document_id,
vault_update_id: last_update_id + 1,
creation_vault_update_id: latest_version.creation_vault_update_id,
relative_path: new_relative_path,
content: merged_content,
updated_date: chrono::Utc::now(),
@ -259,7 +319,7 @@ async fn update_document(
state
.database
.insert_document_version(&vault_id, &new_version, Some(transaction))
.insert_document_version(&vault_id, &new_version, transaction)
.await
.map_err(server_error)?;

View file

@ -1,15 +1,3 @@
use anyhow::Context;
use axum::{
extract::{
Path, State,
ws::{Message, WebSocket, WebSocketUpgrade},
},
response::Response,
};
use futures::stream::StreamExt;
use log::{debug, info};
use serde::Deserialize;
use crate::{
app_state::{
AppState,
@ -24,9 +12,35 @@ use crate::{
},
},
},
consts::{
HANDSHAKE_TIMEOUT, MAX_CURSOR_DOCUMENTS, MAX_CURSORS_PER_DOCUMENT, MAX_RELATIVE_PATH_LEN,
},
errors::{SyncServerError, client_error, server_error},
utils::normalize::normalize,
};
use anyhow::Context;
use axum::{
extract::{
Path, State,
ws::{Message, WebSocket, WebSocketUpgrade},
},
response::Response,
};
use futures::sink::SinkExt;
use futures::stream::StreamExt;
use log::{debug, info, warn};
use serde::Deserialize;
/// Tracks a pending (not yet authenticated) WebSocket connection.
/// Decrements the counter when dropped, ensuring cleanup even if
/// the upgrade never completes or auth fails.
struct PendingWsGuard(std::sync::Arc<std::sync::atomic::AtomicUsize>);
impl Drop for PendingWsGuard {
fn drop(&mut self) {
self.0.fetch_sub(1, std::sync::atomic::Ordering::Relaxed);
}
}
#[derive(Deserialize)]
pub struct WebSocketPathParams {
@ -39,13 +53,31 @@ pub async fn websocket_handler(
Path(WebSocketPathParams { vault_id }): Path<WebSocketPathParams>,
State(state): State<AppState>,
) -> Result<Response, SyncServerError> {
Ok(ws.on_upgrade(move |socket| websocket_wrapped(state, socket, vault_id)))
let current = state
.pending_ws_connections
.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
if current >= state.config.server.max_pending_websocket_connections {
state
.pending_ws_connections
.fetch_sub(1, std::sync::atomic::Ordering::Relaxed);
return Err(client_error(anyhow::anyhow!(
"Too many pending WebSocket connections"
)));
}
let guard = PendingWsGuard(state.pending_ws_connections.clone());
Ok(ws.on_upgrade(move |socket| websocket_wrapped(state, socket, vault_id, guard)))
}
async fn websocket_wrapped(state: AppState, stream: WebSocket, vault_id: VaultId) {
async fn websocket_wrapped(
state: AppState,
stream: WebSocket,
vault_id: VaultId,
pending_guard: PendingWsGuard,
) {
info!("WebSocket connection opened on vault `{vault_id}`");
let result = websocket(state, stream, vault_id.clone()).await;
let result = websocket(state, stream, vault_id.clone(), pending_guard).await;
if let Err(err) = result {
debug!("WebSocket connection error on vault `{vault_id}`: {err}");
@ -57,39 +89,112 @@ async fn websocket(
state: AppState,
stream: WebSocket,
vault_id: VaultId,
pending_guard: PendingWsGuard,
) -> Result<(), SyncServerError> {
let (mut sender, mut websocket_receiver) = stream.split();
let authed_handshake = get_authenticated_handshake(
&state,
&vault_id,
websocket_receiver
.next()
.await
.transpose()
.unwrap_or_default(),
)?;
let handshake_msg = tokio::time::timeout(HANDSHAKE_TIMEOUT, websocket_receiver.next())
.await
.map_err(|_| client_error(anyhow::anyhow!("WebSocket handshake timed out")))?
.transpose()
.map_err(|e| client_error(anyhow::anyhow!("WebSocket error during handshake: {e}")))?;
let authed_handshake = get_authenticated_handshake(&state, &vault_id, handshake_msg)?;
info!(
"WebSocket handshake successful for vault `{vault_id}` for `{}`",
authed_handshake.handshake.device_id
);
let mut broadcast_receiver = state.broadcasts.get_receiver(vault_id.clone()).await;
// Auth complete — no longer a pending connection.
drop(pending_guard);
send_update_over_websocket(
&WebSocketServerMessage::VaultUpdate(WebSocketVaultUpdate {
documents: get_unseen_documents(
&state,
&vault_id,
authed_handshake.handshake.last_seen_vault_update_id,
)
.await?,
is_initial_sync: true,
}),
&mut sender,
let max_clients = state.config.server.max_clients_per_vault;
// Atomic subscribe + cursor snapshot, serialized against in-flight
// broadcasts:
//
// 1. Acquire the per-vault broadcast send lock. While we hold it,
// no `send_document_update` can run, so no broadcast can fire
// between our subscribe and our cursor snapshot.
// 2. Subscribe to the broadcast channel (now we'll see every
// broadcast that fires after we drop the send guard).
// 3. Snapshot `cursor = max committed vault_update_id`. Because
// `insert_document_version` holds the same send lock from
// *before* the commit through *after* the broadcast, every doc
// visible at this cursor has either (a) already had its
// broadcast delivered to all then-existing subscribers — and we
// weren't one of them, so we'll catch it via the snapshot — or
// (b) had its broadcast contend on the lock we're holding, and
// will be delivered to us as soon as we drop the guard, with
// `vault_update_id > cursor`.
// 4. Drop the send guard so writers can resume broadcasting.
// 5. Stream the catch-up bounded by the cursor — i.e. only docs
// with `vault_update_id <= cursor` — exactly once.
// 6. The send task forwards broadcasts but filters to
// `vault_update_id > cursor`, so a doc that's both in the
// catch-up and in a contended-then-released broadcast is
// delivered exactly once (via the catch-up).
let send_guard = state.broadcasts.acquire_send_lock(&vault_id).await;
let mut broadcast_receiver = match state.broadcasts.get_receiver(vault_id.clone(), max_clients)
{
Ok(receiver) => receiver,
Err(err) => {
drop(send_guard);
warn!(
"Vault `{vault_id}` has reached the maximum number of clients ({max_clients}), rejecting connection from `{}`",
authed_handshake.handshake.device_id
);
if let Err(e) = sender
.send(Message::Close(Some(axum::extract::ws::CloseFrame {
code: 4000,
reason: format!(
"Vault has reached the maximum number of clients ({max_clients})"
)
.into(),
})))
.await
{
warn!("Failed to send WebSocket close frame: {e}");
}
return Err(err);
}
};
let cursor = state
.database
.get_max_update_id_in_vault(&vault_id, None)
.await
.map_err(server_error)?;
drop(send_guard);
// Catch-up on versions committed while this client was offline,
// streamed one-at-a-time in ascending `vault_update_id` order, up
// to the snapshot cursor.
let unseen_documents = get_unseen_documents(
&state,
&vault_id,
authed_handshake.handshake.last_seen_vault_update_id,
cursor,
)
.await?;
let unseen_summary: Vec<(i64, bool, String)> = unseen_documents
.iter()
.map(|d| (d.vault_update_id, d.is_deleted, d.relative_path.clone()))
.collect();
info!(
"[CATCHUP] vault={vault_id} device={} last_seen={:?} cursor={cursor} unseen_count={} unseen={:?}",
authed_handshake.handshake.device_id,
authed_handshake.handshake.last_seen_vault_update_id,
unseen_summary.len(),
unseen_summary
);
for document in unseen_documents {
send_update_over_websocket(
&WebSocketServerMessage::VaultUpdate(WebSocketVaultUpdate { document }),
&mut sender,
)
.await?;
}
send_update_over_websocket(
&WebSocketServerMessage::CursorPositions(CursorPositionFromServer {
@ -101,24 +206,57 @@ async fn websocket(
let device_id = authed_handshake.handshake.device_id.clone();
let mut send_task = tokio::spawn(async move {
while let Ok(update) = broadcast_receiver.recv().await {
if Some(&device_id) == update.origin_device_id.as_ref() {
continue;
}
loop {
match broadcast_receiver.recv().await {
Ok(update) => {
// Drop messages this device authored because the HTTP
// response already carried authoritative state back.
// Delete broadcasts are sent without an origin so the
// author also receives them — that's the receipt the
// client needs to drop the doc from its sync queue.
if Some(&device_id) == update.origin_device_id.as_ref() {
continue;
}
let message = match update.message {
WebSocketServerMessage::CursorPositions(CursorPositionFromServer { clients }) => {
WebSocketServerMessage::CursorPositions(CursorPositionFromServer {
clients: clients
.into_iter()
.filter(|client| client.device_id != device_id)
.collect(),
})
// Filter out vault updates already covered by the
// catch-up snapshot. The handshake atomically
// subscribed and snapshotted `cursor` under the
// broadcast send lock, so any broadcast with
// `vault_update_id <= cursor` is one that contended
// on the lock during our subscribe — its row is
// already in the catch-up stream and re-delivering
// it via this channel would duplicate the message.
// Cursor messages aren't versioned and are always
// forwarded.
if let WebSocketServerMessage::VaultUpdate(WebSocketVaultUpdate { document }) =
&update.message
&& document.vault_update_id <= cursor
{
continue;
}
let message = match update.message {
WebSocketServerMessage::CursorPositions(CursorPositionFromServer {
clients,
}) => WebSocketServerMessage::CursorPositions(CursorPositionFromServer {
clients: clients
.into_iter()
.filter(|client| client.device_id != device_id)
.collect(),
}),
WebSocketServerMessage::VaultUpdate(_) => update.message,
};
send_update_over_websocket(&message, &mut sender).await?;
}
WebSocketServerMessage::VaultUpdate(_) => update.message,
};
send_update_over_websocket(&message, &mut sender).await?;
Err(tokio::sync::broadcast::error::RecvError::Lagged(n)) => {
warn!(
"WebSocket receiver lagged, dropped {n} messages — disconnecting client to force full resync"
);
break;
}
Err(tokio::sync::broadcast::error::RecvError::Closed) => break,
}
}
Ok::<(), SyncServerError>(())
@ -128,26 +266,59 @@ async fn websocket(
let vault_id_clone = vault_id.clone();
let cursor_manager = state.cursors.clone();
let mut receive_task = tokio::spawn(async move {
while let Some(Ok(Message::Text(message))) = websocket_receiver.next().await {
let message: WebSocketClientMessage = serde_json::from_str(&message)
.context("Failed to parse WebSocket message from client")
.map_err(server_error)?;
while let Some(msg) = websocket_receiver.next().await {
match msg {
Ok(Message::Text(message)) => {
let message: WebSocketClientMessage = serde_json::from_str(&message)
.context("Failed to parse WebSocket message from client")
.map_err(client_error)?;
match message {
WebSocketClientMessage::Handshake(_) => {
return Err(client_error(anyhow::anyhow!(
"Unexpected handshake message"
)));
match message {
WebSocketClientMessage::Handshake(_) => {
return Err(client_error(anyhow::anyhow!(
"Unexpected handshake message"
)));
}
WebSocketClientMessage::CursorPositions(cursors) => {
let docs = cursors.documents_with_cursors;
if docs.len() > MAX_CURSOR_DOCUMENTS {
warn!(
"Cursor update rejected: {} documents exceeds limit of {MAX_CURSOR_DOCUMENTS}",
docs.len()
);
continue;
}
let valid = docs.iter().all(|doc| {
doc.cursors.len() <= MAX_CURSORS_PER_DOCUMENT
&& doc.relative_path.len() <= MAX_RELATIVE_PATH_LEN
});
if !valid {
warn!(
"Cursor update rejected: a document exceeds cursor or path length limits"
);
continue;
}
cursor_manager
.update_cursors(
vault_id_clone.clone(),
authed_handshake.user.name.clone(),
&device_id,
docs,
)
.await;
}
}
}
WebSocketClientMessage::CursorPositions(cursors) => {
cursor_manager
.update_cursors(
vault_id_clone.clone(),
authed_handshake.user.name.clone(),
&device_id,
cursors.documents_with_cursors,
)
.await;
Ok(Message::Close(_)) => break,
Ok(Message::Binary(_)) => {
warn!("Received unexpected binary WebSocket message, ignoring");
}
Ok(_) => {} // Ping/Pong frames handled by axum
Err(e) => {
debug!("WebSocket receive error: {e}");
break;
}
}
}
@ -155,38 +326,47 @@ async fn websocket(
Ok::<(), SyncServerError>(())
});
tokio::select! {
_ = &mut send_task => receive_task.abort(),
_ = &mut receive_task => send_task.abort(),
let result: Result<(), SyncServerError> = tokio::select! {
send_result = &mut send_task => {
receive_task.abort();
let _ = receive_task.await;
match send_result {
Err(e) => Err(server_error(
anyhow::Error::from(e).context("WebSocket send task failed"),
)),
Ok(inner) => inner,
}
},
receive_result = &mut receive_task => {
send_task.abort();
let _ = send_task.await;
match receive_result {
Err(e) => Err(server_error(
anyhow::Error::from(e).context("WebSocket receive task failed"),
)),
Ok(inner) => inner,
}
},
};
let result: Result<(), SyncServerError> = (async {
send_task
.await
.context("WebSocket send task failed")
.map_err(client_error)
.and_then(|err| err)?;
receive_task
.await
.context("WebSocket receive task failed")
.map_err(client_error)
.and_then(|err| err)?;
Ok(())
})
.await;
state
.cursors
.remove_cursors_of_device(&vault_id, &authed_handshake.handshake.device_id)
.await;
if result.is_err() {
info!(
"WebSocket disconnected on vault `{vault_id}` for `{}`",
authed_handshake.handshake.device_id
);
match &result {
Ok(()) => {
info!(
"WebSocket disconnected on vault `{vault_id}` for `{}`",
authed_handshake.handshake.device_id
);
}
Err(err) => {
warn!(
"WebSocket error on vault `{vault_id}` for `{}`: {err}",
authed_handshake.handshake.device_id
);
}
}
result

View file

@ -1,8 +1,17 @@
use std::sync::LazyLock;
use regex::Regex;
static DEDUP_SUFFIX_REGEX: LazyLock<Regex> =
LazyLock::new(|| Regex::new(r" \((\d+)\)$").expect("invalid regex"));
pub fn dedup_paths(path: &str) -> impl Iterator<Item = String> {
let mut path_parts = path.split('/').collect::<Vec<_>>();
let file_name = path_parts.pop().unwrap().to_owned();
let file_name = path_parts
.pop()
.filter(|s| !s.is_empty())
.unwrap_or(path)
.to_owned();
let mut directory = path_parts.join("/");
if !directory.is_empty() {
@ -29,14 +38,13 @@ pub fn dedup_paths(path: &str) -> impl Iterator<Item = String> {
}
};
let regex = Regex::new(r" \((\d+)\)$").unwrap();
let start_number = regex
let start_number = DEDUP_SUFFIX_REGEX
.captures(&stem)
.and_then(|caps| caps.get(1))
.and_then(|m| m.as_str().parse::<u32>().ok())
.unwrap_or(0);
let clean_stem = regex.replace(&stem, "").to_string();
let clean_stem = DEDUP_SUFFIX_REGEX.replace(&stem, "").to_string();
(start_number..).map(move |dedup_number| {
if dedup_number == 0 {

Some files were not shown because too many files have changed in this diff Show more