Compare commits
5 commits
main
...
asch/split
| Author | SHA1 | Date | |
|---|---|---|---|
| 042233c4d7 | |||
| 4ba439b874 | |||
| 2d5edc6ec5 | |||
| a9ce09b59d | |||
| 70f97c4b16 |
72 changed files with 6258 additions and 3801 deletions
35
.forgejo/workflows/check.yml
Normal file
35
.forgejo/workflows/check.yml
Normal file
|
|
@ -0,0 +1,35 @@
|
|||
name: Check
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: ["main"]
|
||||
pull_request:
|
||||
branches: ["main"]
|
||||
workflow_dispatch:
|
||||
|
||||
env:
|
||||
CARGO_TERM_COLOR: always
|
||||
RUSTFLAGS: "-Dwarnings"
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Setup Node.js environment
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: "25.x"
|
||||
|
||||
- name: Setup Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
toolchain: "1.92.0"
|
||||
components: clippy, rustfmt
|
||||
|
||||
- name: Lint & test
|
||||
run: scripts/check.sh
|
||||
38
.forgejo/workflows/deploy-docs.yml
Normal file
38
.forgejo/workflows/deploy-docs.yml
Normal file
|
|
@ -0,0 +1,38 @@
|
|||
name: Deploy Documentation
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
paths:
|
||||
- "docs/**"
|
||||
- ".forgejo/workflows/deploy-docs.yml"
|
||||
workflow_dispatch:
|
||||
|
||||
concurrency:
|
||||
group: pages
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Setup Node.js environment
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: "25.x"
|
||||
|
||||
- name: Build docs
|
||||
run: scripts/build-docs.sh
|
||||
|
||||
- name: Upload artifact
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: docs
|
||||
path: docs/.vitepress/dist
|
||||
71
.forgejo/workflows/e2e.yml
Normal file
71
.forgejo/workflows/e2e.yml
Normal file
|
|
@ -0,0 +1,71 @@
|
|||
name: E2E tests
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: ["main"]
|
||||
pull_request:
|
||||
branches: ["main"]
|
||||
schedule:
|
||||
- cron: "0 * * * *"
|
||||
workflow_dispatch:
|
||||
|
||||
concurrency:
|
||||
group: e2e-tests
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
RUSTFLAGS: "-Dwarnings"
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Setup Node.js environment
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: "25.x"
|
||||
|
||||
- name: Setup Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
toolchain: "1.92.0"
|
||||
components: clippy, rustfmt
|
||||
|
||||
- name: Setup rust
|
||||
run: |
|
||||
which sqlx || cargo install sqlx-cli
|
||||
cd sync-server
|
||||
sqlx database create --database-url sqlite://db.sqlite3
|
||||
sqlx migrate run --source src/app_state/database/migrations --database-url sqlite://db.sqlite3
|
||||
|
||||
- name: E2E tests
|
||||
run: |
|
||||
cd sync-server
|
||||
cargo run config-e2e.yml --color never &
|
||||
SERVER_PID=$!
|
||||
cd ..
|
||||
|
||||
scripts/e2e.sh 8
|
||||
EXIT_CODE=$?
|
||||
|
||||
kill $SERVER_PID 2>/dev/null || true
|
||||
wait $SERVER_PID 2>/dev/null || true
|
||||
|
||||
exit $EXIT_CODE
|
||||
|
||||
- name: Upload e2e logs
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: e2e-logs
|
||||
path: logs/
|
||||
retention-days: 30
|
||||
|
||||
- name: Cleanup
|
||||
if: always()
|
||||
run: scripts/clean-up.sh
|
||||
51
.forgejo/workflows/publish-cli-docker.yml
Normal file
51
.forgejo/workflows/publish-cli-docker.yml
Normal file
|
|
@ -0,0 +1,51 @@
|
|||
name: Publish CLI
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: ["main"]
|
||||
tags: ["*"]
|
||||
pull_request:
|
||||
branches: ["main"]
|
||||
|
||||
jobs:
|
||||
publish-docker:
|
||||
runs-on: ubuntu-docker
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Extract registry hostname
|
||||
id: registry
|
||||
run: echo "host=$(echo '${{ github.server_url }}' | sed 's|https\?://||')" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Log into container registry
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ${{ steps.registry.outputs.host }}
|
||||
username: ${{ github.repository_owner }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Extract Docker metadata
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: ${{ steps.registry.outputs.host }}/${{ github.repository }}-cli
|
||||
|
||||
- name: Build and push Docker image
|
||||
id: build-and-push
|
||||
uses: docker/build-push-action@v5
|
||||
with:
|
||||
context: frontend
|
||||
file: frontend/local-client-cli/Dockerfile
|
||||
platforms: linux/amd64
|
||||
push: true
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
cache-from: type=registry,ref=${{ steps.registry.outputs.host }}/${{ github.repository }}-cli:buildcache
|
||||
cache-to: type=registry,ref=${{ steps.registry.outputs.host }}/${{ github.repository }}-cli:buildcache,mode=max
|
||||
71
.forgejo/workflows/publish-plugin.yml
Normal file
71
.forgejo/workflows/publish-plugin.yml
Normal file
|
|
@ -0,0 +1,71 @@
|
|||
name: Publish Obsidian plugin
|
||||
|
||||
on:
|
||||
push:
|
||||
tags: ["*"]
|
||||
|
||||
env:
|
||||
CARGO_TERM_COLOR: always
|
||||
|
||||
jobs:
|
||||
publish-plugin:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Setup Node.js environment
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: "25.x"
|
||||
|
||||
- name: Build plugin
|
||||
run: |
|
||||
cd frontend
|
||||
npm ci
|
||||
npm run build
|
||||
|
||||
- name: Setup Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
toolchain: "1.92.0"
|
||||
components: clippy, rustfmt
|
||||
|
||||
- name: Install cross-compilation tools
|
||||
run: |
|
||||
apt update
|
||||
apt install -y gcc-aarch64-linux-gnu musl-tools gcc-mingw-w64-x86-64 jq
|
||||
|
||||
- name: Build Linux and Windows binaries
|
||||
run: ./scripts/build-sync-server-binaries.sh
|
||||
|
||||
- name: Create release
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
SERVER_URL: ${{ github.server_url }}
|
||||
REPO: ${{ github.repository }}
|
||||
run: |
|
||||
tag="${GITHUB_REF#refs/tags/}"
|
||||
|
||||
mkdir -p release
|
||||
cp frontend/obsidian-plugin/dist/* release/
|
||||
cp sync-server/artifacts/sync-server-* release/
|
||||
|
||||
# Create draft release via Forgejo API
|
||||
RELEASE_ID=$(curl -s -X POST \
|
||||
"${SERVER_URL}/api/v1/repos/${REPO}/releases" \
|
||||
-H "Authorization: token ${GITHUB_TOKEN}" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"tag_name\": \"${tag}\", \"name\": \"${tag}\", \"draft\": true}" \
|
||||
| jq -r '.id')
|
||||
|
||||
# Upload release assets
|
||||
for file in release/*; do
|
||||
filename=$(basename "$file")
|
||||
curl -s -X POST \
|
||||
"${SERVER_URL}/api/v1/repos/${REPO}/releases/${RELEASE_ID}/assets?name=${filename}" \
|
||||
-H "Authorization: token ${GITHUB_TOKEN}" \
|
||||
-F "attachment=@${file}"
|
||||
done
|
||||
51
.forgejo/workflows/publish-server-docker.yml
Normal file
51
.forgejo/workflows/publish-server-docker.yml
Normal file
|
|
@ -0,0 +1,51 @@
|
|||
name: Publish server Docker image
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: ["main"]
|
||||
tags: ["*"]
|
||||
pull_request:
|
||||
branches: ["main"]
|
||||
|
||||
jobs:
|
||||
publish-docker:
|
||||
runs-on: ubuntu-docker
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Extract registry hostname
|
||||
id: registry
|
||||
run: echo "host=$(echo '${{ github.server_url }}' | sed 's|https\?://||')" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Log into container registry
|
||||
if: github.ref_type == 'tag'
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ${{ steps.registry.outputs.host }}
|
||||
username: ${{ github.repository_owner }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Extract Docker metadata
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: ${{ steps.registry.outputs.host }}/${{ github.repository }}
|
||||
|
||||
- name: Build and push Docker image
|
||||
id: build-and-push
|
||||
uses: docker/build-push-action@v5
|
||||
with:
|
||||
context: sync-server
|
||||
platforms: linux/amd64,linux/arm64
|
||||
push: ${{ github.ref_type == 'tag' }}
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
cache-from: type=registry,ref=${{ steps.registry.outputs.host }}/${{ github.repository }}:buildcache
|
||||
cache-to: type=registry,ref=${{ steps.registry.outputs.host }}/${{ github.repository }}:buildcache,mode=max
|
||||
4
.github/workflows/check.yml
vendored
4
.github/workflows/check.yml
vendored
|
|
@ -23,13 +23,13 @@ jobs:
|
|||
- name: Setup Node.js environment
|
||||
uses: actions/setup-node@v4.2.0
|
||||
with:
|
||||
node-version: "22.x"
|
||||
node-version: "25.x"
|
||||
check-latest: true
|
||||
|
||||
- name: Setup Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
toolchain: "1.89.0"
|
||||
toolchain: "1.92.0"
|
||||
components: clippy, rustfmt
|
||||
|
||||
- name: Lint & test
|
||||
|
|
|
|||
13
.github/workflows/deploy-docs.yml
vendored
13
.github/workflows/deploy-docs.yml
vendored
|
|
@ -5,8 +5,8 @@ on:
|
|||
branches:
|
||||
- main
|
||||
paths:
|
||||
- 'docs/**'
|
||||
- '.github/workflows/deploy-docs.yml'
|
||||
- "docs/**"
|
||||
- ".github/workflows/deploy-docs.yml"
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
|
|
@ -28,12 +28,11 @@ jobs:
|
|||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Setup Node
|
||||
uses: actions/setup-node@v4
|
||||
- name: Setup Node.js environment
|
||||
uses: actions/setup-node@v4.2.0
|
||||
with:
|
||||
node-version: 22
|
||||
cache: npm
|
||||
cache-dependency-path: docs/package-lock.json
|
||||
node-version: "25.x"
|
||||
check-latest: true
|
||||
|
||||
- name: Setup Pages
|
||||
uses: actions/configure-pages@v4
|
||||
|
|
|
|||
6
.github/workflows/e2e.yml
vendored
6
.github/workflows/e2e.yml
vendored
|
|
@ -6,7 +6,7 @@ on:
|
|||
pull_request:
|
||||
branches: ["main"]
|
||||
schedule:
|
||||
- cron: '0 * * * *'
|
||||
- cron: "0 * * * *"
|
||||
workflow_dispatch:
|
||||
|
||||
concurrency:
|
||||
|
|
@ -28,13 +28,13 @@ jobs:
|
|||
- name: Setup Node.js environment
|
||||
uses: actions/setup-node@v4.2.0
|
||||
with:
|
||||
node-version: "22.x"
|
||||
node-version: "25.x"
|
||||
check-latest: true
|
||||
|
||||
- name: Setup Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
toolchain: "1.89.0"
|
||||
toolchain: "1.92.0"
|
||||
components: clippy, rustfmt
|
||||
|
||||
- name: Setup rust
|
||||
|
|
|
|||
4
.github/workflows/publish-plugin.yml
vendored
4
.github/workflows/publish-plugin.yml
vendored
|
|
@ -19,7 +19,7 @@ jobs:
|
|||
- name: Setup Node.js environment
|
||||
uses: actions/setup-node@v4.2.0
|
||||
with:
|
||||
node-version: "22.x"
|
||||
node-version: "25.x"
|
||||
check-latest: true
|
||||
|
||||
- name: Build plugin
|
||||
|
|
@ -31,7 +31,7 @@ jobs:
|
|||
- name: Setup Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
toolchain: "1.89.0"
|
||||
toolchain: "1.92.0"
|
||||
components: clippy, rustfmt
|
||||
|
||||
- name: Install cross-compilation tools
|
||||
|
|
|
|||
9
.gitignore
vendored
9
.gitignore
vendored
|
|
@ -7,15 +7,18 @@ node_modules
|
|||
# Frontend build folders
|
||||
frontend/*/dist
|
||||
|
||||
sync-server/db.sqlite3*
|
||||
sync-server/databases
|
||||
|
||||
# Rust build folders
|
||||
sync-server/target
|
||||
sync-server/artifacts
|
||||
sync-server/bindings/*.ts
|
||||
|
||||
# build folders
|
||||
sync-server/db.sqlite3*
|
||||
**/databases
|
||||
|
||||
*.log
|
||||
*.sqlx
|
||||
|
||||
target
|
||||
|
||||
.task
|
||||
|
|
|
|||
4
.vscode/settings.json
vendored
4
.vscode/settings.json
vendored
|
|
@ -5,6 +5,6 @@
|
|||
"**/dist": true,
|
||||
"**/node_modules": true,
|
||||
"**/.sqlx": true,
|
||||
"**/target": true,
|
||||
},
|
||||
"**/target": true
|
||||
}
|
||||
}
|
||||
|
|
|
|||
195
CLAUDE.md
195
CLAUDE.md
|
|
@ -2,109 +2,154 @@
|
|||
|
||||
This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
|
||||
|
||||
## Project Overview
|
||||
## Project shape
|
||||
|
||||
VaultLink is a self-hosted Obsidian plugin for real-time collaborative file syncing. The project consists of a Rust-based sync server and a TypeScript frontend with three main components: an Obsidian plugin, a sync client library, and a test client.
|
||||
VaultLink is a self-hosted Obsidian file-sync system. Two halves of one repo:
|
||||
|
||||
## Architecture
|
||||
- `sync-server/` — Rust (axum + sqlx/SQLite). Source of truth for vault state, broadcasts changes via WebSocket.
|
||||
- `frontend/` — npm workspaces. The sync engine (`sync-client`) is consumed by an Obsidian plugin, a standalone CLI, a fuzz E2E harness, a scripted determinism harness, and a history UI.
|
||||
|
||||
### Core Components
|
||||
The HTTP/WS API types are generated from Rust (`ts-rs`) and mirrored into the TS workspaces. **Never hand-edit files in `frontend/sync-client/src/services/types/` or `frontend/history-ui/src/lib/types/`** — run `scripts/update-api-types.sh` after changing anything Serde-derived in the server.
|
||||
|
||||
- **sync-server/**: Rust-based WebSocket server with SQLite database for document versioning and real-time synchronization
|
||||
- **frontend/sync-client/**: TypeScript library providing core sync functionality, WebSocket management, and file operations
|
||||
- **frontend/obsidian-plugin/**: Obsidian plugin that integrates the sync client with Obsidian's API
|
||||
- **frontend/test-client/**: CLI testing tool for the sync functionality
|
||||
### Frontend workspaces
|
||||
|
||||
### Key Technologies
|
||||
- `sync-client` — the sync engine; published to consumers via `dist/`. All other TS workspaces depend on it via `file:../sync-client`.
|
||||
- `obsidian-plugin` — Obsidian plugin built from `sync-client`.
|
||||
- `local-client-cli` — same engine wrapped as a standalone CLI.
|
||||
- `history-ui` — vault-history web UI.
|
||||
- `test-client` — fuzz E2E harness (random ops across N processes).
|
||||
- `deterministic-tests` — scripted multi-client tests with an in-memory FS, run against a real server.
|
||||
|
||||
- **Backend**: Rust with Axum framework, SQLite with SQLx, WebSockets for real-time sync
|
||||
- **Frontend**: TypeScript, Webpack for bundling, Jest for testing
|
||||
- **Sync Algorithm**: Uses reconcile-text library for operational transformation
|
||||
## Common commands
|
||||
|
||||
## Development Commands
|
||||
Pre-push hygiene (formats, lints, runs tests, requires clean git state):
|
||||
|
||||
### Server Development
|
||||
```bash
|
||||
cd sync-server
|
||||
cargo run config-e2e.yml # Start development server
|
||||
cargo test --verbose # Run Rust tests
|
||||
cargo clippy --all-targets --all-features # Lint Rust code
|
||||
cargo clippy --all-targets --all-features --fix --allow-dirty --allow-staged # Auto-fix clippy warnings
|
||||
cargo fmt --all -- --check # Check Rust formatting
|
||||
cargo fmt --all # Auto-format Rust code
|
||||
cargo machete --with-metadata # Detect unused dependencies
|
||||
```sh
|
||||
scripts/check.sh --fix
|
||||
```
|
||||
|
||||
### Frontend Development
|
||||
```bash
|
||||
Run the fuzz E2E (N parallel processes):
|
||||
|
||||
```sh
|
||||
scripts/e2e.sh 12
|
||||
# Logs land in logs/log_<i>.log. Clean with scripts/clean-up.sh
|
||||
```
|
||||
|
||||
Run deterministic tests (require a release-built server in `sync-server/target/release/sync_server` — they spawn it themselves):
|
||||
|
||||
```sh
|
||||
cd sync-server && cargo build --release && cd ..
|
||||
cd frontend
|
||||
npm run dev # Start development mode (watches sync-client and obsidian-plugin)
|
||||
npm run build # Build all workspaces
|
||||
npm run test # Run all tests
|
||||
npm run lint # Lint and format TypeScript code
|
||||
npm run build -w sync-client -w deterministic-tests
|
||||
node deterministic-tests/dist/cli.js # all
|
||||
node deterministic-tests/dist/cli.js --filter=rename # subset
|
||||
node deterministic-tests/dist/cli.js --filter=… -j 4 # cap parallelism
|
||||
```
|
||||
|
||||
### Database Setup (Development)
|
||||
```bash
|
||||
Run a single sync-client unit test by file:
|
||||
|
||||
```sh
|
||||
cd frontend/sync-client && npx tsx --test 'src/**/sync-event-queue.test.ts'
|
||||
```
|
||||
|
||||
Server: dev runs from `sync-server/` against `config-e2e.yml`:
|
||||
|
||||
```sh
|
||||
cd sync-server
|
||||
cargo run config-e2e.yml # dev
|
||||
cargo build --release # used by both e2e harnesses
|
||||
cargo test # unit + ts-rs binding export tests
|
||||
```
|
||||
|
||||
Frontend dev (sync-client + obsidian-plugin watch in parallel):
|
||||
|
||||
```sh
|
||||
cd frontend && npm install && npm run dev
|
||||
```
|
||||
|
||||
Regenerate TS bindings from Rust types (touches `frontend/{sync-client,history-ui}/src/.../types/`):
|
||||
|
||||
```sh
|
||||
scripts/update-api-types.sh
|
||||
```
|
||||
|
||||
## SQLite / sqlx
|
||||
|
||||
The server uses `sqlx::query!` macros that need a prepared `.sqlx` cache to compile offline. Touching any SQL means regenerating it:
|
||||
|
||||
```sh
|
||||
cd sync-server
|
||||
sqlx database create --database-url sqlite://db.sqlite3
|
||||
sqlx migrate run --source src/app_state/database/migrations --database-url sqlite://db.sqlite3
|
||||
cargo sqlx prepare --workspace
|
||||
```
|
||||
|
||||
### Initial Setup
|
||||
```bash
|
||||
# Install required cargo tools
|
||||
cargo install sqlx-cli cargo-machete cargo-edit
|
||||
New migrations: `sqlx migrate add --source src/app_state/database/migrations <name>`.
|
||||
|
||||
## Sync engine architecture
|
||||
|
||||
Read `frontend/sync-client/src/sync-operations/` to follow the sync engine; the rest of `sync-client` is plumbing (filesystem ops, persistence, services, telemetry).
|
||||
|
||||
The engine is **two independent loops with separate invariants**:
|
||||
|
||||
- **Wire loop** (`syncer.ts`) — drains the single-consumer FIFO queue. HTTP and WS handlers update record fields (`remoteRelativePath`, `parentVersionId`, `remoteHash`) and write content to the file at `record.localPath`. They never move files for path placement.
|
||||
- **Path reconciler** (`reconciler.ts`) — runs after every drained event. Best-effort pass that moves files to make `localPath === remoteRelativePath`. The move graph is topologically sorted; cycles are resolved by reading every file in the cycle into memory and writing each back to its new slot (no tmp files). Records with pending local events are skipped on each pass — the reconciler operates only on settled records. Failures (slot occupied by an untracked file, etc.) are silent skips; the next pass retries.
|
||||
|
||||
**`SyncEventQueue`** (`sync-event-queue.ts`) holds:
|
||||
|
||||
- `byDocId: Map<DocumentId, DocumentRecord>` — primary record store.
|
||||
- `byLocalPath: Map<RelativePath, DocumentRecord>` — derived index for path lookups, maintained at every mutation point.
|
||||
- `events: SyncEvent[]` — pending wire ops in FIFO drain order.
|
||||
|
||||
```ts
|
||||
DocumentRecord = {
|
||||
documentId,
|
||||
parentVersionId,
|
||||
remoteHash?,
|
||||
remoteRelativePath,
|
||||
localPath: RelativePath | undefined
|
||||
}
|
||||
```
|
||||
|
||||
### Scripts
|
||||
- `scripts/check.sh`: Full CI check (builds, lints, tests both server and frontend)
|
||||
- `scripts/check.sh --fix`: Same as above but auto-fixes linting and formatting issues
|
||||
- `scripts/e2e.sh`: End-to-end testing
|
||||
- `scripts/clean-up.sh`: Clean logs and database files
|
||||
- `scripts/bump-version.sh patch`: Publish new version
|
||||
- `scripts/update-api-types.sh`: Update TypeScript bindings from Rust types
|
||||
`localPath === undefined` means the doc has no local file yet — typically a remote create whose target slot was occupied at receive time; the reconciler will fetch and place when the slot frees (the bytes wait in `pendingPlacementContent`).
|
||||
|
||||
## Code Structure
|
||||
Local FS events from the watcher update `localPath` synchronously at enqueue time via `setLocalPath` / `upsertRecord`. The wire loop never updates it for path placement; only the reconciler does. A user rename onto a tracked slot enqueues a `LocalDelete` for the displaced doc (the OS rename clobbered its content) and clears that doc's `localPath`.
|
||||
|
||||
### Workspace Configuration
|
||||
The frontend uses npm workspaces with four packages:
|
||||
- `sync-client`: Core synchronization logic
|
||||
- `obsidian-plugin`: Obsidian-specific integration
|
||||
- `test-client`: Testing utilities
|
||||
- `local-client-cli`: Standalone CLI for VaultLink sync client
|
||||
**Pending creates** use a `Promise<DocumentId>` chain to serialize dependent ops (`LocalUpdate`, `LocalDelete`) behind the still-in-flight `LocalCreate`. `resolveCreate` resolves the promise once the server returns a docId, and `replacePendingDocumentId` swaps the resolved id across already-queued events. `findLatestCreateForPath` is the lookup the watcher uses to attach dependents; `updatePendingCreatePath` rewrites a pending create's `event.path` in place when the user renames the file before its create has acked.
|
||||
|
||||
### Type Generation
|
||||
Rust structs generate TypeScript types via ts-rs crate, stored in `sync-server/bindings/` and used by frontend packages.
|
||||
**Watermark.** `lastSeenUpdateId` uses a `MinCovered` (a contiguous-prefix tracker over a stream of integers): we only advance the published min when the next consecutive id has been processed, so out-of-order RemoteChange ids don't fool the WebSocket handshake into requesting a too-recent catch-up.
|
||||
|
||||
### Key Files
|
||||
- `sync-server/src/`: Rust server implementation with WebSocket handlers
|
||||
- `frontend/sync-client/src/sync-client.ts`: Main sync client entry point
|
||||
- `frontend/obsidian-plugin/src/vault-link-plugin.ts`: Main Obsidian plugin class
|
||||
- `frontend/sync-client/src/services/sync-service.ts`: Core synchronization logic
|
||||
**Server catch-up.** The server's WS handshake replays events newer than the client's `last_seen_vault_update_id` from the `latest_document_versions` view (one row per doc, the latest). On those replayed rows `is_new_file` means _new to this client_ (`creation_vault_update_id > last_seen_vault_update_id`), not "this row is the doc's first version" — necessary because the catch-up only carries the latest version; if a doc was created and updated past the watermark, the client never sees its create otherwise.
|
||||
|
||||
## Testing
|
||||
## Edge-case patterns the sync engine has to survive
|
||||
|
||||
### Running Tests
|
||||
- Server: `cargo test --verbose`
|
||||
- Frontend: `npm run test` (runs Jest across all workspaces)
|
||||
- E2E: `scripts/e2e.sh`
|
||||
The two-loop split defuses most of the old race catalogue (slot-collision stashes, conflict-uuid divergence, `MoveOnConflict.NEW`/`EXISTING` policy choices) by separating wire transport from path placement. What's left:
|
||||
|
||||
### Test Structure
|
||||
- Rust: Unit tests alongside source files
|
||||
- TypeScript: `.test.ts` files using Jest
|
||||
- E2E: Uses test-client to simulate multiple concurrent users
|
||||
**Pending-create docId is a `Promise`, not a string, until the create acks.** Any `LocalUpdate` / `LocalDelete` queued behind a still-in-flight `LocalCreate` carries the create's `resolvers.promise` as its `documentId`. `replacePendingDocumentId` swaps the resolved id across queued events when the create resolves; `===` comparisons against the resolved string elsewhere will silently fail until that swap runs. Anything that walks `events[]` looking for a docId match must either run after the swap or be tolerant of `Promise`-typed ids.
|
||||
|
||||
## Code Style
|
||||
**`processCreate` reads `event.path` live, not `event.originalPath`.** The watcher rewrites `event.path` in place via `updatePendingCreatePath` when the user renames a pending-create file. `originalPath` was removed from `LocalCreate` events specifically because reading it would send the stale pre-rename path to the server.
|
||||
|
||||
### Rust
|
||||
- Uses extensive Clippy lints (see Cargo.toml)
|
||||
- Follows pedantic linting rules
|
||||
- Forbids unsafe code
|
||||
- Uses cargo fmt with default settings
|
||||
**`record.localPath` mutates in place across awaits.** When the watcher renames a doc while a drain handler is awaiting an HTTP roundtrip, the queue mutates the in-flight event's record so subsequent reads see the new path. Snapshotting `record.localPath` into a local at function entry and using it after an `await` reads/writes a now-vacated slot. Read `record.localPath` live; only snapshot for the deliberate "did it change while I was awaiting" comparison.
|
||||
|
||||
### TypeScript
|
||||
- Prettier configuration: 4-space tabs, trailing commas removed, LF line endings
|
||||
- ESLint with unused imports plugin
|
||||
- Consistent across all three frontend packages
|
||||
**Reconciler-defer is the wire-loop's contract with the reconciler.** The reconciler skips records where `hasPendingLocalEventsForDocumentId` returns true. Wire-loop handlers can therefore freely write `remoteRelativePath` to whatever the server returned — even if it disagrees with `localPath` — knowing the reconciler won't move the file out from under a queued user rename.
|
||||
|
||||
**Watermark advancement is load-bearing both ways.** Branches that _skip_ a remote event without advancing `lastSeenUpdateId` create permanent gaps that re-deliver forever. Branches that _advance_ without applying the content lose data: the server has no further event to re-deliver, the catch-up only carries the latest version, and any state in between is gone. Don't advance unless the event was actually applied (or deliberately discarded after weighing both halves).
|
||||
|
||||
**`isNewFile` semantics differ between catch-up and real-time.** On WS handshake replay it means _new to this client_ (`creation_vault_update_id > last_seen_vault_update_id`); on real-time broadcasts it means _this version is the create_ (`creation_vault_update_id == vault_update_id`). A handler that decides based on one interpretation will be wrong on the other channel; reasoning about fetch-and-treat-as-new vs. ignore needs to know which channel delivered the event.
|
||||
|
||||
**Pause / disable-sync mid-flight** is the one race the new model doesn't structurally fix. An HTTP that committed server-side but whose response was discarded leaves the server holding a doc the client has no record of. Resume → offline scan → server-side dedupe handles it (the server merges the duplicate create into the existing doc), but if the merge produces a deconflict, the client picks up an extra file. Out of scope for the two-loop split.
|
||||
|
||||
**Cycle reconciliation uses in-memory content swap.** When the move graph contains a cycle, the reconciler reads every file in the cycle into memory and writes each back to its new slot, with no tmp files. A write-ahead marker at `.vaultlink/swap-<uuid>.json` lists each leg; on startup the reconciler reads the marker, hashes each `from` to determine which legs ran, and replays the rest. The `.vaultlink/**` glob is hard-coded as an internal ignore pattern so swap markers don't get sync'd.
|
||||
|
||||
## Two complementary E2E harnesses
|
||||
|
||||
- **`test-client` (fuzz):** random ops across N parallel processes for many minutes. Used by `scripts/e2e.sh`. Catches bugs nobody thought to write a test for, but reproductions are noisy.
|
||||
- **`deterministic-tests`:** scripted scenarios with an in-memory FS pinned to a real server. Used to _capture_ a fuzz-discovered bug as a minimal repro before fixing it. See `frontend/deterministic-tests/README.md` for the step grammar (`pause-server`, `pause-websocket`, `barrier`, `assert-consistent`, etc.).
|
||||
|
||||
When a fuzz failure surfaces, the workflow is: root-cause from logs → write a deterministic test that fails on the bug → fix → confirm both the deterministic test and `e2e.sh` pass.
|
||||
|
||||
## Style
|
||||
|
||||
- TS: 4-space indent, no tabs, LF, prettier (`trailingComma: "none"`). YAML/MD use 2-space indent.
|
||||
- Rust: `rustfmt.toml` enforces 4-space spaces, LF.
|
||||
- Lint: ESLint for TS, Clippy for Rust, `cargo machete` for unused deps. All wired into `scripts/check.sh`.
|
||||
|
|
|
|||
|
|
@ -8,12 +8,12 @@
|
|||
|
||||
## Develop
|
||||
|
||||
### Install [nvm](https://github.com/nvm-sh/nvm)
|
||||
### Set up Node.JS 25 with [nvm](https://github.com/nvm-sh/nvm)
|
||||
|
||||
- `curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh | bash`
|
||||
- `nvm install 22`
|
||||
- `nvm use 22`
|
||||
- Optionally set the system-wide default: `nvm alias default 22`
|
||||
- `nvm install 25`
|
||||
- `nvm use 25`
|
||||
- Optionally, set the system-wide default: `nvm alias default 25`
|
||||
|
||||
### Set up Rust
|
||||
|
||||
|
|
|
|||
|
|
@ -2,12 +2,7 @@
|
|||
"version": "0.2",
|
||||
"language": "en-GB",
|
||||
"dictionaries": ["en-gb"],
|
||||
"ignorePaths": [
|
||||
"node_modules",
|
||||
".vitepress/dist",
|
||||
".vitepress/cache",
|
||||
"package-lock.json"
|
||||
],
|
||||
"ignorePaths": ["node_modules", ".vitepress/dist", ".vitepress/cache", "package-lock.json"],
|
||||
"words": [
|
||||
"VaultLink",
|
||||
"Obsidian",
|
||||
|
|
|
|||
|
|
@ -53,7 +53,7 @@ Central authority for synchronisation. Rust + Axum framework.
|
|||
|
||||
**Technology**:
|
||||
|
||||
- **Language**: Rust 1.89+
|
||||
- **Language**: Rust 1.92+
|
||||
- **Framework**: Axum (async web framework)
|
||||
- **Database**: SQLite with SQLx
|
||||
- **Protocol**: WebSockets for real-time communication
|
||||
|
|
|
|||
|
|
@ -75,7 +75,7 @@ chmod +x sync_server-linux-x86_64
|
|||
|
||||
### Build from Source
|
||||
|
||||
Requirements: Rust 1.89.0+, SQLite development headers, SQLx CLI
|
||||
Requirements: Rust 1.92.0+, SQLite development headers, SQLx CLI
|
||||
|
||||
```bash
|
||||
# Clone the repository
|
||||
|
|
|
|||
6
package-lock.json
generated
Normal file
6
package-lock.json
generated
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
{
|
||||
"name": "vault-link",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {}
|
||||
}
|
||||
11
rustfmt.toml
Normal file
11
rustfmt.toml
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
# Rustfmt configuration
|
||||
# This should match the .editorconfig settings
|
||||
|
||||
# Use spaces for indentation (matches .editorconfig indent_style = space)
|
||||
hard_tabs = false
|
||||
|
||||
# Use 4 spaces for indentation (matches .editorconfig indent_size = 4)
|
||||
tab_spaces = 4
|
||||
|
||||
# Use Unix line endings (matches .editorconfig end_of_line = lf)
|
||||
newline_style = "Unix"
|
||||
|
|
@ -35,7 +35,8 @@ cd ..
|
|||
|
||||
cp frontend/obsidian-plugin/manifest.json manifest.json # for BRAT, otherwise it wouldn't update
|
||||
|
||||
git ls-files | xargs npx eclint fix
|
||||
# Format all files across the project (frontend and backend)
|
||||
npx -C frontend prettier --write "**/*.{ts,js,json,md,yml,yaml}"
|
||||
|
||||
# Commit and tag
|
||||
git add .
|
||||
|
|
|
|||
|
|
@ -30,8 +30,11 @@ fi
|
|||
which cargo-machete || cargo install cargo-machete
|
||||
cargo machete --with-metadata
|
||||
|
||||
cd ..
|
||||
scripts/update-api-types.sh # this will dirty up the git state if not up-to-date
|
||||
|
||||
echo "Running checks in frontend"
|
||||
cd ../frontend
|
||||
cd frontend
|
||||
|
||||
if [[ "$FIX_MODE" == true ]]; then
|
||||
npm install
|
||||
|
|
@ -45,10 +48,11 @@ cd frontend
|
|||
npm run build
|
||||
npm run test
|
||||
npm run lint
|
||||
cd ..
|
||||
|
||||
# Use git ls-files to only check tracked files, respecting .gitignore
|
||||
# We always run in fix mode and then check with git status
|
||||
git ls-files | xargs npx eclint fix
|
||||
# Format all files across the project (frontend and backend)
|
||||
# Prettier respects .gitignore by default
|
||||
npx -C frontend prettier --write "**/*.{ts,js,json,md,yml,yaml}"
|
||||
|
||||
if [[ "$FIX_MODE" == false ]] && [[ $(git status --porcelain) ]]; then
|
||||
git status --porcelain
|
||||
|
|
@ -56,6 +60,4 @@ if [[ "$FIX_MODE" == false ]] && [[ $(git status --porcelain) ]]; then
|
|||
exit 1
|
||||
fi
|
||||
|
||||
cd ..
|
||||
|
||||
echo "Success"
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
#!/bin/bash
|
||||
|
||||
rm -rf sync-server/databases
|
||||
rm -rf /host/tmp/vaultlink-e2e-databases
|
||||
rm -rf logs
|
||||
|
|
|
|||
|
|
@ -19,35 +19,51 @@ process_count=$1
|
|||
|
||||
mkdir -p logs
|
||||
|
||||
# Build and restart the server
|
||||
echo "Building server..."
|
||||
cd sync-server
|
||||
cargo build --release
|
||||
|
||||
# Kill any existing server process
|
||||
echo "Stopping existing server..."
|
||||
pkill -f "sync_server" 2>/dev/null || true
|
||||
sleep 1
|
||||
|
||||
# Clean databases (uses tmpfs via /dev/shm for zero disk I/O)
|
||||
echo "Cleaning databases..."
|
||||
rm -rf /host/tmp/vaultlink-e2e-databases
|
||||
|
||||
# Start the server in the background
|
||||
echo "Starting server..."
|
||||
./target/release/sync_server config-e2e.yml &
|
||||
server_pid=$!
|
||||
echo "Server started with PID: $server_pid"
|
||||
|
||||
# Ensure server is killed on script exit
|
||||
cleanup_server() {
|
||||
if [ -n "$server_pid" ]; then
|
||||
echo "Stopping server (PID: $server_pid)..."
|
||||
kill $server_pid 2>/dev/null || true
|
||||
wait $server_pid 2>/dev/null || true
|
||||
server_pid=""
|
||||
fi
|
||||
}
|
||||
trap cleanup_server EXIT
|
||||
|
||||
cd ..
|
||||
|
||||
cd frontend
|
||||
npm ci
|
||||
npm run build
|
||||
|
||||
../scripts/utils/wait-for-server.sh
|
||||
|
||||
cd ..
|
||||
scripts/update-api-types.sh
|
||||
if [[ $(git status --porcelain) ]]; then
|
||||
git status --porcelain
|
||||
echo "Failing CI because the working directory is not clean after generating api types"
|
||||
exit 1
|
||||
fi
|
||||
cd frontend
|
||||
|
||||
pids=()
|
||||
for i in $(seq 1 $process_count); do
|
||||
# Create a named pipe for this process
|
||||
pipe="/tmp/vaultlink_pipe_$$_$i"
|
||||
mkfifo "$pipe"
|
||||
|
||||
# Start the node process writing to the pipe
|
||||
node test-client/dist/cli.js > "$pipe" 2>&1 &
|
||||
node test-client/dist/cli.js > "../logs/log_${i}.log" 2>&1 &
|
||||
pid=$!
|
||||
pids+=($pid)
|
||||
echo "Started process $i with PID: $pid"
|
||||
|
||||
# Read from pipe, prefix with PID
|
||||
(sed "s/^/[PID $pid] /" < "$pipe" > "../logs/log_${i}.log"; rm "$pipe") &
|
||||
echo "Started process $i with PID: $pid (log: logs/log_${i}.log)"
|
||||
done
|
||||
|
||||
cd ..
|
||||
|
|
@ -75,10 +91,25 @@ print_failed_log() {
|
|||
return 1
|
||||
}
|
||||
|
||||
echo "Monitoring $process_count processes"
|
||||
E2E_TIMEOUT=${2:-3600}
|
||||
start_time=$(date +%s)
|
||||
echo "Monitoring $process_count processes (timeout: ${E2E_TIMEOUT}s)"
|
||||
|
||||
# Monitor processes
|
||||
while true; do
|
||||
# Script-level timeout to prevent indefinite hangs
|
||||
current_time=$(date +%s)
|
||||
elapsed=$((current_time - start_time))
|
||||
if [ $elapsed -ge $E2E_TIMEOUT ]; then
|
||||
echo "E2E timeout reached (${E2E_TIMEOUT}s). Killing remaining processes."
|
||||
for pid in "${pids[@]}"; do
|
||||
if [ -n "$pid" ]; then
|
||||
kill $pid 2>/dev/null || true
|
||||
fi
|
||||
done
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if print_failed_log; then
|
||||
# Kill remaining processes
|
||||
for pid in "${pids[@]}"; do
|
||||
|
|
@ -99,6 +130,7 @@ while true; do
|
|||
done
|
||||
|
||||
if $all_done; then
|
||||
cleanup_server
|
||||
echo "All processes completed successfully"
|
||||
exit 0
|
||||
fi
|
||||
|
|
|
|||
|
|
@ -8,9 +8,15 @@ cd sync-server
|
|||
cargo test export_bindings
|
||||
cd -
|
||||
|
||||
# Both target directories contain only generated bindings — wipe and copy
|
||||
rm -f frontend/sync-client/src/services/types/*.ts
|
||||
rm -f frontend/history-ui/src/lib/types/*.ts
|
||||
cp -r sync-server/bindings/* frontend/sync-client/src/services/types/
|
||||
cp -r sync-server/bindings/* frontend/history-ui/src/lib/types/
|
||||
|
||||
cd frontend
|
||||
npm run lint
|
||||
git ls-files | xargs npx eclint fix
|
||||
cd -
|
||||
cd ..
|
||||
|
||||
# Format all files across the project (frontend and backend)
|
||||
npx -C frontend prettier --write "**/*.{ts,js,json,md,yml,yaml}"
|
||||
|
|
|
|||
|
|
@ -2,8 +2,10 @@
|
|||
|
||||
set -e
|
||||
|
||||
TARGET_NODE_VERSION=25
|
||||
|
||||
node_version=$(node -v | sed 's/^v\([0-9]*\).*/\1/')
|
||||
if [ "$node_version" != "22" ]; then
|
||||
echo "Error: This script requires Node.js version 22, found: $node_version"
|
||||
if [ "$node_version" != "$TARGET_NODE_VERSION" ]; then
|
||||
echo "Error: This script requires Node.js version $TARGET_NODE_VERSION, found: $node_version"
|
||||
exit 1
|
||||
fi
|
||||
|
|
|
|||
|
|
@ -2,14 +2,14 @@
|
|||
|
||||
set -e
|
||||
|
||||
SERVER_URL="http://localhost:3000"
|
||||
SERVER_URL="http://localhost:3010"
|
||||
MAX_RETRIES=30
|
||||
RETRY_INTERVAL_IN_SECONDS=5
|
||||
|
||||
echo "Waiting for $SERVER_URL to become available..."
|
||||
count=0
|
||||
while [ $count -lt $MAX_RETRIES ]; do
|
||||
if curl -s -f -o /dev/null $SERVER_URL; then
|
||||
if curl -s -o /dev/null $SERVER_URL; then
|
||||
echo "$SERVER_URL is now available!"
|
||||
break
|
||||
fi
|
||||
|
|
|
|||
193
sync-server/Cargo.lock
generated
193
sync-server/Cargo.lock
generated
|
|
@ -337,10 +337,11 @@ checksum = "325918d6fe32f23b19878fe4b34794ae41fc19ddbe53b10571a4874d44ffd39b"
|
|||
|
||||
[[package]]
|
||||
name = "cc"
|
||||
version = "1.2.2"
|
||||
version = "1.2.57"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f34d93e62b03caf570cccc334cbc6c2fceca82f39211051345108adcba3eebdc"
|
||||
checksum = "7a0dd1ca384932ff3641c8718a02769f1698e7563dc6974ffd03346116310423"
|
||||
dependencies = [
|
||||
"find-msvc-tools",
|
||||
"shlex",
|
||||
]
|
||||
|
||||
|
|
@ -456,6 +457,15 @@ version = "2.4.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5"
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-channel"
|
||||
version = "0.5.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2"
|
||||
dependencies = [
|
||||
"crossbeam-utils",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-queue"
|
||||
version = "0.3.11"
|
||||
|
|
@ -533,6 +543,15 @@ dependencies = [
|
|||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "deranged"
|
||||
version = "0.5.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c"
|
||||
dependencies = [
|
||||
"powerfmt",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "digest"
|
||||
version = "0.10.7"
|
||||
|
|
@ -624,6 +643,12 @@ version = "2.2.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "486f806e73c5707928240ddc295403b1b93c96a02038563881c4a2fd84b81ac4"
|
||||
|
||||
[[package]]
|
||||
name = "find-msvc-tools"
|
||||
version = "0.1.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582"
|
||||
|
||||
[[package]]
|
||||
name = "flume"
|
||||
version = "0.11.1"
|
||||
|
|
@ -1272,6 +1297,16 @@ version = "0.3.17"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a"
|
||||
|
||||
[[package]]
|
||||
name = "mime_guess"
|
||||
version = "2.0.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e"
|
||||
dependencies = [
|
||||
"mime",
|
||||
"unicase",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "miniz_oxide"
|
||||
version = "0.8.0"
|
||||
|
|
@ -1335,6 +1370,12 @@ dependencies = [
|
|||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num-conv"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c6673768db2d862beb9b39a78fdcb1a69439615d5794a1be50caa9bc92c81967"
|
||||
|
||||
[[package]]
|
||||
name = "num-integer"
|
||||
version = "0.1.46"
|
||||
|
|
@ -1463,6 +1504,12 @@ version = "0.3.31"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2"
|
||||
|
||||
[[package]]
|
||||
name = "powerfmt"
|
||||
version = "0.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391"
|
||||
|
||||
[[package]]
|
||||
name = "ppv-lite86"
|
||||
version = "0.2.20"
|
||||
|
|
@ -1582,12 +1629,12 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "reconcile-text"
|
||||
version = "0.8.0"
|
||||
version = "0.11.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "599cf9539996a2a19e501110404c59ba62f4974009f8fb864a8b7151c15ee5a5"
|
||||
checksum = "52e0cf361887ea64c479ca871c1170dda761f84e122f2616b5579906a38d7557"
|
||||
dependencies = [
|
||||
"serde",
|
||||
"thiserror 2.0.17",
|
||||
"thiserror 2.0.18",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -1648,6 +1695,40 @@ dependencies = [
|
|||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rust-embed"
|
||||
version = "8.11.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "04113cb9355a377d83f06ef1f0a45b8ab8cd7d8b1288160717d66df5c7988d27"
|
||||
dependencies = [
|
||||
"rust-embed-impl",
|
||||
"rust-embed-utils",
|
||||
"walkdir",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rust-embed-impl"
|
||||
version = "8.11.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "da0902e4c7c8e997159ab384e6d0fc91c221375f6894346ae107f47dd0f3ccaa"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"rust-embed-utils",
|
||||
"syn 2.0.90",
|
||||
"walkdir",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rust-embed-utils"
|
||||
version = "8.11.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5bcdef0be6fe7f6fa333b1073c949729274b05f123a0ad7efcb8efd878e5c3b1"
|
||||
dependencies = [
|
||||
"sha2",
|
||||
"walkdir",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustc-demangle"
|
||||
version = "0.1.24"
|
||||
|
|
@ -1679,6 +1760,15 @@ version = "1.0.18"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f"
|
||||
|
||||
[[package]]
|
||||
name = "same-file"
|
||||
version = "1.0.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502"
|
||||
dependencies = [
|
||||
"winapi-util",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sanitize-filename"
|
||||
version = "0.6.0"
|
||||
|
|
@ -1916,7 +2006,7 @@ dependencies = [
|
|||
"serde_json",
|
||||
"sha2",
|
||||
"smallvec",
|
||||
"thiserror 2.0.17",
|
||||
"thiserror 2.0.18",
|
||||
"tokio",
|
||||
"tokio-stream",
|
||||
"tracing",
|
||||
|
|
@ -2000,7 +2090,7 @@ dependencies = [
|
|||
"smallvec",
|
||||
"sqlx-core",
|
||||
"stringprep",
|
||||
"thiserror 2.0.17",
|
||||
"thiserror 2.0.18",
|
||||
"tracing",
|
||||
"uuid",
|
||||
"whoami",
|
||||
|
|
@ -2039,7 +2129,7 @@ dependencies = [
|
|||
"smallvec",
|
||||
"sqlx-core",
|
||||
"stringprep",
|
||||
"thiserror 2.0.17",
|
||||
"thiserror 2.0.18",
|
||||
"tracing",
|
||||
"uuid",
|
||||
"whoami",
|
||||
|
|
@ -2065,7 +2155,7 @@ dependencies = [
|
|||
"serde",
|
||||
"serde_urlencoded",
|
||||
"sqlx-core",
|
||||
"thiserror 2.0.17",
|
||||
"thiserror 2.0.18",
|
||||
"tracing",
|
||||
"url",
|
||||
"uuid",
|
||||
|
|
@ -2100,6 +2190,12 @@ version = "2.6.1"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
|
||||
|
||||
[[package]]
|
||||
name = "symlink"
|
||||
version = "0.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a7973cce6668464ea31f176d85b13c7ab3bba2cb3b77a2ed26abd7801688010a"
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "1.0.109"
|
||||
|
|
@ -2136,18 +2232,22 @@ dependencies = [
|
|||
"futures",
|
||||
"humantime-serde",
|
||||
"log",
|
||||
"mime_guess",
|
||||
"rand 0.9.0",
|
||||
"reconcile-text",
|
||||
"regex",
|
||||
"rust-embed",
|
||||
"sanitize-filename",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"serde_yaml",
|
||||
"sqlx",
|
||||
"thiserror 2.0.17",
|
||||
"subtle",
|
||||
"thiserror 2.0.18",
|
||||
"tokio",
|
||||
"tower-http",
|
||||
"tracing",
|
||||
"tracing-appender",
|
||||
"tracing-subscriber",
|
||||
"ts-rs",
|
||||
"uuid",
|
||||
|
|
@ -2203,11 +2303,11 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "thiserror"
|
||||
version = "2.0.17"
|
||||
version = "2.0.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f63587ca0f12b72a0600bcba1d40081f830876000bb46dd2337a3051618f4fc8"
|
||||
checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4"
|
||||
dependencies = [
|
||||
"thiserror-impl 2.0.17",
|
||||
"thiserror-impl 2.0.18",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -2223,9 +2323,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "thiserror-impl"
|
||||
version = "2.0.17"
|
||||
version = "2.0.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913"
|
||||
checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
|
|
@ -2242,6 +2342,37 @@ dependencies = [
|
|||
"once_cell",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "time"
|
||||
version = "0.3.47"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c"
|
||||
dependencies = [
|
||||
"deranged",
|
||||
"itoa",
|
||||
"num-conv",
|
||||
"powerfmt",
|
||||
"serde_core",
|
||||
"time-core",
|
||||
"time-macros",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "time-core"
|
||||
version = "0.1.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca"
|
||||
|
||||
[[package]]
|
||||
name = "time-macros"
|
||||
version = "0.2.27"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215"
|
||||
dependencies = [
|
||||
"num-conv",
|
||||
"time-core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tinystr"
|
||||
version = "0.7.6"
|
||||
|
|
@ -2276,7 +2407,6 @@ dependencies = [
|
|||
"bytes",
|
||||
"libc",
|
||||
"mio",
|
||||
"parking_lot",
|
||||
"pin-project-lite",
|
||||
"signal-hook-registry",
|
||||
"socket2",
|
||||
|
|
@ -2376,6 +2506,19 @@ dependencies = [
|
|||
"tracing-core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tracing-appender"
|
||||
version = "0.2.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "050686193eb999b4bb3bc2acfa891a13da00f79734704c4b8b4ef1a10b368a3c"
|
||||
dependencies = [
|
||||
"crossbeam-channel",
|
||||
"symlink",
|
||||
"thiserror 2.0.18",
|
||||
"time",
|
||||
"tracing-subscriber",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tracing-attributes"
|
||||
version = "0.1.28"
|
||||
|
|
@ -2434,7 +2577,7 @@ checksum = "e640d9b0964e9d39df633548591090ab92f7a4567bc31d3891af23471a3365c6"
|
|||
dependencies = [
|
||||
"chrono",
|
||||
"lazy_static",
|
||||
"thiserror 2.0.17",
|
||||
"thiserror 2.0.18",
|
||||
"ts-rs-macros",
|
||||
"uuid",
|
||||
]
|
||||
|
|
@ -2481,6 +2624,12 @@ version = "0.10.4"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f720def6ce1ee2fc44d40ac9ed6d3a59c361c80a75a7aa8e75bb9baed31cf2ea"
|
||||
|
||||
[[package]]
|
||||
name = "unicase"
|
||||
version = "2.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-bidi"
|
||||
version = "0.3.17"
|
||||
|
|
@ -2577,6 +2726,16 @@ version = "0.9.5"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
|
||||
|
||||
[[package]]
|
||||
name = "walkdir"
|
||||
version = "2.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b"
|
||||
dependencies = [
|
||||
"same-file",
|
||||
"winapi-util",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wasi"
|
||||
version = "0.11.0+wasi-snapshot-preview1"
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
[package]
|
||||
name = "sync_server"
|
||||
rust-version = "1.89.0"
|
||||
rust-version = "1.94.0"
|
||||
authors = ["Andras Schmelczer <andras@schmelczer.dev>"]
|
||||
edition = "2024"
|
||||
license = "MIT"
|
||||
|
|
@ -10,7 +10,7 @@ version = "0.14.0"
|
|||
[dependencies]
|
||||
serde = { version = "1.0.219", default-features = false, features = ["derive"] }
|
||||
thiserror = { version = "2.0.12", default-features = false }
|
||||
tokio = { version = "1.48.0", features = ["full"]}
|
||||
tokio = { version = "1.48.0", features = ["macros", "rt-multi-thread", "sync", "time", "net", "fs", "signal"]}
|
||||
uuid = { version = "1.16.0", features = ["v4", "serde"] }
|
||||
log = { version = "0.4.28" }
|
||||
anyhow = { version = "1.0.100", features = ["backtrace"] }
|
||||
|
|
@ -20,6 +20,7 @@ axum_typed_multipart = "0.11.0"
|
|||
tower-http = { version = "0.6.1", features = ["cors", "trace", "limit", "timeout"] }
|
||||
tracing = "0.1.41"
|
||||
tracing-subscriber = { version = "0.3.20", features = ["fmt", "env-filter"]}
|
||||
tracing-appender = "0.2.5"
|
||||
humantime-serde = "1.1.1"
|
||||
sqlx = { version = "0.8.6", features = ["sqlite", "runtime-tokio", "uuid", "chrono"] }
|
||||
chrono = { version = "0.4.41", features = ["serde"] }
|
||||
|
|
@ -33,7 +34,10 @@ serde_json = "1.0.140"
|
|||
bimap = "0.6.3"
|
||||
ts-rs = { version = "10.1", features = ["uuid-impl", "chrono-impl"] }
|
||||
base64 = "0.22.1"
|
||||
reconcile-text = { version = "0.8.0", features = ["serde"] }
|
||||
reconcile-text = { version = "0.11.0", features = ["serde"] }
|
||||
rust-embed = "8.5"
|
||||
mime_guess = "2.0"
|
||||
subtle = "2.6.1"
|
||||
|
||||
[profile.release]
|
||||
codegen-units = 1
|
||||
|
|
|
|||
|
|
@ -1,5 +1,16 @@
|
|||
// generated by `sqlx migrate build-script`
|
||||
fn main() {
|
||||
// trigger recompilation when a new migration is added
|
||||
println!("cargo:rerun-if-changed=migrations");
|
||||
|
||||
// Ensure the history-ui dist directory exists so rust-embed can compile
|
||||
// even when the frontend hasn't been built yet.
|
||||
let dist_path = std::path::Path::new("../frontend/history-ui/dist");
|
||||
if !dist_path.exists() {
|
||||
std::fs::create_dir_all(dist_path).expect("Failed to create history-ui dist directory");
|
||||
std::fs::write(
|
||||
dist_path.join("index.html"),
|
||||
"<!DOCTYPE html><html><body><p>Run <code>npm run build -w history-ui</code> first.</p></body></html>",
|
||||
)
|
||||
.expect("Failed to write placeholder index.html");
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,12 +1,14 @@
|
|||
database:
|
||||
databases_directory_path: databases
|
||||
max_connections_per_vault: 12
|
||||
databases_directory_path: /host/tmp/vaultlink-e2e-databases
|
||||
max_connections_per_vault: 8
|
||||
cursor_timeout: 1m
|
||||
server:
|
||||
host: 0.0.0.0
|
||||
port: 3000
|
||||
port: 3010
|
||||
max_body_size_mb: 512
|
||||
max_clients_per_vault: 256
|
||||
max_pending_websocket_connections: 4096
|
||||
broadcast_channel_capacity: 1024
|
||||
response_timeout: 30m
|
||||
mergeable_file_extensions:
|
||||
- md
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
[toolchain]
|
||||
channel = "1.89.0"
|
||||
channel = "1.94.0"
|
||||
targets = [
|
||||
"x86_64-unknown-linux-gnu",
|
||||
"x86_64-unknown-linux-musl",
|
||||
|
|
|
|||
|
|
@ -2,6 +2,8 @@ pub mod cursors;
|
|||
pub mod database;
|
||||
pub mod websocket;
|
||||
|
||||
use std::sync::{Arc, atomic::AtomicUsize};
|
||||
|
||||
use anyhow::Result;
|
||||
use cursors::Cursors;
|
||||
use database::Database;
|
||||
|
|
@ -15,21 +17,42 @@ pub struct AppState {
|
|||
pub database: Database,
|
||||
pub cursors: Cursors,
|
||||
pub broadcasts: Broadcasts,
|
||||
/// Tracks WebSocket connections that have upgraded but not yet completed
|
||||
/// the authentication handshake
|
||||
pub pending_ws_connections: Arc<AtomicUsize>,
|
||||
/// Send on this channel to stop background tasks (cursor cleanup,
|
||||
/// idle-pool cleanup)
|
||||
shutdown_tx: Arc<tokio::sync::watch::Sender<()>>,
|
||||
}
|
||||
|
||||
impl AppState {
|
||||
pub async fn try_new(config: Config) -> Result<Self> {
|
||||
let (shutdown_tx, shutdown_rx) = tokio::sync::watch::channel(());
|
||||
|
||||
let broadcasts = Broadcasts::new(&config.server);
|
||||
let database = Database::try_new(&config.database, &broadcasts).await?;
|
||||
let database =
|
||||
Database::try_new(&config.database, &broadcasts, shutdown_rx.clone()).await?;
|
||||
let cursors: Cursors = Cursors::new(&config.database, &broadcasts);
|
||||
|
||||
Cursors::start_background_task(cursors.clone());
|
||||
Cursors::start_background_task(cursors.clone(), shutdown_rx);
|
||||
|
||||
Ok(Self {
|
||||
config,
|
||||
database,
|
||||
cursors,
|
||||
broadcasts,
|
||||
pending_ws_connections: Arc::new(AtomicUsize::new(0)),
|
||||
shutdown_tx: Arc::new(shutdown_tx),
|
||||
})
|
||||
}
|
||||
|
||||
/// Signal all background tasks (idle pool cleanup, cursor cleanup) to stop
|
||||
pub fn shutdown(&self) {
|
||||
let _ = self.shutdown_tx.send(());
|
||||
}
|
||||
|
||||
/// Get a receiver to be notified when shutdown is triggered
|
||||
pub fn subscribe_shutdown(&self) -> tokio::sync::watch::Receiver<()> {
|
||||
self.shutdown_tx.subscribe()
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -42,7 +42,9 @@ impl Cursors {
|
|||
) {
|
||||
let mut vault_to_cursors = self.vault_to_cursors.lock().await;
|
||||
|
||||
let all_device_cursors = vault_to_cursors.entry(vault_id).or_insert_with(Vec::new);
|
||||
let all_device_cursors = vault_to_cursors
|
||||
.entry(vault_id.clone())
|
||||
.or_insert_with(Vec::new);
|
||||
|
||||
all_device_cursors.retain(|c| &c.client_cursors.device_id != device_id);
|
||||
all_device_cursors.push(ClientCursorsWithTimeToLive::new(ClientCursors {
|
||||
|
|
@ -52,7 +54,7 @@ impl Cursors {
|
|||
}));
|
||||
|
||||
drop(vault_to_cursors); // Explicitly drop the lock before broadcasting to avoid deadlock
|
||||
self.broadcast_cursors().await;
|
||||
self.broadcast_cursors_for_vault(&vault_id).await;
|
||||
}
|
||||
|
||||
pub async fn get_cursors(&self, vault_id: &VaultId) -> Vec<ClientCursors> {
|
||||
|
|
@ -69,45 +71,81 @@ impl Cursors {
|
|||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
pub fn start_background_task(self) {
|
||||
pub fn start_background_task(self, mut shutdown: tokio::sync::watch::Receiver<()>) {
|
||||
tokio::spawn(async move {
|
||||
loop {
|
||||
tokio::select! {
|
||||
() = tokio::time::sleep(Duration::from_secs(1)) => {
|
||||
self.remove_expired_cursors().await;
|
||||
tokio::time::sleep(Duration::from_secs(1)).await;
|
||||
}
|
||||
Ok(()) = shutdown.changed() => break,
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async fn remove_expired_cursors(&self) {
|
||||
let changed_vaults: Vec<VaultId> = {
|
||||
let mut vault_to_cursors = self.vault_to_cursors.lock().await;
|
||||
|
||||
for (_vault_id, cursors) in vault_to_cursors.iter_mut() {
|
||||
let mut changed = Vec::new();
|
||||
for (vault_id, cursors) in vault_to_cursors.iter_mut() {
|
||||
let before = cursors.len();
|
||||
cursors.retain(|cursor| !cursor.is_expired(self.config.cursor_timeout));
|
||||
if cursors.len() != before {
|
||||
changed.push(vault_id.clone());
|
||||
}
|
||||
}
|
||||
|
||||
async fn broadcast_cursors(&self) {
|
||||
// Remove empty vault entries to prevent unbounded growth
|
||||
vault_to_cursors.retain(|_, cursors| !cursors.is_empty());
|
||||
|
||||
changed
|
||||
};
|
||||
|
||||
for vault_id in &changed_vaults {
|
||||
self.broadcast_cursors_for_vault(vault_id).await;
|
||||
}
|
||||
}
|
||||
|
||||
async fn broadcast_cursors_for_vault(&self, vault_id: &VaultId) {
|
||||
let client_cursors: Vec<ClientCursors> = {
|
||||
let vault_to_cursors = self.vault_to_cursors.lock().await;
|
||||
vault_to_cursors
|
||||
.get(vault_id)
|
||||
.map(|cursors| cursors.iter().map(|c| c.client_cursors.clone()).collect())
|
||||
.unwrap_or_default()
|
||||
};
|
||||
|
||||
for (vault_id, cursors) in vault_to_cursors.iter() {
|
||||
self.broadcasts
|
||||
.send_document_update(
|
||||
self.broadcasts.send_document_update(
|
||||
vault_id.clone(),
|
||||
WebSocketServerMessageWithOrigin::new(WebSocketServerMessage::CursorPositions(
|
||||
CursorPositionFromServer {
|
||||
clients: cursors.iter().map(|c| c.client_cursors.clone()).collect(),
|
||||
clients: client_cursors,
|
||||
},
|
||||
)),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
pub async fn remove_cursors_of_device(&self, vault_id: &str, device_id: &str) {
|
||||
pub async fn remove_cursors_of_device(&self, vault_id: &VaultId, device_id: &DeviceId) {
|
||||
let changed = {
|
||||
let mut vault_to_cursors = self.vault_to_cursors.lock().await;
|
||||
|
||||
if let Some(cursors) = vault_to_cursors.get_mut(vault_id) {
|
||||
cursors.retain(|c| c.client_cursors.device_id != device_id);
|
||||
let before = cursors.len();
|
||||
cursors.retain(|c| c.client_cursors.device_id != *device_id);
|
||||
let changed = cursors.len() != before;
|
||||
if cursors.is_empty() {
|
||||
vault_to_cursors.remove(vault_id);
|
||||
}
|
||||
changed
|
||||
} else {
|
||||
false
|
||||
}
|
||||
};
|
||||
|
||||
if changed {
|
||||
self.broadcast_cursors_for_vault(vault_id).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -0,0 +1,2 @@
|
|||
CREATE INDEX IF NOT EXISTS idx_documents_document_id
|
||||
ON documents (document_id, vault_update_id);
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
ALTER TABLE documents ADD COLUMN creation_vault_update_id INTEGER NOT NULL DEFAULT 0;
|
||||
|
||||
UPDATE documents
|
||||
SET creation_vault_update_id = (
|
||||
SELECT MIN(d2.vault_update_id)
|
||||
FROM documents d2
|
||||
WHERE d2.document_id = documents.document_id
|
||||
);
|
||||
|
||||
DROP VIEW latest_document_versions;
|
||||
|
||||
CREATE VIEW IF NOT EXISTS latest_document_versions AS --recreate view as it now includes one more field
|
||||
SELECT d.*
|
||||
FROM documents d
|
||||
INNER JOIN (
|
||||
SELECT MAX(vault_update_id) AS max_version_id
|
||||
FROM documents
|
||||
GROUP BY document_id
|
||||
) max_versions
|
||||
ON d.vault_update_id = max_versions.max_version_id;
|
||||
|
|
@ -13,6 +13,7 @@ pub type DeviceId = String;
|
|||
#[derive(Debug, Clone)]
|
||||
pub struct StoredDocumentVersion {
|
||||
pub vault_update_id: VaultUpdateId,
|
||||
pub creation_vault_update_id: VaultUpdateId,
|
||||
pub document_id: DocumentId,
|
||||
pub relative_path: String,
|
||||
pub updated_date: DateTime<Utc>,
|
||||
|
|
@ -33,7 +34,7 @@ impl PartialEq<Self> for StoredDocumentVersion {
|
|||
#[derive(TS, Debug, Clone, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct DocumentVersionWithoutContent {
|
||||
#[ts(as = "i32")]
|
||||
#[ts(type = "number")]
|
||||
pub vault_update_id: VaultUpdateId,
|
||||
|
||||
pub document_id: DocumentId,
|
||||
|
|
@ -43,12 +44,16 @@ pub struct DocumentVersionWithoutContent {
|
|||
pub user_id: UserId,
|
||||
pub device_id: DeviceId,
|
||||
|
||||
#[ts(as = "i32")]
|
||||
#[ts(type = "number")]
|
||||
pub content_size: u64,
|
||||
|
||||
/// True iff this is the first version of the document
|
||||
pub is_new_file: bool,
|
||||
}
|
||||
|
||||
impl From<StoredDocumentVersion> for DocumentVersionWithoutContent {
|
||||
fn from(value: StoredDocumentVersion) -> Self {
|
||||
let is_new_file = value.creation_vault_update_id == value.vault_update_id;
|
||||
Self {
|
||||
vault_update_id: value.vault_update_id,
|
||||
document_id: value.document_id,
|
||||
|
|
@ -58,6 +63,7 @@ impl From<StoredDocumentVersion> for DocumentVersionWithoutContent {
|
|||
user_id: value.user_id,
|
||||
device_id: value.device_id,
|
||||
content_size: value.content.len() as u64,
|
||||
is_new_file,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -65,7 +71,7 @@ impl From<StoredDocumentVersion> for DocumentVersionWithoutContent {
|
|||
#[derive(TS, Debug, Clone, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct DocumentVersion {
|
||||
#[ts(as = "i32")]
|
||||
#[ts(type = "number")]
|
||||
pub vault_update_id: VaultUpdateId,
|
||||
|
||||
pub document_id: DocumentId,
|
||||
|
|
@ -77,6 +83,25 @@ pub struct DocumentVersion {
|
|||
pub device_id: DeviceId,
|
||||
}
|
||||
|
||||
/// Row struct for vault history queries (used by `sqlx::query_as!`)
|
||||
#[derive(Debug)]
|
||||
pub struct VaultHistoryRow {
|
||||
pub vault_update_id: VaultUpdateId,
|
||||
pub creation_vault_update_id: VaultUpdateId,
|
||||
pub document_id: DocumentId,
|
||||
pub relative_path: String,
|
||||
pub updated_date: DateTime<Utc>,
|
||||
pub is_deleted: bool,
|
||||
pub user_id: String,
|
||||
pub device_id: String,
|
||||
pub content_size: Option<u64>,
|
||||
}
|
||||
|
||||
pub struct VaultStats {
|
||||
pub created_at: Option<DateTime<Utc>>,
|
||||
pub document_count: u32,
|
||||
}
|
||||
|
||||
impl From<StoredDocumentVersion> for DocumentVersion {
|
||||
fn from(value: StoredDocumentVersion) -> Self {
|
||||
Self {
|
||||
|
|
|
|||
|
|
@ -1,69 +1,147 @@
|
|||
use std::{collections::HashMap, sync::Arc};
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
sync::{Arc, Mutex as StdMutex},
|
||||
};
|
||||
|
||||
use anyhow::Context;
|
||||
use log::{debug, warn};
|
||||
use log::{debug, info, warn};
|
||||
use tokio::sync::{Mutex, broadcast};
|
||||
|
||||
use super::models::WebSocketServerMessageWithOrigin;
|
||||
use crate::{
|
||||
app_state::database::models::VaultId, config::server_config::ServerConfig, errors::server_error,
|
||||
};
|
||||
use super::models::{WebSocketServerMessage, WebSocketServerMessageWithOrigin};
|
||||
use crate::{app_state::database::models::VaultId, config::server_config::ServerConfig};
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Broadcasts {
|
||||
max_clients_per_vault: usize,
|
||||
tx: Arc<Mutex<HashMap<VaultId, broadcast::Sender<WebSocketServerMessageWithOrigin>>>>,
|
||||
broadcast_channel_capacity: usize,
|
||||
// `tx` uses a blocking std::sync::Mutex because the critical section is
|
||||
// a HashMap lookup plus a synchronous `broadcast::Sender::send`. Making
|
||||
// this non-async lets `send_document_update` run without an `.await`,
|
||||
// so an axum handler that is cancelled between `transaction.commit()`
|
||||
// and the broadcast can never drop the notification mid-flight.
|
||||
tx: Arc<StdMutex<HashMap<VaultId, broadcast::Sender<WebSocketServerMessageWithOrigin>>>>,
|
||||
send_locks: Arc<Mutex<HashMap<VaultId, Arc<tokio::sync::Mutex<()>>>>>,
|
||||
}
|
||||
|
||||
type TxMap = HashMap<VaultId, broadcast::Sender<WebSocketServerMessageWithOrigin>>;
|
||||
|
||||
impl Broadcasts {
|
||||
pub fn new(server_config: &ServerConfig) -> Self {
|
||||
Self {
|
||||
max_clients_per_vault: server_config.max_clients_per_vault,
|
||||
tx: Arc::new(Mutex::new(HashMap::new())),
|
||||
broadcast_channel_capacity: server_config.broadcast_channel_capacity,
|
||||
tx: Arc::new(StdMutex::new(HashMap::new())),
|
||||
send_locks: Arc::new(Mutex::new(HashMap::new())),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn get_receiver(
|
||||
/// Acquire a per-vault lock that serializes broadcasts in commit order.
|
||||
/// Must be acquired before the insert, held through commit and broadcast.
|
||||
pub async fn acquire_send_lock(&self, vault: &VaultId) -> tokio::sync::OwnedMutexGuard<()> {
|
||||
let lock = {
|
||||
let mut locks = self.send_locks.lock().await;
|
||||
locks
|
||||
.entry(vault.clone())
|
||||
.or_insert_with(|| Arc::new(tokio::sync::Mutex::new(())))
|
||||
.clone()
|
||||
};
|
||||
lock.lock_owned().await
|
||||
}
|
||||
|
||||
/// Remove senders for vaults with no active receivers
|
||||
fn prune_inactive_vaults(tx_map: &mut TxMap) -> Vec<VaultId> {
|
||||
let mut pruned = Vec::new();
|
||||
tx_map.retain(|vault, sender| {
|
||||
let alive = sender.receiver_count() > 0;
|
||||
if !alive {
|
||||
pruned.push(vault.clone());
|
||||
}
|
||||
alive
|
||||
});
|
||||
pruned
|
||||
}
|
||||
|
||||
pub fn get_receiver(
|
||||
&self,
|
||||
vault: VaultId,
|
||||
) -> broadcast::Receiver<WebSocketServerMessageWithOrigin> {
|
||||
let tx = self.get_or_create(vault).await;
|
||||
max_clients: usize,
|
||||
) -> Result<broadcast::Receiver<WebSocketServerMessageWithOrigin>, crate::errors::SyncServerError>
|
||||
{
|
||||
let mut tx_map = self
|
||||
.tx
|
||||
.lock()
|
||||
.expect("broadcasts.tx mutex poisoned — a previous holder panicked");
|
||||
|
||||
tx.subscribe()
|
||||
let count_before_prune = tx_map
|
||||
.get(&vault)
|
||||
.map_or(0, tokio::sync::broadcast::Sender::receiver_count);
|
||||
let pruned = Self::prune_inactive_vaults(&mut tx_map);
|
||||
let pruned_self = pruned.contains(&vault);
|
||||
|
||||
let sender = tx_map
|
||||
.entry(vault.clone())
|
||||
.or_insert_with(|| broadcast::channel(self.broadcast_channel_capacity).0);
|
||||
|
||||
// Hold the lock across the count check *and* the subscribe so the
|
||||
// `max_clients` cap is atomic: two concurrent callers can't both
|
||||
// observe `receiver_count() < max_clients` and both subscribe.
|
||||
if sender.receiver_count() >= max_clients {
|
||||
return Err(crate::errors::client_error(anyhow::anyhow!(
|
||||
"Vault has reached the maximum number of clients ({max_clients})"
|
||||
)));
|
||||
}
|
||||
|
||||
let receiver = sender.subscribe();
|
||||
let count_after = sender.receiver_count();
|
||||
info!(
|
||||
"[BCAST] get_receiver vault={vault} count_before_prune={count_before_prune} pruned_self={pruned_self} pruned_total={} count_after_subscribe={count_after}",
|
||||
pruned.len()
|
||||
);
|
||||
Ok(receiver)
|
||||
}
|
||||
|
||||
/// Notify all clients (who are subscribed to the vault) about an update.
|
||||
/// We only log failures and don't propagate them.
|
||||
pub async fn send_document_update(
|
||||
&self,
|
||||
vault: VaultId,
|
||||
document: WebSocketServerMessageWithOrigin,
|
||||
) {
|
||||
let tx = self.get_or_create(vault.clone()).await;
|
||||
/// Synchronous: safe to invoke from a handler between `commit()` and
|
||||
/// function return without worrying about task cancellation dropping
|
||||
/// the broadcast mid-flight. Failures are logged, never propagated.
|
||||
pub fn send_document_update(&self, vault: VaultId, document: WebSocketServerMessageWithOrigin) {
|
||||
let vault_update_id = match &document.message {
|
||||
WebSocketServerMessage::VaultUpdate(u) => Some(u.document.vault_update_id),
|
||||
WebSocketServerMessage::CursorPositions(_) => None,
|
||||
};
|
||||
let is_deleted = match &document.message {
|
||||
WebSocketServerMessage::VaultUpdate(u) => Some(u.document.is_deleted),
|
||||
WebSocketServerMessage::CursorPositions(_) => None,
|
||||
};
|
||||
let mut tx_map = self
|
||||
.tx
|
||||
.lock()
|
||||
.expect("broadcasts.tx mutex poisoned — a previous holder panicked");
|
||||
let count_before_prune = tx_map
|
||||
.get(&vault)
|
||||
.map_or(0, tokio::sync::broadcast::Sender::receiver_count);
|
||||
let pruned = Self::prune_inactive_vaults(&mut tx_map);
|
||||
let pruned_self = pruned.contains(&vault);
|
||||
|
||||
if tx.receiver_count() == 0 {
|
||||
let sender = tx_map
|
||||
.entry(vault.clone())
|
||||
.or_insert_with(|| broadcast::channel(self.broadcast_channel_capacity).0);
|
||||
|
||||
let count_before_send = sender.receiver_count();
|
||||
|
||||
if count_before_send == 0 {
|
||||
info!(
|
||||
"[BCAST] send_document_update vault={vault} vuid={vault_update_id:?} is_deleted={is_deleted:?} count_before_prune={count_before_prune} pruned_self={pruned_self} count_before_send=0 SKIPPED"
|
||||
);
|
||||
debug!("Skipping broadcast, no clients connected for vault `{vault}`");
|
||||
return;
|
||||
}
|
||||
|
||||
let result = tx
|
||||
.send(document)
|
||||
.context("Cannot broadcast server message to websocket listeners")
|
||||
.map_err(server_error);
|
||||
|
||||
if result.is_err() {
|
||||
warn!("Failed to send message: {result:?}");
|
||||
let send_result = sender.send(document);
|
||||
match &send_result {
|
||||
Ok(n) => info!(
|
||||
"[BCAST] send_document_update vault={vault} vuid={vault_update_id:?} is_deleted={is_deleted:?} count_before_prune={count_before_prune} pruned_self={pruned_self} count_before_send={count_before_send} SENT delivered_to={n}"
|
||||
),
|
||||
Err(e) => warn!(
|
||||
"[BCAST] send_document_update vault={vault} vuid={vault_update_id:?} is_deleted={is_deleted:?} count_before_prune={count_before_prune} pruned_self={pruned_self} count_before_send={count_before_send} FAILED err={e}"
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
async fn get_or_create(
|
||||
&self,
|
||||
vault: VaultId,
|
||||
) -> broadcast::Sender<WebSocketServerMessageWithOrigin> {
|
||||
let mut tx = self.tx.lock().await;
|
||||
|
||||
tx.entry(vault)
|
||||
.or_insert_with(|| broadcast::channel(self.max_clients_per_vault).0.clone())
|
||||
.clone()
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ pub struct WebSocketHandshake {
|
|||
pub token: String,
|
||||
pub device_id: DeviceId,
|
||||
|
||||
#[ts(as = "Option<i32>")]
|
||||
#[ts(type = "number | null")]
|
||||
pub last_seen_vault_update_id: Option<VaultUpdateId>,
|
||||
}
|
||||
|
||||
|
|
@ -22,13 +22,14 @@ pub struct CursorPositionFromClient {
|
|||
}
|
||||
|
||||
#[derive(TS, Serialize, Deserialize, Clone, Debug)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct DocumentWithCursors {
|
||||
// It's None in case the document is dirty.
|
||||
// We still want to sync the cursor to mark
|
||||
// that it exists and can be client-side
|
||||
// interpolated. However, the actual
|
||||
// position is meaningless.
|
||||
#[ts(as = "Option<u32>")]
|
||||
#[ts(type = "number | null")]
|
||||
pub vault_update_id: Option<VaultUpdateId>,
|
||||
|
||||
pub document_id: DocumentId,
|
||||
|
|
@ -57,11 +58,19 @@ pub struct CursorPositionFromServer {
|
|||
pub clients: Vec<ClientCursors>,
|
||||
}
|
||||
|
||||
// One committed version. Non-delete updates are broadcast to every
|
||||
// connected client *except* the device that authored them — that
|
||||
// device already has the new state via its HTTP response. Deletes are
|
||||
// broadcast to every client including the author: the author keeps
|
||||
// the document in its sync queue until this receipt arrives so a late
|
||||
// remote update can't sneak in between the HTTP response and the
|
||||
// queue cleanup. The server also emits these one-at-a-time to catch
|
||||
// up a freshly-connected client on versions committed while it was
|
||||
// offline, in ascending `vault_update_id` order.
|
||||
#[derive(TS, Serialize, Clone, Debug)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct WebSocketVaultUpdate {
|
||||
pub documents: Vec<DocumentVersionWithoutContent>,
|
||||
pub is_initial_sync: bool,
|
||||
pub document: DocumentVersionWithoutContent,
|
||||
}
|
||||
|
||||
#[derive(TS, Deserialize, Clone, Debug)]
|
||||
|
|
@ -80,6 +89,10 @@ pub enum WebSocketServerMessage {
|
|||
CursorPositions(CursorPositionFromServer),
|
||||
}
|
||||
|
||||
/// Broadcast envelope carrying the message plus the device that produced
|
||||
/// it. The per-recipient send task compares `origin_device_id` against
|
||||
/// its own device id to fill in `originates_from_self` before the message
|
||||
/// is serialized on the wire.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct WebSocketServerMessageWithOrigin {
|
||||
pub origin_device_id: Option<DeviceId>,
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ use crate::{
|
|||
database::models::{DocumentVersionWithoutContent, VaultId, VaultUpdateId},
|
||||
},
|
||||
config::user_config::User,
|
||||
errors::{SyncServerError, server_error, unauthenticated_error},
|
||||
errors::{SyncServerError, client_error, server_error, unauthenticated_error},
|
||||
server::auth::auth,
|
||||
};
|
||||
|
||||
|
|
@ -26,7 +26,7 @@ pub fn get_authenticated_handshake(
|
|||
if let Some(Message::Text(message)) = message {
|
||||
let message: WebSocketClientMessage = serde_json::from_str(&message)
|
||||
.context("Failed to parse message")
|
||||
.map_err(server_error)?;
|
||||
.map_err(client_error)?;
|
||||
|
||||
match message {
|
||||
WebSocketClientMessage::Handshake(handshake) => {
|
||||
|
|
@ -44,21 +44,29 @@ pub fn get_authenticated_handshake(
|
|||
}
|
||||
}
|
||||
|
||||
/// Stream the documents the client missed while offline, bounded above
|
||||
/// by `up_to_vault_update_id` so the catch-up is a stable snapshot at
|
||||
/// exactly that cursor. The WebSocket handshake atomically subscribes
|
||||
/// to the broadcast channel and snapshots this cursor under the per-
|
||||
/// vault send lock; commits past the cursor are then delivered solely
|
||||
/// through the broadcast channel (filtered by the same cursor on the
|
||||
/// receive side), so every committed update is delivered exactly once.
|
||||
pub async fn get_unseen_documents(
|
||||
state: &AppState,
|
||||
vault_id: &VaultId,
|
||||
last_seen_vault_update_id: Option<VaultUpdateId>,
|
||||
up_to_vault_update_id: VaultUpdateId,
|
||||
) -> Result<Vec<DocumentVersionWithoutContent>, SyncServerError> {
|
||||
if let Some(update_id) = last_seen_vault_update_id {
|
||||
state
|
||||
.database
|
||||
.get_latest_documents_since(vault_id, update_id, None)
|
||||
.get_latest_documents_since(vault_id, update_id, Some(up_to_vault_update_id), None)
|
||||
.await
|
||||
.map_err(server_error)
|
||||
} else {
|
||||
state
|
||||
.database
|
||||
.get_latest_documents(vault_id, None)
|
||||
.get_latest_documents(vault_id, Some(up_to_vault_update_id), None)
|
||||
.await
|
||||
.map_err(server_error)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -27,25 +27,35 @@ pub struct Config {
|
|||
}
|
||||
|
||||
impl Config {
|
||||
pub async fn read_or_create(path: &Path) -> Result<Self> {
|
||||
let config = if path.exists() {
|
||||
info!(
|
||||
"Loading configuration from `{}`",
|
||||
path.canonicalize().unwrap().display()
|
||||
);
|
||||
Self::load_from_file(path).await?
|
||||
} else {
|
||||
Self::default()
|
||||
};
|
||||
pub fn validate(&self) -> Result<()> {
|
||||
self.server
|
||||
.validate()
|
||||
.context("Invalid server configuration")?;
|
||||
self.logging
|
||||
.validate()
|
||||
.context("Invalid logging configuration")?;
|
||||
self.database
|
||||
.validate()
|
||||
.context("Invalid database configuration")?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn read_or_create(path: &Path) -> Result<Self> {
|
||||
let display_path = path.canonicalize().unwrap_or_else(|_| path.to_path_buf());
|
||||
|
||||
if path.exists() {
|
||||
info!("Loading configuration from `{}`", display_path.display());
|
||||
Self::load_from_file(path).await
|
||||
} else {
|
||||
let config = Self::default();
|
||||
config.write(path).await?;
|
||||
info!(
|
||||
"Updated configuration at `{}`",
|
||||
path.canonicalize().unwrap().display()
|
||||
"Created default configuration at `{}`",
|
||||
display_path.display()
|
||||
);
|
||||
|
||||
Ok(config)
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn load_from_file(path: &Path) -> Result<Self> {
|
||||
let contents = fs::read_to_string(path).await.with_context(|| {
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
use std::{path::PathBuf, time::Duration};
|
||||
|
||||
use anyhow::{Result, ensure};
|
||||
use log::debug;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
|
|
@ -34,6 +35,24 @@ fn default_cursor_timeout() -> Duration {
|
|||
DEFAULT_CURSOR_TIMEOUT
|
||||
}
|
||||
|
||||
impl DatabaseConfig {
|
||||
pub fn validate(&self) -> Result<()> {
|
||||
ensure!(
|
||||
!self.databases_directory_path.as_os_str().is_empty(),
|
||||
"databases_directory_path must not be empty"
|
||||
);
|
||||
ensure!(
|
||||
self.max_connections_per_vault > 0,
|
||||
"max_connections_per_vault must be greater than 0"
|
||||
);
|
||||
ensure!(
|
||||
!self.cursor_timeout.is_zero(),
|
||||
"cursor_timeout must be greater than 0"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for DatabaseConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
|
|
|
|||
|
|
@ -1,10 +1,13 @@
|
|||
use std::time::Duration;
|
||||
|
||||
use anyhow::{Result, ensure};
|
||||
use log::debug;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::{
|
||||
consts::{DEFAULT_LOG_DIRECTORY, DEFAULT_LOG_LEVEL, DEFAULT_LOG_ROTATION_INTERVAL},
|
||||
consts::{
|
||||
DEFAULT_LOG_DIRECTORY, DEFAULT_LOG_LEVEL, DEFAULT_LOG_ROTATION_INTERVAL, DURATION_ZERO,
|
||||
},
|
||||
utils::log_level::LogLevel,
|
||||
};
|
||||
|
||||
|
|
@ -20,6 +23,20 @@ pub struct LoggingConfig {
|
|||
pub log_level: LogLevel,
|
||||
}
|
||||
|
||||
impl LoggingConfig {
|
||||
pub fn validate(&self) -> Result<()> {
|
||||
ensure!(
|
||||
!self.log_directory.is_empty(),
|
||||
"log_directory must not be an empty string"
|
||||
);
|
||||
ensure!(
|
||||
self.log_rotation > DURATION_ZERO,
|
||||
"log_rotation must be greater than 0"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for LoggingConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
|
|
|
|||
|
|
@ -1,10 +1,13 @@
|
|||
use anyhow::{Result, ensure};
|
||||
use log::debug;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::time::Duration;
|
||||
|
||||
use crate::consts::{
|
||||
DEFAULT_HOST, DEFAULT_MAX_BODY_SIZE_MB, DEFAULT_MAX_CLIENTS_PER_VAULT,
|
||||
DEFAULT_MERGEABLE_FILE_EXTENSIONS, DEFAULT_PORT, DEFAULT_RESPONSE_TIMEOUT_SECONDS,
|
||||
DEFAULT_ALLOWED_ORIGINS, DEFAULT_BROADCAST_CHANNEL_CAPACITY, DEFAULT_HOST,
|
||||
DEFAULT_MAX_BODY_SIZE_MB, DEFAULT_MAX_CLIENTS_PER_VAULT, DEFAULT_MAX_PENDING_WS_CONNECTIONS,
|
||||
DEFAULT_MERGEABLE_FILE_EXTENSIONS, DEFAULT_PORT, DEFAULT_RATE_LIMIT_PER_USER_PER_SECOND,
|
||||
DEFAULT_RESPONSE_TIMEOUT_SECONDS, DURATION_ZERO,
|
||||
};
|
||||
|
||||
#[derive(Debug, Deserialize, Serialize, Clone, Default)]
|
||||
|
|
@ -21,11 +24,56 @@ pub struct ServerConfig {
|
|||
#[serde(default = "default_max_clients_per_vault")]
|
||||
pub max_clients_per_vault: usize,
|
||||
|
||||
#[serde(default = "default_broadcast_channel_capacity")]
|
||||
pub broadcast_channel_capacity: usize,
|
||||
|
||||
#[serde(default = "default_response_timeout", with = "humantime_serde")]
|
||||
pub response_timeout: Duration,
|
||||
|
||||
#[serde(default = "default_mergeable_file_extensions")]
|
||||
pub mergeable_file_extensions: Vec<String>,
|
||||
|
||||
/// Per-user maximum requests per second (keyed by bearer token).
|
||||
/// `None` disables rate limiting.
|
||||
#[serde(default = "default_rate_limit_per_user_per_second")]
|
||||
pub rate_limit_per_user_per_second: Option<u64>,
|
||||
|
||||
/// Allowed CORS origins. Default: `["*"]` (allow all).
|
||||
#[serde(default = "default_allowed_origins")]
|
||||
pub allowed_origins: Vec<String>,
|
||||
|
||||
/// Maximum concurrent unauthenticated WebSocket connections waiting for
|
||||
/// handshake. Limits resource consumption from clients that connect but
|
||||
/// never authenticate.
|
||||
#[serde(default = "default_max_pending_websocket_connections")]
|
||||
pub max_pending_websocket_connections: usize,
|
||||
}
|
||||
|
||||
impl ServerConfig {
|
||||
pub fn validate(&self) -> Result<()> {
|
||||
ensure!(
|
||||
self.response_timeout > DURATION_ZERO,
|
||||
"response_timeout must be greater than 0"
|
||||
);
|
||||
ensure!(
|
||||
self.max_body_size_mb > 0,
|
||||
"max_body_size_mb must be greater than 0"
|
||||
);
|
||||
ensure!(
|
||||
self.max_clients_per_vault > 0,
|
||||
"max_clients_per_vault must be greater than 0"
|
||||
);
|
||||
ensure!(
|
||||
self.broadcast_channel_capacity > 0,
|
||||
"broadcast_channel_capacity must be greater than 0"
|
||||
);
|
||||
ensure!(
|
||||
self.max_pending_websocket_connections > 0,
|
||||
"max_pending_websocket_connections must be greater than 0"
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn default_host() -> String {
|
||||
|
|
@ -48,6 +96,11 @@ fn default_max_clients_per_vault() -> usize {
|
|||
DEFAULT_MAX_CLIENTS_PER_VAULT
|
||||
}
|
||||
|
||||
fn default_broadcast_channel_capacity() -> usize {
|
||||
debug!("Using default broadcast channel capacity: {DEFAULT_BROADCAST_CHANNEL_CAPACITY}");
|
||||
DEFAULT_BROADCAST_CHANNEL_CAPACITY
|
||||
}
|
||||
|
||||
fn default_response_timeout() -> Duration {
|
||||
debug!("Using default response timeout: {DEFAULT_RESPONSE_TIMEOUT_SECONDS:?}");
|
||||
DEFAULT_RESPONSE_TIMEOUT_SECONDS
|
||||
|
|
@ -60,3 +113,21 @@ fn default_mergeable_file_extensions() -> Vec<String> {
|
|||
.map(|s| (*s).to_owned())
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn default_rate_limit_per_user_per_second() -> Option<u64> {
|
||||
debug!("Using default rate limit per second: {DEFAULT_RATE_LIMIT_PER_USER_PER_SECOND:?}");
|
||||
DEFAULT_RATE_LIMIT_PER_USER_PER_SECOND
|
||||
}
|
||||
|
||||
fn default_allowed_origins() -> Vec<String> {
|
||||
debug!("Using default allowed origins: {DEFAULT_ALLOWED_ORIGINS:?}");
|
||||
DEFAULT_ALLOWED_ORIGINS
|
||||
.iter()
|
||||
.map(|s| (*s).to_owned())
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn default_max_pending_websocket_connections() -> usize {
|
||||
debug!("Using default max pending WebSocket connections: {DEFAULT_MAX_PENDING_WS_CONNECTIONS}");
|
||||
DEFAULT_MAX_PENDING_WS_CONNECTIONS
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
use bimap::BiHashMap;
|
||||
use rand::{Rng, distr::Alphanumeric, rng};
|
||||
use serde::{Deserialize, Deserializer, Serialize, de::Error};
|
||||
use subtle::ConstantTimeEq;
|
||||
|
||||
use crate::app_state::database::models::VaultId;
|
||||
|
||||
|
|
@ -19,10 +20,19 @@ where
|
|||
let mut user_token_map = BiHashMap::new();
|
||||
for user in &users {
|
||||
if let Some(existing_name) = user_token_map.get_by_right(&user.token) {
|
||||
let redacted = if user.token.len() > 6 {
|
||||
format!(
|
||||
"{}...{}",
|
||||
&user.token[..3],
|
||||
&user.token[user.token.len() - 3..]
|
||||
)
|
||||
} else {
|
||||
"***".to_owned()
|
||||
};
|
||||
return Err(D::Error::custom(format!(
|
||||
"Duplicate user token found: `{}` for users `{}` and `{}`. User tokens must be \
|
||||
unique.",
|
||||
user.token, existing_name, user.name
|
||||
"Duplicate user token found: `{redacted}` for users `{}` and `{}`. User tokens \
|
||||
must be unique.",
|
||||
existing_name, user.name
|
||||
)));
|
||||
}
|
||||
|
||||
|
|
@ -41,7 +51,9 @@ where
|
|||
|
||||
impl UserConfig {
|
||||
pub fn get_user(&self, token: &str) -> Option<&User> {
|
||||
self.user_configs.iter().find(|u| u.token == token)
|
||||
self.user_configs
|
||||
.iter()
|
||||
.find(|u| u.token.as_bytes().ct_eq(token.as_bytes()).into())
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -2,22 +2,36 @@ use std::time::Duration;
|
|||
|
||||
use crate::utils::log_level::LogLevel;
|
||||
|
||||
pub const DURATION_ZERO: Duration = Duration::from_secs(0);
|
||||
|
||||
pub const DEFAULT_CONFIG_PATH: &str = "config.yml";
|
||||
|
||||
pub const DEFAULT_DATABASES_DIRECTORY_PATH: &str = "databases";
|
||||
pub const DEFAULT_MAX_CONNECTIONS_PER_VAULT: u32 = 12;
|
||||
pub const DEFAULT_MAX_CONNECTIONS_PER_VAULT: u32 = 6;
|
||||
pub const DEFAULT_CURSOR_TIMEOUT: Duration = Duration::from_secs(60);
|
||||
|
||||
pub const DEFAULT_HOST: &str = "127.0.0.1";
|
||||
pub const DEFAULT_PORT: u16 = 3000;
|
||||
pub const DEFAULT_MAX_BODY_SIZE_MB: usize = 4096;
|
||||
pub const DEFAULT_RESPONSE_TIMEOUT_SECONDS: Duration = Duration::from_secs(1800);
|
||||
pub const DEFAULT_RESPONSE_TIMEOUT_SECONDS: Duration = Duration::from_mins(30);
|
||||
pub const DEFAULT_MAX_CLIENTS_PER_VAULT: usize = 256;
|
||||
pub const DEFAULT_BROADCAST_CHANNEL_CAPACITY: usize = 4096;
|
||||
pub const DEFAULT_MAX_PENDING_WS_CONNECTIONS: usize = 128;
|
||||
|
||||
pub const DEFAULT_LOG_DIRECTORY: &str = "logs";
|
||||
pub const DEFAULT_LOG_ROTATION_INTERVAL: Duration = Duration::from_secs(60 * 60 * 24); // 1 day
|
||||
pub const DEFAULT_LOG_ROTATION_INTERVAL: Duration = Duration::from_hours(24);
|
||||
pub const IDLE_POOL_TIMEOUT: Duration = Duration::from_mins(5);
|
||||
pub const GRACEFUL_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(10);
|
||||
pub const HANDSHAKE_TIMEOUT: Duration = Duration::from_secs(10);
|
||||
|
||||
pub const MAX_CURSOR_DOCUMENTS: usize = 1000;
|
||||
pub const MAX_CURSORS_PER_DOCUMENT: usize = 100;
|
||||
pub const MAX_RELATIVE_PATH_LEN: usize = 4096;
|
||||
|
||||
pub const DEFAULT_LOG_LEVEL: LogLevel = LogLevel::Info;
|
||||
|
||||
pub const DEFAULT_MERGEABLE_FILE_EXTENSIONS: &[&str] = &["md", "txt"];
|
||||
|
||||
pub const SUPPORTED_API_VERSION: u32 = 2;
|
||||
pub const DEFAULT_RATE_LIMIT_PER_USER_PER_SECOND: Option<u64> = None;
|
||||
pub const DEFAULT_ALLOWED_ORIGINS: &[&str] = &["*"];
|
||||
pub const SUPPORTED_API_VERSION: u32 = 3;
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ use axum::{
|
|||
http::StatusCode,
|
||||
response::{IntoResponse, Response},
|
||||
};
|
||||
use log::{debug, error};
|
||||
use log::{debug, error, warn};
|
||||
use serde::Serialize;
|
||||
use thiserror::Error;
|
||||
use ts_rs::TS;
|
||||
|
|
@ -29,6 +29,9 @@ pub enum SyncServerError {
|
|||
|
||||
#[error("Permission denied error: {0}")]
|
||||
PermissionDeniedError(#[source] anyhow::Error),
|
||||
|
||||
#[error("Too many requests: {0}")]
|
||||
TooManyRequests(#[source] anyhow::Error),
|
||||
}
|
||||
|
||||
impl SyncServerError {
|
||||
|
|
@ -39,7 +42,8 @@ impl SyncServerError {
|
|||
| Self::ServerError(error)
|
||||
| Self::NotFound(error)
|
||||
| Self::Unauthenticated(error)
|
||||
| Self::PermissionDeniedError(error) => error.into(),
|
||||
| Self::PermissionDeniedError(error)
|
||||
| Self::TooManyRequests(error) => error.into(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -69,7 +73,22 @@ impl Display for SerializedError {
|
|||
|
||||
impl IntoResponse for SyncServerError {
|
||||
fn into_response(self) -> Response {
|
||||
let body = Json(self.serialize());
|
||||
let serialized = self.serialize();
|
||||
|
||||
match &self {
|
||||
Self::InitError(_) | Self::ServerError(_) => {
|
||||
error!("{serialized}");
|
||||
}
|
||||
Self::ClientError(_) | Self::NotFound(_) => {
|
||||
warn!("{serialized}");
|
||||
}
|
||||
Self::TooManyRequests(_) => {
|
||||
warn!("{serialized}");
|
||||
}
|
||||
Self::Unauthenticated(_) | Self::PermissionDeniedError(_) => {}
|
||||
}
|
||||
|
||||
let body = Json(serialized);
|
||||
|
||||
match self {
|
||||
Self::InitError(_) | Self::ServerError(_) => {
|
||||
|
|
@ -79,6 +98,7 @@ impl IntoResponse for SyncServerError {
|
|||
Self::NotFound(_) => (StatusCode::NOT_FOUND, body).into_response(),
|
||||
Self::Unauthenticated(_) => (StatusCode::UNAUTHORIZED, body).into_response(),
|
||||
Self::PermissionDeniedError(_) => (StatusCode::FORBIDDEN, body).into_response(),
|
||||
Self::TooManyRequests(_) => (StatusCode::TOO_MANY_REQUESTS, body).into_response(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -102,6 +122,7 @@ impl From<&anyhow::Error> for SerializedError {
|
|||
SyncServerError::NotFound(_) => "NotFound",
|
||||
SyncServerError::Unauthenticated(_) => "Unauthenticated",
|
||||
SyncServerError::PermissionDeniedError(_) => "PermissionDeniedError",
|
||||
SyncServerError::TooManyRequests(_) => "TooManyRequests",
|
||||
},
|
||||
),
|
||||
message: error.to_string(),
|
||||
|
|
@ -139,3 +160,21 @@ pub fn permission_denied_error(error: anyhow::Error) -> SyncServerError {
|
|||
debug!("Permission denied: {error:?}");
|
||||
SyncServerError::PermissionDeniedError(error)
|
||||
}
|
||||
|
||||
pub fn too_many_requests_error(error: anyhow::Error) -> SyncServerError {
|
||||
debug!("Too many requests: {error:?}");
|
||||
SyncServerError::TooManyRequests(error)
|
||||
}
|
||||
|
||||
/// Maps a `create_write_transaction` error to 429 if the database is busy,
|
||||
/// or 500 for all other failures.
|
||||
pub fn write_transaction_error(error: anyhow::Error) -> SyncServerError {
|
||||
if error
|
||||
.downcast_ref::<crate::app_state::database::WriteBusyError>()
|
||||
.is_some()
|
||||
{
|
||||
too_many_requests_error(error)
|
||||
} else {
|
||||
server_error(error)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ use consts::DEFAULT_CONFIG_PATH;
|
|||
use errors::{SyncServerError, init_error};
|
||||
use log::info;
|
||||
use server::create_server;
|
||||
use tracing_appender::non_blocking::WorkerGuard;
|
||||
use tracing_subscriber::{EnvFilter, fmt::format, layer::SubscriberExt, util::SubscriberInitExt};
|
||||
use utils::rotating_file_writer::RotatingFileWriter;
|
||||
|
||||
|
|
@ -41,11 +42,14 @@ async fn main() -> ExitCode {
|
|||
}
|
||||
};
|
||||
|
||||
let mut result = set_up_logging(&args, &config.logging);
|
||||
|
||||
if result.is_ok() {
|
||||
result = start_server(config).await;
|
||||
let result = async {
|
||||
config.validate().map_err(init_error)?;
|
||||
// Hold the non-blocking writer guards until shutdown so the
|
||||
// dedicated writer threads stay alive and flush queued log lines.
|
||||
let _log_guards = set_up_logging(&args, &config.logging)?;
|
||||
start_server(config).await
|
||||
}
|
||||
.await;
|
||||
|
||||
match result {
|
||||
Ok(()) => ExitCode::SUCCESS,
|
||||
|
|
@ -59,7 +63,7 @@ async fn main() -> ExitCode {
|
|||
fn set_up_logging(
|
||||
args: &Args,
|
||||
logging_config: &config::logging_config::LoggingConfig,
|
||||
) -> Result<(), SyncServerError> {
|
||||
) -> Result<[WorkerGuard; 2], SyncServerError> {
|
||||
let level_filter = logging_config.log_level.as_tracing_level();
|
||||
|
||||
let env_filter = EnvFilter::builder()
|
||||
|
|
@ -80,6 +84,14 @@ fn set_up_logging(
|
|||
.context("Failed to create rotating file writer")
|
||||
.map_err(init_error)?;
|
||||
|
||||
// Decouple log emission from disk/stderr I/O. Without this, a tokio
|
||||
// worker that holds the writer's std::sync::Mutex while a `write(2)`
|
||||
// is throttled by the kernel (e.g. btrfs writeback) cascades the
|
||||
// stall to every other worker that tries to log, freezing the whole
|
||||
// runtime. The guards must outlive every emitter.
|
||||
let (file_writer, file_guard) = tracing_appender::non_blocking(file_appender);
|
||||
let (stderr_writer, stderr_guard) = tracing_appender::non_blocking(std::io::stderr());
|
||||
|
||||
let format = format()
|
||||
.with_target(is_debug_mode)
|
||||
.with_line_number(is_debug_mode)
|
||||
|
|
@ -87,12 +99,12 @@ fn set_up_logging(
|
|||
|
||||
let stderr_layer = tracing_subscriber::fmt::layer()
|
||||
.with_ansi(use_colors)
|
||||
.with_writer(std::io::stderr)
|
||||
.with_writer(stderr_writer)
|
||||
.event_format(format.clone());
|
||||
|
||||
let file_layer = tracing_subscriber::fmt::layer()
|
||||
.with_ansi(false)
|
||||
.with_writer(file_appender)
|
||||
.with_writer(file_writer)
|
||||
.event_format(format);
|
||||
|
||||
tracing_subscriber::registry()
|
||||
|
|
@ -103,7 +115,7 @@ fn set_up_logging(
|
|||
.context("Failed to initialise tracing")
|
||||
.map_err(init_error)?;
|
||||
|
||||
Ok(())
|
||||
Ok([file_guard, stderr_guard])
|
||||
}
|
||||
|
||||
async fn start_server(config: Config) -> Result<(), SyncServerError> {
|
||||
|
|
|
|||
|
|
@ -4,27 +4,30 @@ mod delete_document;
|
|||
mod device_id_header;
|
||||
mod fetch_document_version;
|
||||
mod fetch_document_version_content;
|
||||
mod fetch_document_versions;
|
||||
mod fetch_latest_document_version;
|
||||
mod fetch_latest_documents;
|
||||
mod fetch_vault_history;
|
||||
mod index;
|
||||
mod list_vaults;
|
||||
mod ping;
|
||||
mod rate_limit;
|
||||
mod requests;
|
||||
mod responses;
|
||||
mod update_document;
|
||||
mod websocket;
|
||||
|
||||
use anyhow::{Context as _, Result, anyhow};
|
||||
use anyhow::{Context as _, Result};
|
||||
use auth::auth_middleware;
|
||||
use axum::{
|
||||
Router,
|
||||
extract::{DefaultBodyLimit, Request},
|
||||
http::{self, HeaderValue, Method},
|
||||
middleware,
|
||||
response::IntoResponse,
|
||||
routing::{IntoMakeService, delete, get, post, put},
|
||||
};
|
||||
use device_id_header::DEVICE_ID_HEADER_NAME;
|
||||
use log::info;
|
||||
use log::{info, warn};
|
||||
use tokio::signal;
|
||||
use tower_http::{
|
||||
LatencyUnit,
|
||||
|
|
@ -41,7 +44,7 @@ use tracing::{Level, info_span};
|
|||
use crate::{
|
||||
app_state::AppState,
|
||||
config::{Config, server_config::ServerConfig},
|
||||
errors::{client_error, not_found_error},
|
||||
consts::GRACEFUL_SHUTDOWN_TIMEOUT,
|
||||
};
|
||||
|
||||
pub async fn create_server(config: Config) -> Result<()> {
|
||||
|
|
@ -51,26 +54,33 @@ pub async fn create_server(config: Config) -> Result<()> {
|
|||
|
||||
let server_config = app_state.config.server.clone();
|
||||
|
||||
let app = Router::new()
|
||||
let mut app = Router::new()
|
||||
.nest("/", get_authed_routes(app_state.clone()))
|
||||
.route("/", get(index::index))
|
||||
.route("/assets/*path", get(index::spa_assets))
|
||||
.route("/vaults", get(list_vaults::list_vaults))
|
||||
.route("/vaults/:vault_id/ping", get(ping::ping))
|
||||
.route("/vaults/:vault_id/ws", get(websocket::websocket_handler))
|
||||
.fallback(index::spa_fallback);
|
||||
|
||||
let cors_layer = build_cors_layer(&server_config).context("Invalid CORS configuration")?;
|
||||
|
||||
if let Some(rate_limit) = server_config.rate_limit_per_user_per_second {
|
||||
info!("Rate limiting enabled: {rate_limit} requests/second per user");
|
||||
let limiter = rate_limit::RateLimiter::new(rate_limit);
|
||||
app = app.layer(middleware::from_fn_with_state(
|
||||
limiter,
|
||||
rate_limit::rate_limit_middleware,
|
||||
));
|
||||
}
|
||||
|
||||
let app = app
|
||||
.layer(DefaultBodyLimit::disable())
|
||||
.layer(RequestBodyLimitLayer::new(
|
||||
app_state.config.server.max_body_size_mb * 1024 * 1024,
|
||||
))
|
||||
.layer(TimeoutLayer::new(server_config.response_timeout))
|
||||
.layer(
|
||||
CorsLayer::new()
|
||||
.allow_origin("*".parse::<HeaderValue>().expect("Failed to parse origin"))
|
||||
.allow_headers([
|
||||
http::header::CONTENT_TYPE,
|
||||
http::header::AUTHORIZATION,
|
||||
DEVICE_ID_HEADER_NAME.clone(),
|
||||
])
|
||||
.allow_methods([Method::GET, Method::POST, Method::PUT, Method::DELETE]),
|
||||
)
|
||||
.layer(cors_layer)
|
||||
.layer(
|
||||
TraceLayer::new_for_http()
|
||||
.make_span_with(|request: &Request<_>| {
|
||||
|
|
@ -90,12 +100,39 @@ pub async fn create_server(config: Config) -> Result<()> {
|
|||
.on_eos(DefaultOnEos::new())
|
||||
.on_failure(DefaultOnFailure::new().level(Level::ERROR)),
|
||||
)
|
||||
.with_state(app_state)
|
||||
.fallback(handle_404)
|
||||
.fallback(handle_405)
|
||||
.with_state(app_state.clone())
|
||||
.into_make_service();
|
||||
|
||||
start_server(app, &server_config).await
|
||||
start_server(app, &server_config, app_state).await
|
||||
}
|
||||
|
||||
fn build_cors_layer(server_config: &ServerConfig) -> Result<CorsLayer> {
|
||||
let origins = &server_config.allowed_origins;
|
||||
|
||||
let cors = if origins.len() == 1 && origins[0] == "*" {
|
||||
info!("CORS: allowing all origins");
|
||||
let header: HeaderValue = "*"
|
||||
.parse()
|
||||
.context("Failed to parse wildcard CORS origin")?;
|
||||
CorsLayer::new().allow_origin(header)
|
||||
} else {
|
||||
let parsed: Vec<HeaderValue> = origins
|
||||
.iter()
|
||||
.map(|o| {
|
||||
o.parse::<HeaderValue>()
|
||||
.with_context(|| format!("Failed to parse CORS origin: `{o}`"))
|
||||
})
|
||||
.collect::<Result<Vec<_>>>()?;
|
||||
CorsLayer::new().allow_origin(parsed)
|
||||
};
|
||||
|
||||
Ok(cors
|
||||
.allow_headers([
|
||||
http::header::CONTENT_TYPE,
|
||||
http::header::AUTHORIZATION,
|
||||
DEVICE_ID_HEADER_NAME.clone(),
|
||||
])
|
||||
.allow_methods([Method::GET, Method::POST, Method::PUT, Method::DELETE]))
|
||||
}
|
||||
|
||||
fn get_authed_routes(app_state: AppState) -> Router<AppState> {
|
||||
|
|
@ -120,6 +157,10 @@ fn get_authed_routes(app_state: AppState) -> Router<AppState> {
|
|||
"/vaults/:vault_id/documents/:document_id/text",
|
||||
put(update_document::update_text),
|
||||
)
|
||||
.route(
|
||||
"/vaults/:vault_id/documents/:document_id/versions",
|
||||
get(fetch_document_versions::fetch_document_versions),
|
||||
)
|
||||
.route(
|
||||
"/vaults/:vault_id/documents/:document_id/versions/:vault_update_id",
|
||||
get(fetch_document_version::fetch_document_version),
|
||||
|
|
@ -132,10 +173,18 @@ fn get_authed_routes(app_state: AppState) -> Router<AppState> {
|
|||
"/vaults/:vault_id/documents/:document_id",
|
||||
delete(delete_document::delete_document),
|
||||
)
|
||||
.route(
|
||||
"/vaults/:vault_id/history",
|
||||
get(fetch_vault_history::fetch_vault_history),
|
||||
)
|
||||
.layer(middleware::from_fn_with_state(app_state, auth_middleware))
|
||||
}
|
||||
|
||||
async fn start_server(app: IntoMakeService<axum::Router>, config: &ServerConfig) -> Result<()> {
|
||||
async fn start_server(
|
||||
app: IntoMakeService<axum::Router>,
|
||||
config: &ServerConfig,
|
||||
app_state: AppState,
|
||||
) -> Result<()> {
|
||||
let address = format!("{}:{}", config.host, config.port);
|
||||
let listener = tokio::net::TcpListener::bind(address.clone())
|
||||
.await
|
||||
|
|
@ -148,26 +197,46 @@ async fn start_server(app: IntoMakeService<axum::Router>, config: &ServerConfig)
|
|||
.context("Failed to get local address")?
|
||||
);
|
||||
|
||||
axum::serve(listener, app)
|
||||
.with_graceful_shutdown(shutdown_signal())
|
||||
.tcp_nodelay(true)
|
||||
.await
|
||||
.context("Failed to start server")
|
||||
let mut shutdown_rx = app_state.subscribe_shutdown();
|
||||
|
||||
let server = axum::serve(listener, app)
|
||||
.with_graceful_shutdown(async move {
|
||||
shutdown_signal().await;
|
||||
app_state.shutdown();
|
||||
})
|
||||
.tcp_nodelay(true);
|
||||
|
||||
tokio::select! {
|
||||
result = server => result.context("Failed to start server"),
|
||||
() = async {
|
||||
let _ = shutdown_rx.changed().await;
|
||||
info!(
|
||||
"Shutdown signal received, waiting up to {}s for in-flight requests to complete...",
|
||||
GRACEFUL_SHUTDOWN_TIMEOUT.as_secs()
|
||||
);
|
||||
tokio::time::sleep(GRACEFUL_SHUTDOWN_TIMEOUT).await;
|
||||
warn!("Graceful shutdown timed out, forcing exit");
|
||||
} => Ok(()),
|
||||
}
|
||||
}
|
||||
|
||||
async fn shutdown_signal() {
|
||||
let ctrl_c = async {
|
||||
signal::ctrl_c()
|
||||
.await
|
||||
.expect("failed to install Ctrl+C handler");
|
||||
if let Err(e) = signal::ctrl_c().await {
|
||||
log::error!("Failed to install Ctrl+C handler: {e}");
|
||||
}
|
||||
};
|
||||
|
||||
#[cfg(unix)]
|
||||
let terminate = async {
|
||||
signal::unix::signal(signal::unix::SignalKind::terminate())
|
||||
.expect("failed to install signal handler")
|
||||
.recv()
|
||||
.await;
|
||||
match signal::unix::signal(signal::unix::SignalKind::terminate()) {
|
||||
Ok(mut signal) => {
|
||||
signal.recv().await;
|
||||
}
|
||||
Err(e) => {
|
||||
log::error!("Failed to install SIGTERM handler: {e}");
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
#[cfg(not(unix))]
|
||||
|
|
@ -178,11 +247,3 @@ async fn shutdown_signal() {
|
|||
() = terminate => {},
|
||||
}
|
||||
}
|
||||
|
||||
async fn handle_404() -> impl IntoResponse {
|
||||
not_found_error(anyhow!("Page not found"))
|
||||
}
|
||||
|
||||
async fn handle_405() -> impl IntoResponse {
|
||||
client_error(anyhow!("Method not allowed"))
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ use axum_extra::{
|
|||
TypedHeader,
|
||||
headers::{Authorization, authorization::Bearer},
|
||||
};
|
||||
use log::info;
|
||||
use log::{debug, info};
|
||||
|
||||
use crate::{
|
||||
app_state::{AppState, database::models::VaultId},
|
||||
|
|
@ -21,10 +21,12 @@ use crate::{
|
|||
pub async fn auth_middleware(
|
||||
State(state): State<AppState>,
|
||||
Path(path_params): Path<HashMap<String, String>>,
|
||||
TypedHeader(auth_header): TypedHeader<Authorization<Bearer>>,
|
||||
auth_header: Option<TypedHeader<Authorization<Bearer>>>,
|
||||
mut req: Request,
|
||||
next: Next,
|
||||
) -> Result<Response, SyncServerError> {
|
||||
let auth_header = auth_header
|
||||
.ok_or_else(|| unauthenticated_error(anyhow::anyhow!("Missing Authorization header")))?;
|
||||
let token = auth_header.token().trim();
|
||||
let vault_id = normalize_string(
|
||||
path_params
|
||||
|
|
@ -39,20 +41,24 @@ pub async fn auth_middleware(
|
|||
Ok(next.run(req).await)
|
||||
}
|
||||
|
||||
pub fn auth(state: &AppState, token: &str, vault_id: &VaultId) -> Result<User, SyncServerError> {
|
||||
let user = state
|
||||
pub fn authenticate(state: &AppState, token: &str) -> Result<User, SyncServerError> {
|
||||
state
|
||||
.config
|
||||
.users
|
||||
.get_user(token)
|
||||
.cloned()
|
||||
.ok_or_else(|| unauthenticated_error(anyhow::anyhow!("Invalid token")))?;
|
||||
.ok_or_else(|| unauthenticated_error(anyhow::anyhow!("Invalid token")))
|
||||
}
|
||||
|
||||
pub fn auth(state: &AppState, token: &str, vault_id: &VaultId) -> Result<User, SyncServerError> {
|
||||
let user = authenticate(state, token)?;
|
||||
|
||||
if match user.vault_access {
|
||||
VaultAccess::AllowAccessToAll => true,
|
||||
VaultAccess::AllowList(AllowListedVaults { ref allowed }) => allowed.contains(vault_id),
|
||||
} {
|
||||
info!(
|
||||
"User `{}` is authenticated and is authorised to access to vault `{vault_id}`",
|
||||
debug!(
|
||||
"User `{}` is authenticated and is authorised to access vault `{vault_id}`",
|
||||
user.name
|
||||
);
|
||||
|
||||
|
|
|
|||
|
|
@ -11,12 +11,14 @@ use super::{device_id_header::DeviceIdHeader, requests::CreateDocumentVersion};
|
|||
use crate::{
|
||||
app_state::{
|
||||
AppState,
|
||||
database::models::{DocumentVersionWithoutContent, StoredDocumentVersion, VaultId},
|
||||
database::models::{StoredDocumentVersion, VaultId},
|
||||
},
|
||||
config::user_config::User,
|
||||
errors::{SyncServerError, client_error, server_error},
|
||||
errors::{SyncServerError, client_error, server_error, write_transaction_error},
|
||||
server::{responses::DocumentUpdateResponse, update_document},
|
||||
utils::{
|
||||
find_first_available_path::find_first_available_path, normalize::normalize,
|
||||
find_first_available_path::find_first_available_path, is_binary::is_binary,
|
||||
is_file_type_mergable::is_file_type_mergable, normalize::normalize,
|
||||
sanitize_path::sanitize_path,
|
||||
},
|
||||
};
|
||||
|
|
@ -30,48 +32,137 @@ pub struct CreateDocumentPathParams {
|
|||
/// Create a new document in case a document with the same doesn't exist
|
||||
/// already. If a document with the same path exists, a new version is created
|
||||
/// with their content merged.
|
||||
///
|
||||
/// Text content must be UTF-8 encoded. Clients are responsible for
|
||||
/// transcoding other encodings (e.g. UTF-16) to UTF-8 before sending.
|
||||
#[axum::debug_handler]
|
||||
#[allow(clippy::too_many_lines)]
|
||||
pub async fn create_document(
|
||||
Path(CreateDocumentPathParams { vault_id }): Path<CreateDocumentPathParams>,
|
||||
Extension(user): Extension<User>,
|
||||
TypedHeader(device_id): TypedHeader<DeviceIdHeader>,
|
||||
State(state): State<AppState>,
|
||||
TypedMultipart(request): TypedMultipart<CreateDocumentVersion>,
|
||||
) -> Result<Json<DocumentVersionWithoutContent>, SyncServerError> {
|
||||
) -> Result<Json<DocumentUpdateResponse>, SyncServerError> {
|
||||
debug!("Creating document in vault `{vault_id}`");
|
||||
|
||||
let mut transaction = state
|
||||
.database
|
||||
.create_write_transaction(&vault_id)
|
||||
.await
|
||||
.map_err(server_error)?;
|
||||
.map_err(write_transaction_error)?;
|
||||
|
||||
let document_id = match request.document_id {
|
||||
Some(document_id) => {
|
||||
let existing_version = state
|
||||
let sanitized_relative_path = sanitize_path(&request.relative_path).map_err(client_error)?;
|
||||
let new_content = request.content.contents.to_vec();
|
||||
|
||||
let latest_version = state
|
||||
.database
|
||||
.get_latest_document(&vault_id, &document_id, Some(&mut transaction))
|
||||
.get_latest_non_deleted_document_by_path(
|
||||
&vault_id,
|
||||
&sanitized_relative_path,
|
||||
Some(&mut *transaction),
|
||||
)
|
||||
.await
|
||||
.map_err(server_error)?;
|
||||
|
||||
if existing_version.is_some() {
|
||||
return Err(client_error(anyhow::anyhow!(
|
||||
"Document with the same ID `{document_id}` already exists"
|
||||
)));
|
||||
if let Some(latest_version) = latest_version {
|
||||
// Only merge with an existing document the client couldn't have
|
||||
// known about: its creation is newer than the client's last seen
|
||||
// vault update to avoid creating cycles by merging two documents into one.
|
||||
// This could happen if both clients know of document A at path P1,
|
||||
// but client 2 moves it to P2 while client 1 creates a new document at P2,
|
||||
// then client 1 would merge its new document with the moved version of A at P2
|
||||
// that client 2 resulting in two files (P1 and P2) with the same doc id (A).
|
||||
if latest_version.creation_vault_update_id > request.last_seen_vault_update_id
|
||||
&& latest_version.creation_vault_update_id == latest_version.vault_update_id
|
||||
// can't allow merging with a moved document as that could create a cycle
|
||||
{
|
||||
let is_mergeable_text = is_file_type_mergable(
|
||||
&sanitized_relative_path,
|
||||
&state.config.server.mergeable_file_extensions,
|
||||
) && !is_binary(&latest_version.content)
|
||||
&& !is_binary(&new_content);
|
||||
|
||||
if is_mergeable_text || new_content == latest_version.content {
|
||||
return update_document::update_document(
|
||||
&sanitized_relative_path,
|
||||
Vec::new(),
|
||||
vault_id,
|
||||
latest_version.document_id,
|
||||
Some(&request.relative_path),
|
||||
new_content,
|
||||
user,
|
||||
device_id,
|
||||
state,
|
||||
transaction,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
document_id
|
||||
// For non-mergeable (binary) files with different content, don't
|
||||
// merge, create a separate document at a deconflicted path so
|
||||
// neither client's data is silently overwritten.
|
||||
}
|
||||
None => uuid::Uuid::new_v4(),
|
||||
};
|
||||
}
|
||||
|
||||
// Lost-create + local rename recovery. If this device has a doc
|
||||
// the requesting client hasn't seen yet (its create succeeded
|
||||
// server-side but the response was discarded — e.g. a sync
|
||||
// reset mid-flight) and the new request carries the same content
|
||||
// at a different path (the user renamed the file before the
|
||||
// retry), bind the retry to that existing doc instead of
|
||||
// creating a duplicate. The dedup is scoped tightly:
|
||||
// - same `device_id` (only this client's own lost create),
|
||||
// - `creation_vault_update_id > last_seen` (client never saw
|
||||
// this doc, so it can't be deliberately creating another
|
||||
// copy with matching content),
|
||||
// - `creation == latest` (the doc has only its create version,
|
||||
// nobody else has touched it; safe to relocate),
|
||||
// - exact content match.
|
||||
// Outside that window we fall through to the normal deconflict
|
||||
// path, so legitimate "this device created a duplicate of an
|
||||
// already-acknowledged file" flows still produce a new doc.
|
||||
if let Some(lost_create) = state
|
||||
.database
|
||||
.find_unseen_lost_create_by_device_and_content(
|
||||
&vault_id,
|
||||
&device_id.0,
|
||||
request.last_seen_vault_update_id,
|
||||
&new_content,
|
||||
Some(&mut *transaction),
|
||||
)
|
||||
.await
|
||||
.map_err(server_error)?
|
||||
{
|
||||
info!(
|
||||
"Lost-create recovery: binding retry at `{sanitized_relative_path}` to existing doc {} (was at `{}`) in vault `{vault_id}` for device `{}`",
|
||||
lost_create.document_id,
|
||||
lost_create.relative_path,
|
||||
device_id.0
|
||||
);
|
||||
return update_document::update_document(
|
||||
&sanitized_relative_path,
|
||||
Vec::new(),
|
||||
vault_id,
|
||||
lost_create.document_id,
|
||||
Some(&request.relative_path),
|
||||
new_content,
|
||||
user,
|
||||
device_id,
|
||||
state,
|
||||
transaction,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
let document_id = uuid::Uuid::new_v4();
|
||||
|
||||
let last_update_id = state
|
||||
.database
|
||||
.get_max_update_id_in_vault(&vault_id, Some(&mut transaction))
|
||||
.get_max_update_id_in_vault(&vault_id, Some(&mut *transaction))
|
||||
.await
|
||||
.map_err(server_error)?;
|
||||
|
||||
let sanitized_relative_path = sanitize_path(&request.relative_path);
|
||||
let deduped_path = find_first_available_path(
|
||||
&vault_id,
|
||||
&sanitized_relative_path,
|
||||
|
|
@ -87,11 +178,13 @@ pub async fn create_document(
|
|||
);
|
||||
}
|
||||
|
||||
let new_vault_update_id = last_update_id + 1;
|
||||
let new_version = StoredDocumentVersion {
|
||||
vault_update_id: last_update_id + 1,
|
||||
vault_update_id: new_vault_update_id,
|
||||
creation_vault_update_id: new_vault_update_id,
|
||||
document_id,
|
||||
relative_path: deduped_path,
|
||||
content: request.content.contents.to_vec(),
|
||||
content: new_content,
|
||||
updated_date: chrono::Utc::now(),
|
||||
is_deleted: false,
|
||||
user_id: user.name,
|
||||
|
|
@ -101,9 +194,11 @@ pub async fn create_document(
|
|||
|
||||
state
|
||||
.database
|
||||
.insert_document_version(&vault_id, &new_version, Some(transaction))
|
||||
.insert_document_version(&vault_id, &new_version, transaction)
|
||||
.await
|
||||
.map_err(server_error)?;
|
||||
|
||||
Ok(Json(new_version.into()))
|
||||
Ok(Json(DocumentUpdateResponse::FastForwardUpdate(
|
||||
new_version.into(),
|
||||
)))
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
use anyhow::Context;
|
||||
use anyhow::{Context, anyhow};
|
||||
use axum::{
|
||||
Extension, Json,
|
||||
extract::{Path, State},
|
||||
|
|
@ -7,7 +7,7 @@ use axum_extra::TypedHeader;
|
|||
use log::{debug, info};
|
||||
use serde::Deserialize;
|
||||
|
||||
use super::{device_id_header::DeviceIdHeader, requests::DeleteDocumentVersion};
|
||||
use super::device_id_header::DeviceIdHeader;
|
||||
use crate::{
|
||||
app_state::{
|
||||
AppState,
|
||||
|
|
@ -16,8 +16,8 @@ use crate::{
|
|||
},
|
||||
},
|
||||
config::user_config::User,
|
||||
errors::{SyncServerError, server_error},
|
||||
utils::{normalize::normalize, sanitize_path::sanitize_path},
|
||||
errors::{SyncServerError, not_found_error, server_error, write_transaction_error},
|
||||
utils::normalize::normalize,
|
||||
};
|
||||
|
||||
#[derive(Deserialize)]
|
||||
|
|
@ -37,7 +37,6 @@ pub async fn delete_document(
|
|||
Extension(user): Extension<User>,
|
||||
TypedHeader(device_id): TypedHeader<DeviceIdHeader>,
|
||||
State(state): State<AppState>,
|
||||
Json(request): Json<DeleteDocumentVersion>,
|
||||
) -> Result<Json<DocumentVersionWithoutContent>, SyncServerError> {
|
||||
debug!("Deleting document `{document_id}` in vault `{vault_id}`");
|
||||
|
||||
|
|
@ -45,7 +44,7 @@ pub async fn delete_document(
|
|||
.database
|
||||
.create_write_transaction(&vault_id)
|
||||
.await
|
||||
.map_err(server_error)?;
|
||||
.map_err(write_transaction_error)?;
|
||||
|
||||
let last_update_id = state
|
||||
.database
|
||||
|
|
@ -59,9 +58,18 @@ pub async fn delete_document(
|
|||
.await
|
||||
.map_err(server_error)?;
|
||||
|
||||
if let Some(latest_version) = &latest_version
|
||||
&& latest_version.is_deleted
|
||||
{
|
||||
let Some(latest_version) = latest_version else {
|
||||
transaction
|
||||
.rollback()
|
||||
.await
|
||||
.context("Failed to roll back transaction")
|
||||
.map_err(server_error)?;
|
||||
return Err(not_found_error(anyhow!(
|
||||
"Document `{document_id}` not found in vault `{vault_id}`"
|
||||
)));
|
||||
};
|
||||
|
||||
if latest_version.is_deleted {
|
||||
transaction
|
||||
.rollback()
|
||||
.await
|
||||
|
|
@ -69,15 +77,19 @@ pub async fn delete_document(
|
|||
.map_err(server_error)?;
|
||||
|
||||
info!("Document `{document_id}` has already been deleted",);
|
||||
return Ok(Json(latest_version.clone().into()));
|
||||
return Ok(Json(latest_version.into()));
|
||||
}
|
||||
|
||||
let latest_content = latest_version.map_or_else(Vec::new, |version| version.content); // in case the document has never existed before deleting it
|
||||
let new_vault_update_id = last_update_id + 1;
|
||||
let latest_relative_path = latest_version.relative_path;
|
||||
let latest_content = latest_version.content;
|
||||
let creation_vault_update_id = latest_version.creation_vault_update_id;
|
||||
|
||||
let new_version = StoredDocumentVersion {
|
||||
vault_update_id: last_update_id + 1,
|
||||
vault_update_id: new_vault_update_id,
|
||||
creation_vault_update_id,
|
||||
document_id,
|
||||
relative_path: sanitize_path(&request.relative_path),
|
||||
relative_path: latest_relative_path,
|
||||
content: latest_content, // copy the content from the latest version
|
||||
updated_date: chrono::Utc::now(),
|
||||
is_deleted: true,
|
||||
|
|
@ -88,7 +100,7 @@ pub async fn delete_document(
|
|||
|
||||
state
|
||||
.database
|
||||
.insert_document_version(&vault_id, &new_version, Some(transaction))
|
||||
.insert_document_version(&vault_id, &new_version, transaction)
|
||||
.await
|
||||
.map_err(server_error)?;
|
||||
|
||||
|
|
|
|||
|
|
@ -16,20 +16,31 @@ impl Header for DeviceIdHeader {
|
|||
{
|
||||
let value = values.next().ok_or_else(headers::Error::invalid)?;
|
||||
|
||||
Ok(DeviceIdHeader(
|
||||
value
|
||||
.to_str()
|
||||
.map_err(|_| headers::Error::invalid())?
|
||||
.to_owned(),
|
||||
))
|
||||
let s = value.to_str().map_err(|_| headers::Error::invalid())?;
|
||||
|
||||
if s.is_empty() || s.len() > 256 {
|
||||
return Err(headers::Error::invalid());
|
||||
}
|
||||
|
||||
// Only allow safe characters to prevent log injection and similar attacks.
|
||||
// Covers UUIDs, user-agent strings like "vault-link/1.0 (12345; linux)",
|
||||
// and human-readable device names.
|
||||
if !s
|
||||
.chars()
|
||||
.all(|c| c.is_ascii_alphanumeric() || "-_./ ();:@+,".contains(c))
|
||||
{
|
||||
return Err(headers::Error::invalid());
|
||||
}
|
||||
|
||||
Ok(DeviceIdHeader(s.to_owned()))
|
||||
}
|
||||
|
||||
fn encode<E>(&self, values: &mut E)
|
||||
where
|
||||
E: Extend<HeaderValue>,
|
||||
{
|
||||
let value = HeaderValue::from_static(Box::leak(self.0.clone().into_boxed_str()));
|
||||
|
||||
if let Ok(value) = HeaderValue::from_str(&self.0) {
|
||||
values.extend(std::iter::once(value));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ use crate::{
|
|||
AppState,
|
||||
database::models::{DocumentId, DocumentVersion, VaultId, VaultUpdateId},
|
||||
},
|
||||
errors::{SyncServerError, not_found_error, server_error},
|
||||
errors::{SyncServerError, client_error, not_found_error, server_error},
|
||||
utils::normalize::normalize,
|
||||
};
|
||||
|
||||
|
|
@ -52,7 +52,7 @@ pub async fn fetch_document_version(
|
|||
)?;
|
||||
|
||||
if result.document_id != document_id {
|
||||
return Err(not_found_error(anyhow!(
|
||||
return Err(client_error(anyhow!(
|
||||
"Document with document id `{document_id}` does not have a version with id \
|
||||
`{vault_update_id}`",
|
||||
)));
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ use crate::{
|
|||
AppState,
|
||||
database::models::{DocumentId, VaultId, VaultUpdateId},
|
||||
},
|
||||
errors::{SyncServerError, not_found_error, server_error},
|
||||
errors::{SyncServerError, client_error, not_found_error, server_error},
|
||||
utils::normalize::normalize,
|
||||
};
|
||||
|
||||
|
|
@ -52,7 +52,7 @@ pub async fn fetch_document_version_content(
|
|||
)?;
|
||||
|
||||
if result.document_id != document_id {
|
||||
return Err(not_found_error(anyhow!(
|
||||
return Err(client_error(anyhow!(
|
||||
"Document with document id `{document_id}` does not have a version with id \
|
||||
`{vault_update_id}`",
|
||||
)));
|
||||
|
|
|
|||
42
sync-server/src/server/fetch_document_versions.rs
Normal file
42
sync-server/src/server/fetch_document_versions.rs
Normal file
|
|
@ -0,0 +1,42 @@
|
|||
use axum::{
|
||||
Json,
|
||||
extract::{Path, State},
|
||||
};
|
||||
use log::debug;
|
||||
use serde::Deserialize;
|
||||
|
||||
use crate::{
|
||||
app_state::{
|
||||
AppState,
|
||||
database::models::{DocumentId, DocumentVersionWithoutContent, VaultId},
|
||||
},
|
||||
errors::{SyncServerError, server_error},
|
||||
utils::normalize::normalize,
|
||||
};
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub struct FetchDocumentVersionsPathParams {
|
||||
#[serde(deserialize_with = "normalize")]
|
||||
vault_id: VaultId,
|
||||
|
||||
document_id: DocumentId,
|
||||
}
|
||||
|
||||
#[axum::debug_handler]
|
||||
pub async fn fetch_document_versions(
|
||||
Path(FetchDocumentVersionsPathParams {
|
||||
vault_id,
|
||||
document_id,
|
||||
}): Path<FetchDocumentVersionsPathParams>,
|
||||
State(state): State<AppState>,
|
||||
) -> Result<Json<Vec<DocumentVersionWithoutContent>>, SyncServerError> {
|
||||
debug!("Fetching all versions for document `{document_id}` in vault `{vault_id}`");
|
||||
|
||||
let versions = state
|
||||
.database
|
||||
.get_document_versions(&vault_id, &document_id, None)
|
||||
.await
|
||||
.map_err(server_error)?;
|
||||
|
||||
Ok(Json(versions))
|
||||
}
|
||||
|
|
@ -37,13 +37,13 @@ pub async fn fetch_latest_documents(
|
|||
let documents = if let Some(since_update_id) = since_update_id {
|
||||
state
|
||||
.database
|
||||
.get_latest_documents_since(&vault_id, since_update_id, None)
|
||||
.get_latest_documents_since(&vault_id, since_update_id, None, None)
|
||||
.await
|
||||
.map_err(server_error)
|
||||
} else {
|
||||
state
|
||||
.database
|
||||
.get_latest_documents(&vault_id, None)
|
||||
.get_latest_documents(&vault_id, None, None)
|
||||
.await
|
||||
.map_err(server_error)
|
||||
}?;
|
||||
|
|
|
|||
70
sync-server/src/server/fetch_vault_history.rs
Normal file
70
sync-server/src/server/fetch_vault_history.rs
Normal file
|
|
@ -0,0 +1,70 @@
|
|||
use axum::{
|
||||
Json,
|
||||
extract::{Path, Query, State},
|
||||
};
|
||||
use log::debug;
|
||||
use serde::Deserialize;
|
||||
|
||||
use super::responses::VaultHistoryResponse;
|
||||
use crate::{
|
||||
app_state::{
|
||||
AppState,
|
||||
database::models::{VaultId, VaultUpdateId},
|
||||
},
|
||||
errors::{SyncServerError, client_error, server_error},
|
||||
utils::normalize::normalize,
|
||||
};
|
||||
|
||||
const DEFAULT_LIMIT: i64 = 50;
|
||||
const MAX_LIMIT: i64 = 500;
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub struct FetchVaultHistoryPathParams {
|
||||
#[serde(deserialize_with = "normalize")]
|
||||
vault_id: VaultId,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub struct QueryParams {
|
||||
limit: Option<i64>,
|
||||
before_update_id: Option<VaultUpdateId>,
|
||||
}
|
||||
|
||||
#[axum::debug_handler]
|
||||
pub async fn fetch_vault_history(
|
||||
Path(FetchVaultHistoryPathParams { vault_id }): Path<FetchVaultHistoryPathParams>,
|
||||
Query(QueryParams {
|
||||
limit,
|
||||
before_update_id,
|
||||
}): Query<QueryParams>,
|
||||
State(state): State<AppState>,
|
||||
) -> Result<Json<VaultHistoryResponse>, SyncServerError> {
|
||||
if let Some(id) = before_update_id
|
||||
&& id <= 0
|
||||
{
|
||||
return Err(client_error(anyhow::anyhow!(
|
||||
"before_update_id must be a positive integer"
|
||||
)));
|
||||
}
|
||||
|
||||
let limit = limit.unwrap_or(DEFAULT_LIMIT).clamp(1, MAX_LIMIT);
|
||||
|
||||
debug!(
|
||||
"Fetching vault history for vault `{vault_id}` (limit={limit}, before={before_update_id:?})"
|
||||
);
|
||||
|
||||
// Fetch one extra row to determine if there are more results
|
||||
let mut versions = state
|
||||
.database
|
||||
.get_vault_history(&vault_id, limit + 1, before_update_id, None)
|
||||
.await
|
||||
.map_err(server_error)?;
|
||||
|
||||
#[allow(clippy::cast_sign_loss)] // limit is clamped to [1, 500] above
|
||||
let has_more = versions.len() > limit as usize;
|
||||
if has_more {
|
||||
versions.pop();
|
||||
}
|
||||
|
||||
Ok(Json(VaultHistoryResponse { versions, has_more }))
|
||||
}
|
||||
|
|
@ -1,7 +1,77 @@
|
|||
use axum::response::{Html, IntoResponse};
|
||||
use axum::{
|
||||
body::Body,
|
||||
extract::{Path, State},
|
||||
http::{StatusCode, header},
|
||||
response::{Html, IntoResponse, Response},
|
||||
};
|
||||
use log::warn;
|
||||
use rust_embed::Embed;
|
||||
|
||||
pub async fn index() -> impl IntoResponse {
|
||||
const HTML_CONTENT: &str = include_str!("./assets/index.html");
|
||||
let html_content = HTML_CONTENT;
|
||||
Html(html_content)
|
||||
use crate::app_state::AppState;
|
||||
|
||||
#[derive(Embed)]
|
||||
#[folder = "../frontend/history-ui/dist/"]
|
||||
struct HistoryUiAssets;
|
||||
|
||||
pub async fn index(State(_state): State<AppState>) -> impl IntoResponse {
|
||||
if let Some(content) = HistoryUiAssets::get("index.html") {
|
||||
Html(
|
||||
std::str::from_utf8(content.data.as_ref())
|
||||
.inspect_err(|e| warn!("Embedded index.html is not valid UTF-8: {e}"))
|
||||
.unwrap_or("<h1>VaultLink</h1>")
|
||||
.to_owned(),
|
||||
)
|
||||
.into_response()
|
||||
} else {
|
||||
warn!("No embedded index.html found — history UI may not have been built");
|
||||
Html("<h1>VaultLink server</h1>".to_owned()).into_response()
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn spa_assets(Path(path): Path<String>) -> impl IntoResponse {
|
||||
// The route is /assets/*path so path is relative to assets/.
|
||||
// The embedded files include the assets/ prefix from the dist directory.
|
||||
let full_path = format!("assets/{path}");
|
||||
if let Some(content) = HistoryUiAssets::get(&full_path) {
|
||||
let mime = mime_guess::from_path(&full_path).first_or_octet_stream();
|
||||
return Response::builder()
|
||||
.status(StatusCode::OK)
|
||||
.header(header::CONTENT_TYPE, mime.as_ref())
|
||||
.body(Body::from(content.data.to_vec()))
|
||||
.unwrap_or_else(|_| {
|
||||
Response::builder()
|
||||
.status(StatusCode::INTERNAL_SERVER_ERROR)
|
||||
.body(Body::empty())
|
||||
.unwrap_or_else(|_| Response::new(Body::empty()))
|
||||
});
|
||||
}
|
||||
|
||||
// Asset paths must match an embedded file — no SPA fallback.
|
||||
// Serving index.html here would return 200 with text/html for missing
|
||||
// .css/.js files, causing the browser to silently ignore the content.
|
||||
Response::builder()
|
||||
.status(StatusCode::NOT_FOUND)
|
||||
.body(Body::from("Not found"))
|
||||
.unwrap_or_else(|_| Response::new(Body::from("Not found")))
|
||||
}
|
||||
|
||||
/// SPA fallback for production: serves index.html for client-side routes
|
||||
/// (e.g. `/documents/123`).
|
||||
pub async fn spa_fallback() -> impl IntoResponse {
|
||||
match HistoryUiAssets::get("index.html") {
|
||||
Some(content) => Response::builder()
|
||||
.status(StatusCode::OK)
|
||||
.header(header::CONTENT_TYPE, "text/html")
|
||||
.body(Body::from(content.data.to_vec()))
|
||||
.unwrap_or_else(|_| {
|
||||
Response::builder()
|
||||
.status(StatusCode::INTERNAL_SERVER_ERROR)
|
||||
.body(Body::empty())
|
||||
.unwrap_or_else(|_| Response::new(Body::empty()))
|
||||
}),
|
||||
None => Response::builder()
|
||||
.status(StatusCode::NOT_FOUND)
|
||||
.body(Body::from("Not found"))
|
||||
.unwrap_or_else(|_| Response::new(Body::from("Not found"))),
|
||||
}
|
||||
}
|
||||
|
|
|
|||
82
sync-server/src/server/list_vaults.rs
Normal file
82
sync-server/src/server/list_vaults.rs
Normal file
|
|
@ -0,0 +1,82 @@
|
|||
use axum::{
|
||||
Json,
|
||||
extract::{Query, State},
|
||||
};
|
||||
use axum_extra::{
|
||||
TypedHeader,
|
||||
headers::{Authorization, authorization::Bearer},
|
||||
};
|
||||
use log::debug;
|
||||
use serde::Deserialize;
|
||||
|
||||
use super::{
|
||||
auth::authenticate,
|
||||
responses::{ListVaultsResponse, VaultInfo},
|
||||
};
|
||||
use crate::{
|
||||
app_state::AppState,
|
||||
config::user_config::{AllowListedVaults, VaultAccess},
|
||||
errors::{SyncServerError, server_error, unauthenticated_error},
|
||||
};
|
||||
|
||||
const DEFAULT_LIMIT: usize = 50;
|
||||
const MAX_LIMIT: usize = 200;
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub struct QueryParams {
|
||||
limit: Option<usize>,
|
||||
after: Option<String>,
|
||||
}
|
||||
|
||||
#[axum::debug_handler]
|
||||
pub async fn list_vaults(
|
||||
auth_header: Option<TypedHeader<Authorization<Bearer>>>,
|
||||
Query(QueryParams { limit, after }): Query<QueryParams>,
|
||||
State(state): State<AppState>,
|
||||
) -> Result<Json<ListVaultsResponse>, SyncServerError> {
|
||||
let auth_header = auth_header
|
||||
.ok_or_else(|| unauthenticated_error(anyhow::anyhow!("Missing Authorization header")))?;
|
||||
|
||||
let user = authenticate(&state, auth_header.token().trim())?;
|
||||
|
||||
debug!("User `{}` listing accessible vaults", user.name);
|
||||
|
||||
let existing_vaults = state.database.list_vaults().await.map_err(server_error)?;
|
||||
|
||||
let mut accessible: Vec<String> = match user.vault_access {
|
||||
VaultAccess::AllowAccessToAll => existing_vaults,
|
||||
VaultAccess::AllowList(AllowListedVaults { ref allowed }) => existing_vaults
|
||||
.into_iter()
|
||||
.filter(|v| allowed.contains(v))
|
||||
.collect(),
|
||||
};
|
||||
|
||||
// Cursor-based pagination: skip vaults up to and including `after`
|
||||
if let Some(ref cursor) = after {
|
||||
accessible.retain(|v| v.as_str() > cursor.as_str());
|
||||
}
|
||||
|
||||
let limit = limit.unwrap_or(DEFAULT_LIMIT).clamp(1, MAX_LIMIT);
|
||||
let has_more = accessible.len() > limit;
|
||||
accessible.truncate(limit);
|
||||
|
||||
let mut vaults = Vec::with_capacity(accessible.len());
|
||||
for name in accessible {
|
||||
let stats = state
|
||||
.database
|
||||
.get_vault_stats(&name)
|
||||
.await
|
||||
.map_err(server_error)?;
|
||||
vaults.push(VaultInfo {
|
||||
name,
|
||||
document_count: stats.document_count,
|
||||
created_at: stats.created_at,
|
||||
});
|
||||
}
|
||||
|
||||
Ok(Json(ListVaultsResponse {
|
||||
vaults,
|
||||
has_more,
|
||||
user_name: user.name,
|
||||
}))
|
||||
}
|
||||
102
sync-server/src/server/rate_limit.rs
Normal file
102
sync-server/src/server/rate_limit.rs
Normal file
|
|
@ -0,0 +1,102 @@
|
|||
use std::{
|
||||
collections::HashMap,
|
||||
sync::{Arc, Mutex},
|
||||
time::Instant,
|
||||
};
|
||||
|
||||
use axum::{extract::Request, http::StatusCode, middleware::Next, response::Response};
|
||||
use axum_extra::{
|
||||
TypedHeader,
|
||||
headers::{Authorization, authorization::Bearer},
|
||||
};
|
||||
|
||||
/// Per-user token-bucket rate limiter. Each bearer token gets its own bucket
|
||||
/// that refills to `max_per_second` tokens every second.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct RateLimiter {
|
||||
max_per_second: u64,
|
||||
buckets: Arc<Mutex<HashMap<String, Arc<TokenBucket>>>>,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct TokenBucket {
|
||||
state: Mutex<BucketState>,
|
||||
max_tokens: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct BucketState {
|
||||
tokens: u64,
|
||||
last_refill: Instant,
|
||||
}
|
||||
|
||||
impl RateLimiter {
|
||||
/// Create a new per-user rate limiter.
|
||||
///
|
||||
/// # Panics
|
||||
///
|
||||
/// Panics if `max_per_second` is 0.
|
||||
pub fn new(max_per_second: u64) -> Self {
|
||||
assert!(
|
||||
max_per_second > 0,
|
||||
"max_per_second must be > 0 (set rate_limit_per_user_per_second to null in config to disable)"
|
||||
);
|
||||
|
||||
Self {
|
||||
max_per_second,
|
||||
buckets: Arc::new(Mutex::new(HashMap::new())),
|
||||
}
|
||||
}
|
||||
|
||||
fn get_or_create_bucket(&self, token: &str) -> Arc<TokenBucket> {
|
||||
self.buckets
|
||||
.lock()
|
||||
.expect("rate limiter lock poisoned")
|
||||
.entry(token.to_owned())
|
||||
.or_insert_with(|| {
|
||||
Arc::new(TokenBucket {
|
||||
state: Mutex::new(BucketState {
|
||||
tokens: self.max_per_second,
|
||||
last_refill: Instant::now(),
|
||||
}),
|
||||
max_tokens: self.max_per_second,
|
||||
})
|
||||
})
|
||||
.clone()
|
||||
}
|
||||
}
|
||||
|
||||
impl TokenBucket {
|
||||
fn try_acquire(&self) -> bool {
|
||||
let mut state = self.state.lock().expect("token bucket lock poisoned");
|
||||
let now = Instant::now();
|
||||
if now.duration_since(state.last_refill).as_secs() >= 1 {
|
||||
state.tokens = self.max_tokens;
|
||||
state.last_refill = now;
|
||||
}
|
||||
if state.tokens > 0 {
|
||||
state.tokens -= 1;
|
||||
true
|
||||
} else {
|
||||
false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn rate_limit_middleware(
|
||||
axum::extract::State(limiter): axum::extract::State<RateLimiter>,
|
||||
auth_header: Option<TypedHeader<Authorization<Bearer>>>,
|
||||
req: Request,
|
||||
next: Next,
|
||||
) -> Result<Response, StatusCode> {
|
||||
let Some(TypedHeader(auth)) = auth_header else {
|
||||
return Ok(next.run(req).await);
|
||||
};
|
||||
|
||||
let bucket = limiter.get_or_create_bucket(auth.token());
|
||||
if bucket.try_acquire() {
|
||||
Ok(next.run(req).await)
|
||||
} else {
|
||||
Err(StatusCode::TOO_MANY_REQUESTS)
|
||||
}
|
||||
}
|
||||
|
|
@ -4,18 +4,16 @@ use reconcile_text::NumberOrText;
|
|||
use serde::{self, Deserialize};
|
||||
use ts_rs::TS;
|
||||
|
||||
use crate::app_state::database::models::{DocumentId, VaultUpdateId};
|
||||
use crate::app_state::database::models::VaultUpdateId;
|
||||
|
||||
#[derive(TS, Debug, TryFromMultipart)]
|
||||
#[ts(export)]
|
||||
pub struct CreateDocumentVersion {
|
||||
/// The client can decide the document id (if it wishes to) in order
|
||||
/// to help with syncing. If the client does not provide a document id,
|
||||
/// the server will generate one. If the client provides a document id
|
||||
/// it must not already exist in the database.
|
||||
pub document_id: Option<DocumentId>,
|
||||
pub relative_path: String,
|
||||
|
||||
#[ts(type = "number")]
|
||||
pub last_seen_vault_update_id: VaultUpdateId,
|
||||
|
||||
#[ts(as = "Vec<u8>")]
|
||||
#[form_data(limit = "unlimited")]
|
||||
pub content: FieldData<Bytes>,
|
||||
|
|
@ -24,7 +22,9 @@ pub struct CreateDocumentVersion {
|
|||
#[derive(Debug, TryFromMultipart)]
|
||||
pub struct UpdateBinaryDocumentVersion {
|
||||
pub parent_version_id: VaultUpdateId,
|
||||
pub relative_path: String,
|
||||
// None on a content-only edit; Some on a user rename. When None,
|
||||
// the server keeps the document at its current path.
|
||||
pub relative_path: Option<String>,
|
||||
|
||||
#[form_data(limit = "unlimited")]
|
||||
pub content: FieldData<Bytes>,
|
||||
|
|
@ -34,18 +34,13 @@ pub struct UpdateBinaryDocumentVersion {
|
|||
#[serde(rename_all = "camelCase")]
|
||||
#[ts(export)]
|
||||
pub struct UpdateTextDocumentVersion {
|
||||
#[ts(as = "i32")]
|
||||
#[ts(type = "number")]
|
||||
pub parent_version_id: VaultUpdateId,
|
||||
|
||||
pub relative_path: String,
|
||||
// None on a content-only edit; Some on a user rename. When None,
|
||||
// the server keeps the document at its current path.
|
||||
pub relative_path: Option<String>,
|
||||
|
||||
#[ts(type = "Array<number | string>")]
|
||||
pub content: Vec<NumberOrText>,
|
||||
}
|
||||
|
||||
#[derive(TS, Debug, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
#[ts(export)]
|
||||
pub struct DeleteDocumentVersion {
|
||||
pub relative_path: String,
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
use chrono::{DateTime, Utc};
|
||||
use serde::{self, Serialize};
|
||||
use ts_rs::TS;
|
||||
|
||||
|
|
@ -36,7 +37,36 @@ pub struct FetchLatestDocumentsResponse {
|
|||
pub last_update_id: VaultUpdateId,
|
||||
}
|
||||
|
||||
/// Response to an update document request.
|
||||
/// Response to a vault history request (paginated).
|
||||
#[derive(TS, Debug, Clone, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
#[ts(export)]
|
||||
pub struct VaultHistoryResponse {
|
||||
pub versions: Vec<DocumentVersionWithoutContent>,
|
||||
pub has_more: bool,
|
||||
}
|
||||
|
||||
/// Summary of a single vault returned by the list-vaults endpoint.
|
||||
#[derive(TS, Debug, Clone, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
#[ts(export)]
|
||||
pub struct VaultInfo {
|
||||
pub name: String,
|
||||
pub document_count: u32,
|
||||
pub created_at: Option<DateTime<Utc>>,
|
||||
}
|
||||
|
||||
/// Response to listing vaults accessible to the authenticated user.
|
||||
#[derive(TS, Debug, Clone, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
#[ts(export)]
|
||||
pub struct ListVaultsResponse {
|
||||
pub vaults: Vec<VaultInfo>,
|
||||
pub has_more: bool,
|
||||
pub user_name: String,
|
||||
}
|
||||
|
||||
/// Response to a create/update document request.
|
||||
#[derive(TS, Debug, Clone, Serialize)]
|
||||
#[serde(tag = "type")]
|
||||
#[ts(export)]
|
||||
|
|
|
|||
|
|
@ -16,10 +16,15 @@ use super::{
|
|||
use crate::{
|
||||
app_state::{
|
||||
AppState,
|
||||
database::models::{DocumentId, StoredDocumentVersion, VaultId, VaultUpdateId},
|
||||
database::{
|
||||
WriteTransaction,
|
||||
models::{DocumentId, StoredDocumentVersion, VaultId, VaultUpdateId},
|
||||
},
|
||||
},
|
||||
config::user_config::User,
|
||||
errors::{SyncServerError, client_error, not_found_error, server_error},
|
||||
errors::{
|
||||
SyncServerError, client_error, not_found_error, server_error, write_transaction_error,
|
||||
},
|
||||
server::requests::UpdateBinaryDocumentVersion,
|
||||
utils::{
|
||||
find_first_available_path::find_first_available_path, is_binary::is_binary,
|
||||
|
|
@ -46,18 +51,27 @@ pub async fn update_binary(
|
|||
State(state): State<AppState>,
|
||||
TypedMultipart(request): TypedMultipart<UpdateBinaryDocumentVersion>,
|
||||
) -> Result<Json<DocumentUpdateResponse>, SyncServerError> {
|
||||
let parent_document = get_parent_document(&state, &vault_id, request.parent_version_id).await?;
|
||||
let parent_document =
|
||||
get_parent_document(&state, &vault_id, &document_id, request.parent_version_id).await?;
|
||||
let content = request.content.contents.to_vec();
|
||||
|
||||
let transaction = state
|
||||
.database
|
||||
.create_write_transaction(&vault_id)
|
||||
.await
|
||||
.map_err(write_transaction_error)?;
|
||||
|
||||
update_document(
|
||||
parent_document,
|
||||
&parent_document.relative_path,
|
||||
parent_document.content,
|
||||
vault_id,
|
||||
document_id,
|
||||
request.relative_path.as_deref(),
|
||||
content,
|
||||
user,
|
||||
device_id,
|
||||
state,
|
||||
&request.relative_path,
|
||||
content,
|
||||
transaction,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
|
@ -74,28 +88,36 @@ pub async fn update_text(
|
|||
State(state): State<AppState>,
|
||||
Json(request): Json<UpdateTextDocumentVersion>,
|
||||
) -> Result<Json<DocumentUpdateResponse>, SyncServerError> {
|
||||
let parent_document = get_parent_document(&state, &vault_id, request.parent_version_id).await?;
|
||||
let parent_document =
|
||||
get_parent_document(&state, &vault_id, &document_id, request.parent_version_id).await?;
|
||||
|
||||
let edited_text = EditedText::from_diff(
|
||||
str::from_utf8(&parent_document.content)
|
||||
.expect("parent must be valid UTF-8 because it's a text document"),
|
||||
request.content,
|
||||
&*BuiltinTokenizer::Word,
|
||||
)
|
||||
let parent_text = str::from_utf8(&parent_document.content)
|
||||
.context("Parent version contains binary content; use putBinary instead of putText")
|
||||
.map_err(client_error)?;
|
||||
|
||||
let edited_text = EditedText::from_diff(parent_text, request.content, &*BuiltinTokenizer::Word)
|
||||
.context("Failed to apply given diff to parent document")
|
||||
.map_err(client_error)?;
|
||||
|
||||
let content = edited_text.apply().text().into_bytes();
|
||||
|
||||
let transaction = state
|
||||
.database
|
||||
.create_write_transaction(&vault_id)
|
||||
.await
|
||||
.map_err(write_transaction_error)?;
|
||||
|
||||
update_document(
|
||||
parent_document,
|
||||
&parent_document.relative_path,
|
||||
parent_document.content,
|
||||
vault_id,
|
||||
document_id,
|
||||
request.relative_path.as_deref(),
|
||||
content,
|
||||
user,
|
||||
device_id,
|
||||
state,
|
||||
&request.relative_path,
|
||||
content,
|
||||
transaction,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
|
@ -103,9 +125,10 @@ pub async fn update_text(
|
|||
async fn get_parent_document(
|
||||
state: &AppState,
|
||||
vault_id: &VaultId,
|
||||
document_id: &DocumentId,
|
||||
parent_version_id: VaultUpdateId,
|
||||
) -> Result<StoredDocumentVersion, SyncServerError> {
|
||||
state
|
||||
let parent = state
|
||||
.database
|
||||
.get_document_version(vault_id, parent_version_id, None)
|
||||
.await
|
||||
|
|
@ -117,29 +140,36 @@ async fn get_parent_document(
|
|||
)))
|
||||
},
|
||||
Ok,
|
||||
)
|
||||
)?;
|
||||
|
||||
if &parent.document_id != document_id {
|
||||
return Err(client_error(anyhow!(
|
||||
"Parent version `{parent_version_id}` does not belong to document `{document_id}`"
|
||||
)));
|
||||
}
|
||||
|
||||
Ok(parent)
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_lines, clippy::too_many_arguments)]
|
||||
async fn update_document(
|
||||
parent_document: StoredDocumentVersion,
|
||||
pub async fn update_document(
|
||||
parent_relative_path: &str,
|
||||
parent_content: Vec<u8>,
|
||||
vault_id: VaultId,
|
||||
document_id: DocumentId,
|
||||
relative_path: Option<&str>,
|
||||
content: Vec<u8>,
|
||||
user: User,
|
||||
device_id: DeviceIdHeader,
|
||||
state: AppState,
|
||||
relative_path: &str,
|
||||
content: Vec<u8>,
|
||||
mut transaction: WriteTransaction,
|
||||
) -> Result<Json<DocumentUpdateResponse>, SyncServerError> {
|
||||
debug!("Updating document `{document_id}` in vault `{vault_id}`");
|
||||
|
||||
let sanitized_relative_path = sanitize_path(relative_path);
|
||||
|
||||
let mut transaction = state
|
||||
.database
|
||||
.create_write_transaction(&vault_id)
|
||||
.await
|
||||
.map_err(server_error)?;
|
||||
let sanitized_relative_path = relative_path
|
||||
.map(sanitize_path)
|
||||
.transpose()
|
||||
.map_err(client_error)?;
|
||||
|
||||
let last_update_id = state
|
||||
.database
|
||||
|
|
@ -175,9 +205,12 @@ async fn update_document(
|
|||
}
|
||||
|
||||
// Return the latest version if the content and path are the same as the latest
|
||||
// version
|
||||
if content == latest_version.content && sanitized_relative_path == latest_version.relative_path
|
||||
{
|
||||
// version. A missing relative_path means "keep current path", so the path
|
||||
// is implicitly unchanged.
|
||||
let path_unchanged = sanitized_relative_path
|
||||
.as_deref()
|
||||
.is_none_or(|p| p == latest_version.relative_path);
|
||||
if content == latest_version.content && path_unchanged {
|
||||
info!(
|
||||
"Document content is the same as the latest version for `{document_id}`, skipping update"
|
||||
);
|
||||
|
|
@ -192,62 +225,89 @@ async fn update_document(
|
|||
)));
|
||||
}
|
||||
|
||||
// For mergability, use whichever path the new version will live at — the
|
||||
// requested rename target if the client sent one, otherwise the existing
|
||||
// server-side path.
|
||||
let mergable_check_path = sanitized_relative_path
|
||||
.as_deref()
|
||||
.unwrap_or(&latest_version.relative_path);
|
||||
let are_all_participants_mergable = is_file_type_mergable(
|
||||
&sanitized_relative_path,
|
||||
mergable_check_path,
|
||||
&state.config.server.mergeable_file_extensions,
|
||||
) && !is_binary(&parent_document.content)
|
||||
) && !is_binary(&parent_content)
|
||||
&& !is_binary(&latest_version.content)
|
||||
&& !is_binary(&content);
|
||||
|
||||
let merged_content = if are_all_participants_mergable {
|
||||
let (merged_content, is_different_from_request_content) = if are_all_participants_mergable {
|
||||
info!("Merging changes for document `{document_id}` in vault `{vault_id}`");
|
||||
reconcile(
|
||||
str::from_utf8(&parent_document.content)
|
||||
.expect("parent must be valid UTF-8 because it's not binary"),
|
||||
&str::from_utf8(&latest_version.content)
|
||||
.expect("latest_version must be valid UTF-8 because it's not binary")
|
||||
.into(),
|
||||
&str::from_utf8(&content)
|
||||
.expect("content must be valid UTF-8 because it's not binary")
|
||||
.into(),
|
||||
let parent_text = str::from_utf8(&parent_content)
|
||||
.context("Parent document content is not valid UTF-8")
|
||||
.map_err(client_error)?;
|
||||
let latest_text = str::from_utf8(&latest_version.content)
|
||||
.context("Latest version content is not valid UTF-8")
|
||||
.map_err(client_error)?;
|
||||
let new_text = str::from_utf8(&content)
|
||||
.context("New content is not valid UTF-8")
|
||||
.map_err(client_error)?;
|
||||
let parent_owned = parent_text.to_owned();
|
||||
let latest_owned = latest_text.to_owned();
|
||||
let new_owned = new_text.to_owned();
|
||||
let content_clone = content.clone();
|
||||
|
||||
let (merged, is_different) = tokio::task::spawn_blocking(move || {
|
||||
let merged = reconcile(
|
||||
&parent_owned,
|
||||
&latest_owned.into(),
|
||||
&new_owned.into(),
|
||||
&*BuiltinTokenizer::Word,
|
||||
)
|
||||
.apply()
|
||||
.text()
|
||||
.into_bytes()
|
||||
.into_bytes();
|
||||
let is_different = merged != content_clone;
|
||||
(merged, is_different)
|
||||
})
|
||||
.await
|
||||
.map_err(|e| server_error(anyhow::anyhow!("Reconcile task failed: {e}")))?;
|
||||
|
||||
(merged, is_different)
|
||||
} else {
|
||||
content.clone()
|
||||
(content, false) // false means that the client doesn't need to refetch the file as we can ensure the remote and local versions are the same as LWW is the merging method for binary files
|
||||
};
|
||||
|
||||
let is_different_from_request_content = merged_content != content;
|
||||
|
||||
// We can only update the relative path if we're the first one to do so
|
||||
let new_relative_path = if parent_document.relative_path == latest_version.relative_path
|
||||
&& latest_version.relative_path != sanitized_relative_path
|
||||
// Rename resolution: only apply the client's rename if (a) the client
|
||||
// requested one (`sanitized_relative_path` is `Some`) and (b) the
|
||||
// document's path hasn't changed since this client's parent version.
|
||||
// If the parent and latest paths differ, another client already renamed
|
||||
// the document — keep the latest path (first rename wins). Content
|
||||
// changes from both clients are still merged correctly via the 3-way
|
||||
// reconcile above, independent of which rename wins. A missing
|
||||
// relative_path means "keep current path" (content-only edit).
|
||||
let new_relative_path = match sanitized_relative_path.as_deref() {
|
||||
Some(requested)
|
||||
if parent_relative_path == latest_version.relative_path
|
||||
&& requested != latest_version.relative_path =>
|
||||
{
|
||||
let new_path = find_first_available_path(
|
||||
&vault_id,
|
||||
&sanitized_relative_path,
|
||||
&state.database,
|
||||
&mut transaction,
|
||||
)
|
||||
let new_path =
|
||||
find_first_available_path(&vault_id, requested, &state.database, &mut transaction)
|
||||
.await
|
||||
.map_err(server_error)?;
|
||||
|
||||
if new_path != sanitized_relative_path {
|
||||
if new_path != requested {
|
||||
info!(
|
||||
"Document already exists at new location: `{sanitized_relative_path}` when trying to update it in vault `{vault_id}`, deconflicting by creating at `{new_path}`"
|
||||
"Document already exists at new location: `{requested}` when trying to update it in vault `{vault_id}`, deconflicting by creating at `{new_path}`"
|
||||
);
|
||||
}
|
||||
|
||||
new_path
|
||||
} else {
|
||||
latest_version.relative_path.clone()
|
||||
}
|
||||
_ => latest_version.relative_path.clone(),
|
||||
};
|
||||
|
||||
let new_version = StoredDocumentVersion {
|
||||
document_id,
|
||||
vault_update_id: last_update_id + 1,
|
||||
creation_vault_update_id: latest_version.creation_vault_update_id,
|
||||
relative_path: new_relative_path,
|
||||
content: merged_content,
|
||||
updated_date: chrono::Utc::now(),
|
||||
|
|
@ -259,7 +319,7 @@ async fn update_document(
|
|||
|
||||
state
|
||||
.database
|
||||
.insert_document_version(&vault_id, &new_version, Some(transaction))
|
||||
.insert_document_version(&vault_id, &new_version, transaction)
|
||||
.await
|
||||
.map_err(server_error)?;
|
||||
|
||||
|
|
|
|||
|
|
@ -1,15 +1,3 @@
|
|||
use anyhow::Context;
|
||||
use axum::{
|
||||
extract::{
|
||||
Path, State,
|
||||
ws::{Message, WebSocket, WebSocketUpgrade},
|
||||
},
|
||||
response::Response,
|
||||
};
|
||||
use futures::stream::StreamExt;
|
||||
use log::{debug, info};
|
||||
use serde::Deserialize;
|
||||
|
||||
use crate::{
|
||||
app_state::{
|
||||
AppState,
|
||||
|
|
@ -24,9 +12,35 @@ use crate::{
|
|||
},
|
||||
},
|
||||
},
|
||||
consts::{
|
||||
HANDSHAKE_TIMEOUT, MAX_CURSOR_DOCUMENTS, MAX_CURSORS_PER_DOCUMENT, MAX_RELATIVE_PATH_LEN,
|
||||
},
|
||||
errors::{SyncServerError, client_error, server_error},
|
||||
utils::normalize::normalize,
|
||||
};
|
||||
use anyhow::Context;
|
||||
use axum::{
|
||||
extract::{
|
||||
Path, State,
|
||||
ws::{Message, WebSocket, WebSocketUpgrade},
|
||||
},
|
||||
response::Response,
|
||||
};
|
||||
use futures::sink::SinkExt;
|
||||
use futures::stream::StreamExt;
|
||||
use log::{debug, info, warn};
|
||||
use serde::Deserialize;
|
||||
|
||||
/// Tracks a pending (not yet authenticated) WebSocket connection.
|
||||
/// Decrements the counter when dropped, ensuring cleanup even if
|
||||
/// the upgrade never completes or auth fails.
|
||||
struct PendingWsGuard(std::sync::Arc<std::sync::atomic::AtomicUsize>);
|
||||
|
||||
impl Drop for PendingWsGuard {
|
||||
fn drop(&mut self) {
|
||||
self.0.fetch_sub(1, std::sync::atomic::Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub struct WebSocketPathParams {
|
||||
|
|
@ -39,13 +53,31 @@ pub async fn websocket_handler(
|
|||
Path(WebSocketPathParams { vault_id }): Path<WebSocketPathParams>,
|
||||
State(state): State<AppState>,
|
||||
) -> Result<Response, SyncServerError> {
|
||||
Ok(ws.on_upgrade(move |socket| websocket_wrapped(state, socket, vault_id)))
|
||||
let current = state
|
||||
.pending_ws_connections
|
||||
.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
|
||||
if current >= state.config.server.max_pending_websocket_connections {
|
||||
state
|
||||
.pending_ws_connections
|
||||
.fetch_sub(1, std::sync::atomic::Ordering::Relaxed);
|
||||
return Err(client_error(anyhow::anyhow!(
|
||||
"Too many pending WebSocket connections"
|
||||
)));
|
||||
}
|
||||
|
||||
async fn websocket_wrapped(state: AppState, stream: WebSocket, vault_id: VaultId) {
|
||||
let guard = PendingWsGuard(state.pending_ws_connections.clone());
|
||||
Ok(ws.on_upgrade(move |socket| websocket_wrapped(state, socket, vault_id, guard)))
|
||||
}
|
||||
|
||||
async fn websocket_wrapped(
|
||||
state: AppState,
|
||||
stream: WebSocket,
|
||||
vault_id: VaultId,
|
||||
pending_guard: PendingWsGuard,
|
||||
) {
|
||||
info!("WebSocket connection opened on vault `{vault_id}`");
|
||||
|
||||
let result = websocket(state, stream, vault_id.clone()).await;
|
||||
let result = websocket(state, stream, vault_id.clone(), pending_guard).await;
|
||||
|
||||
if let Err(err) = result {
|
||||
debug!("WebSocket connection error on vault `{vault_id}`: {err}");
|
||||
|
|
@ -57,39 +89,112 @@ async fn websocket(
|
|||
state: AppState,
|
||||
stream: WebSocket,
|
||||
vault_id: VaultId,
|
||||
pending_guard: PendingWsGuard,
|
||||
) -> Result<(), SyncServerError> {
|
||||
let (mut sender, mut websocket_receiver) = stream.split();
|
||||
|
||||
let authed_handshake = get_authenticated_handshake(
|
||||
&state,
|
||||
&vault_id,
|
||||
websocket_receiver
|
||||
.next()
|
||||
let handshake_msg = tokio::time::timeout(HANDSHAKE_TIMEOUT, websocket_receiver.next())
|
||||
.await
|
||||
.map_err(|_| client_error(anyhow::anyhow!("WebSocket handshake timed out")))?
|
||||
.transpose()
|
||||
.unwrap_or_default(),
|
||||
)?;
|
||||
.map_err(|e| client_error(anyhow::anyhow!("WebSocket error during handshake: {e}")))?;
|
||||
|
||||
let authed_handshake = get_authenticated_handshake(&state, &vault_id, handshake_msg)?;
|
||||
|
||||
info!(
|
||||
"WebSocket handshake successful for vault `{vault_id}` for `{}`",
|
||||
authed_handshake.handshake.device_id
|
||||
);
|
||||
|
||||
let mut broadcast_receiver = state.broadcasts.get_receiver(vault_id.clone()).await;
|
||||
// Auth complete — no longer a pending connection.
|
||||
drop(pending_guard);
|
||||
|
||||
send_update_over_websocket(
|
||||
&WebSocketServerMessage::VaultUpdate(WebSocketVaultUpdate {
|
||||
documents: get_unseen_documents(
|
||||
let max_clients = state.config.server.max_clients_per_vault;
|
||||
|
||||
// Atomic subscribe + cursor snapshot, serialized against in-flight
|
||||
// broadcasts:
|
||||
//
|
||||
// 1. Acquire the per-vault broadcast send lock. While we hold it,
|
||||
// no `send_document_update` can run, so no broadcast can fire
|
||||
// between our subscribe and our cursor snapshot.
|
||||
// 2. Subscribe to the broadcast channel (now we'll see every
|
||||
// broadcast that fires after we drop the send guard).
|
||||
// 3. Snapshot `cursor = max committed vault_update_id`. Because
|
||||
// `insert_document_version` holds the same send lock from
|
||||
// *before* the commit through *after* the broadcast, every doc
|
||||
// visible at this cursor has either (a) already had its
|
||||
// broadcast delivered to all then-existing subscribers — and we
|
||||
// weren't one of them, so we'll catch it via the snapshot — or
|
||||
// (b) had its broadcast contend on the lock we're holding, and
|
||||
// will be delivered to us as soon as we drop the guard, with
|
||||
// `vault_update_id > cursor`.
|
||||
// 4. Drop the send guard so writers can resume broadcasting.
|
||||
// 5. Stream the catch-up bounded by the cursor — i.e. only docs
|
||||
// with `vault_update_id <= cursor` — exactly once.
|
||||
// 6. The send task forwards broadcasts but filters to
|
||||
// `vault_update_id > cursor`, so a doc that's both in the
|
||||
// catch-up and in a contended-then-released broadcast is
|
||||
// delivered exactly once (via the catch-up).
|
||||
let send_guard = state.broadcasts.acquire_send_lock(&vault_id).await;
|
||||
let mut broadcast_receiver = match state.broadcasts.get_receiver(vault_id.clone(), max_clients)
|
||||
{
|
||||
Ok(receiver) => receiver,
|
||||
Err(err) => {
|
||||
drop(send_guard);
|
||||
warn!(
|
||||
"Vault `{vault_id}` has reached the maximum number of clients ({max_clients}), rejecting connection from `{}`",
|
||||
authed_handshake.handshake.device_id
|
||||
);
|
||||
if let Err(e) = sender
|
||||
.send(Message::Close(Some(axum::extract::ws::CloseFrame {
|
||||
code: 4000,
|
||||
reason: format!(
|
||||
"Vault has reached the maximum number of clients ({max_clients})"
|
||||
)
|
||||
.into(),
|
||||
})))
|
||||
.await
|
||||
{
|
||||
warn!("Failed to send WebSocket close frame: {e}");
|
||||
}
|
||||
return Err(err);
|
||||
}
|
||||
};
|
||||
let cursor = state
|
||||
.database
|
||||
.get_max_update_id_in_vault(&vault_id, None)
|
||||
.await
|
||||
.map_err(server_error)?;
|
||||
drop(send_guard);
|
||||
|
||||
// Catch-up on versions committed while this client was offline,
|
||||
// streamed one-at-a-time in ascending `vault_update_id` order, up
|
||||
// to the snapshot cursor.
|
||||
let unseen_documents = get_unseen_documents(
|
||||
&state,
|
||||
&vault_id,
|
||||
authed_handshake.handshake.last_seen_vault_update_id,
|
||||
cursor,
|
||||
)
|
||||
.await?,
|
||||
is_initial_sync: true,
|
||||
}),
|
||||
.await?;
|
||||
let unseen_summary: Vec<(i64, bool, String)> = unseen_documents
|
||||
.iter()
|
||||
.map(|d| (d.vault_update_id, d.is_deleted, d.relative_path.clone()))
|
||||
.collect();
|
||||
info!(
|
||||
"[CATCHUP] vault={vault_id} device={} last_seen={:?} cursor={cursor} unseen_count={} unseen={:?}",
|
||||
authed_handshake.handshake.device_id,
|
||||
authed_handshake.handshake.last_seen_vault_update_id,
|
||||
unseen_summary.len(),
|
||||
unseen_summary
|
||||
);
|
||||
for document in unseen_documents {
|
||||
send_update_over_websocket(
|
||||
&WebSocketServerMessage::VaultUpdate(WebSocketVaultUpdate { document }),
|
||||
&mut sender,
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
|
||||
send_update_over_websocket(
|
||||
&WebSocketServerMessage::CursorPositions(CursorPositionFromServer {
|
||||
|
|
@ -101,25 +206,58 @@ async fn websocket(
|
|||
|
||||
let device_id = authed_handshake.handshake.device_id.clone();
|
||||
let mut send_task = tokio::spawn(async move {
|
||||
while let Ok(update) = broadcast_receiver.recv().await {
|
||||
loop {
|
||||
match broadcast_receiver.recv().await {
|
||||
Ok(update) => {
|
||||
// Drop messages this device authored because the HTTP
|
||||
// response already carried authoritative state back.
|
||||
// Delete broadcasts are sent without an origin so the
|
||||
// author also receives them — that's the receipt the
|
||||
// client needs to drop the doc from its sync queue.
|
||||
if Some(&device_id) == update.origin_device_id.as_ref() {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Filter out vault updates already covered by the
|
||||
// catch-up snapshot. The handshake atomically
|
||||
// subscribed and snapshotted `cursor` under the
|
||||
// broadcast send lock, so any broadcast with
|
||||
// `vault_update_id <= cursor` is one that contended
|
||||
// on the lock during our subscribe — its row is
|
||||
// already in the catch-up stream and re-delivering
|
||||
// it via this channel would duplicate the message.
|
||||
// Cursor messages aren't versioned and are always
|
||||
// forwarded.
|
||||
if let WebSocketServerMessage::VaultUpdate(WebSocketVaultUpdate { document }) =
|
||||
&update.message
|
||||
&& document.vault_update_id <= cursor
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
let message = match update.message {
|
||||
WebSocketServerMessage::CursorPositions(CursorPositionFromServer { clients }) => {
|
||||
WebSocketServerMessage::CursorPositions(CursorPositionFromServer {
|
||||
clients,
|
||||
}) => WebSocketServerMessage::CursorPositions(CursorPositionFromServer {
|
||||
clients: clients
|
||||
.into_iter()
|
||||
.filter(|client| client.device_id != device_id)
|
||||
.collect(),
|
||||
})
|
||||
}
|
||||
}),
|
||||
WebSocketServerMessage::VaultUpdate(_) => update.message,
|
||||
};
|
||||
|
||||
send_update_over_websocket(&message, &mut sender).await?;
|
||||
}
|
||||
Err(tokio::sync::broadcast::error::RecvError::Lagged(n)) => {
|
||||
warn!(
|
||||
"WebSocket receiver lagged, dropped {n} messages — disconnecting client to force full resync"
|
||||
);
|
||||
break;
|
||||
}
|
||||
Err(tokio::sync::broadcast::error::RecvError::Closed) => break,
|
||||
}
|
||||
}
|
||||
|
||||
Ok::<(), SyncServerError>(())
|
||||
});
|
||||
|
|
@ -128,10 +266,12 @@ async fn websocket(
|
|||
let vault_id_clone = vault_id.clone();
|
||||
let cursor_manager = state.cursors.clone();
|
||||
let mut receive_task = tokio::spawn(async move {
|
||||
while let Some(Ok(Message::Text(message))) = websocket_receiver.next().await {
|
||||
while let Some(msg) = websocket_receiver.next().await {
|
||||
match msg {
|
||||
Ok(Message::Text(message)) => {
|
||||
let message: WebSocketClientMessage = serde_json::from_str(&message)
|
||||
.context("Failed to parse WebSocket message from client")
|
||||
.map_err(server_error)?;
|
||||
.map_err(client_error)?;
|
||||
|
||||
match message {
|
||||
WebSocketClientMessage::Handshake(_) => {
|
||||
|
|
@ -140,54 +280,94 @@ async fn websocket(
|
|||
)));
|
||||
}
|
||||
WebSocketClientMessage::CursorPositions(cursors) => {
|
||||
let docs = cursors.documents_with_cursors;
|
||||
if docs.len() > MAX_CURSOR_DOCUMENTS {
|
||||
warn!(
|
||||
"Cursor update rejected: {} documents exceeds limit of {MAX_CURSOR_DOCUMENTS}",
|
||||
docs.len()
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
let valid = docs.iter().all(|doc| {
|
||||
doc.cursors.len() <= MAX_CURSORS_PER_DOCUMENT
|
||||
&& doc.relative_path.len() <= MAX_RELATIVE_PATH_LEN
|
||||
});
|
||||
if !valid {
|
||||
warn!(
|
||||
"Cursor update rejected: a document exceeds cursor or path length limits"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
cursor_manager
|
||||
.update_cursors(
|
||||
vault_id_clone.clone(),
|
||||
authed_handshake.user.name.clone(),
|
||||
&device_id,
|
||||
cursors.documents_with_cursors,
|
||||
docs,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(Message::Close(_)) => break,
|
||||
Ok(Message::Binary(_)) => {
|
||||
warn!("Received unexpected binary WebSocket message, ignoring");
|
||||
}
|
||||
Ok(_) => {} // Ping/Pong frames handled by axum
|
||||
Err(e) => {
|
||||
debug!("WebSocket receive error: {e}");
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok::<(), SyncServerError>(())
|
||||
});
|
||||
|
||||
tokio::select! {
|
||||
_ = &mut send_task => receive_task.abort(),
|
||||
_ = &mut receive_task => send_task.abort(),
|
||||
let result: Result<(), SyncServerError> = tokio::select! {
|
||||
send_result = &mut send_task => {
|
||||
receive_task.abort();
|
||||
let _ = receive_task.await;
|
||||
match send_result {
|
||||
Err(e) => Err(server_error(
|
||||
anyhow::Error::from(e).context("WebSocket send task failed"),
|
||||
)),
|
||||
Ok(inner) => inner,
|
||||
}
|
||||
},
|
||||
receive_result = &mut receive_task => {
|
||||
send_task.abort();
|
||||
let _ = send_task.await;
|
||||
match receive_result {
|
||||
Err(e) => Err(server_error(
|
||||
anyhow::Error::from(e).context("WebSocket receive task failed"),
|
||||
)),
|
||||
Ok(inner) => inner,
|
||||
}
|
||||
},
|
||||
};
|
||||
|
||||
let result: Result<(), SyncServerError> = (async {
|
||||
send_task
|
||||
.await
|
||||
.context("WebSocket send task failed")
|
||||
.map_err(client_error)
|
||||
.and_then(|err| err)?;
|
||||
|
||||
receive_task
|
||||
.await
|
||||
.context("WebSocket receive task failed")
|
||||
.map_err(client_error)
|
||||
.and_then(|err| err)?;
|
||||
|
||||
Ok(())
|
||||
})
|
||||
.await;
|
||||
|
||||
state
|
||||
.cursors
|
||||
.remove_cursors_of_device(&vault_id, &authed_handshake.handshake.device_id)
|
||||
.await;
|
||||
|
||||
if result.is_err() {
|
||||
match &result {
|
||||
Ok(()) => {
|
||||
info!(
|
||||
"WebSocket disconnected on vault `{vault_id}` for `{}`",
|
||||
authed_handshake.handshake.device_id
|
||||
);
|
||||
}
|
||||
Err(err) => {
|
||||
warn!(
|
||||
"WebSocket error on vault `{vault_id}` for `{}`: {err}",
|
||||
authed_handshake.handshake.device_id
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
result
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,8 +1,17 @@
|
|||
use std::sync::LazyLock;
|
||||
|
||||
use regex::Regex;
|
||||
|
||||
static DEDUP_SUFFIX_REGEX: LazyLock<Regex> =
|
||||
LazyLock::new(|| Regex::new(r" \((\d+)\)$").expect("invalid regex"));
|
||||
|
||||
pub fn dedup_paths(path: &str) -> impl Iterator<Item = String> {
|
||||
let mut path_parts = path.split('/').collect::<Vec<_>>();
|
||||
let file_name = path_parts.pop().unwrap().to_owned();
|
||||
let file_name = path_parts
|
||||
.pop()
|
||||
.filter(|s| !s.is_empty())
|
||||
.unwrap_or(path)
|
||||
.to_owned();
|
||||
|
||||
let mut directory = path_parts.join("/");
|
||||
if !directory.is_empty() {
|
||||
|
|
@ -29,14 +38,13 @@ pub fn dedup_paths(path: &str) -> impl Iterator<Item = String> {
|
|||
}
|
||||
};
|
||||
|
||||
let regex = Regex::new(r" \((\d+)\)$").unwrap();
|
||||
let start_number = regex
|
||||
let start_number = DEDUP_SUFFIX_REGEX
|
||||
.captures(&stem)
|
||||
.and_then(|caps| caps.get(1))
|
||||
.and_then(|m| m.as_str().parse::<u32>().ok())
|
||||
.unwrap_or(0);
|
||||
|
||||
let clean_stem = regex.replace(&stem, "").to_string();
|
||||
let clean_stem = DEDUP_SUFFIX_REGEX.replace(&stem, "").to_string();
|
||||
|
||||
(start_number..).map(move |dedup_number| {
|
||||
if dedup_number == 0 {
|
||||
|
|
|
|||
|
|
@ -1,25 +1,30 @@
|
|||
use crate::app_state::database::models::VaultId;
|
||||
use crate::{app_state::database::Transaction, utils::dedup_paths::dedup_paths};
|
||||
use crate::utils::dedup_paths::dedup_paths;
|
||||
use anyhow::Result;
|
||||
use log::{debug, info};
|
||||
use sqlx::sqlite::SqliteConnection;
|
||||
|
||||
pub async fn find_first_available_path(
|
||||
vault_id: &VaultId,
|
||||
sanitized_relative_path: &str,
|
||||
database: &crate::app_state::database::Database,
|
||||
transaction: &mut Transaction<'_>,
|
||||
connection: &mut SqliteConnection,
|
||||
) -> Result<String> {
|
||||
info!("Finding first available path for `{sanitized_relative_path}` in vault `{vault_id}`");
|
||||
for candidate in dedup_paths(sanitized_relative_path) {
|
||||
debug!("Checking candidate path for deconflicting names: `{candidate}`");
|
||||
if database
|
||||
.get_latest_document_by_path(vault_id, &candidate, Some(transaction))
|
||||
.get_latest_non_deleted_document_by_path(vault_id, &candidate, Some(connection))
|
||||
.await?
|
||||
.is_none()
|
||||
{
|
||||
info!("Selected available path: `{candidate}`");
|
||||
return Ok(candidate);
|
||||
}
|
||||
|
||||
info!(
|
||||
"Finding first available path for `{sanitized_relative_path}` in vault `{vault_id}` as `{candidate}` is already taken"
|
||||
);
|
||||
}
|
||||
|
||||
unreachable!("dedup_paths produces infinite paths");
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ use std::{
|
|||
time::{Duration, SystemTime, UNIX_EPOCH},
|
||||
};
|
||||
|
||||
use chrono::{Local, NaiveDateTime};
|
||||
use chrono::NaiveDateTime;
|
||||
use tracing_subscriber::fmt::MakeWriter;
|
||||
|
||||
#[derive(Clone)]
|
||||
|
|
@ -55,7 +55,7 @@ impl RotatingFileWriter {
|
|||
let timestamp_str = filename.get(prefix_len..filename.len().checked_sub(4)?)?;
|
||||
|
||||
let dt = NaiveDateTime::parse_from_str(timestamp_str, "%Y-%m-%d_%H-%M-%S").ok()?;
|
||||
let timestamp = dt.and_local_timezone(Local).single()?;
|
||||
let timestamp = dt.and_utc();
|
||||
let secs: u64 = timestamp.timestamp().try_into().ok()?;
|
||||
|
||||
Some(UNIX_EPOCH + Duration::from_secs(secs))
|
||||
|
|
@ -114,7 +114,7 @@ impl RotatingFileWriter {
|
|||
}
|
||||
|
||||
fn rotate(inner: &mut RotatingFileWriterInner) -> io::Result<()> {
|
||||
let timestamp = Local::now().format("%Y-%m-%d_%H-%M-%S");
|
||||
let timestamp = chrono::Utc::now().format("%Y-%m-%d_%H-%M-%S");
|
||||
let filename = format!("{}.{}.log", inner.file_prefix, timestamp);
|
||||
let filepath = inner.directory.join(filename);
|
||||
|
||||
|
|
@ -132,8 +132,14 @@ impl RotatingFileWriter {
|
|||
|
||||
impl Write for RotatingFileWriter {
|
||||
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
|
||||
let mut inner = self.inner.lock().unwrap();
|
||||
let mut inner = self.inner.lock().unwrap_or_else(|poisoned| {
|
||||
eprintln!("RotatingFileWriter mutex was poisoned, recovering");
|
||||
poisoned.into_inner()
|
||||
});
|
||||
|
||||
// Reset file handle after poison recovery so the next branch
|
||||
// re-opens a valid file rather than writing to a potentially
|
||||
// half-closed handle.
|
||||
if inner.current_file.is_none() {
|
||||
Self::open_or_create_log_file(&mut inner)?;
|
||||
} else if Self::should_rotate(&inner) {
|
||||
|
|
@ -148,7 +154,10 @@ impl Write for RotatingFileWriter {
|
|||
}
|
||||
|
||||
fn flush(&mut self) -> io::Result<()> {
|
||||
let mut inner = self.inner.lock().unwrap();
|
||||
let mut inner = self.inner.lock().unwrap_or_else(|poisoned| {
|
||||
eprintln!("RotatingFileWriter mutex was poisoned, recovering");
|
||||
poisoned.into_inner()
|
||||
});
|
||||
if let Some(ref mut file) = inner.current_file {
|
||||
file.flush()
|
||||
} else {
|
||||
|
|
@ -267,7 +276,7 @@ mod tests {
|
|||
// Parse the expected time
|
||||
let expected_dt =
|
||||
NaiveDateTime::parse_from_str(timestamp_str, "%Y-%m-%d_%H-%M-%S").unwrap();
|
||||
let expected_timestamp = expected_dt.and_local_timezone(Local).single().unwrap();
|
||||
let expected_timestamp = expected_dt.and_utc();
|
||||
let expected_duration =
|
||||
Duration::from_secs(expected_timestamp.timestamp().try_into().unwrap());
|
||||
let expected_next = UNIX_EPOCH + expected_duration + rotation_duration;
|
||||
|
|
@ -306,7 +315,7 @@ mod tests {
|
|||
// Should use the latest file (2025-10-26_14-00-00)
|
||||
let expected_dt =
|
||||
NaiveDateTime::parse_from_str("2025-10-26_14-00-00", "%Y-%m-%d_%H-%M-%S").unwrap();
|
||||
let expected_timestamp = expected_dt.and_local_timezone(Local).single().unwrap();
|
||||
let expected_timestamp = expected_dt.and_utc();
|
||||
let expected_duration =
|
||||
Duration::from_secs(expected_timestamp.timestamp().try_into().unwrap());
|
||||
let expected_next = UNIX_EPOCH + expected_duration + rotation_duration;
|
||||
|
|
|
|||
|
|
@ -1,14 +1,28 @@
|
|||
use anyhow::{Result, ensure};
|
||||
|
||||
use crate::consts::MAX_RELATIVE_PATH_LEN;
|
||||
|
||||
/// Sanitize the document's path to allow all clients to create the same path in
|
||||
/// their filesystem. If we didn't do this server-side, client's would need to
|
||||
/// deal with mapping invalid names to valid ones and then back.
|
||||
pub fn sanitize_path(path: &str) -> String {
|
||||
pub fn sanitize_path(path: &str) -> Result<String> {
|
||||
// Enforce the length cap at the single chokepoint every create/update
|
||||
// handler goes through, so clients can't blow up axum's JSON/multipart
|
||||
// parser with a 1 MB `relative_path` before the handler ever runs.
|
||||
// The WebSocket cursor handler enforces this separately.
|
||||
ensure!(
|
||||
path.len() <= MAX_RELATIVE_PATH_LEN,
|
||||
"Relative path exceeds the maximum length of {MAX_RELATIVE_PATH_LEN} bytes"
|
||||
);
|
||||
|
||||
let options = sanitize_filename::Options {
|
||||
truncate: true,
|
||||
windows: true, // Windows is the lowest common denominator
|
||||
replacement: "",
|
||||
};
|
||||
|
||||
path.split('/')
|
||||
let result = path
|
||||
.split('/')
|
||||
.map(|part| {
|
||||
let proposal = sanitize_filename::sanitize_with_options(part, options.clone());
|
||||
if !part.is_empty() && proposal.is_empty() {
|
||||
|
|
@ -18,7 +32,13 @@ pub fn sanitize_path(path: &str) -> String {
|
|||
}
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
.join("/")
|
||||
.join("/");
|
||||
|
||||
ensure!(
|
||||
!result.is_empty(),
|
||||
"Relative path is empty after sanitization"
|
||||
);
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
|
@ -27,8 +47,32 @@ mod test {
|
|||
|
||||
#[test]
|
||||
fn test_sanitize_path() {
|
||||
assert_eq!(sanitize_path("/my/path/what?"), "/my/path/what");
|
||||
assert_eq!(sanitize_path("file (1).md"), "file (1).md");
|
||||
assert_eq!(sanitize_path("/my/path/\\\\:?"), "/my/path/_");
|
||||
assert_eq!(sanitize_path("/my/path/what?").unwrap(), "/my/path/what");
|
||||
assert_eq!(sanitize_path("file (1).md").unwrap(), "file (1).md");
|
||||
assert_eq!(sanitize_path("/my/path/\\\\:?").unwrap(), "/my/path/_");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_sanitize_path_empty() {
|
||||
assert!(sanitize_path("").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_sanitize_path_idempotent_simple() {
|
||||
let mut result = sanitize_path("notes/my file.md").unwrap();
|
||||
for _ in 0..5 {
|
||||
result = sanitize_path(&result).unwrap();
|
||||
}
|
||||
assert_eq!(result, "notes/my file.md");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_sanitize_path_idempotent_special_chars() {
|
||||
let first = sanitize_path("/my/path/what?/file:name<>.md").unwrap();
|
||||
let mut result = first.clone();
|
||||
for _ in 0..5 {
|
||||
result = sanitize_path(&result).unwrap();
|
||||
}
|
||||
assert_eq!(result, first);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue