Compare commits
3 commits
main
...
asch/split
| Author | SHA1 | Date | |
|---|---|---|---|
| 2d5edc6ec5 | |||
| a9ce09b59d | |||
| 70f97c4b16 |
51 changed files with 4933 additions and 3429 deletions
35
.forgejo/workflows/check.yml
Normal file
35
.forgejo/workflows/check.yml
Normal file
|
|
@ -0,0 +1,35 @@
|
||||||
|
name: Check
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
branches: ["main"]
|
||||||
|
pull_request:
|
||||||
|
branches: ["main"]
|
||||||
|
workflow_dispatch:
|
||||||
|
|
||||||
|
env:
|
||||||
|
CARGO_TERM_COLOR: always
|
||||||
|
RUSTFLAGS: "-Dwarnings"
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
build:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
with:
|
||||||
|
fetch-depth: 0
|
||||||
|
|
||||||
|
- name: Setup Node.js environment
|
||||||
|
uses: actions/setup-node@v4
|
||||||
|
with:
|
||||||
|
node-version: "25.x"
|
||||||
|
|
||||||
|
- name: Setup Rust toolchain
|
||||||
|
uses: dtolnay/rust-toolchain@stable
|
||||||
|
with:
|
||||||
|
toolchain: "1.92.0"
|
||||||
|
components: clippy, rustfmt
|
||||||
|
|
||||||
|
- name: Lint & test
|
||||||
|
run: scripts/check.sh
|
||||||
38
.forgejo/workflows/deploy-docs.yml
Normal file
38
.forgejo/workflows/deploy-docs.yml
Normal file
|
|
@ -0,0 +1,38 @@
|
||||||
|
name: Deploy Documentation
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
branches:
|
||||||
|
- main
|
||||||
|
paths:
|
||||||
|
- "docs/**"
|
||||||
|
- ".forgejo/workflows/deploy-docs.yml"
|
||||||
|
workflow_dispatch:
|
||||||
|
|
||||||
|
concurrency:
|
||||||
|
group: pages
|
||||||
|
cancel-in-progress: false
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
build:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- name: Checkout
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
with:
|
||||||
|
fetch-depth: 0
|
||||||
|
|
||||||
|
- name: Setup Node.js environment
|
||||||
|
uses: actions/setup-node@v4
|
||||||
|
with:
|
||||||
|
node-version: "25.x"
|
||||||
|
|
||||||
|
- name: Build docs
|
||||||
|
run: scripts/build-docs.sh
|
||||||
|
|
||||||
|
- name: Upload artifact
|
||||||
|
uses: actions/upload-artifact@v4
|
||||||
|
with:
|
||||||
|
name: docs
|
||||||
|
path: docs/.vitepress/dist
|
||||||
71
.forgejo/workflows/e2e.yml
Normal file
71
.forgejo/workflows/e2e.yml
Normal file
|
|
@ -0,0 +1,71 @@
|
||||||
|
name: E2E tests
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
branches: ["main"]
|
||||||
|
pull_request:
|
||||||
|
branches: ["main"]
|
||||||
|
schedule:
|
||||||
|
- cron: "0 * * * *"
|
||||||
|
workflow_dispatch:
|
||||||
|
|
||||||
|
concurrency:
|
||||||
|
group: e2e-tests
|
||||||
|
cancel-in-progress: false
|
||||||
|
|
||||||
|
env:
|
||||||
|
RUSTFLAGS: "-Dwarnings"
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
build:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
with:
|
||||||
|
fetch-depth: 0
|
||||||
|
|
||||||
|
- name: Setup Node.js environment
|
||||||
|
uses: actions/setup-node@v4
|
||||||
|
with:
|
||||||
|
node-version: "25.x"
|
||||||
|
|
||||||
|
- name: Setup Rust toolchain
|
||||||
|
uses: dtolnay/rust-toolchain@stable
|
||||||
|
with:
|
||||||
|
toolchain: "1.92.0"
|
||||||
|
components: clippy, rustfmt
|
||||||
|
|
||||||
|
- name: Setup rust
|
||||||
|
run: |
|
||||||
|
which sqlx || cargo install sqlx-cli
|
||||||
|
cd sync-server
|
||||||
|
sqlx database create --database-url sqlite://db.sqlite3
|
||||||
|
sqlx migrate run --source src/app_state/database/migrations --database-url sqlite://db.sqlite3
|
||||||
|
|
||||||
|
- name: E2E tests
|
||||||
|
run: |
|
||||||
|
cd sync-server
|
||||||
|
cargo run config-e2e.yml --color never &
|
||||||
|
SERVER_PID=$!
|
||||||
|
cd ..
|
||||||
|
|
||||||
|
scripts/e2e.sh 8
|
||||||
|
EXIT_CODE=$?
|
||||||
|
|
||||||
|
kill $SERVER_PID 2>/dev/null || true
|
||||||
|
wait $SERVER_PID 2>/dev/null || true
|
||||||
|
|
||||||
|
exit $EXIT_CODE
|
||||||
|
|
||||||
|
- name: Upload e2e logs
|
||||||
|
if: always()
|
||||||
|
uses: actions/upload-artifact@v4
|
||||||
|
with:
|
||||||
|
name: e2e-logs
|
||||||
|
path: logs/
|
||||||
|
retention-days: 30
|
||||||
|
|
||||||
|
- name: Cleanup
|
||||||
|
if: always()
|
||||||
|
run: scripts/clean-up.sh
|
||||||
51
.forgejo/workflows/publish-cli-docker.yml
Normal file
51
.forgejo/workflows/publish-cli-docker.yml
Normal file
|
|
@ -0,0 +1,51 @@
|
||||||
|
name: Publish CLI
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
branches: ["main"]
|
||||||
|
tags: ["*"]
|
||||||
|
pull_request:
|
||||||
|
branches: ["main"]
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
publish-docker:
|
||||||
|
runs-on: ubuntu-docker
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- name: Checkout repository
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
with:
|
||||||
|
fetch-depth: 0
|
||||||
|
|
||||||
|
- name: Extract registry hostname
|
||||||
|
id: registry
|
||||||
|
run: echo "host=$(echo '${{ github.server_url }}' | sed 's|https\?://||')" >> "$GITHUB_OUTPUT"
|
||||||
|
|
||||||
|
- name: Set up Docker Buildx
|
||||||
|
uses: docker/setup-buildx-action@v3
|
||||||
|
|
||||||
|
- name: Log into container registry
|
||||||
|
uses: docker/login-action@v3
|
||||||
|
with:
|
||||||
|
registry: ${{ steps.registry.outputs.host }}
|
||||||
|
username: ${{ github.repository_owner }}
|
||||||
|
password: ${{ secrets.GITHUB_TOKEN }}
|
||||||
|
|
||||||
|
- name: Extract Docker metadata
|
||||||
|
id: meta
|
||||||
|
uses: docker/metadata-action@v5
|
||||||
|
with:
|
||||||
|
images: ${{ steps.registry.outputs.host }}/${{ github.repository }}-cli
|
||||||
|
|
||||||
|
- name: Build and push Docker image
|
||||||
|
id: build-and-push
|
||||||
|
uses: docker/build-push-action@v5
|
||||||
|
with:
|
||||||
|
context: frontend
|
||||||
|
file: frontend/local-client-cli/Dockerfile
|
||||||
|
platforms: linux/amd64
|
||||||
|
push: true
|
||||||
|
tags: ${{ steps.meta.outputs.tags }}
|
||||||
|
labels: ${{ steps.meta.outputs.labels }}
|
||||||
|
cache-from: type=registry,ref=${{ steps.registry.outputs.host }}/${{ github.repository }}-cli:buildcache
|
||||||
|
cache-to: type=registry,ref=${{ steps.registry.outputs.host }}/${{ github.repository }}-cli:buildcache,mode=max
|
||||||
71
.forgejo/workflows/publish-plugin.yml
Normal file
71
.forgejo/workflows/publish-plugin.yml
Normal file
|
|
@ -0,0 +1,71 @@
|
||||||
|
name: Publish Obsidian plugin
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
tags: ["*"]
|
||||||
|
|
||||||
|
env:
|
||||||
|
CARGO_TERM_COLOR: always
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
publish-plugin:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
with:
|
||||||
|
fetch-depth: 0
|
||||||
|
|
||||||
|
- name: Setup Node.js environment
|
||||||
|
uses: actions/setup-node@v4
|
||||||
|
with:
|
||||||
|
node-version: "25.x"
|
||||||
|
|
||||||
|
- name: Build plugin
|
||||||
|
run: |
|
||||||
|
cd frontend
|
||||||
|
npm ci
|
||||||
|
npm run build
|
||||||
|
|
||||||
|
- name: Setup Rust toolchain
|
||||||
|
uses: dtolnay/rust-toolchain@stable
|
||||||
|
with:
|
||||||
|
toolchain: "1.92.0"
|
||||||
|
components: clippy, rustfmt
|
||||||
|
|
||||||
|
- name: Install cross-compilation tools
|
||||||
|
run: |
|
||||||
|
apt update
|
||||||
|
apt install -y gcc-aarch64-linux-gnu musl-tools gcc-mingw-w64-x86-64 jq
|
||||||
|
|
||||||
|
- name: Build Linux and Windows binaries
|
||||||
|
run: ./scripts/build-sync-server-binaries.sh
|
||||||
|
|
||||||
|
- name: Create release
|
||||||
|
env:
|
||||||
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||||
|
SERVER_URL: ${{ github.server_url }}
|
||||||
|
REPO: ${{ github.repository }}
|
||||||
|
run: |
|
||||||
|
tag="${GITHUB_REF#refs/tags/}"
|
||||||
|
|
||||||
|
mkdir -p release
|
||||||
|
cp frontend/obsidian-plugin/dist/* release/
|
||||||
|
cp sync-server/artifacts/sync-server-* release/
|
||||||
|
|
||||||
|
# Create draft release via Forgejo API
|
||||||
|
RELEASE_ID=$(curl -s -X POST \
|
||||||
|
"${SERVER_URL}/api/v1/repos/${REPO}/releases" \
|
||||||
|
-H "Authorization: token ${GITHUB_TOKEN}" \
|
||||||
|
-H "Content-Type: application/json" \
|
||||||
|
-d "{\"tag_name\": \"${tag}\", \"name\": \"${tag}\", \"draft\": true}" \
|
||||||
|
| jq -r '.id')
|
||||||
|
|
||||||
|
# Upload release assets
|
||||||
|
for file in release/*; do
|
||||||
|
filename=$(basename "$file")
|
||||||
|
curl -s -X POST \
|
||||||
|
"${SERVER_URL}/api/v1/repos/${REPO}/releases/${RELEASE_ID}/assets?name=${filename}" \
|
||||||
|
-H "Authorization: token ${GITHUB_TOKEN}" \
|
||||||
|
-F "attachment=@${file}"
|
||||||
|
done
|
||||||
51
.forgejo/workflows/publish-server-docker.yml
Normal file
51
.forgejo/workflows/publish-server-docker.yml
Normal file
|
|
@ -0,0 +1,51 @@
|
||||||
|
name: Publish server Docker image
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
branches: ["main"]
|
||||||
|
tags: ["*"]
|
||||||
|
pull_request:
|
||||||
|
branches: ["main"]
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
publish-docker:
|
||||||
|
runs-on: ubuntu-docker
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- name: Checkout repository
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
with:
|
||||||
|
fetch-depth: 0
|
||||||
|
|
||||||
|
- name: Extract registry hostname
|
||||||
|
id: registry
|
||||||
|
run: echo "host=$(echo '${{ github.server_url }}' | sed 's|https\?://||')" >> "$GITHUB_OUTPUT"
|
||||||
|
|
||||||
|
- name: Set up Docker Buildx
|
||||||
|
uses: docker/setup-buildx-action@v3
|
||||||
|
|
||||||
|
- name: Log into container registry
|
||||||
|
if: github.ref_type == 'tag'
|
||||||
|
uses: docker/login-action@v3
|
||||||
|
with:
|
||||||
|
registry: ${{ steps.registry.outputs.host }}
|
||||||
|
username: ${{ github.repository_owner }}
|
||||||
|
password: ${{ secrets.GITHUB_TOKEN }}
|
||||||
|
|
||||||
|
- name: Extract Docker metadata
|
||||||
|
id: meta
|
||||||
|
uses: docker/metadata-action@v5
|
||||||
|
with:
|
||||||
|
images: ${{ steps.registry.outputs.host }}/${{ github.repository }}
|
||||||
|
|
||||||
|
- name: Build and push Docker image
|
||||||
|
id: build-and-push
|
||||||
|
uses: docker/build-push-action@v5
|
||||||
|
with:
|
||||||
|
context: sync-server
|
||||||
|
platforms: linux/amd64,linux/arm64
|
||||||
|
push: ${{ github.ref_type == 'tag' }}
|
||||||
|
tags: ${{ steps.meta.outputs.tags }}
|
||||||
|
labels: ${{ steps.meta.outputs.labels }}
|
||||||
|
cache-from: type=registry,ref=${{ steps.registry.outputs.host }}/${{ github.repository }}:buildcache
|
||||||
|
cache-to: type=registry,ref=${{ steps.registry.outputs.host }}/${{ github.repository }}:buildcache,mode=max
|
||||||
4
.github/workflows/check.yml
vendored
4
.github/workflows/check.yml
vendored
|
|
@ -23,13 +23,13 @@ jobs:
|
||||||
- name: Setup Node.js environment
|
- name: Setup Node.js environment
|
||||||
uses: actions/setup-node@v4.2.0
|
uses: actions/setup-node@v4.2.0
|
||||||
with:
|
with:
|
||||||
node-version: "22.x"
|
node-version: "25.x"
|
||||||
check-latest: true
|
check-latest: true
|
||||||
|
|
||||||
- name: Setup Rust toolchain
|
- name: Setup Rust toolchain
|
||||||
uses: dtolnay/rust-toolchain@stable
|
uses: dtolnay/rust-toolchain@stable
|
||||||
with:
|
with:
|
||||||
toolchain: "1.89.0"
|
toolchain: "1.92.0"
|
||||||
components: clippy, rustfmt
|
components: clippy, rustfmt
|
||||||
|
|
||||||
- name: Lint & test
|
- name: Lint & test
|
||||||
|
|
|
||||||
13
.github/workflows/deploy-docs.yml
vendored
13
.github/workflows/deploy-docs.yml
vendored
|
|
@ -5,8 +5,8 @@ on:
|
||||||
branches:
|
branches:
|
||||||
- main
|
- main
|
||||||
paths:
|
paths:
|
||||||
- 'docs/**'
|
- "docs/**"
|
||||||
- '.github/workflows/deploy-docs.yml'
|
- ".github/workflows/deploy-docs.yml"
|
||||||
workflow_dispatch:
|
workflow_dispatch:
|
||||||
|
|
||||||
permissions:
|
permissions:
|
||||||
|
|
@ -28,12 +28,11 @@ jobs:
|
||||||
with:
|
with:
|
||||||
fetch-depth: 0
|
fetch-depth: 0
|
||||||
|
|
||||||
- name: Setup Node
|
- name: Setup Node.js environment
|
||||||
uses: actions/setup-node@v4
|
uses: actions/setup-node@v4.2.0
|
||||||
with:
|
with:
|
||||||
node-version: 22
|
node-version: "25.x"
|
||||||
cache: npm
|
check-latest: true
|
||||||
cache-dependency-path: docs/package-lock.json
|
|
||||||
|
|
||||||
- name: Setup Pages
|
- name: Setup Pages
|
||||||
uses: actions/configure-pages@v4
|
uses: actions/configure-pages@v4
|
||||||
|
|
|
||||||
6
.github/workflows/e2e.yml
vendored
6
.github/workflows/e2e.yml
vendored
|
|
@ -6,7 +6,7 @@ on:
|
||||||
pull_request:
|
pull_request:
|
||||||
branches: ["main"]
|
branches: ["main"]
|
||||||
schedule:
|
schedule:
|
||||||
- cron: '0 * * * *'
|
- cron: "0 * * * *"
|
||||||
workflow_dispatch:
|
workflow_dispatch:
|
||||||
|
|
||||||
concurrency:
|
concurrency:
|
||||||
|
|
@ -28,13 +28,13 @@ jobs:
|
||||||
- name: Setup Node.js environment
|
- name: Setup Node.js environment
|
||||||
uses: actions/setup-node@v4.2.0
|
uses: actions/setup-node@v4.2.0
|
||||||
with:
|
with:
|
||||||
node-version: "22.x"
|
node-version: "25.x"
|
||||||
check-latest: true
|
check-latest: true
|
||||||
|
|
||||||
- name: Setup Rust toolchain
|
- name: Setup Rust toolchain
|
||||||
uses: dtolnay/rust-toolchain@stable
|
uses: dtolnay/rust-toolchain@stable
|
||||||
with:
|
with:
|
||||||
toolchain: "1.89.0"
|
toolchain: "1.92.0"
|
||||||
components: clippy, rustfmt
|
components: clippy, rustfmt
|
||||||
|
|
||||||
- name: Setup rust
|
- name: Setup rust
|
||||||
|
|
|
||||||
4
.github/workflows/publish-plugin.yml
vendored
4
.github/workflows/publish-plugin.yml
vendored
|
|
@ -19,7 +19,7 @@ jobs:
|
||||||
- name: Setup Node.js environment
|
- name: Setup Node.js environment
|
||||||
uses: actions/setup-node@v4.2.0
|
uses: actions/setup-node@v4.2.0
|
||||||
with:
|
with:
|
||||||
node-version: "22.x"
|
node-version: "25.x"
|
||||||
check-latest: true
|
check-latest: true
|
||||||
|
|
||||||
- name: Build plugin
|
- name: Build plugin
|
||||||
|
|
@ -31,7 +31,7 @@ jobs:
|
||||||
- name: Setup Rust toolchain
|
- name: Setup Rust toolchain
|
||||||
uses: dtolnay/rust-toolchain@stable
|
uses: dtolnay/rust-toolchain@stable
|
||||||
with:
|
with:
|
||||||
toolchain: "1.89.0"
|
toolchain: "1.92.0"
|
||||||
components: clippy, rustfmt
|
components: clippy, rustfmt
|
||||||
|
|
||||||
- name: Install cross-compilation tools
|
- name: Install cross-compilation tools
|
||||||
|
|
|
||||||
9
.gitignore
vendored
9
.gitignore
vendored
|
|
@ -7,15 +7,18 @@ node_modules
|
||||||
# Frontend build folders
|
# Frontend build folders
|
||||||
frontend/*/dist
|
frontend/*/dist
|
||||||
|
|
||||||
sync-server/db.sqlite3*
|
|
||||||
sync-server/databases
|
|
||||||
|
|
||||||
# Rust build folders
|
# Rust build folders
|
||||||
sync-server/target
|
sync-server/target
|
||||||
sync-server/artifacts
|
sync-server/artifacts
|
||||||
sync-server/bindings/*.ts
|
sync-server/bindings/*.ts
|
||||||
|
|
||||||
|
# build folders
|
||||||
|
sync-server/db.sqlite3*
|
||||||
|
**/databases
|
||||||
|
|
||||||
*.log
|
*.log
|
||||||
*.sqlx
|
*.sqlx
|
||||||
|
|
||||||
target
|
target
|
||||||
|
|
||||||
|
.task
|
||||||
|
|
|
||||||
4
.vscode/settings.json
vendored
4
.vscode/settings.json
vendored
|
|
@ -5,6 +5,6 @@
|
||||||
"**/dist": true,
|
"**/dist": true,
|
||||||
"**/node_modules": true,
|
"**/node_modules": true,
|
||||||
"**/.sqlx": true,
|
"**/.sqlx": true,
|
||||||
"**/target": true,
|
"**/target": true
|
||||||
},
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
195
CLAUDE.md
195
CLAUDE.md
|
|
@ -2,109 +2,154 @@
|
||||||
|
|
||||||
This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
|
This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
|
||||||
|
|
||||||
## Project Overview
|
## Project shape
|
||||||
|
|
||||||
VaultLink is a self-hosted Obsidian plugin for real-time collaborative file syncing. The project consists of a Rust-based sync server and a TypeScript frontend with three main components: an Obsidian plugin, a sync client library, and a test client.
|
VaultLink is a self-hosted Obsidian file-sync system. Two halves of one repo:
|
||||||
|
|
||||||
## Architecture
|
- `sync-server/` — Rust (axum + sqlx/SQLite). Source of truth for vault state, broadcasts changes via WebSocket.
|
||||||
|
- `frontend/` — npm workspaces. The sync engine (`sync-client`) is consumed by an Obsidian plugin, a standalone CLI, a fuzz E2E harness, a scripted determinism harness, and a history UI.
|
||||||
|
|
||||||
### Core Components
|
The HTTP/WS API types are generated from Rust (`ts-rs`) and mirrored into the TS workspaces. **Never hand-edit files in `frontend/sync-client/src/services/types/` or `frontend/history-ui/src/lib/types/`** — run `scripts/update-api-types.sh` after changing anything Serde-derived in the server.
|
||||||
|
|
||||||
- **sync-server/**: Rust-based WebSocket server with SQLite database for document versioning and real-time synchronization
|
### Frontend workspaces
|
||||||
- **frontend/sync-client/**: TypeScript library providing core sync functionality, WebSocket management, and file operations
|
|
||||||
- **frontend/obsidian-plugin/**: Obsidian plugin that integrates the sync client with Obsidian's API
|
|
||||||
- **frontend/test-client/**: CLI testing tool for the sync functionality
|
|
||||||
|
|
||||||
### Key Technologies
|
- `sync-client` — the sync engine; published to consumers via `dist/`. All other TS workspaces depend on it via `file:../sync-client`.
|
||||||
|
- `obsidian-plugin` — Obsidian plugin built from `sync-client`.
|
||||||
|
- `local-client-cli` — same engine wrapped as a standalone CLI.
|
||||||
|
- `history-ui` — vault-history web UI.
|
||||||
|
- `test-client` — fuzz E2E harness (random ops across N processes).
|
||||||
|
- `deterministic-tests` — scripted multi-client tests with an in-memory FS, run against a real server.
|
||||||
|
|
||||||
- **Backend**: Rust with Axum framework, SQLite with SQLx, WebSockets for real-time sync
|
## Common commands
|
||||||
- **Frontend**: TypeScript, Webpack for bundling, Jest for testing
|
|
||||||
- **Sync Algorithm**: Uses reconcile-text library for operational transformation
|
|
||||||
|
|
||||||
## Development Commands
|
Pre-push hygiene (formats, lints, runs tests, requires clean git state):
|
||||||
|
|
||||||
### Server Development
|
```sh
|
||||||
```bash
|
scripts/check.sh --fix
|
||||||
cd sync-server
|
|
||||||
cargo run config-e2e.yml # Start development server
|
|
||||||
cargo test --verbose # Run Rust tests
|
|
||||||
cargo clippy --all-targets --all-features # Lint Rust code
|
|
||||||
cargo clippy --all-targets --all-features --fix --allow-dirty --allow-staged # Auto-fix clippy warnings
|
|
||||||
cargo fmt --all -- --check # Check Rust formatting
|
|
||||||
cargo fmt --all # Auto-format Rust code
|
|
||||||
cargo machete --with-metadata # Detect unused dependencies
|
|
||||||
```
|
```
|
||||||
|
|
||||||
### Frontend Development
|
Run the fuzz E2E (N parallel processes):
|
||||||
```bash
|
|
||||||
|
```sh
|
||||||
|
scripts/e2e.sh 12
|
||||||
|
# Logs land in logs/log_<i>.log. Clean with scripts/clean-up.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
Run deterministic tests (require a release-built server in `sync-server/target/release/sync_server` — they spawn it themselves):
|
||||||
|
|
||||||
|
```sh
|
||||||
|
cd sync-server && cargo build --release && cd ..
|
||||||
cd frontend
|
cd frontend
|
||||||
npm run dev # Start development mode (watches sync-client and obsidian-plugin)
|
npm run build -w sync-client -w deterministic-tests
|
||||||
npm run build # Build all workspaces
|
node deterministic-tests/dist/cli.js # all
|
||||||
npm run test # Run all tests
|
node deterministic-tests/dist/cli.js --filter=rename # subset
|
||||||
npm run lint # Lint and format TypeScript code
|
node deterministic-tests/dist/cli.js --filter=… -j 4 # cap parallelism
|
||||||
```
|
```
|
||||||
|
|
||||||
### Database Setup (Development)
|
Run a single sync-client unit test by file:
|
||||||
```bash
|
|
||||||
|
```sh
|
||||||
|
cd frontend/sync-client && npx tsx --test 'src/**/sync-event-queue.test.ts'
|
||||||
|
```
|
||||||
|
|
||||||
|
Server: dev runs from `sync-server/` against `config-e2e.yml`:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
cd sync-server
|
||||||
|
cargo run config-e2e.yml # dev
|
||||||
|
cargo build --release # used by both e2e harnesses
|
||||||
|
cargo test # unit + ts-rs binding export tests
|
||||||
|
```
|
||||||
|
|
||||||
|
Frontend dev (sync-client + obsidian-plugin watch in parallel):
|
||||||
|
|
||||||
|
```sh
|
||||||
|
cd frontend && npm install && npm run dev
|
||||||
|
```
|
||||||
|
|
||||||
|
Regenerate TS bindings from Rust types (touches `frontend/{sync-client,history-ui}/src/.../types/`):
|
||||||
|
|
||||||
|
```sh
|
||||||
|
scripts/update-api-types.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
## SQLite / sqlx
|
||||||
|
|
||||||
|
The server uses `sqlx::query!` macros that need a prepared `.sqlx` cache to compile offline. Touching any SQL means regenerating it:
|
||||||
|
|
||||||
|
```sh
|
||||||
cd sync-server
|
cd sync-server
|
||||||
sqlx database create --database-url sqlite://db.sqlite3
|
sqlx database create --database-url sqlite://db.sqlite3
|
||||||
sqlx migrate run --source src/app_state/database/migrations --database-url sqlite://db.sqlite3
|
sqlx migrate run --source src/app_state/database/migrations --database-url sqlite://db.sqlite3
|
||||||
cargo sqlx prepare --workspace
|
cargo sqlx prepare --workspace
|
||||||
```
|
```
|
||||||
|
|
||||||
### Initial Setup
|
New migrations: `sqlx migrate add --source src/app_state/database/migrations <name>`.
|
||||||
```bash
|
|
||||||
# Install required cargo tools
|
## Sync engine architecture
|
||||||
cargo install sqlx-cli cargo-machete cargo-edit
|
|
||||||
|
Read `frontend/sync-client/src/sync-operations/` to follow the sync engine; the rest of `sync-client` is plumbing (filesystem ops, persistence, services, telemetry).
|
||||||
|
|
||||||
|
The engine is **two independent loops with separate invariants**:
|
||||||
|
|
||||||
|
- **Wire loop** (`syncer.ts`) — drains the single-consumer FIFO queue. HTTP and WS handlers update record fields (`remoteRelativePath`, `parentVersionId`, `remoteHash`) and write content to the file at `record.localPath`. They never move files for path placement.
|
||||||
|
- **Path reconciler** (`reconciler.ts`) — runs after every drained event. Best-effort pass that moves files to make `localPath === remoteRelativePath`. The move graph is topologically sorted; cycles are resolved by reading every file in the cycle into memory and writing each back to its new slot (no tmp files). Records with pending local events are skipped on each pass — the reconciler operates only on settled records. Failures (slot occupied by an untracked file, etc.) are silent skips; the next pass retries.
|
||||||
|
|
||||||
|
**`SyncEventQueue`** (`sync-event-queue.ts`) holds:
|
||||||
|
|
||||||
|
- `byDocId: Map<DocumentId, DocumentRecord>` — primary record store.
|
||||||
|
- `byLocalPath: Map<RelativePath, DocumentRecord>` — derived index for path lookups, maintained at every mutation point.
|
||||||
|
- `events: SyncEvent[]` — pending wire ops in FIFO drain order.
|
||||||
|
|
||||||
|
```ts
|
||||||
|
DocumentRecord = {
|
||||||
|
documentId,
|
||||||
|
parentVersionId,
|
||||||
|
remoteHash?,
|
||||||
|
remoteRelativePath,
|
||||||
|
localPath: RelativePath | undefined
|
||||||
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
### Scripts
|
`localPath === undefined` means the doc has no local file yet — typically a remote create whose target slot was occupied at receive time; the reconciler will fetch and place when the slot frees (the bytes wait in `pendingPlacementContent`).
|
||||||
- `scripts/check.sh`: Full CI check (builds, lints, tests both server and frontend)
|
|
||||||
- `scripts/check.sh --fix`: Same as above but auto-fixes linting and formatting issues
|
|
||||||
- `scripts/e2e.sh`: End-to-end testing
|
|
||||||
- `scripts/clean-up.sh`: Clean logs and database files
|
|
||||||
- `scripts/bump-version.sh patch`: Publish new version
|
|
||||||
- `scripts/update-api-types.sh`: Update TypeScript bindings from Rust types
|
|
||||||
|
|
||||||
## Code Structure
|
Local FS events from the watcher update `localPath` synchronously at enqueue time via `setLocalPath` / `upsertRecord`. The wire loop never updates it for path placement; only the reconciler does. A user rename onto a tracked slot enqueues a `LocalDelete` for the displaced doc (the OS rename clobbered its content) and clears that doc's `localPath`.
|
||||||
|
|
||||||
### Workspace Configuration
|
**Pending creates** use a `Promise<DocumentId>` chain to serialize dependent ops (`LocalUpdate`, `LocalDelete`) behind the still-in-flight `LocalCreate`. `resolveCreate` resolves the promise once the server returns a docId, and `replacePendingDocumentId` swaps the resolved id across already-queued events. `findLatestCreateForPath` is the lookup the watcher uses to attach dependents; `updatePendingCreatePath` rewrites a pending create's `event.path` in place when the user renames the file before its create has acked.
|
||||||
The frontend uses npm workspaces with four packages:
|
|
||||||
- `sync-client`: Core synchronization logic
|
|
||||||
- `obsidian-plugin`: Obsidian-specific integration
|
|
||||||
- `test-client`: Testing utilities
|
|
||||||
- `local-client-cli`: Standalone CLI for VaultLink sync client
|
|
||||||
|
|
||||||
### Type Generation
|
**Watermark.** `lastSeenUpdateId` uses a `MinCovered` (a contiguous-prefix tracker over a stream of integers): we only advance the published min when the next consecutive id has been processed, so out-of-order RemoteChange ids don't fool the WebSocket handshake into requesting a too-recent catch-up.
|
||||||
Rust structs generate TypeScript types via ts-rs crate, stored in `sync-server/bindings/` and used by frontend packages.
|
|
||||||
|
|
||||||
### Key Files
|
**Server catch-up.** The server's WS handshake replays events newer than the client's `last_seen_vault_update_id` from the `latest_document_versions` view (one row per doc, the latest). On those replayed rows `is_new_file` means _new to this client_ (`creation_vault_update_id > last_seen_vault_update_id`), not "this row is the doc's first version" — necessary because the catch-up only carries the latest version; if a doc was created and updated past the watermark, the client never sees its create otherwise.
|
||||||
- `sync-server/src/`: Rust server implementation with WebSocket handlers
|
|
||||||
- `frontend/sync-client/src/sync-client.ts`: Main sync client entry point
|
|
||||||
- `frontend/obsidian-plugin/src/vault-link-plugin.ts`: Main Obsidian plugin class
|
|
||||||
- `frontend/sync-client/src/services/sync-service.ts`: Core synchronization logic
|
|
||||||
|
|
||||||
## Testing
|
## Edge-case patterns the sync engine has to survive
|
||||||
|
|
||||||
### Running Tests
|
The two-loop split defuses most of the old race catalogue (slot-collision stashes, conflict-uuid divergence, `MoveOnConflict.NEW`/`EXISTING` policy choices) by separating wire transport from path placement. What's left:
|
||||||
- Server: `cargo test --verbose`
|
|
||||||
- Frontend: `npm run test` (runs Jest across all workspaces)
|
|
||||||
- E2E: `scripts/e2e.sh`
|
|
||||||
|
|
||||||
### Test Structure
|
**Pending-create docId is a `Promise`, not a string, until the create acks.** Any `LocalUpdate` / `LocalDelete` queued behind a still-in-flight `LocalCreate` carries the create's `resolvers.promise` as its `documentId`. `replacePendingDocumentId` swaps the resolved id across queued events when the create resolves; `===` comparisons against the resolved string elsewhere will silently fail until that swap runs. Anything that walks `events[]` looking for a docId match must either run after the swap or be tolerant of `Promise`-typed ids.
|
||||||
- Rust: Unit tests alongside source files
|
|
||||||
- TypeScript: `.test.ts` files using Jest
|
|
||||||
- E2E: Uses test-client to simulate multiple concurrent users
|
|
||||||
|
|
||||||
## Code Style
|
**`processCreate` reads `event.path` live, not `event.originalPath`.** The watcher rewrites `event.path` in place via `updatePendingCreatePath` when the user renames a pending-create file. `originalPath` was removed from `LocalCreate` events specifically because reading it would send the stale pre-rename path to the server.
|
||||||
|
|
||||||
### Rust
|
**`record.localPath` mutates in place across awaits.** When the watcher renames a doc while a drain handler is awaiting an HTTP roundtrip, the queue mutates the in-flight event's record so subsequent reads see the new path. Snapshotting `record.localPath` into a local at function entry and using it after an `await` reads/writes a now-vacated slot. Read `record.localPath` live; only snapshot for the deliberate "did it change while I was awaiting" comparison.
|
||||||
- Uses extensive Clippy lints (see Cargo.toml)
|
|
||||||
- Follows pedantic linting rules
|
|
||||||
- Forbids unsafe code
|
|
||||||
- Uses cargo fmt with default settings
|
|
||||||
|
|
||||||
### TypeScript
|
**Reconciler-defer is the wire-loop's contract with the reconciler.** The reconciler skips records where `hasPendingLocalEventsForDocumentId` returns true. Wire-loop handlers can therefore freely write `remoteRelativePath` to whatever the server returned — even if it disagrees with `localPath` — knowing the reconciler won't move the file out from under a queued user rename.
|
||||||
- Prettier configuration: 4-space tabs, trailing commas removed, LF line endings
|
|
||||||
- ESLint with unused imports plugin
|
**Watermark advancement is load-bearing both ways.** Branches that _skip_ a remote event without advancing `lastSeenUpdateId` create permanent gaps that re-deliver forever. Branches that _advance_ without applying the content lose data: the server has no further event to re-deliver, the catch-up only carries the latest version, and any state in between is gone. Don't advance unless the event was actually applied (or deliberately discarded after weighing both halves).
|
||||||
- Consistent across all three frontend packages
|
|
||||||
|
**`isNewFile` semantics differ between catch-up and real-time.** On WS handshake replay it means _new to this client_ (`creation_vault_update_id > last_seen_vault_update_id`); on real-time broadcasts it means _this version is the create_ (`creation_vault_update_id == vault_update_id`). A handler that decides based on one interpretation will be wrong on the other channel; reasoning about fetch-and-treat-as-new vs. ignore needs to know which channel delivered the event.
|
||||||
|
|
||||||
|
**Pause / disable-sync mid-flight** is the one race the new model doesn't structurally fix. An HTTP that committed server-side but whose response was discarded leaves the server holding a doc the client has no record of. Resume → offline scan → server-side dedupe handles it (the server merges the duplicate create into the existing doc), but if the merge produces a deconflict, the client picks up an extra file. Out of scope for the two-loop split.
|
||||||
|
|
||||||
|
**Cycle reconciliation uses in-memory content swap.** When the move graph contains a cycle, the reconciler reads every file in the cycle into memory and writes each back to its new slot, with no tmp files. A write-ahead marker at `.vaultlink/swap-<uuid>.json` lists each leg; on startup the reconciler reads the marker, hashes each `from` to determine which legs ran, and replays the rest. The `.vaultlink/**` glob is hard-coded as an internal ignore pattern so swap markers don't get sync'd.
|
||||||
|
|
||||||
|
## Two complementary E2E harnesses
|
||||||
|
|
||||||
|
- **`test-client` (fuzz):** random ops across N parallel processes for many minutes. Used by `scripts/e2e.sh`. Catches bugs nobody thought to write a test for, but reproductions are noisy.
|
||||||
|
- **`deterministic-tests`:** scripted scenarios with an in-memory FS pinned to a real server. Used to _capture_ a fuzz-discovered bug as a minimal repro before fixing it. See `frontend/deterministic-tests/README.md` for the step grammar (`pause-server`, `pause-websocket`, `barrier`, `assert-consistent`, etc.).
|
||||||
|
|
||||||
|
When a fuzz failure surfaces, the workflow is: root-cause from logs → write a deterministic test that fails on the bug → fix → confirm both the deterministic test and `e2e.sh` pass.
|
||||||
|
|
||||||
|
## Style
|
||||||
|
|
||||||
|
- TS: 4-space indent, no tabs, LF, prettier (`trailingComma: "none"`). YAML/MD use 2-space indent.
|
||||||
|
- Rust: `rustfmt.toml` enforces 4-space spaces, LF.
|
||||||
|
- Lint: ESLint for TS, Clippy for Rust, `cargo machete` for unused deps. All wired into `scripts/check.sh`.
|
||||||
|
|
|
||||||
|
|
@ -8,12 +8,12 @@
|
||||||
|
|
||||||
## Develop
|
## Develop
|
||||||
|
|
||||||
### Install [nvm](https://github.com/nvm-sh/nvm)
|
### Set up Node.JS 25 with [nvm](https://github.com/nvm-sh/nvm)
|
||||||
|
|
||||||
- `curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh | bash`
|
- `curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh | bash`
|
||||||
- `nvm install 22`
|
- `nvm install 25`
|
||||||
- `nvm use 22`
|
- `nvm use 25`
|
||||||
- Optionally set the system-wide default: `nvm alias default 22`
|
- Optionally, set the system-wide default: `nvm alias default 25`
|
||||||
|
|
||||||
### Set up Rust
|
### Set up Rust
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -2,12 +2,7 @@
|
||||||
"version": "0.2",
|
"version": "0.2",
|
||||||
"language": "en-GB",
|
"language": "en-GB",
|
||||||
"dictionaries": ["en-gb"],
|
"dictionaries": ["en-gb"],
|
||||||
"ignorePaths": [
|
"ignorePaths": ["node_modules", ".vitepress/dist", ".vitepress/cache", "package-lock.json"],
|
||||||
"node_modules",
|
|
||||||
".vitepress/dist",
|
|
||||||
".vitepress/cache",
|
|
||||||
"package-lock.json"
|
|
||||||
],
|
|
||||||
"words": [
|
"words": [
|
||||||
"VaultLink",
|
"VaultLink",
|
||||||
"Obsidian",
|
"Obsidian",
|
||||||
|
|
|
||||||
|
|
@ -53,7 +53,7 @@ Central authority for synchronisation. Rust + Axum framework.
|
||||||
|
|
||||||
**Technology**:
|
**Technology**:
|
||||||
|
|
||||||
- **Language**: Rust 1.89+
|
- **Language**: Rust 1.92+
|
||||||
- **Framework**: Axum (async web framework)
|
- **Framework**: Axum (async web framework)
|
||||||
- **Database**: SQLite with SQLx
|
- **Database**: SQLite with SQLx
|
||||||
- **Protocol**: WebSockets for real-time communication
|
- **Protocol**: WebSockets for real-time communication
|
||||||
|
|
|
||||||
|
|
@ -75,7 +75,7 @@ chmod +x sync_server-linux-x86_64
|
||||||
|
|
||||||
### Build from Source
|
### Build from Source
|
||||||
|
|
||||||
Requirements: Rust 1.89.0+, SQLite development headers, SQLx CLI
|
Requirements: Rust 1.92.0+, SQLite development headers, SQLx CLI
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Clone the repository
|
# Clone the repository
|
||||||
|
|
|
||||||
6
package-lock.json
generated
Normal file
6
package-lock.json
generated
Normal file
|
|
@ -0,0 +1,6 @@
|
||||||
|
{
|
||||||
|
"name": "vault-link",
|
||||||
|
"lockfileVersion": 3,
|
||||||
|
"requires": true,
|
||||||
|
"packages": {}
|
||||||
|
}
|
||||||
11
rustfmt.toml
Normal file
11
rustfmt.toml
Normal file
|
|
@ -0,0 +1,11 @@
|
||||||
|
# Rustfmt configuration
|
||||||
|
# This should match the .editorconfig settings
|
||||||
|
|
||||||
|
# Use spaces for indentation (matches .editorconfig indent_style = space)
|
||||||
|
hard_tabs = false
|
||||||
|
|
||||||
|
# Use 4 spaces for indentation (matches .editorconfig indent_size = 4)
|
||||||
|
tab_spaces = 4
|
||||||
|
|
||||||
|
# Use Unix line endings (matches .editorconfig end_of_line = lf)
|
||||||
|
newline_style = "Unix"
|
||||||
|
|
@ -35,7 +35,8 @@ cd ..
|
||||||
|
|
||||||
cp frontend/obsidian-plugin/manifest.json manifest.json # for BRAT, otherwise it wouldn't update
|
cp frontend/obsidian-plugin/manifest.json manifest.json # for BRAT, otherwise it wouldn't update
|
||||||
|
|
||||||
git ls-files | xargs npx eclint fix
|
# Format all files across the project (frontend and backend)
|
||||||
|
npx -C frontend prettier --write "**/*.{ts,js,json,md,yml,yaml}"
|
||||||
|
|
||||||
# Commit and tag
|
# Commit and tag
|
||||||
git add .
|
git add .
|
||||||
|
|
|
||||||
|
|
@ -30,8 +30,11 @@ fi
|
||||||
which cargo-machete || cargo install cargo-machete
|
which cargo-machete || cargo install cargo-machete
|
||||||
cargo machete --with-metadata
|
cargo machete --with-metadata
|
||||||
|
|
||||||
|
cd ..
|
||||||
|
scripts/update-api-types.sh # this will dirty up the git state if not up-to-date
|
||||||
|
|
||||||
echo "Running checks in frontend"
|
echo "Running checks in frontend"
|
||||||
cd ../frontend
|
cd frontend
|
||||||
|
|
||||||
if [[ "$FIX_MODE" == true ]]; then
|
if [[ "$FIX_MODE" == true ]]; then
|
||||||
npm install
|
npm install
|
||||||
|
|
@ -45,10 +48,11 @@ cd frontend
|
||||||
npm run build
|
npm run build
|
||||||
npm run test
|
npm run test
|
||||||
npm run lint
|
npm run lint
|
||||||
|
cd ..
|
||||||
|
|
||||||
# Use git ls-files to only check tracked files, respecting .gitignore
|
# Format all files across the project (frontend and backend)
|
||||||
# We always run in fix mode and then check with git status
|
# Prettier respects .gitignore by default
|
||||||
git ls-files | xargs npx eclint fix
|
npx -C frontend prettier --write "**/*.{ts,js,json,md,yml,yaml}"
|
||||||
|
|
||||||
if [[ "$FIX_MODE" == false ]] && [[ $(git status --porcelain) ]]; then
|
if [[ "$FIX_MODE" == false ]] && [[ $(git status --porcelain) ]]; then
|
||||||
git status --porcelain
|
git status --porcelain
|
||||||
|
|
@ -56,6 +60,4 @@ if [[ "$FIX_MODE" == false ]] && [[ $(git status --porcelain) ]]; then
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
cd ..
|
|
||||||
|
|
||||||
echo "Success"
|
echo "Success"
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,4 @@
|
||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
|
|
||||||
rm -rf sync-server/databases
|
rm -rf /host/tmp/vaultlink-e2e-databases
|
||||||
rm -rf logs
|
rm -rf logs
|
||||||
|
|
|
||||||
|
|
@ -19,35 +19,51 @@ process_count=$1
|
||||||
|
|
||||||
mkdir -p logs
|
mkdir -p logs
|
||||||
|
|
||||||
|
# Build and restart the server
|
||||||
|
echo "Building server..."
|
||||||
|
cd sync-server
|
||||||
|
cargo build --release
|
||||||
|
|
||||||
|
# Kill any existing server process
|
||||||
|
echo "Stopping existing server..."
|
||||||
|
pkill -f "sync_server" 2>/dev/null || true
|
||||||
|
sleep 1
|
||||||
|
|
||||||
|
# Clean databases (uses tmpfs via /dev/shm for zero disk I/O)
|
||||||
|
echo "Cleaning databases..."
|
||||||
|
rm -rf /host/tmp/vaultlink-e2e-databases
|
||||||
|
|
||||||
|
# Start the server in the background
|
||||||
|
echo "Starting server..."
|
||||||
|
./target/release/sync_server config-e2e.yml &
|
||||||
|
server_pid=$!
|
||||||
|
echo "Server started with PID: $server_pid"
|
||||||
|
|
||||||
|
# Ensure server is killed on script exit
|
||||||
|
cleanup_server() {
|
||||||
|
if [ -n "$server_pid" ]; then
|
||||||
|
echo "Stopping server (PID: $server_pid)..."
|
||||||
|
kill $server_pid 2>/dev/null || true
|
||||||
|
wait $server_pid 2>/dev/null || true
|
||||||
|
server_pid=""
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
trap cleanup_server EXIT
|
||||||
|
|
||||||
|
cd ..
|
||||||
|
|
||||||
cd frontend
|
cd frontend
|
||||||
npm ci
|
npm ci
|
||||||
npm run build
|
npm run build
|
||||||
|
|
||||||
../scripts/utils/wait-for-server.sh
|
../scripts/utils/wait-for-server.sh
|
||||||
|
|
||||||
cd ..
|
|
||||||
scripts/update-api-types.sh
|
|
||||||
if [[ $(git status --porcelain) ]]; then
|
|
||||||
git status --porcelain
|
|
||||||
echo "Failing CI because the working directory is not clean after generating api types"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
cd frontend
|
|
||||||
|
|
||||||
pids=()
|
pids=()
|
||||||
for i in $(seq 1 $process_count); do
|
for i in $(seq 1 $process_count); do
|
||||||
# Create a named pipe for this process
|
node test-client/dist/cli.js > "../logs/log_${i}.log" 2>&1 &
|
||||||
pipe="/tmp/vaultlink_pipe_$$_$i"
|
|
||||||
mkfifo "$pipe"
|
|
||||||
|
|
||||||
# Start the node process writing to the pipe
|
|
||||||
node test-client/dist/cli.js > "$pipe" 2>&1 &
|
|
||||||
pid=$!
|
pid=$!
|
||||||
pids+=($pid)
|
pids+=($pid)
|
||||||
echo "Started process $i with PID: $pid"
|
echo "Started process $i with PID: $pid (log: logs/log_${i}.log)"
|
||||||
|
|
||||||
# Read from pipe, prefix with PID
|
|
||||||
(sed "s/^/[PID $pid] /" < "$pipe" > "../logs/log_${i}.log"; rm "$pipe") &
|
|
||||||
done
|
done
|
||||||
|
|
||||||
cd ..
|
cd ..
|
||||||
|
|
@ -75,10 +91,25 @@ print_failed_log() {
|
||||||
return 1
|
return 1
|
||||||
}
|
}
|
||||||
|
|
||||||
echo "Monitoring $process_count processes"
|
E2E_TIMEOUT=${2:-3600}
|
||||||
|
start_time=$(date +%s)
|
||||||
|
echo "Monitoring $process_count processes (timeout: ${E2E_TIMEOUT}s)"
|
||||||
|
|
||||||
# Monitor processes
|
# Monitor processes
|
||||||
while true; do
|
while true; do
|
||||||
|
# Script-level timeout to prevent indefinite hangs
|
||||||
|
current_time=$(date +%s)
|
||||||
|
elapsed=$((current_time - start_time))
|
||||||
|
if [ $elapsed -ge $E2E_TIMEOUT ]; then
|
||||||
|
echo "E2E timeout reached (${E2E_TIMEOUT}s). Killing remaining processes."
|
||||||
|
for pid in "${pids[@]}"; do
|
||||||
|
if [ -n "$pid" ]; then
|
||||||
|
kill $pid 2>/dev/null || true
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
if print_failed_log; then
|
if print_failed_log; then
|
||||||
# Kill remaining processes
|
# Kill remaining processes
|
||||||
for pid in "${pids[@]}"; do
|
for pid in "${pids[@]}"; do
|
||||||
|
|
@ -99,6 +130,7 @@ while true; do
|
||||||
done
|
done
|
||||||
|
|
||||||
if $all_done; then
|
if $all_done; then
|
||||||
|
cleanup_server
|
||||||
echo "All processes completed successfully"
|
echo "All processes completed successfully"
|
||||||
exit 0
|
exit 0
|
||||||
fi
|
fi
|
||||||
|
|
|
||||||
|
|
@ -8,9 +8,15 @@ cd sync-server
|
||||||
cargo test export_bindings
|
cargo test export_bindings
|
||||||
cd -
|
cd -
|
||||||
|
|
||||||
|
# Both target directories contain only generated bindings — wipe and copy
|
||||||
|
rm -f frontend/sync-client/src/services/types/*.ts
|
||||||
|
rm -f frontend/history-ui/src/lib/types/*.ts
|
||||||
cp -r sync-server/bindings/* frontend/sync-client/src/services/types/
|
cp -r sync-server/bindings/* frontend/sync-client/src/services/types/
|
||||||
|
cp -r sync-server/bindings/* frontend/history-ui/src/lib/types/
|
||||||
|
|
||||||
cd frontend
|
cd frontend
|
||||||
npm run lint
|
npm run lint
|
||||||
git ls-files | xargs npx eclint fix
|
cd ..
|
||||||
cd -
|
|
||||||
|
# Format all files across the project (frontend and backend)
|
||||||
|
npx -C frontend prettier --write "**/*.{ts,js,json,md,yml,yaml}"
|
||||||
|
|
|
||||||
|
|
@ -2,8 +2,10 @@
|
||||||
|
|
||||||
set -e
|
set -e
|
||||||
|
|
||||||
|
TARGET_NODE_VERSION=25
|
||||||
|
|
||||||
node_version=$(node -v | sed 's/^v\([0-9]*\).*/\1/')
|
node_version=$(node -v | sed 's/^v\([0-9]*\).*/\1/')
|
||||||
if [ "$node_version" != "22" ]; then
|
if [ "$node_version" != "$TARGET_NODE_VERSION" ]; then
|
||||||
echo "Error: This script requires Node.js version 22, found: $node_version"
|
echo "Error: This script requires Node.js version $TARGET_NODE_VERSION, found: $node_version"
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
|
||||||
|
|
@ -2,14 +2,14 @@
|
||||||
|
|
||||||
set -e
|
set -e
|
||||||
|
|
||||||
SERVER_URL="http://localhost:3000"
|
SERVER_URL="http://localhost:3010"
|
||||||
MAX_RETRIES=30
|
MAX_RETRIES=30
|
||||||
RETRY_INTERVAL_IN_SECONDS=5
|
RETRY_INTERVAL_IN_SECONDS=5
|
||||||
|
|
||||||
echo "Waiting for $SERVER_URL to become available..."
|
echo "Waiting for $SERVER_URL to become available..."
|
||||||
count=0
|
count=0
|
||||||
while [ $count -lt $MAX_RETRIES ]; do
|
while [ $count -lt $MAX_RETRIES ]; do
|
||||||
if curl -s -f -o /dev/null $SERVER_URL; then
|
if curl -s -o /dev/null $SERVER_URL; then
|
||||||
echo "$SERVER_URL is now available!"
|
echo "$SERVER_URL is now available!"
|
||||||
break
|
break
|
||||||
fi
|
fi
|
||||||
|
|
|
||||||
193
sync-server/Cargo.lock
generated
193
sync-server/Cargo.lock
generated
|
|
@ -337,10 +337,11 @@ checksum = "325918d6fe32f23b19878fe4b34794ae41fc19ddbe53b10571a4874d44ffd39b"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "cc"
|
name = "cc"
|
||||||
version = "1.2.2"
|
version = "1.2.57"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "f34d93e62b03caf570cccc334cbc6c2fceca82f39211051345108adcba3eebdc"
|
checksum = "7a0dd1ca384932ff3641c8718a02769f1698e7563dc6974ffd03346116310423"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
|
"find-msvc-tools",
|
||||||
"shlex",
|
"shlex",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
@ -456,6 +457,15 @@ version = "2.4.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5"
|
checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "crossbeam-channel"
|
||||||
|
version = "0.5.15"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2"
|
||||||
|
dependencies = [
|
||||||
|
"crossbeam-utils",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "crossbeam-queue"
|
name = "crossbeam-queue"
|
||||||
version = "0.3.11"
|
version = "0.3.11"
|
||||||
|
|
@ -533,6 +543,15 @@ dependencies = [
|
||||||
"zeroize",
|
"zeroize",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "deranged"
|
||||||
|
version = "0.5.8"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c"
|
||||||
|
dependencies = [
|
||||||
|
"powerfmt",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "digest"
|
name = "digest"
|
||||||
version = "0.10.7"
|
version = "0.10.7"
|
||||||
|
|
@ -624,6 +643,12 @@ version = "2.2.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "486f806e73c5707928240ddc295403b1b93c96a02038563881c4a2fd84b81ac4"
|
checksum = "486f806e73c5707928240ddc295403b1b93c96a02038563881c4a2fd84b81ac4"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "find-msvc-tools"
|
||||||
|
version = "0.1.9"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "flume"
|
name = "flume"
|
||||||
version = "0.11.1"
|
version = "0.11.1"
|
||||||
|
|
@ -1272,6 +1297,16 @@ version = "0.3.17"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a"
|
checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "mime_guess"
|
||||||
|
version = "2.0.5"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e"
|
||||||
|
dependencies = [
|
||||||
|
"mime",
|
||||||
|
"unicase",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "miniz_oxide"
|
name = "miniz_oxide"
|
||||||
version = "0.8.0"
|
version = "0.8.0"
|
||||||
|
|
@ -1335,6 +1370,12 @@ dependencies = [
|
||||||
"zeroize",
|
"zeroize",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "num-conv"
|
||||||
|
version = "0.2.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "c6673768db2d862beb9b39a78fdcb1a69439615d5794a1be50caa9bc92c81967"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "num-integer"
|
name = "num-integer"
|
||||||
version = "0.1.46"
|
version = "0.1.46"
|
||||||
|
|
@ -1463,6 +1504,12 @@ version = "0.3.31"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2"
|
checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "powerfmt"
|
||||||
|
version = "0.2.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "ppv-lite86"
|
name = "ppv-lite86"
|
||||||
version = "0.2.20"
|
version = "0.2.20"
|
||||||
|
|
@ -1582,12 +1629,12 @@ dependencies = [
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "reconcile-text"
|
name = "reconcile-text"
|
||||||
version = "0.8.0"
|
version = "0.11.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "599cf9539996a2a19e501110404c59ba62f4974009f8fb864a8b7151c15ee5a5"
|
checksum = "52e0cf361887ea64c479ca871c1170dda761f84e122f2616b5579906a38d7557"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"serde",
|
"serde",
|
||||||
"thiserror 2.0.17",
|
"thiserror 2.0.18",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
|
|
@ -1648,6 +1695,40 @@ dependencies = [
|
||||||
"zeroize",
|
"zeroize",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "rust-embed"
|
||||||
|
version = "8.11.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "04113cb9355a377d83f06ef1f0a45b8ab8cd7d8b1288160717d66df5c7988d27"
|
||||||
|
dependencies = [
|
||||||
|
"rust-embed-impl",
|
||||||
|
"rust-embed-utils",
|
||||||
|
"walkdir",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "rust-embed-impl"
|
||||||
|
version = "8.11.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "da0902e4c7c8e997159ab384e6d0fc91c221375f6894346ae107f47dd0f3ccaa"
|
||||||
|
dependencies = [
|
||||||
|
"proc-macro2",
|
||||||
|
"quote",
|
||||||
|
"rust-embed-utils",
|
||||||
|
"syn 2.0.90",
|
||||||
|
"walkdir",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "rust-embed-utils"
|
||||||
|
version = "8.11.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "5bcdef0be6fe7f6fa333b1073c949729274b05f123a0ad7efcb8efd878e5c3b1"
|
||||||
|
dependencies = [
|
||||||
|
"sha2",
|
||||||
|
"walkdir",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "rustc-demangle"
|
name = "rustc-demangle"
|
||||||
version = "0.1.24"
|
version = "0.1.24"
|
||||||
|
|
@ -1679,6 +1760,15 @@ version = "1.0.18"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f"
|
checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "same-file"
|
||||||
|
version = "1.0.6"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502"
|
||||||
|
dependencies = [
|
||||||
|
"winapi-util",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "sanitize-filename"
|
name = "sanitize-filename"
|
||||||
version = "0.6.0"
|
version = "0.6.0"
|
||||||
|
|
@ -1916,7 +2006,7 @@ dependencies = [
|
||||||
"serde_json",
|
"serde_json",
|
||||||
"sha2",
|
"sha2",
|
||||||
"smallvec",
|
"smallvec",
|
||||||
"thiserror 2.0.17",
|
"thiserror 2.0.18",
|
||||||
"tokio",
|
"tokio",
|
||||||
"tokio-stream",
|
"tokio-stream",
|
||||||
"tracing",
|
"tracing",
|
||||||
|
|
@ -2000,7 +2090,7 @@ dependencies = [
|
||||||
"smallvec",
|
"smallvec",
|
||||||
"sqlx-core",
|
"sqlx-core",
|
||||||
"stringprep",
|
"stringprep",
|
||||||
"thiserror 2.0.17",
|
"thiserror 2.0.18",
|
||||||
"tracing",
|
"tracing",
|
||||||
"uuid",
|
"uuid",
|
||||||
"whoami",
|
"whoami",
|
||||||
|
|
@ -2039,7 +2129,7 @@ dependencies = [
|
||||||
"smallvec",
|
"smallvec",
|
||||||
"sqlx-core",
|
"sqlx-core",
|
||||||
"stringprep",
|
"stringprep",
|
||||||
"thiserror 2.0.17",
|
"thiserror 2.0.18",
|
||||||
"tracing",
|
"tracing",
|
||||||
"uuid",
|
"uuid",
|
||||||
"whoami",
|
"whoami",
|
||||||
|
|
@ -2065,7 +2155,7 @@ dependencies = [
|
||||||
"serde",
|
"serde",
|
||||||
"serde_urlencoded",
|
"serde_urlencoded",
|
||||||
"sqlx-core",
|
"sqlx-core",
|
||||||
"thiserror 2.0.17",
|
"thiserror 2.0.18",
|
||||||
"tracing",
|
"tracing",
|
||||||
"url",
|
"url",
|
||||||
"uuid",
|
"uuid",
|
||||||
|
|
@ -2100,6 +2190,12 @@ version = "2.6.1"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
|
checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "symlink"
|
||||||
|
version = "0.1.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "a7973cce6668464ea31f176d85b13c7ab3bba2cb3b77a2ed26abd7801688010a"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "syn"
|
name = "syn"
|
||||||
version = "1.0.109"
|
version = "1.0.109"
|
||||||
|
|
@ -2136,18 +2232,22 @@ dependencies = [
|
||||||
"futures",
|
"futures",
|
||||||
"humantime-serde",
|
"humantime-serde",
|
||||||
"log",
|
"log",
|
||||||
|
"mime_guess",
|
||||||
"rand 0.9.0",
|
"rand 0.9.0",
|
||||||
"reconcile-text",
|
"reconcile-text",
|
||||||
"regex",
|
"regex",
|
||||||
|
"rust-embed",
|
||||||
"sanitize-filename",
|
"sanitize-filename",
|
||||||
"serde",
|
"serde",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
"serde_yaml",
|
"serde_yaml",
|
||||||
"sqlx",
|
"sqlx",
|
||||||
"thiserror 2.0.17",
|
"subtle",
|
||||||
|
"thiserror 2.0.18",
|
||||||
"tokio",
|
"tokio",
|
||||||
"tower-http",
|
"tower-http",
|
||||||
"tracing",
|
"tracing",
|
||||||
|
"tracing-appender",
|
||||||
"tracing-subscriber",
|
"tracing-subscriber",
|
||||||
"ts-rs",
|
"ts-rs",
|
||||||
"uuid",
|
"uuid",
|
||||||
|
|
@ -2203,11 +2303,11 @@ dependencies = [
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "thiserror"
|
name = "thiserror"
|
||||||
version = "2.0.17"
|
version = "2.0.18"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "f63587ca0f12b72a0600bcba1d40081f830876000bb46dd2337a3051618f4fc8"
|
checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"thiserror-impl 2.0.17",
|
"thiserror-impl 2.0.18",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
|
|
@ -2223,9 +2323,9 @@ dependencies = [
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "thiserror-impl"
|
name = "thiserror-impl"
|
||||||
version = "2.0.17"
|
version = "2.0.18"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913"
|
checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"proc-macro2",
|
"proc-macro2",
|
||||||
"quote",
|
"quote",
|
||||||
|
|
@ -2242,6 +2342,37 @@ dependencies = [
|
||||||
"once_cell",
|
"once_cell",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "time"
|
||||||
|
version = "0.3.47"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c"
|
||||||
|
dependencies = [
|
||||||
|
"deranged",
|
||||||
|
"itoa",
|
||||||
|
"num-conv",
|
||||||
|
"powerfmt",
|
||||||
|
"serde_core",
|
||||||
|
"time-core",
|
||||||
|
"time-macros",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "time-core"
|
||||||
|
version = "0.1.8"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "time-macros"
|
||||||
|
version = "0.2.27"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215"
|
||||||
|
dependencies = [
|
||||||
|
"num-conv",
|
||||||
|
"time-core",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "tinystr"
|
name = "tinystr"
|
||||||
version = "0.7.6"
|
version = "0.7.6"
|
||||||
|
|
@ -2276,7 +2407,6 @@ dependencies = [
|
||||||
"bytes",
|
"bytes",
|
||||||
"libc",
|
"libc",
|
||||||
"mio",
|
"mio",
|
||||||
"parking_lot",
|
|
||||||
"pin-project-lite",
|
"pin-project-lite",
|
||||||
"signal-hook-registry",
|
"signal-hook-registry",
|
||||||
"socket2",
|
"socket2",
|
||||||
|
|
@ -2376,6 +2506,19 @@ dependencies = [
|
||||||
"tracing-core",
|
"tracing-core",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "tracing-appender"
|
||||||
|
version = "0.2.5"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "050686193eb999b4bb3bc2acfa891a13da00f79734704c4b8b4ef1a10b368a3c"
|
||||||
|
dependencies = [
|
||||||
|
"crossbeam-channel",
|
||||||
|
"symlink",
|
||||||
|
"thiserror 2.0.18",
|
||||||
|
"time",
|
||||||
|
"tracing-subscriber",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "tracing-attributes"
|
name = "tracing-attributes"
|
||||||
version = "0.1.28"
|
version = "0.1.28"
|
||||||
|
|
@ -2434,7 +2577,7 @@ checksum = "e640d9b0964e9d39df633548591090ab92f7a4567bc31d3891af23471a3365c6"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"chrono",
|
"chrono",
|
||||||
"lazy_static",
|
"lazy_static",
|
||||||
"thiserror 2.0.17",
|
"thiserror 2.0.18",
|
||||||
"ts-rs-macros",
|
"ts-rs-macros",
|
||||||
"uuid",
|
"uuid",
|
||||||
]
|
]
|
||||||
|
|
@ -2481,6 +2624,12 @@ version = "0.10.4"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "f720def6ce1ee2fc44d40ac9ed6d3a59c361c80a75a7aa8e75bb9baed31cf2ea"
|
checksum = "f720def6ce1ee2fc44d40ac9ed6d3a59c361c80a75a7aa8e75bb9baed31cf2ea"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "unicase"
|
||||||
|
version = "2.9.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "unicode-bidi"
|
name = "unicode-bidi"
|
||||||
version = "0.3.17"
|
version = "0.3.17"
|
||||||
|
|
@ -2577,6 +2726,16 @@ version = "0.9.5"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
|
checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "walkdir"
|
||||||
|
version = "2.5.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b"
|
||||||
|
dependencies = [
|
||||||
|
"same-file",
|
||||||
|
"winapi-util",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "wasi"
|
name = "wasi"
|
||||||
version = "0.11.0+wasi-snapshot-preview1"
|
version = "0.11.0+wasi-snapshot-preview1"
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
[package]
|
[package]
|
||||||
name = "sync_server"
|
name = "sync_server"
|
||||||
rust-version = "1.89.0"
|
rust-version = "1.94.0"
|
||||||
authors = ["Andras Schmelczer <andras@schmelczer.dev>"]
|
authors = ["Andras Schmelczer <andras@schmelczer.dev>"]
|
||||||
edition = "2024"
|
edition = "2024"
|
||||||
license = "MIT"
|
license = "MIT"
|
||||||
|
|
@ -10,7 +10,7 @@ version = "0.14.0"
|
||||||
[dependencies]
|
[dependencies]
|
||||||
serde = { version = "1.0.219", default-features = false, features = ["derive"] }
|
serde = { version = "1.0.219", default-features = false, features = ["derive"] }
|
||||||
thiserror = { version = "2.0.12", default-features = false }
|
thiserror = { version = "2.0.12", default-features = false }
|
||||||
tokio = { version = "1.48.0", features = ["full"]}
|
tokio = { version = "1.48.0", features = ["macros", "rt-multi-thread", "sync", "time", "net", "fs", "signal"]}
|
||||||
uuid = { version = "1.16.0", features = ["v4", "serde"] }
|
uuid = { version = "1.16.0", features = ["v4", "serde"] }
|
||||||
log = { version = "0.4.28" }
|
log = { version = "0.4.28" }
|
||||||
anyhow = { version = "1.0.100", features = ["backtrace"] }
|
anyhow = { version = "1.0.100", features = ["backtrace"] }
|
||||||
|
|
@ -20,6 +20,7 @@ axum_typed_multipart = "0.11.0"
|
||||||
tower-http = { version = "0.6.1", features = ["cors", "trace", "limit", "timeout"] }
|
tower-http = { version = "0.6.1", features = ["cors", "trace", "limit", "timeout"] }
|
||||||
tracing = "0.1.41"
|
tracing = "0.1.41"
|
||||||
tracing-subscriber = { version = "0.3.20", features = ["fmt", "env-filter"]}
|
tracing-subscriber = { version = "0.3.20", features = ["fmt", "env-filter"]}
|
||||||
|
tracing-appender = "0.2.5"
|
||||||
humantime-serde = "1.1.1"
|
humantime-serde = "1.1.1"
|
||||||
sqlx = { version = "0.8.6", features = ["sqlite", "runtime-tokio", "uuid", "chrono"] }
|
sqlx = { version = "0.8.6", features = ["sqlite", "runtime-tokio", "uuid", "chrono"] }
|
||||||
chrono = { version = "0.4.41", features = ["serde"] }
|
chrono = { version = "0.4.41", features = ["serde"] }
|
||||||
|
|
@ -33,7 +34,10 @@ serde_json = "1.0.140"
|
||||||
bimap = "0.6.3"
|
bimap = "0.6.3"
|
||||||
ts-rs = { version = "10.1", features = ["uuid-impl", "chrono-impl"] }
|
ts-rs = { version = "10.1", features = ["uuid-impl", "chrono-impl"] }
|
||||||
base64 = "0.22.1"
|
base64 = "0.22.1"
|
||||||
reconcile-text = { version = "0.8.0", features = ["serde"] }
|
reconcile-text = { version = "0.11.0", features = ["serde"] }
|
||||||
|
rust-embed = "8.5"
|
||||||
|
mime_guess = "2.0"
|
||||||
|
subtle = "2.6.1"
|
||||||
|
|
||||||
[profile.release]
|
[profile.release]
|
||||||
codegen-units = 1
|
codegen-units = 1
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,16 @@
|
||||||
// generated by `sqlx migrate build-script`
|
|
||||||
fn main() {
|
fn main() {
|
||||||
// trigger recompilation when a new migration is added
|
// trigger recompilation when a new migration is added
|
||||||
println!("cargo:rerun-if-changed=migrations");
|
println!("cargo:rerun-if-changed=migrations");
|
||||||
|
|
||||||
|
// Ensure the history-ui dist directory exists so rust-embed can compile
|
||||||
|
// even when the frontend hasn't been built yet.
|
||||||
|
let dist_path = std::path::Path::new("../frontend/history-ui/dist");
|
||||||
|
if !dist_path.exists() {
|
||||||
|
std::fs::create_dir_all(dist_path).expect("Failed to create history-ui dist directory");
|
||||||
|
std::fs::write(
|
||||||
|
dist_path.join("index.html"),
|
||||||
|
"<!DOCTYPE html><html><body><p>Run <code>npm run build -w history-ui</code> first.</p></body></html>",
|
||||||
|
)
|
||||||
|
.expect("Failed to write placeholder index.html");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,12 +1,14 @@
|
||||||
database:
|
database:
|
||||||
databases_directory_path: databases
|
databases_directory_path: /host/tmp/vaultlink-e2e-databases
|
||||||
max_connections_per_vault: 12
|
max_connections_per_vault: 8
|
||||||
cursor_timeout: 1m
|
cursor_timeout: 1m
|
||||||
server:
|
server:
|
||||||
host: 0.0.0.0
|
host: 0.0.0.0
|
||||||
port: 3000
|
port: 3010
|
||||||
max_body_size_mb: 512
|
max_body_size_mb: 512
|
||||||
max_clients_per_vault: 256
|
max_clients_per_vault: 256
|
||||||
|
max_pending_websocket_connections: 4096
|
||||||
|
broadcast_channel_capacity: 1024
|
||||||
response_timeout: 30m
|
response_timeout: 30m
|
||||||
mergeable_file_extensions:
|
mergeable_file_extensions:
|
||||||
- md
|
- md
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,5 @@
|
||||||
[toolchain]
|
[toolchain]
|
||||||
channel = "1.89.0"
|
channel = "1.94.0"
|
||||||
targets = [
|
targets = [
|
||||||
"x86_64-unknown-linux-gnu",
|
"x86_64-unknown-linux-gnu",
|
||||||
"x86_64-unknown-linux-musl",
|
"x86_64-unknown-linux-musl",
|
||||||
|
|
|
||||||
|
|
@ -2,6 +2,8 @@ pub mod cursors;
|
||||||
pub mod database;
|
pub mod database;
|
||||||
pub mod websocket;
|
pub mod websocket;
|
||||||
|
|
||||||
|
use std::sync::{Arc, atomic::AtomicUsize};
|
||||||
|
|
||||||
use anyhow::Result;
|
use anyhow::Result;
|
||||||
use cursors::Cursors;
|
use cursors::Cursors;
|
||||||
use database::Database;
|
use database::Database;
|
||||||
|
|
@ -15,21 +17,42 @@ pub struct AppState {
|
||||||
pub database: Database,
|
pub database: Database,
|
||||||
pub cursors: Cursors,
|
pub cursors: Cursors,
|
||||||
pub broadcasts: Broadcasts,
|
pub broadcasts: Broadcasts,
|
||||||
|
/// Tracks WebSocket connections that have upgraded but not yet completed
|
||||||
|
/// the authentication handshake
|
||||||
|
pub pending_ws_connections: Arc<AtomicUsize>,
|
||||||
|
/// Send on this channel to stop background tasks (cursor cleanup,
|
||||||
|
/// idle-pool cleanup)
|
||||||
|
shutdown_tx: Arc<tokio::sync::watch::Sender<()>>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl AppState {
|
impl AppState {
|
||||||
pub async fn try_new(config: Config) -> Result<Self> {
|
pub async fn try_new(config: Config) -> Result<Self> {
|
||||||
|
let (shutdown_tx, shutdown_rx) = tokio::sync::watch::channel(());
|
||||||
|
|
||||||
let broadcasts = Broadcasts::new(&config.server);
|
let broadcasts = Broadcasts::new(&config.server);
|
||||||
let database = Database::try_new(&config.database, &broadcasts).await?;
|
let database =
|
||||||
|
Database::try_new(&config.database, &broadcasts, shutdown_rx.clone()).await?;
|
||||||
let cursors: Cursors = Cursors::new(&config.database, &broadcasts);
|
let cursors: Cursors = Cursors::new(&config.database, &broadcasts);
|
||||||
|
|
||||||
Cursors::start_background_task(cursors.clone());
|
Cursors::start_background_task(cursors.clone(), shutdown_rx);
|
||||||
|
|
||||||
Ok(Self {
|
Ok(Self {
|
||||||
config,
|
config,
|
||||||
database,
|
database,
|
||||||
cursors,
|
cursors,
|
||||||
broadcasts,
|
broadcasts,
|
||||||
|
pending_ws_connections: Arc::new(AtomicUsize::new(0)),
|
||||||
|
shutdown_tx: Arc::new(shutdown_tx),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Signal all background tasks (idle pool cleanup, cursor cleanup) to stop
|
||||||
|
pub fn shutdown(&self) {
|
||||||
|
let _ = self.shutdown_tx.send(());
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Get a receiver to be notified when shutdown is triggered
|
||||||
|
pub fn subscribe_shutdown(&self) -> tokio::sync::watch::Receiver<()> {
|
||||||
|
self.shutdown_tx.subscribe()
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
File diff suppressed because it is too large
Load diff
|
|
@ -0,0 +1,2 @@
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_documents_document_id
|
||||||
|
ON documents (document_id, vault_update_id);
|
||||||
|
|
@ -0,0 +1,20 @@
|
||||||
|
ALTER TABLE documents ADD COLUMN creation_vault_update_id INTEGER NOT NULL DEFAULT 0;
|
||||||
|
|
||||||
|
UPDATE documents
|
||||||
|
SET creation_vault_update_id = (
|
||||||
|
SELECT MIN(d2.vault_update_id)
|
||||||
|
FROM documents d2
|
||||||
|
WHERE d2.document_id = documents.document_id
|
||||||
|
);
|
||||||
|
|
||||||
|
DROP VIEW latest_document_versions;
|
||||||
|
|
||||||
|
CREATE VIEW IF NOT EXISTS latest_document_versions AS --recreate view as it now includes one more field
|
||||||
|
SELECT d.*
|
||||||
|
FROM documents d
|
||||||
|
INNER JOIN (
|
||||||
|
SELECT MAX(vault_update_id) AS max_version_id
|
||||||
|
FROM documents
|
||||||
|
GROUP BY document_id
|
||||||
|
) max_versions
|
||||||
|
ON d.vault_update_id = max_versions.max_version_id;
|
||||||
|
|
@ -13,6 +13,7 @@ pub type DeviceId = String;
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
pub struct StoredDocumentVersion {
|
pub struct StoredDocumentVersion {
|
||||||
pub vault_update_id: VaultUpdateId,
|
pub vault_update_id: VaultUpdateId,
|
||||||
|
pub creation_vault_update_id: VaultUpdateId,
|
||||||
pub document_id: DocumentId,
|
pub document_id: DocumentId,
|
||||||
pub relative_path: String,
|
pub relative_path: String,
|
||||||
pub updated_date: DateTime<Utc>,
|
pub updated_date: DateTime<Utc>,
|
||||||
|
|
@ -33,7 +34,7 @@ impl PartialEq<Self> for StoredDocumentVersion {
|
||||||
#[derive(TS, Debug, Clone, Serialize)]
|
#[derive(TS, Debug, Clone, Serialize)]
|
||||||
#[serde(rename_all = "camelCase")]
|
#[serde(rename_all = "camelCase")]
|
||||||
pub struct DocumentVersionWithoutContent {
|
pub struct DocumentVersionWithoutContent {
|
||||||
#[ts(as = "i32")]
|
#[ts(type = "number")]
|
||||||
pub vault_update_id: VaultUpdateId,
|
pub vault_update_id: VaultUpdateId,
|
||||||
|
|
||||||
pub document_id: DocumentId,
|
pub document_id: DocumentId,
|
||||||
|
|
@ -43,12 +44,16 @@ pub struct DocumentVersionWithoutContent {
|
||||||
pub user_id: UserId,
|
pub user_id: UserId,
|
||||||
pub device_id: DeviceId,
|
pub device_id: DeviceId,
|
||||||
|
|
||||||
#[ts(as = "i32")]
|
#[ts(type = "number")]
|
||||||
pub content_size: u64,
|
pub content_size: u64,
|
||||||
|
|
||||||
|
/// True iff this is the first version of the document
|
||||||
|
pub is_new_file: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl From<StoredDocumentVersion> for DocumentVersionWithoutContent {
|
impl From<StoredDocumentVersion> for DocumentVersionWithoutContent {
|
||||||
fn from(value: StoredDocumentVersion) -> Self {
|
fn from(value: StoredDocumentVersion) -> Self {
|
||||||
|
let is_new_file = value.creation_vault_update_id == value.vault_update_id;
|
||||||
Self {
|
Self {
|
||||||
vault_update_id: value.vault_update_id,
|
vault_update_id: value.vault_update_id,
|
||||||
document_id: value.document_id,
|
document_id: value.document_id,
|
||||||
|
|
@ -58,6 +63,7 @@ impl From<StoredDocumentVersion> for DocumentVersionWithoutContent {
|
||||||
user_id: value.user_id,
|
user_id: value.user_id,
|
||||||
device_id: value.device_id,
|
device_id: value.device_id,
|
||||||
content_size: value.content.len() as u64,
|
content_size: value.content.len() as u64,
|
||||||
|
is_new_file,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -65,7 +71,7 @@ impl From<StoredDocumentVersion> for DocumentVersionWithoutContent {
|
||||||
#[derive(TS, Debug, Clone, Serialize)]
|
#[derive(TS, Debug, Clone, Serialize)]
|
||||||
#[serde(rename_all = "camelCase")]
|
#[serde(rename_all = "camelCase")]
|
||||||
pub struct DocumentVersion {
|
pub struct DocumentVersion {
|
||||||
#[ts(as = "i32")]
|
#[ts(type = "number")]
|
||||||
pub vault_update_id: VaultUpdateId,
|
pub vault_update_id: VaultUpdateId,
|
||||||
|
|
||||||
pub document_id: DocumentId,
|
pub document_id: DocumentId,
|
||||||
|
|
@ -77,6 +83,25 @@ pub struct DocumentVersion {
|
||||||
pub device_id: DeviceId,
|
pub device_id: DeviceId,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Row struct for vault history queries (used by `sqlx::query_as!`)
|
||||||
|
#[derive(Debug)]
|
||||||
|
pub struct VaultHistoryRow {
|
||||||
|
pub vault_update_id: VaultUpdateId,
|
||||||
|
pub creation_vault_update_id: VaultUpdateId,
|
||||||
|
pub document_id: DocumentId,
|
||||||
|
pub relative_path: String,
|
||||||
|
pub updated_date: DateTime<Utc>,
|
||||||
|
pub is_deleted: bool,
|
||||||
|
pub user_id: String,
|
||||||
|
pub device_id: String,
|
||||||
|
pub content_size: Option<u64>,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub struct VaultStats {
|
||||||
|
pub created_at: Option<DateTime<Utc>>,
|
||||||
|
pub document_count: u32,
|
||||||
|
}
|
||||||
|
|
||||||
impl From<StoredDocumentVersion> for DocumentVersion {
|
impl From<StoredDocumentVersion> for DocumentVersion {
|
||||||
fn from(value: StoredDocumentVersion) -> Self {
|
fn from(value: StoredDocumentVersion) -> Self {
|
||||||
Self {
|
Self {
|
||||||
|
|
|
||||||
|
|
@ -27,25 +27,35 @@ pub struct Config {
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Config {
|
impl Config {
|
||||||
pub async fn read_or_create(path: &Path) -> Result<Self> {
|
pub fn validate(&self) -> Result<()> {
|
||||||
let config = if path.exists() {
|
self.server
|
||||||
info!(
|
.validate()
|
||||||
"Loading configuration from `{}`",
|
.context("Invalid server configuration")?;
|
||||||
path.canonicalize().unwrap().display()
|
self.logging
|
||||||
);
|
.validate()
|
||||||
Self::load_from_file(path).await?
|
.context("Invalid logging configuration")?;
|
||||||
} else {
|
self.database
|
||||||
Self::default()
|
.validate()
|
||||||
};
|
.context("Invalid database configuration")?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn read_or_create(path: &Path) -> Result<Self> {
|
||||||
|
let display_path = path.canonicalize().unwrap_or_else(|_| path.to_path_buf());
|
||||||
|
|
||||||
|
if path.exists() {
|
||||||
|
info!("Loading configuration from `{}`", display_path.display());
|
||||||
|
Self::load_from_file(path).await
|
||||||
|
} else {
|
||||||
|
let config = Self::default();
|
||||||
config.write(path).await?;
|
config.write(path).await?;
|
||||||
info!(
|
info!(
|
||||||
"Updated configuration at `{}`",
|
"Created default configuration at `{}`",
|
||||||
path.canonicalize().unwrap().display()
|
display_path.display()
|
||||||
);
|
);
|
||||||
|
|
||||||
Ok(config)
|
Ok(config)
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
pub async fn load_from_file(path: &Path) -> Result<Self> {
|
pub async fn load_from_file(path: &Path) -> Result<Self> {
|
||||||
let contents = fs::read_to_string(path).await.with_context(|| {
|
let contents = fs::read_to_string(path).await.with_context(|| {
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,6 @@
|
||||||
use std::{path::PathBuf, time::Duration};
|
use std::{path::PathBuf, time::Duration};
|
||||||
|
|
||||||
|
use anyhow::{Result, ensure};
|
||||||
use log::debug;
|
use log::debug;
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
|
|
@ -34,6 +35,24 @@ fn default_cursor_timeout() -> Duration {
|
||||||
DEFAULT_CURSOR_TIMEOUT
|
DEFAULT_CURSOR_TIMEOUT
|
||||||
}
|
}
|
||||||
|
|
||||||
|
impl DatabaseConfig {
|
||||||
|
pub fn validate(&self) -> Result<()> {
|
||||||
|
ensure!(
|
||||||
|
!self.databases_directory_path.as_os_str().is_empty(),
|
||||||
|
"databases_directory_path must not be empty"
|
||||||
|
);
|
||||||
|
ensure!(
|
||||||
|
self.max_connections_per_vault > 0,
|
||||||
|
"max_connections_per_vault must be greater than 0"
|
||||||
|
);
|
||||||
|
ensure!(
|
||||||
|
!self.cursor_timeout.is_zero(),
|
||||||
|
"cursor_timeout must be greater than 0"
|
||||||
|
);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
impl Default for DatabaseConfig {
|
impl Default for DatabaseConfig {
|
||||||
fn default() -> Self {
|
fn default() -> Self {
|
||||||
Self {
|
Self {
|
||||||
|
|
|
||||||
|
|
@ -1,10 +1,13 @@
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
|
|
||||||
|
use anyhow::{Result, ensure};
|
||||||
use log::debug;
|
use log::debug;
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
use crate::{
|
use crate::{
|
||||||
consts::{DEFAULT_LOG_DIRECTORY, DEFAULT_LOG_LEVEL, DEFAULT_LOG_ROTATION_INTERVAL},
|
consts::{
|
||||||
|
DEFAULT_LOG_DIRECTORY, DEFAULT_LOG_LEVEL, DEFAULT_LOG_ROTATION_INTERVAL, DURATION_ZERO,
|
||||||
|
},
|
||||||
utils::log_level::LogLevel,
|
utils::log_level::LogLevel,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
@ -20,6 +23,20 @@ pub struct LoggingConfig {
|
||||||
pub log_level: LogLevel,
|
pub log_level: LogLevel,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
impl LoggingConfig {
|
||||||
|
pub fn validate(&self) -> Result<()> {
|
||||||
|
ensure!(
|
||||||
|
!self.log_directory.is_empty(),
|
||||||
|
"log_directory must not be an empty string"
|
||||||
|
);
|
||||||
|
ensure!(
|
||||||
|
self.log_rotation > DURATION_ZERO,
|
||||||
|
"log_rotation must be greater than 0"
|
||||||
|
);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
impl Default for LoggingConfig {
|
impl Default for LoggingConfig {
|
||||||
fn default() -> Self {
|
fn default() -> Self {
|
||||||
Self {
|
Self {
|
||||||
|
|
|
||||||
|
|
@ -1,10 +1,13 @@
|
||||||
|
use anyhow::{Result, ensure};
|
||||||
use log::debug;
|
use log::debug;
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
|
|
||||||
use crate::consts::{
|
use crate::consts::{
|
||||||
DEFAULT_HOST, DEFAULT_MAX_BODY_SIZE_MB, DEFAULT_MAX_CLIENTS_PER_VAULT,
|
DEFAULT_ALLOWED_ORIGINS, DEFAULT_BROADCAST_CHANNEL_CAPACITY, DEFAULT_HOST,
|
||||||
DEFAULT_MERGEABLE_FILE_EXTENSIONS, DEFAULT_PORT, DEFAULT_RESPONSE_TIMEOUT_SECONDS,
|
DEFAULT_MAX_BODY_SIZE_MB, DEFAULT_MAX_CLIENTS_PER_VAULT, DEFAULT_MAX_PENDING_WS_CONNECTIONS,
|
||||||
|
DEFAULT_MERGEABLE_FILE_EXTENSIONS, DEFAULT_PORT, DEFAULT_RATE_LIMIT_PER_USER_PER_SECOND,
|
||||||
|
DEFAULT_RESPONSE_TIMEOUT_SECONDS, DURATION_ZERO,
|
||||||
};
|
};
|
||||||
|
|
||||||
#[derive(Debug, Deserialize, Serialize, Clone, Default)]
|
#[derive(Debug, Deserialize, Serialize, Clone, Default)]
|
||||||
|
|
@ -21,11 +24,56 @@ pub struct ServerConfig {
|
||||||
#[serde(default = "default_max_clients_per_vault")]
|
#[serde(default = "default_max_clients_per_vault")]
|
||||||
pub max_clients_per_vault: usize,
|
pub max_clients_per_vault: usize,
|
||||||
|
|
||||||
|
#[serde(default = "default_broadcast_channel_capacity")]
|
||||||
|
pub broadcast_channel_capacity: usize,
|
||||||
|
|
||||||
#[serde(default = "default_response_timeout", with = "humantime_serde")]
|
#[serde(default = "default_response_timeout", with = "humantime_serde")]
|
||||||
pub response_timeout: Duration,
|
pub response_timeout: Duration,
|
||||||
|
|
||||||
#[serde(default = "default_mergeable_file_extensions")]
|
#[serde(default = "default_mergeable_file_extensions")]
|
||||||
pub mergeable_file_extensions: Vec<String>,
|
pub mergeable_file_extensions: Vec<String>,
|
||||||
|
|
||||||
|
/// Per-user maximum requests per second (keyed by bearer token).
|
||||||
|
/// `None` disables rate limiting.
|
||||||
|
#[serde(default = "default_rate_limit_per_user_per_second")]
|
||||||
|
pub rate_limit_per_user_per_second: Option<u64>,
|
||||||
|
|
||||||
|
/// Allowed CORS origins. Default: `["*"]` (allow all).
|
||||||
|
#[serde(default = "default_allowed_origins")]
|
||||||
|
pub allowed_origins: Vec<String>,
|
||||||
|
|
||||||
|
/// Maximum concurrent unauthenticated WebSocket connections waiting for
|
||||||
|
/// handshake. Limits resource consumption from clients that connect but
|
||||||
|
/// never authenticate.
|
||||||
|
#[serde(default = "default_max_pending_websocket_connections")]
|
||||||
|
pub max_pending_websocket_connections: usize,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ServerConfig {
|
||||||
|
pub fn validate(&self) -> Result<()> {
|
||||||
|
ensure!(
|
||||||
|
self.response_timeout > DURATION_ZERO,
|
||||||
|
"response_timeout must be greater than 0"
|
||||||
|
);
|
||||||
|
ensure!(
|
||||||
|
self.max_body_size_mb > 0,
|
||||||
|
"max_body_size_mb must be greater than 0"
|
||||||
|
);
|
||||||
|
ensure!(
|
||||||
|
self.max_clients_per_vault > 0,
|
||||||
|
"max_clients_per_vault must be greater than 0"
|
||||||
|
);
|
||||||
|
ensure!(
|
||||||
|
self.broadcast_channel_capacity > 0,
|
||||||
|
"broadcast_channel_capacity must be greater than 0"
|
||||||
|
);
|
||||||
|
ensure!(
|
||||||
|
self.max_pending_websocket_connections > 0,
|
||||||
|
"max_pending_websocket_connections must be greater than 0"
|
||||||
|
);
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn default_host() -> String {
|
fn default_host() -> String {
|
||||||
|
|
@ -48,6 +96,11 @@ fn default_max_clients_per_vault() -> usize {
|
||||||
DEFAULT_MAX_CLIENTS_PER_VAULT
|
DEFAULT_MAX_CLIENTS_PER_VAULT
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn default_broadcast_channel_capacity() -> usize {
|
||||||
|
debug!("Using default broadcast channel capacity: {DEFAULT_BROADCAST_CHANNEL_CAPACITY}");
|
||||||
|
DEFAULT_BROADCAST_CHANNEL_CAPACITY
|
||||||
|
}
|
||||||
|
|
||||||
fn default_response_timeout() -> Duration {
|
fn default_response_timeout() -> Duration {
|
||||||
debug!("Using default response timeout: {DEFAULT_RESPONSE_TIMEOUT_SECONDS:?}");
|
debug!("Using default response timeout: {DEFAULT_RESPONSE_TIMEOUT_SECONDS:?}");
|
||||||
DEFAULT_RESPONSE_TIMEOUT_SECONDS
|
DEFAULT_RESPONSE_TIMEOUT_SECONDS
|
||||||
|
|
@ -60,3 +113,21 @@ fn default_mergeable_file_extensions() -> Vec<String> {
|
||||||
.map(|s| (*s).to_owned())
|
.map(|s| (*s).to_owned())
|
||||||
.collect()
|
.collect()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn default_rate_limit_per_user_per_second() -> Option<u64> {
|
||||||
|
debug!("Using default rate limit per second: {DEFAULT_RATE_LIMIT_PER_USER_PER_SECOND:?}");
|
||||||
|
DEFAULT_RATE_LIMIT_PER_USER_PER_SECOND
|
||||||
|
}
|
||||||
|
|
||||||
|
fn default_allowed_origins() -> Vec<String> {
|
||||||
|
debug!("Using default allowed origins: {DEFAULT_ALLOWED_ORIGINS:?}");
|
||||||
|
DEFAULT_ALLOWED_ORIGINS
|
||||||
|
.iter()
|
||||||
|
.map(|s| (*s).to_owned())
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn default_max_pending_websocket_connections() -> usize {
|
||||||
|
debug!("Using default max pending WebSocket connections: {DEFAULT_MAX_PENDING_WS_CONNECTIONS}");
|
||||||
|
DEFAULT_MAX_PENDING_WS_CONNECTIONS
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,7 @@
|
||||||
use bimap::BiHashMap;
|
use bimap::BiHashMap;
|
||||||
use rand::{Rng, distr::Alphanumeric, rng};
|
use rand::{Rng, distr::Alphanumeric, rng};
|
||||||
use serde::{Deserialize, Deserializer, Serialize, de::Error};
|
use serde::{Deserialize, Deserializer, Serialize, de::Error};
|
||||||
|
use subtle::ConstantTimeEq;
|
||||||
|
|
||||||
use crate::app_state::database::models::VaultId;
|
use crate::app_state::database::models::VaultId;
|
||||||
|
|
||||||
|
|
@ -19,10 +20,19 @@ where
|
||||||
let mut user_token_map = BiHashMap::new();
|
let mut user_token_map = BiHashMap::new();
|
||||||
for user in &users {
|
for user in &users {
|
||||||
if let Some(existing_name) = user_token_map.get_by_right(&user.token) {
|
if let Some(existing_name) = user_token_map.get_by_right(&user.token) {
|
||||||
|
let redacted = if user.token.len() > 6 {
|
||||||
|
format!(
|
||||||
|
"{}...{}",
|
||||||
|
&user.token[..3],
|
||||||
|
&user.token[user.token.len() - 3..]
|
||||||
|
)
|
||||||
|
} else {
|
||||||
|
"***".to_owned()
|
||||||
|
};
|
||||||
return Err(D::Error::custom(format!(
|
return Err(D::Error::custom(format!(
|
||||||
"Duplicate user token found: `{}` for users `{}` and `{}`. User tokens must be \
|
"Duplicate user token found: `{redacted}` for users `{}` and `{}`. User tokens \
|
||||||
unique.",
|
must be unique.",
|
||||||
user.token, existing_name, user.name
|
existing_name, user.name
|
||||||
)));
|
)));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -41,7 +51,9 @@ where
|
||||||
|
|
||||||
impl UserConfig {
|
impl UserConfig {
|
||||||
pub fn get_user(&self, token: &str) -> Option<&User> {
|
pub fn get_user(&self, token: &str) -> Option<&User> {
|
||||||
self.user_configs.iter().find(|u| u.token == token)
|
self.user_configs
|
||||||
|
.iter()
|
||||||
|
.find(|u| u.token.as_bytes().ct_eq(token.as_bytes()).into())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -2,22 +2,36 @@ use std::time::Duration;
|
||||||
|
|
||||||
use crate::utils::log_level::LogLevel;
|
use crate::utils::log_level::LogLevel;
|
||||||
|
|
||||||
|
pub const DURATION_ZERO: Duration = Duration::from_secs(0);
|
||||||
|
|
||||||
pub const DEFAULT_CONFIG_PATH: &str = "config.yml";
|
pub const DEFAULT_CONFIG_PATH: &str = "config.yml";
|
||||||
|
|
||||||
pub const DEFAULT_DATABASES_DIRECTORY_PATH: &str = "databases";
|
pub const DEFAULT_DATABASES_DIRECTORY_PATH: &str = "databases";
|
||||||
pub const DEFAULT_MAX_CONNECTIONS_PER_VAULT: u32 = 12;
|
pub const DEFAULT_MAX_CONNECTIONS_PER_VAULT: u32 = 6;
|
||||||
pub const DEFAULT_CURSOR_TIMEOUT: Duration = Duration::from_secs(60);
|
pub const DEFAULT_CURSOR_TIMEOUT: Duration = Duration::from_secs(60);
|
||||||
|
|
||||||
pub const DEFAULT_HOST: &str = "127.0.0.1";
|
pub const DEFAULT_HOST: &str = "127.0.0.1";
|
||||||
pub const DEFAULT_PORT: u16 = 3000;
|
pub const DEFAULT_PORT: u16 = 3000;
|
||||||
pub const DEFAULT_MAX_BODY_SIZE_MB: usize = 4096;
|
pub const DEFAULT_MAX_BODY_SIZE_MB: usize = 4096;
|
||||||
pub const DEFAULT_RESPONSE_TIMEOUT_SECONDS: Duration = Duration::from_secs(1800);
|
pub const DEFAULT_RESPONSE_TIMEOUT_SECONDS: Duration = Duration::from_mins(30);
|
||||||
pub const DEFAULT_MAX_CLIENTS_PER_VAULT: usize = 256;
|
pub const DEFAULT_MAX_CLIENTS_PER_VAULT: usize = 256;
|
||||||
|
pub const DEFAULT_BROADCAST_CHANNEL_CAPACITY: usize = 4096;
|
||||||
|
pub const DEFAULT_MAX_PENDING_WS_CONNECTIONS: usize = 128;
|
||||||
|
|
||||||
pub const DEFAULT_LOG_DIRECTORY: &str = "logs";
|
pub const DEFAULT_LOG_DIRECTORY: &str = "logs";
|
||||||
pub const DEFAULT_LOG_ROTATION_INTERVAL: Duration = Duration::from_secs(60 * 60 * 24); // 1 day
|
pub const DEFAULT_LOG_ROTATION_INTERVAL: Duration = Duration::from_hours(24);
|
||||||
|
pub const IDLE_POOL_TIMEOUT: Duration = Duration::from_mins(5);
|
||||||
|
pub const GRACEFUL_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(10);
|
||||||
|
pub const HANDSHAKE_TIMEOUT: Duration = Duration::from_secs(10);
|
||||||
|
|
||||||
|
pub const MAX_CURSOR_DOCUMENTS: usize = 1000;
|
||||||
|
pub const MAX_CURSORS_PER_DOCUMENT: usize = 100;
|
||||||
|
pub const MAX_RELATIVE_PATH_LEN: usize = 4096;
|
||||||
|
|
||||||
pub const DEFAULT_LOG_LEVEL: LogLevel = LogLevel::Info;
|
pub const DEFAULT_LOG_LEVEL: LogLevel = LogLevel::Info;
|
||||||
|
|
||||||
pub const DEFAULT_MERGEABLE_FILE_EXTENSIONS: &[&str] = &["md", "txt"];
|
pub const DEFAULT_MERGEABLE_FILE_EXTENSIONS: &[&str] = &["md", "txt"];
|
||||||
|
|
||||||
pub const SUPPORTED_API_VERSION: u32 = 2;
|
pub const DEFAULT_RATE_LIMIT_PER_USER_PER_SECOND: Option<u64> = None;
|
||||||
|
pub const DEFAULT_ALLOWED_ORIGINS: &[&str] = &["*"];
|
||||||
|
pub const SUPPORTED_API_VERSION: u32 = 3;
|
||||||
|
|
|
||||||
|
|
@ -5,7 +5,7 @@ use axum::{
|
||||||
http::StatusCode,
|
http::StatusCode,
|
||||||
response::{IntoResponse, Response},
|
response::{IntoResponse, Response},
|
||||||
};
|
};
|
||||||
use log::{debug, error};
|
use log::{debug, error, warn};
|
||||||
use serde::Serialize;
|
use serde::Serialize;
|
||||||
use thiserror::Error;
|
use thiserror::Error;
|
||||||
use ts_rs::TS;
|
use ts_rs::TS;
|
||||||
|
|
@ -29,6 +29,9 @@ pub enum SyncServerError {
|
||||||
|
|
||||||
#[error("Permission denied error: {0}")]
|
#[error("Permission denied error: {0}")]
|
||||||
PermissionDeniedError(#[source] anyhow::Error),
|
PermissionDeniedError(#[source] anyhow::Error),
|
||||||
|
|
||||||
|
#[error("Too many requests: {0}")]
|
||||||
|
TooManyRequests(#[source] anyhow::Error),
|
||||||
}
|
}
|
||||||
|
|
||||||
impl SyncServerError {
|
impl SyncServerError {
|
||||||
|
|
@ -39,7 +42,8 @@ impl SyncServerError {
|
||||||
| Self::ServerError(error)
|
| Self::ServerError(error)
|
||||||
| Self::NotFound(error)
|
| Self::NotFound(error)
|
||||||
| Self::Unauthenticated(error)
|
| Self::Unauthenticated(error)
|
||||||
| Self::PermissionDeniedError(error) => error.into(),
|
| Self::PermissionDeniedError(error)
|
||||||
|
| Self::TooManyRequests(error) => error.into(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -69,7 +73,22 @@ impl Display for SerializedError {
|
||||||
|
|
||||||
impl IntoResponse for SyncServerError {
|
impl IntoResponse for SyncServerError {
|
||||||
fn into_response(self) -> Response {
|
fn into_response(self) -> Response {
|
||||||
let body = Json(self.serialize());
|
let serialized = self.serialize();
|
||||||
|
|
||||||
|
match &self {
|
||||||
|
Self::InitError(_) | Self::ServerError(_) => {
|
||||||
|
error!("{serialized}");
|
||||||
|
}
|
||||||
|
Self::ClientError(_) | Self::NotFound(_) => {
|
||||||
|
warn!("{serialized}");
|
||||||
|
}
|
||||||
|
Self::TooManyRequests(_) => {
|
||||||
|
warn!("{serialized}");
|
||||||
|
}
|
||||||
|
Self::Unauthenticated(_) | Self::PermissionDeniedError(_) => {}
|
||||||
|
}
|
||||||
|
|
||||||
|
let body = Json(serialized);
|
||||||
|
|
||||||
match self {
|
match self {
|
||||||
Self::InitError(_) | Self::ServerError(_) => {
|
Self::InitError(_) | Self::ServerError(_) => {
|
||||||
|
|
@ -79,6 +98,7 @@ impl IntoResponse for SyncServerError {
|
||||||
Self::NotFound(_) => (StatusCode::NOT_FOUND, body).into_response(),
|
Self::NotFound(_) => (StatusCode::NOT_FOUND, body).into_response(),
|
||||||
Self::Unauthenticated(_) => (StatusCode::UNAUTHORIZED, body).into_response(),
|
Self::Unauthenticated(_) => (StatusCode::UNAUTHORIZED, body).into_response(),
|
||||||
Self::PermissionDeniedError(_) => (StatusCode::FORBIDDEN, body).into_response(),
|
Self::PermissionDeniedError(_) => (StatusCode::FORBIDDEN, body).into_response(),
|
||||||
|
Self::TooManyRequests(_) => (StatusCode::TOO_MANY_REQUESTS, body).into_response(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -102,6 +122,7 @@ impl From<&anyhow::Error> for SerializedError {
|
||||||
SyncServerError::NotFound(_) => "NotFound",
|
SyncServerError::NotFound(_) => "NotFound",
|
||||||
SyncServerError::Unauthenticated(_) => "Unauthenticated",
|
SyncServerError::Unauthenticated(_) => "Unauthenticated",
|
||||||
SyncServerError::PermissionDeniedError(_) => "PermissionDeniedError",
|
SyncServerError::PermissionDeniedError(_) => "PermissionDeniedError",
|
||||||
|
SyncServerError::TooManyRequests(_) => "TooManyRequests",
|
||||||
},
|
},
|
||||||
),
|
),
|
||||||
message: error.to_string(),
|
message: error.to_string(),
|
||||||
|
|
@ -139,3 +160,21 @@ pub fn permission_denied_error(error: anyhow::Error) -> SyncServerError {
|
||||||
debug!("Permission denied: {error:?}");
|
debug!("Permission denied: {error:?}");
|
||||||
SyncServerError::PermissionDeniedError(error)
|
SyncServerError::PermissionDeniedError(error)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn too_many_requests_error(error: anyhow::Error) -> SyncServerError {
|
||||||
|
debug!("Too many requests: {error:?}");
|
||||||
|
SyncServerError::TooManyRequests(error)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Maps a `create_write_transaction` error to 429 if the database is busy,
|
||||||
|
/// or 500 for all other failures.
|
||||||
|
pub fn write_transaction_error(error: anyhow::Error) -> SyncServerError {
|
||||||
|
if error
|
||||||
|
.downcast_ref::<crate::app_state::database::WriteBusyError>()
|
||||||
|
.is_some()
|
||||||
|
{
|
||||||
|
too_many_requests_error(error)
|
||||||
|
} else {
|
||||||
|
server_error(error)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -16,6 +16,7 @@ use consts::DEFAULT_CONFIG_PATH;
|
||||||
use errors::{SyncServerError, init_error};
|
use errors::{SyncServerError, init_error};
|
||||||
use log::info;
|
use log::info;
|
||||||
use server::create_server;
|
use server::create_server;
|
||||||
|
use tracing_appender::non_blocking::WorkerGuard;
|
||||||
use tracing_subscriber::{EnvFilter, fmt::format, layer::SubscriberExt, util::SubscriberInitExt};
|
use tracing_subscriber::{EnvFilter, fmt::format, layer::SubscriberExt, util::SubscriberInitExt};
|
||||||
use utils::rotating_file_writer::RotatingFileWriter;
|
use utils::rotating_file_writer::RotatingFileWriter;
|
||||||
|
|
||||||
|
|
@ -41,11 +42,14 @@ async fn main() -> ExitCode {
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
let mut result = set_up_logging(&args, &config.logging);
|
let result = async {
|
||||||
|
config.validate().map_err(init_error)?;
|
||||||
if result.is_ok() {
|
// Hold the non-blocking writer guards until shutdown so the
|
||||||
result = start_server(config).await;
|
// dedicated writer threads stay alive and flush queued log lines.
|
||||||
|
let _log_guards = set_up_logging(&args, &config.logging)?;
|
||||||
|
start_server(config).await
|
||||||
}
|
}
|
||||||
|
.await;
|
||||||
|
|
||||||
match result {
|
match result {
|
||||||
Ok(()) => ExitCode::SUCCESS,
|
Ok(()) => ExitCode::SUCCESS,
|
||||||
|
|
@ -59,7 +63,7 @@ async fn main() -> ExitCode {
|
||||||
fn set_up_logging(
|
fn set_up_logging(
|
||||||
args: &Args,
|
args: &Args,
|
||||||
logging_config: &config::logging_config::LoggingConfig,
|
logging_config: &config::logging_config::LoggingConfig,
|
||||||
) -> Result<(), SyncServerError> {
|
) -> Result<[WorkerGuard; 2], SyncServerError> {
|
||||||
let level_filter = logging_config.log_level.as_tracing_level();
|
let level_filter = logging_config.log_level.as_tracing_level();
|
||||||
|
|
||||||
let env_filter = EnvFilter::builder()
|
let env_filter = EnvFilter::builder()
|
||||||
|
|
@ -80,6 +84,14 @@ fn set_up_logging(
|
||||||
.context("Failed to create rotating file writer")
|
.context("Failed to create rotating file writer")
|
||||||
.map_err(init_error)?;
|
.map_err(init_error)?;
|
||||||
|
|
||||||
|
// Decouple log emission from disk/stderr I/O. Without this, a tokio
|
||||||
|
// worker that holds the writer's std::sync::Mutex while a `write(2)`
|
||||||
|
// is throttled by the kernel (e.g. btrfs writeback) cascades the
|
||||||
|
// stall to every other worker that tries to log, freezing the whole
|
||||||
|
// runtime. The guards must outlive every emitter.
|
||||||
|
let (file_writer, file_guard) = tracing_appender::non_blocking(file_appender);
|
||||||
|
let (stderr_writer, stderr_guard) = tracing_appender::non_blocking(std::io::stderr());
|
||||||
|
|
||||||
let format = format()
|
let format = format()
|
||||||
.with_target(is_debug_mode)
|
.with_target(is_debug_mode)
|
||||||
.with_line_number(is_debug_mode)
|
.with_line_number(is_debug_mode)
|
||||||
|
|
@ -87,12 +99,12 @@ fn set_up_logging(
|
||||||
|
|
||||||
let stderr_layer = tracing_subscriber::fmt::layer()
|
let stderr_layer = tracing_subscriber::fmt::layer()
|
||||||
.with_ansi(use_colors)
|
.with_ansi(use_colors)
|
||||||
.with_writer(std::io::stderr)
|
.with_writer(stderr_writer)
|
||||||
.event_format(format.clone());
|
.event_format(format.clone());
|
||||||
|
|
||||||
let file_layer = tracing_subscriber::fmt::layer()
|
let file_layer = tracing_subscriber::fmt::layer()
|
||||||
.with_ansi(false)
|
.with_ansi(false)
|
||||||
.with_writer(file_appender)
|
.with_writer(file_writer)
|
||||||
.event_format(format);
|
.event_format(format);
|
||||||
|
|
||||||
tracing_subscriber::registry()
|
tracing_subscriber::registry()
|
||||||
|
|
@ -103,7 +115,7 @@ fn set_up_logging(
|
||||||
.context("Failed to initialise tracing")
|
.context("Failed to initialise tracing")
|
||||||
.map_err(init_error)?;
|
.map_err(init_error)?;
|
||||||
|
|
||||||
Ok(())
|
Ok([file_guard, stderr_guard])
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn start_server(config: Config) -> Result<(), SyncServerError> {
|
async fn start_server(config: Config) -> Result<(), SyncServerError> {
|
||||||
|
|
|
||||||
|
|
@ -1,8 +1,17 @@
|
||||||
|
use std::sync::LazyLock;
|
||||||
|
|
||||||
use regex::Regex;
|
use regex::Regex;
|
||||||
|
|
||||||
|
static DEDUP_SUFFIX_REGEX: LazyLock<Regex> =
|
||||||
|
LazyLock::new(|| Regex::new(r" \((\d+)\)$").expect("invalid regex"));
|
||||||
|
|
||||||
pub fn dedup_paths(path: &str) -> impl Iterator<Item = String> {
|
pub fn dedup_paths(path: &str) -> impl Iterator<Item = String> {
|
||||||
let mut path_parts = path.split('/').collect::<Vec<_>>();
|
let mut path_parts = path.split('/').collect::<Vec<_>>();
|
||||||
let file_name = path_parts.pop().unwrap().to_owned();
|
let file_name = path_parts
|
||||||
|
.pop()
|
||||||
|
.filter(|s| !s.is_empty())
|
||||||
|
.unwrap_or(path)
|
||||||
|
.to_owned();
|
||||||
|
|
||||||
let mut directory = path_parts.join("/");
|
let mut directory = path_parts.join("/");
|
||||||
if !directory.is_empty() {
|
if !directory.is_empty() {
|
||||||
|
|
@ -29,14 +38,13 @@ pub fn dedup_paths(path: &str) -> impl Iterator<Item = String> {
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
let regex = Regex::new(r" \((\d+)\)$").unwrap();
|
let start_number = DEDUP_SUFFIX_REGEX
|
||||||
let start_number = regex
|
|
||||||
.captures(&stem)
|
.captures(&stem)
|
||||||
.and_then(|caps| caps.get(1))
|
.and_then(|caps| caps.get(1))
|
||||||
.and_then(|m| m.as_str().parse::<u32>().ok())
|
.and_then(|m| m.as_str().parse::<u32>().ok())
|
||||||
.unwrap_or(0);
|
.unwrap_or(0);
|
||||||
|
|
||||||
let clean_stem = regex.replace(&stem, "").to_string();
|
let clean_stem = DEDUP_SUFFIX_REGEX.replace(&stem, "").to_string();
|
||||||
|
|
||||||
(start_number..).map(move |dedup_number| {
|
(start_number..).map(move |dedup_number| {
|
||||||
if dedup_number == 0 {
|
if dedup_number == 0 {
|
||||||
|
|
|
||||||
|
|
@ -1,25 +1,30 @@
|
||||||
use crate::app_state::database::models::VaultId;
|
use crate::app_state::database::models::VaultId;
|
||||||
use crate::{app_state::database::Transaction, utils::dedup_paths::dedup_paths};
|
use crate::utils::dedup_paths::dedup_paths;
|
||||||
use anyhow::Result;
|
use anyhow::Result;
|
||||||
use log::{debug, info};
|
use log::{debug, info};
|
||||||
|
use sqlx::sqlite::SqliteConnection;
|
||||||
|
|
||||||
pub async fn find_first_available_path(
|
pub async fn find_first_available_path(
|
||||||
vault_id: &VaultId,
|
vault_id: &VaultId,
|
||||||
sanitized_relative_path: &str,
|
sanitized_relative_path: &str,
|
||||||
database: &crate::app_state::database::Database,
|
database: &crate::app_state::database::Database,
|
||||||
transaction: &mut Transaction<'_>,
|
connection: &mut SqliteConnection,
|
||||||
) -> Result<String> {
|
) -> Result<String> {
|
||||||
info!("Finding first available path for `{sanitized_relative_path}` in vault `{vault_id}`");
|
info!("Finding first available path for `{sanitized_relative_path}` in vault `{vault_id}`");
|
||||||
for candidate in dedup_paths(sanitized_relative_path) {
|
for candidate in dedup_paths(sanitized_relative_path) {
|
||||||
debug!("Checking candidate path for deconflicting names: `{candidate}`");
|
debug!("Checking candidate path for deconflicting names: `{candidate}`");
|
||||||
if database
|
if database
|
||||||
.get_latest_document_by_path(vault_id, &candidate, Some(transaction))
|
.get_latest_non_deleted_document_by_path(vault_id, &candidate, Some(connection))
|
||||||
.await?
|
.await?
|
||||||
.is_none()
|
.is_none()
|
||||||
{
|
{
|
||||||
info!("Selected available path: `{candidate}`");
|
info!("Selected available path: `{candidate}`");
|
||||||
return Ok(candidate);
|
return Ok(candidate);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
info!(
|
||||||
|
"Finding first available path for `{sanitized_relative_path}` in vault `{vault_id}` as `{candidate}` is already taken"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
unreachable!("dedup_paths produces infinite paths");
|
unreachable!("dedup_paths produces infinite paths");
|
||||||
|
|
|
||||||
|
|
@ -6,7 +6,7 @@ use std::{
|
||||||
time::{Duration, SystemTime, UNIX_EPOCH},
|
time::{Duration, SystemTime, UNIX_EPOCH},
|
||||||
};
|
};
|
||||||
|
|
||||||
use chrono::{Local, NaiveDateTime};
|
use chrono::NaiveDateTime;
|
||||||
use tracing_subscriber::fmt::MakeWriter;
|
use tracing_subscriber::fmt::MakeWriter;
|
||||||
|
|
||||||
#[derive(Clone)]
|
#[derive(Clone)]
|
||||||
|
|
@ -55,7 +55,7 @@ impl RotatingFileWriter {
|
||||||
let timestamp_str = filename.get(prefix_len..filename.len().checked_sub(4)?)?;
|
let timestamp_str = filename.get(prefix_len..filename.len().checked_sub(4)?)?;
|
||||||
|
|
||||||
let dt = NaiveDateTime::parse_from_str(timestamp_str, "%Y-%m-%d_%H-%M-%S").ok()?;
|
let dt = NaiveDateTime::parse_from_str(timestamp_str, "%Y-%m-%d_%H-%M-%S").ok()?;
|
||||||
let timestamp = dt.and_local_timezone(Local).single()?;
|
let timestamp = dt.and_utc();
|
||||||
let secs: u64 = timestamp.timestamp().try_into().ok()?;
|
let secs: u64 = timestamp.timestamp().try_into().ok()?;
|
||||||
|
|
||||||
Some(UNIX_EPOCH + Duration::from_secs(secs))
|
Some(UNIX_EPOCH + Duration::from_secs(secs))
|
||||||
|
|
@ -114,7 +114,7 @@ impl RotatingFileWriter {
|
||||||
}
|
}
|
||||||
|
|
||||||
fn rotate(inner: &mut RotatingFileWriterInner) -> io::Result<()> {
|
fn rotate(inner: &mut RotatingFileWriterInner) -> io::Result<()> {
|
||||||
let timestamp = Local::now().format("%Y-%m-%d_%H-%M-%S");
|
let timestamp = chrono::Utc::now().format("%Y-%m-%d_%H-%M-%S");
|
||||||
let filename = format!("{}.{}.log", inner.file_prefix, timestamp);
|
let filename = format!("{}.{}.log", inner.file_prefix, timestamp);
|
||||||
let filepath = inner.directory.join(filename);
|
let filepath = inner.directory.join(filename);
|
||||||
|
|
||||||
|
|
@ -132,8 +132,14 @@ impl RotatingFileWriter {
|
||||||
|
|
||||||
impl Write for RotatingFileWriter {
|
impl Write for RotatingFileWriter {
|
||||||
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
|
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
|
||||||
let mut inner = self.inner.lock().unwrap();
|
let mut inner = self.inner.lock().unwrap_or_else(|poisoned| {
|
||||||
|
eprintln!("RotatingFileWriter mutex was poisoned, recovering");
|
||||||
|
poisoned.into_inner()
|
||||||
|
});
|
||||||
|
|
||||||
|
// Reset file handle after poison recovery so the next branch
|
||||||
|
// re-opens a valid file rather than writing to a potentially
|
||||||
|
// half-closed handle.
|
||||||
if inner.current_file.is_none() {
|
if inner.current_file.is_none() {
|
||||||
Self::open_or_create_log_file(&mut inner)?;
|
Self::open_or_create_log_file(&mut inner)?;
|
||||||
} else if Self::should_rotate(&inner) {
|
} else if Self::should_rotate(&inner) {
|
||||||
|
|
@ -148,7 +154,10 @@ impl Write for RotatingFileWriter {
|
||||||
}
|
}
|
||||||
|
|
||||||
fn flush(&mut self) -> io::Result<()> {
|
fn flush(&mut self) -> io::Result<()> {
|
||||||
let mut inner = self.inner.lock().unwrap();
|
let mut inner = self.inner.lock().unwrap_or_else(|poisoned| {
|
||||||
|
eprintln!("RotatingFileWriter mutex was poisoned, recovering");
|
||||||
|
poisoned.into_inner()
|
||||||
|
});
|
||||||
if let Some(ref mut file) = inner.current_file {
|
if let Some(ref mut file) = inner.current_file {
|
||||||
file.flush()
|
file.flush()
|
||||||
} else {
|
} else {
|
||||||
|
|
@ -267,7 +276,7 @@ mod tests {
|
||||||
// Parse the expected time
|
// Parse the expected time
|
||||||
let expected_dt =
|
let expected_dt =
|
||||||
NaiveDateTime::parse_from_str(timestamp_str, "%Y-%m-%d_%H-%M-%S").unwrap();
|
NaiveDateTime::parse_from_str(timestamp_str, "%Y-%m-%d_%H-%M-%S").unwrap();
|
||||||
let expected_timestamp = expected_dt.and_local_timezone(Local).single().unwrap();
|
let expected_timestamp = expected_dt.and_utc();
|
||||||
let expected_duration =
|
let expected_duration =
|
||||||
Duration::from_secs(expected_timestamp.timestamp().try_into().unwrap());
|
Duration::from_secs(expected_timestamp.timestamp().try_into().unwrap());
|
||||||
let expected_next = UNIX_EPOCH + expected_duration + rotation_duration;
|
let expected_next = UNIX_EPOCH + expected_duration + rotation_duration;
|
||||||
|
|
@ -306,7 +315,7 @@ mod tests {
|
||||||
// Should use the latest file (2025-10-26_14-00-00)
|
// Should use the latest file (2025-10-26_14-00-00)
|
||||||
let expected_dt =
|
let expected_dt =
|
||||||
NaiveDateTime::parse_from_str("2025-10-26_14-00-00", "%Y-%m-%d_%H-%M-%S").unwrap();
|
NaiveDateTime::parse_from_str("2025-10-26_14-00-00", "%Y-%m-%d_%H-%M-%S").unwrap();
|
||||||
let expected_timestamp = expected_dt.and_local_timezone(Local).single().unwrap();
|
let expected_timestamp = expected_dt.and_utc();
|
||||||
let expected_duration =
|
let expected_duration =
|
||||||
Duration::from_secs(expected_timestamp.timestamp().try_into().unwrap());
|
Duration::from_secs(expected_timestamp.timestamp().try_into().unwrap());
|
||||||
let expected_next = UNIX_EPOCH + expected_duration + rotation_duration;
|
let expected_next = UNIX_EPOCH + expected_duration + rotation_duration;
|
||||||
|
|
|
||||||
|
|
@ -1,14 +1,28 @@
|
||||||
|
use anyhow::{Result, ensure};
|
||||||
|
|
||||||
|
use crate::consts::MAX_RELATIVE_PATH_LEN;
|
||||||
|
|
||||||
/// Sanitize the document's path to allow all clients to create the same path in
|
/// Sanitize the document's path to allow all clients to create the same path in
|
||||||
/// their filesystem. If we didn't do this server-side, client's would need to
|
/// their filesystem. If we didn't do this server-side, client's would need to
|
||||||
/// deal with mapping invalid names to valid ones and then back.
|
/// deal with mapping invalid names to valid ones and then back.
|
||||||
pub fn sanitize_path(path: &str) -> String {
|
pub fn sanitize_path(path: &str) -> Result<String> {
|
||||||
|
// Enforce the length cap at the single chokepoint every create/update
|
||||||
|
// handler goes through, so clients can't blow up axum's JSON/multipart
|
||||||
|
// parser with a 1 MB `relative_path` before the handler ever runs.
|
||||||
|
// The WebSocket cursor handler enforces this separately.
|
||||||
|
ensure!(
|
||||||
|
path.len() <= MAX_RELATIVE_PATH_LEN,
|
||||||
|
"Relative path exceeds the maximum length of {MAX_RELATIVE_PATH_LEN} bytes"
|
||||||
|
);
|
||||||
|
|
||||||
let options = sanitize_filename::Options {
|
let options = sanitize_filename::Options {
|
||||||
truncate: true,
|
truncate: true,
|
||||||
windows: true, // Windows is the lowest common denominator
|
windows: true, // Windows is the lowest common denominator
|
||||||
replacement: "",
|
replacement: "",
|
||||||
};
|
};
|
||||||
|
|
||||||
path.split('/')
|
let result = path
|
||||||
|
.split('/')
|
||||||
.map(|part| {
|
.map(|part| {
|
||||||
let proposal = sanitize_filename::sanitize_with_options(part, options.clone());
|
let proposal = sanitize_filename::sanitize_with_options(part, options.clone());
|
||||||
if !part.is_empty() && proposal.is_empty() {
|
if !part.is_empty() && proposal.is_empty() {
|
||||||
|
|
@ -18,7 +32,13 @@ pub fn sanitize_path(path: &str) -> String {
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
.collect::<Vec<_>>()
|
.collect::<Vec<_>>()
|
||||||
.join("/")
|
.join("/");
|
||||||
|
|
||||||
|
ensure!(
|
||||||
|
!result.is_empty(),
|
||||||
|
"Relative path is empty after sanitization"
|
||||||
|
);
|
||||||
|
Ok(result)
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
|
|
@ -27,8 +47,32 @@ mod test {
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_sanitize_path() {
|
fn test_sanitize_path() {
|
||||||
assert_eq!(sanitize_path("/my/path/what?"), "/my/path/what");
|
assert_eq!(sanitize_path("/my/path/what?").unwrap(), "/my/path/what");
|
||||||
assert_eq!(sanitize_path("file (1).md"), "file (1).md");
|
assert_eq!(sanitize_path("file (1).md").unwrap(), "file (1).md");
|
||||||
assert_eq!(sanitize_path("/my/path/\\\\:?"), "/my/path/_");
|
assert_eq!(sanitize_path("/my/path/\\\\:?").unwrap(), "/my/path/_");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_sanitize_path_empty() {
|
||||||
|
assert!(sanitize_path("").is_err());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_sanitize_path_idempotent_simple() {
|
||||||
|
let mut result = sanitize_path("notes/my file.md").unwrap();
|
||||||
|
for _ in 0..5 {
|
||||||
|
result = sanitize_path(&result).unwrap();
|
||||||
|
}
|
||||||
|
assert_eq!(result, "notes/my file.md");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_sanitize_path_idempotent_special_chars() {
|
||||||
|
let first = sanitize_path("/my/path/what?/file:name<>.md").unwrap();
|
||||||
|
let mut result = first.clone();
|
||||||
|
for _ in 0..5 {
|
||||||
|
result = sanitize_path(&result).unwrap();
|
||||||
|
}
|
||||||
|
assert_eq!(result, first);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue