diff --git a/.github/dependabot.yml b/.github/dependabot.yml index b445fda5..7d56669b 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -6,7 +6,7 @@ version: 2 updates: - package-ecosystem: "npm" - directories: ["/frontend"] + directories: ["/frontend", "/docs"] schedule: interval: "daily" diff --git a/.github/workflows/deploy-docs.yml b/.github/workflows/deploy-docs.yml new file mode 100644 index 00000000..e1c3bcf8 --- /dev/null +++ b/.github/workflows/deploy-docs.yml @@ -0,0 +1,75 @@ +name: Deploy Documentation + +on: + push: + branches: + - main + paths: + - 'docs/**' + - '.github/workflows/deploy-docs.yml' + workflow_dispatch: + +permissions: + contents: read + pages: write + id-token: write + +concurrency: + group: pages + cancel-in-progress: false + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Setup Node + uses: actions/setup-node@v4 + with: + node-version: 20 + cache: npm + cache-dependency-path: docs/package-lock.json + + - name: Setup Pages + uses: actions/configure-pages@v4 + + - name: Install dependencies + run: | + cd docs + npm ci + + - name: Check formatting + run: | + cd docs + npm run format:check + + - name: Check spelling + run: | + cd docs + npm run spell:check + + - name: Build documentation + run: | + cd docs + npm run build + + - name: Upload artifact + uses: actions/upload-pages-artifact@v3 + with: + path: docs/.vitepress/dist + + deploy: + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + needs: build + runs-on: ubuntu-latest + name: Deploy + steps: + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v4 diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index c540f1e4..0ec25803 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -5,6 +5,12 @@ on: branches: ["main"] pull_request: branches: ["main"] + schedule: + - cron: '0 */4 * * *' + +concurrency: + group: e2e-tests + cancel-in-progress: false env: CARGO_TERM_COLOR: always @@ -42,4 +48,4 @@ jobs: cargo run config-e2e.yml --color never & cd .. - scripts/e2e.sh 32 + scripts/e2e.sh 8 diff --git a/CLAUDE.md b/CLAUDE.md index e05e784a..6f1bff23 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -29,7 +29,10 @@ cd sync-server cargo run config-e2e.yml # Start development server cargo test --verbose # Run Rust tests cargo clippy --all-targets --all-features # Lint Rust code +cargo clippy --all-targets --all-features --fix --allow-dirty --allow-staged # Auto-fix clippy warnings cargo fmt --all -- --check # Check Rust formatting +cargo fmt --all # Auto-format Rust code +cargo machete --with-metadata # Detect unused dependencies ``` ### Frontend Development @@ -49,8 +52,15 @@ sqlx migrate run --source src/app_state/database/migrations --database-url sqlit cargo sqlx prepare --workspace ``` +### Initial Setup +```bash +# Install required cargo tools +cargo install sqlx-cli cargo-machete cargo-edit +``` + ### Scripts - `scripts/check.sh`: Full CI check (builds, lints, tests both server and frontend) +- `scripts/check.sh --fix`: Same as above but auto-fixes linting and formatting issues - `scripts/e2e.sh`: End-to-end testing - `scripts/clean-up.sh`: Clean logs and database files - `scripts/bump-version.sh patch`: Publish new version @@ -59,10 +69,11 @@ cargo sqlx prepare --workspace ## Code Structure ### Workspace Configuration -The frontend uses npm workspaces with three packages: +The frontend uses npm workspaces with four packages: - `sync-client`: Core synchronization logic - `obsidian-plugin`: Obsidian-specific integration - `test-client`: Testing utilities +- `local-client-cli`: Standalone CLI for VaultLink sync client ### Type Generation Rust structs generate TypeScript types via ts-rs crate, stored in `sync-server/bindings/` and used by frontend packages. diff --git a/docs/.cspell.json b/docs/.cspell.json new file mode 100644 index 00000000..4967ec16 --- /dev/null +++ b/docs/.cspell.json @@ -0,0 +1,92 @@ +{ + "version": "0.2", + "language": "en-GB", + "dictionaries": ["en-gb"], + "ignorePaths": [ + "node_modules", + ".vitepress/dist", + ".vitepress/cache", + "package-lock.json" + ], + "words": [ + "VaultLink", + "Obsidian", + "WebSocket", + "SQLite", + "codebase", + "CRDT", + "CRDTs", + "YAML", + "nginx", + "Caddy", + "Traefik", + "systemd", + "localhost", + "vaultlink", + "Axum", + "Tokio", + "SQLx", + "reconcile", + "postgresql", + "VitePress", + "markdownlint", + "filesystem", + "backend", + "frontend", + "macOS", + "CLI", + "API", + "JSON", + "HTTP", + "HTTPS", + "SSL", + "TLS", + "WSS", + "TCP", + "VPS", + "Docker", + "Github", + "Dockerfile", + "dockerignore", + "Rustup", + "PostgreSQL", + "UUID", + "CORS", + "HSTS", + "CI", + "CD", + "OpenSSL", + "README", + "config", + "submodule", + "repo", + "autocomplete", + "autoformat", + "dedupe", + "diff", + "grep", + "stdout", + "stderr", + "chmod", + "mkdir", + "rclone", + "uuidgen", + "letsencrypt", + "fullchain", + "privkey", + "schmelczer", + "Schmelczer", + "ghcr", + "keepalive", + "healthcheck", + "writable", + "Cloudant", + "Syncthing", + "cadvisor", + "Caddyfile", + "nodelay", + "websecure", + "certresolver", + "rootfs" + ] +} diff --git a/docs/.gitignore b/docs/.gitignore new file mode 100644 index 00000000..da61f8d6 --- /dev/null +++ b/docs/.gitignore @@ -0,0 +1,4 @@ +node_modules/ +.vitepress/dist/ +.vitepress/cache/ +package-lock.json diff --git a/docs/.prettierignore b/docs/.prettierignore new file mode 100644 index 00000000..da61f8d6 --- /dev/null +++ b/docs/.prettierignore @@ -0,0 +1,4 @@ +node_modules/ +.vitepress/dist/ +.vitepress/cache/ +package-lock.json diff --git a/docs/.prettierrc b/docs/.prettierrc new file mode 100644 index 00000000..ea125e10 --- /dev/null +++ b/docs/.prettierrc @@ -0,0 +1,19 @@ +{ + "printWidth": 120, + "tabWidth": 4, + "useTabs": true, + "semi": false, + "singleQuote": false, + "trailingComma": "none", + "endOfLine": "lf", + "proseWrap": "preserve", + "overrides": [ + { + "files": "*.md", + "options": { + "proseWrap": "preserve", + "printWidth": 120 + } + } + ] +} diff --git a/docs/.vitepress/config.mts b/docs/.vitepress/config.mts new file mode 100644 index 00000000..d009127a --- /dev/null +++ b/docs/.vitepress/config.mts @@ -0,0 +1,60 @@ +import { defineConfig } from "vitepress" + +export default defineConfig({ + title: "VaultLink", + description: "Self-hosted real-time synchronisation for Obsidian", + base: "/vault-link/", + themeConfig: { + logo: "/logo.svg", + nav: [ + { text: "Home", link: "/" }, + { text: "Guide", link: "/guide/getting-started" }, + { text: "Architecture", link: "/architecture/" }, + { text: "GitHub", link: "https://github.com/schmelczer/vault-link" } + ], + sidebar: [ + { + text: "Introduction", + items: [ + { text: "What is VaultLink?", link: "/guide/what-is-vaultlink" }, + { text: "Getting Started", link: "/guide/getting-started" }, + { text: "Limitations", link: "/guide/limitations" }, + { text: "Comparison with Alternatives", link: "/guide/alternatives" } + ] + }, + { + text: "Setup", + items: [ + { text: "Server Setup", link: "/guide/server-setup" }, + { text: "Obsidian Plugin", link: "/guide/obsidian-plugin" }, + { text: "CLI Client", link: "/guide/cli-client" } + ] + }, + { + text: "Configuration", + items: [ + { text: "Server Configuration", link: "/config/server" }, + { text: "Authentication", link: "/config/authentication" }, + { text: "Advanced Options", link: "/config/advanced" } + ] + }, + { + text: "Architecture", + items: [ + { text: "Overview", link: "/architecture/" }, + { text: "Sync Algorithm", link: "/architecture/sync-algorithm" }, + { text: "Data Flow", link: "/architecture/data-flow" } + ] + } + ], + socialLinks: [{ icon: "github", link: "https://github.com/schmelczer/vault-link" }], + footer: { + message: "Released under the MIT License.", + copyright: "Copyright © 2024-present Andras Schmelczer" + }, + search: { + provider: "local" + } + }, + head: [["link", { rel: "icon", type: "image/svg+xml", href: "/vault-link/logo.svg" }]] +}) diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 00000000..bfeb0ee7 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,159 @@ +# VaultLink Documentation + +This directory contains the VaultLink documentation site built with [VitePress](https://vitepress.dev/). + +## Development + +### Prerequisites + +- Node.js 18+ +- npm + +### Setup + +```bash +cd docs +npm install +``` + +### Local Development + +Start the development server with hot reload: + +```bash +npm run dev +``` + +The site will be available at `http://localhost:5173/vault-link/` + +### Build + +Build the static site: + +```bash +npm run build +``` + +Output will be in `.vitepress/dist/` + +### Preview + +Preview the built site: + +```bash +npm run preview +``` + +### Format + +Format all markdown and TypeScript files: + +```bash +npm run format +``` + +Check formatting without making changes: + +```bash +npm run format:check +``` + +### Spell Check + +Check spelling (British English): + +```bash +npm run spell +``` + +The spell checker enforces British English spellings (e.g., "synchronisation", "optimise", "behaviour"). + +## Deployment + +The documentation is automatically deployed to GitHub Pages when changes are pushed to the `main` branch. + +The deployment workflow is configured in `.github/workflows/deploy-docs.yml`. + +## Structure + +``` +docs/ +├── .vitepress/ +│ └── config.ts # VitePress configuration +├── public/ # Static assets +│ └── logo.svg # VaultLink logo +├── guide/ # User guides +│ ├── what-is-vaultlink.md +│ ├── getting-started.md +│ ├── server-setup.md +│ ├── obsidian-plugin.md +│ └── cli-client.md +├── architecture/ # Architecture documentation +│ ├── index.md +│ ├── sync-algorithm.md +│ └── data-flow.md +├── config/ # Configuration reference +│ ├── server.md +│ ├── authentication.md +│ └── advanced.md +└── index.md # Home page + +``` + +## Writing Documentation + +### Language + +All documentation uses **British English**. The spell checker enforces this in CI. + +### Markdown Features + +VitePress supports: + +- GitHub Flavoured Markdown +- Custom containers (tip, warning, danger) +- Code syntax highlighting +- Mermaid diagrams +- Emoji :rocket: + +### Custom Containers + +```markdown +::: tip +This is a tip +::: + +::: warning +This is a warning +::: + +::: danger +This is a danger message +::: +``` + +### Code Blocks + +````markdown +```bash +npm install +``` + +```yaml +server: + port: 3000 +``` +```` + +## Contributing + +When adding new pages: + +1. Create the markdown file in the appropriate directory +2. Add it to the sidebar in `.vitepress/config.ts` +3. Test locally with `npm run dev` +4. Submit a pull request + +## License + +MIT - Same as VaultLink diff --git a/docs/architecture/data-flow.md b/docs/architecture/data-flow.md new file mode 100644 index 00000000..5b256f1d --- /dev/null +++ b/docs/architecture/data-flow.md @@ -0,0 +1,553 @@ +# Data Flow + +How data flows through VaultLink, from client to server and back. + +## Connection Lifecycle + +### 1. Initial Connection + +```mermaid +sequenceDiagram + participant C as Client + participant S as Server + participant DB as Database + + C->>S: WebSocket connect + S->>S: Accept connection + C->>S: Auth message (token + vault) + S->>S: Validate token + S->>S: Check vault access + S-->>C: Auth success + Note over C,S: Connection established +``` + +**Steps**: + +1. Client initiates WebSocket connection to server +2. Server accepts connection +3. Client sends authentication message with token and vault name +4. Server validates token against `config.yml` +5. Server checks if user has access to requested vault +6. Server responds with success or error +7. Connection is ready for syncing + +### 2. Initial Sync + +After authentication, the client performs initial synchronisation: + +```mermaid +sequenceDiagram + participant C as Client + participant S as Server + participant DB as SQLite + + C->>C: Scan local filesystem + C->>S: Request file list + S->>DB: Query all files + DB-->>S: File metadata + S-->>C: File list with versions + + loop For each local file + C->>C: Check if file on server + alt File not on server + C->>S: Upload file + S->>DB: Store file + metadata + else File on server (different version) + C->>C: Compare versions + C->>S: Upload newer or merge + end + end + + loop For each server file + C->>C: Check if file local + alt File not local + C->>S: Download file + S->>DB: Retrieve file + DB-->>S: File content + S-->>C: File content + C->>C: Write to disk + end + end + + S-->>C: Sync complete message +``` + +**Process**: + +1. Client scans local filesystem +2. Client requests file list from server +3. Server queries database and returns metadata +4. Client uploads missing or changed local files +5. Client downloads missing files from server +6. Server sends sync complete notification + +### 3. Real-Time Synchronization + +After initial sync, changes are pushed in real-time: + +```mermaid +sequenceDiagram + participant FS as Filesystem + participant C1 as Client 1 + participant S as Server + participant DB as Database + participant C2 as Client 2 + + FS->>C1: File changed (fs.watch) + C1->>C1: Read file content + C1->>S: Upload file + S->>DB: Store new version + S->>S: Apply OT if needed + S-->>C1: Upload ACK + S->>C2: File update notification + C2->>S: Download file + S->>DB: Retrieve file + DB-->>S: File content + S-->>C2: File content + C2->>FS: Write to disk +``` + +**Flow**: + +1. Filesystem watcher detects local change +2. Client reads file content +3. Client uploads file via WebSocket +4. Server stores in database +5. Server applies operational transformation if concurrent edits +6. Server acknowledges upload to sender +7. Server broadcasts update to other clients +8. Other clients download and apply changes + +## File Operations + +### Upload + +``` +┌─────────┐ +│ Client │ +└────┬────┘ + │ 1. Detect file change + │ + ├─► 2. Read file content + │ + ├─► 3. Create upload message + │ { + │ type: "upload_file", + │ path: "notes/daily.md", + │ content: "...", + │ version: 42, + │ timestamp: "2024-01-01T12:00:00Z" + │ } + │ + ▼ +┌─────────┐ +│ Server │ +└────┬────┘ + │ 4. Validate message + │ + ├─► 5. Check permissions + │ + ├─► 6. Apply OT (if conflicts) + │ + ├─► 7. Store in database + │ + ├─► 8. Update version + │ + ├─► 9. Broadcast to clients + │ + └─► 10. Send ACK to uploader +``` + +### Download + +``` +┌─────────┐ +│ Server │ +└────┬────┘ + │ 1. File updated by another client + │ + ├─► 2. Broadcast notification + │ { + │ type: "file_updated", + │ path: "notes/daily.md", + │ version: 43 + │ } + │ + ▼ +┌─────────┐ +│ Client │ +└────┬────┘ + │ 3. Receive notification + │ + ├─► 4. Request file download + │ { + │ type: "download_file", + │ path: "notes/daily.md", + │ version: 43 + │ } + │ + ▼ +┌─────────┐ +│ Server │ +└────┬────┘ + │ 5. Retrieve from database + │ + └─► 6. Send file content + { + type: "file_content", + path: "notes/daily.md", + content: "...", + version: 43 + } + │ + ▼ + ┌─────────┐ + │ Client │ + └────┬────┘ + │ 7. Write to filesystem + │ + └─► 8. Update local metadata +``` + +### Delete + +``` +┌─────────┐ +│ Client │ +└────┬────┘ + │ 1. File deleted locally + │ + ├─► 2. Send delete message + │ { + │ type: "delete_file", + │ path: "notes/old.md" + │ } + │ + ▼ +┌─────────┐ +│ Server │ +└────┬────┘ + │ 3. Mark as deleted in DB + │ (soft delete for history) + │ + ├─► 4. Broadcast deletion + │ + └─► 5. ACK to sender + │ + ▼ + ┌─────────┐ + │ Other │ + │ Clients │ + └────┬────┘ + │ 6. Delete local file + │ + └─► 7. Update metadata +``` + +## Conflict Resolution Flow + +### Concurrent Edits Scenario + +``` +Time → + +Client A Server Client B + │ │ │ + │ Edit file v10 │ │ + │ "Add line A" │ │ Edit file v10 + │ │ │ "Add line B" + │ │ │ + ├─── Upload @ t1 ─────────►│ │ + │ │◄────── Upload @ t2 ────────┤ + │ │ │ + │ │ 1. Receive both edits │ + │ │ (based on v10) │ + │ │ │ + │ │ 2. Apply first edit │ + │ │ → v11 (line A added) │ + │ │ │ + │ │ 3. Transform second edit │ + │ │ against first │ + │ │ │ + │ │ 4. Apply transformed edit │ + │ │ → v12 (both lines) │ + │ │ │ + │◄──── v12 content ────────┤ │ + │ ├───── v12 content ─────────►│ + │ │ │ + │ Apply v12 │ │ Apply v12 + │ (has both lines) │ │ (has both lines) + │ │ │ +``` + +### Conflict Resolution Steps + +1. **Detection**: Server receives two edits based on the same version +2. **Ordering**: Determine which edit to apply first (by timestamp or client ID) +3. **First edit**: Apply directly to database +4. **Transformation**: Transform second edit against first using OT +5. **Second edit**: Apply transformed edit to database +6. **Broadcast**: Send merged result to all clients +7. **Application**: Clients apply merged version locally + +## Database Schema + +### Core Tables + +```sql +-- Document metadata +CREATE TABLE documents ( + id INTEGER PRIMARY KEY, + path TEXT NOT NULL, + version INTEGER NOT NULL, + content_hash TEXT, + size INTEGER, + created_at TIMESTAMP, + updated_at TIMESTAMP, + deleted BOOLEAN DEFAULT FALSE +); + +-- Version history +CREATE TABLE versions ( + id INTEGER PRIMARY KEY, + document_id INTEGER, + version INTEGER, + content BLOB, + created_at TIMESTAMP, + FOREIGN KEY (document_id) REFERENCES documents(id) +); + +-- Client sync cursors +CREATE TABLE cursors ( + client_id TEXT PRIMARY KEY, + last_version INTEGER, + last_updated TIMESTAMP +); +``` + +### Queries + +**Get files since version**: + +```sql +SELECT * FROM documents +WHERE version > ? AND deleted = FALSE +ORDER BY version ASC; +``` + +**Store new version**: + +```sql +INSERT INTO versions (document_id, version, content, created_at) +VALUES (?, ?, ?, ?); + +UPDATE documents +SET version = ?, updated_at = ? +WHERE id = ?; +``` + +**Update cursor**: + +```sql +INSERT OR REPLACE INTO cursors (client_id, last_version, last_updated) +VALUES (?, ?, ?); +``` + +## Message Protocol + +### Client → Server Messages + +**Upload File**: + +```json +{ + "type": "upload_file", + "path": "notes/example.md", + "content": "File content here...", + "base_version": 10, + "timestamp": "2024-01-01T12:00:00Z" +} +``` + +**Download File**: + +```json +{ + "type": "download_file", + "path": "notes/example.md" +} +``` + +**Delete File**: + +```json +{ + "type": "delete_file", + "path": "notes/old.md" +} +``` + +**List Files**: + +```json +{ + "type": "list_files", + "since_version": 0 +} +``` + +### Server → Client Messages + +**File Updated**: + +```json +{ + "type": "file_updated", + "path": "notes/example.md", + "version": 11, + "size": 1024, + "hash": "abc123..." +} +``` + +**File Content**: + +```json +{ + "type": "file_content", + "path": "notes/example.md", + "content": "Updated content...", + "version": 11 +} +``` + +**File Deleted**: + +```json +{ + "type": "file_deleted", + "path": "notes/old.md", + "version": 12 +} +``` + +**Sync Complete**: + +```json +{ + "type": "sync_complete", + "total_files": 150, + "current_version": 200 +} +``` + +**Error**: + +```json +{ + "type": "error", + "message": "File too large", + "code": "FILE_TOO_LARGE" +} +``` + +## Error Handling + +### Client-Side Errors + +**Network failure**: + +1. Detect WebSocket disconnect +2. Queue pending operations +3. Retry connection with exponential backoff +4. Replay queued operations on reconnect + +**File read error**: + +1. Log error +2. Skip file +3. Continue with other files +4. Report to user + +**Write conflict**: + +1. Receive updated version from server +2. Apply OT merge locally +3. Overwrite local file +4. Continue syncing + +### Server-Side Errors + +**Database error**: + +1. Log error +2. Return error to client +3. Client retries operation + +**Invalid operation**: + +1. Validate message format +2. Return specific error code +3. Client handles error appropriately + +**Authentication failure**: + +1. Reject connection +2. Send auth error +3. Client prompts for new credentials + +## Performance Optimizations + +### Batching + +- Small, rapid changes are batched together +- Reduces message overhead +- Applied as single atomic update + +### Compression + +- Large files compressed before transmission +- Reduces bandwidth usage +- Transparent to application layer + +### Incremental Sync + +- Only changed portions of files sent +- Uses content-based diffing +- Significantly reduces data transfer + +### Caching + +- Server caches recent file versions +- Reduces database queries +- Improves response time + +## Monitoring Data Flow + +### Server Logs + +``` +2024-01-01 12:00:00 INFO WebSocket connection from 192.168.1.100 +2024-01-01 12:00:01 INFO User 'alice' authenticated for vault 'personal' +2024-01-01 12:00:05 INFO Upload: notes/daily.md (v10 -> v11) +2024-01-01 12:00:06 INFO Broadcast to 3 clients +2024-01-01 12:00:10 INFO Conflict resolved: notes/shared.md (v12) +``` + +### Client Logs + +``` +2024-01-01 12:00:00 INFO Connecting to ws://sync.example.com +2024-01-01 12:00:01 INFO Connected, authenticating... +2024-01-01 12:00:01 INFO Authentication successful +2024-01-01 12:00:02 INFO Starting initial sync +2024-01-01 12:00:10 INFO Sync complete: 150 files, 200 MB +2024-01-01 12:00:15 INFO Uploaded: notes/daily.md +2024-01-01 12:00:20 INFO Downloaded: notes/shared.md (merged) +``` + +## Next Steps + +- [Understand the sync algorithm →](/architecture/sync-algorithm) +- [Configure the server →](/config/server) +- [Deploy VaultLink →](/guide/getting-started) diff --git a/docs/architecture/index.md b/docs/architecture/index.md new file mode 100644 index 00000000..5d4c6d73 --- /dev/null +++ b/docs/architecture/index.md @@ -0,0 +1,334 @@ +# Architecture Overview + +Central sync server with multiple clients. High-level architecture and design decisions. + +## System Components + +``` +┌─────────────────────────────────────────────────────────────┐ +│ Clients │ +├─────────────────────┬───────────────────┬───────────────────┤ +│ Obsidian Plugin │ Obsidian Plugin │ CLI Client │ +│ (User A - Device1) │ (User A - Device2│ (Server/Backup) │ +└──────────┬──────────┴─────────┬─────────┴──────────┬────────┘ + │ │ │ + │ WebSocket │ WebSocket │ WebSocket + │ │ │ + └────────────────────┼────────────────────┘ + │ + ┌───────────▼───────────┐ + │ Sync Server │ + │ (Rust + Axum) │ + │ │ + │ ┌─────────────────┐ │ + │ │ WebSocket Hub │ │ + │ └────────┬────────┘ │ + │ │ │ + │ ┌────────▼────────┐ │ + │ │ Sync Engine │ │ + │ │ (OT Algorithm) │ │ + │ └────────┬────────┘ │ + │ │ │ + │ ┌────────▼────────┐ │ + │ │ SQLite Database │ │ + │ │ (Per Vault) │ │ + │ └─────────────────┘ │ + └───────────────────────┘ +``` + +## Core Components + +### Sync Server + +Central authority for synchronisation. Rust + Axum framework. + +**Responsibilities**: + +- Accept WebSocket connections from clients +- Authenticate users via token-based auth +- Store document versions in SQLite +- Coordinate real-time updates between clients +- Apply operational transformation for conflict resolution +- Manage vault access control + +**Technology**: + +- **Language**: Rust 1.89+ +- **Framework**: Axum (async web framework) +- **Database**: SQLite with SQLx +- **Protocol**: WebSockets for real-time communication +- **Sync Algorithm**: reconcile-text (operational transformation) + +### Sync Client Library + +TypeScript library with core sync logic. Used by Obsidian plugin and CLI client. + +**Responsibilities**: + +- Manage WebSocket connection to server +- Watch local filesystem for changes +- Upload and download files +- Apply remote changes locally +- Handle conflict resolution +- Maintain sync metadata + +**Technology**: + +- **Language**: TypeScript +- **Build**: Webpack +- **Protocol**: WebSocket client +- **File System**: Node.js `fs` API / Obsidian API + +### Obsidian Plugin + +Integration layer between sync client and Obsidian. + +**Responsibilities**: + +- Provide UI for configuration +- Bridge sync client with Obsidian's file system API +- Handle Obsidian lifecycle events +- Display sync status to users + +**Technology**: + +- **Platform**: Obsidian Plugin API +- **Core**: sync-client library +- **UI**: Obsidian settings UI + +### CLI Client + +Standalone executable for syncing vaults without Obsidian. + +**Responsibilities**: + +- Command-line interface +- File system access via Node.js +- Daemon mode for continuous sync +- Health check endpoint for monitoring + +**Technology**: + +- **Language**: TypeScript +- **Runtime**: Node.js +- **CLI**: Commander.js +- **Core**: sync-client library + +## Data Flow + +### Initial Connection + +1. Client connects via WebSocket to server +2. Server authenticates using provided token +3. Server verifies user has access to requested vault +4. Connection established, sync begins + +### File Upload Flow + +``` +Client Server + │ │ + │ 1. File changed locally │ + │ │ + │ 2. Read file content │ + │ │ + │ 3. WebSocket: Upload file │ + ├──────────────────────────────►│ + │ │ 4. Store in SQLite + │ │ + │ │ 5. Broadcast to other clients + │ ├───────────────────────► + │ 6. Ack upload │ + │◄──────────────────────────────┤ +``` + +### File Download Flow + +``` +Client A Server Client B + │ │ │ + │ │ 1. File uploaded │ + │ │◄────────────────────────┤ + │ │ │ + │ │ 2. Store in DB │ + │ │ │ + │ 3. Push notification │ │ + │◄────────────────────────┤ │ + │ │ │ + │ 4. Download file │ │ + ├────────────────────────►│ │ + │ │ │ + │ 5. Write locally │ │ + │ │ │ +``` + +### Conflict Resolution + +When two clients edit the same file simultaneously: + +``` +Client A Server Client B + │ │ │ + │ 1. Edit file │ │ 1. Edit same file + │ │ │ + │ 2. Upload changes │ │ 2. Upload changes + ├────────────────────────►│◄────────────────────────┤ + │ │ │ + │ │ 3. Apply OT algorithm │ + │ │ - Merge both edits │ + │ │ - Preserve all changes│ + │ │ │ + │ 4. Receive merged ver. │ 5. Receive merged ver. │ + │◄────────────────────────┤────────────────────────►│ + │ │ │ + │ 6. Apply locally │ │ 6. Apply locally +``` + +## Storage Architecture + +### Server Storage + +Each vault has its own SQLite database: + +``` +databases/ +├── vault-1.db +├── vault-2.db +└── shared-team.db +``` + +**Database Schema** (simplified): + +- **documents**: File metadata (path, size, modified time) +- **versions**: Document content with version history +- **cursors**: Client sync state + +### Client Storage + +Clients maintain sync metadata: + +``` +.vaultlink/ +├── metadata.json # Sync state +└── cache/ # Optional local cache +``` + +The `.vaultlink` directory tracks which files have been synced and their versions to enable efficient synchronisation. + +## Communication Protocol + +### WebSocket Messages + +Client-server communication uses JSON messages over WebSocket. + +**Message Types**: + +- `upload_file`: Client → Server (file upload) +- `download_file`: Client → Server (request file) +- `file_updated`: Server → Client (file changed notification) +- `file_deleted`: Server → Client (file deleted notification) +- `sync_complete`: Server → Client (initial sync finished) + +### Authentication + +Token-based authentication on connection: + +```typescript +// Client sends token on connect +{ + type: "auth", + token: "user-auth-token", + vault: "vault-name" +} + +// Server responds +{ + type: "auth_success" +} +// or +{ + type: "auth_error", + message: "Invalid token" +} +``` + +## Scalability Considerations + +### Current Architecture + +- **SQLite per vault**: Simple, performant, limited to single server +- **WebSocket connections**: Stateful, requires sticky sessions for load balancing +- **Operational transformation**: Centralized on server + +### Scaling Approaches + +**Vertical Scaling**: + +- Increase server resources (CPU, RAM, storage) +- Optimize database queries and indexing +- Tune connection limits + +**Horizontal Scaling** (future): + +- Separate vault servers (vault sharding) +- Load balancer with sticky sessions +- Shared storage layer for SQLite databases +- Consider alternative databases (PostgreSQL) for multi-server setups + +### Performance Characteristics + +- **Small vaults** (< 1000 files): Excellent performance +- **Medium vaults** (1000-10000 files): Good performance with tuning +- **Large vaults** (> 10000 files): May require optimisation +- **Concurrent users**: Tested with dozens of simultaneous clients per vault + +## Security Model + +### Authentication + +- Token-based authentication +- Tokens configured in server `config.yml` +- No password hashing (tokens are secrets) + +### Authorization + +- Per-user vault access control +- Allow-list or deny-list patterns +- Global access or vault-specific access + +### Network Security + +- WebSocket over TLS (WSS) for encrypted transport +- No built-in SSL (use reverse proxy) +- CORS configured for web clients + +### Data Security + +- No encryption at rest (use encrypted filesystems if needed) +- No end-to-end encryption (server sees all content) +- Self-hosted model: you control the data + +## Technology Choices + +**Rust**: Low latency, memory safe, excellent async with Tokio, compile-time SQL verification + +**SQLite**: No separate database server, fast for reads, single file per vault, backups are file copies + +**WebSocket**: Bidirectional push, no polling overhead, built-in browser/Node.js support + +**Operational Transformation**: Automatic conflict resolution, preserves all edits, real-time collaboration + +## Design Principles + +1. **Self-hosted first**: Users control their data and infrastructure +2. **Simplicity**: Easy to deploy and operate +3. **Real-time**: Changes appear immediately +4. **Reliability**: Handle network failures gracefully +5. **Performance**: Fast sync for typical vault sizes +6. **Privacy**: No third-party services or telemetry + +## Next Steps + +- [Learn about the sync algorithm →](/architecture/sync-algorithm) +- [Understand data flow in detail →](/architecture/data-flow) +- [Deploy the server →](/guide/server-setup) diff --git a/docs/architecture/sync-algorithm.md b/docs/architecture/sync-algorithm.md new file mode 100644 index 00000000..35e63d50 --- /dev/null +++ b/docs/architecture/sync-algorithm.md @@ -0,0 +1,438 @@ +# Sync Algorithm + +VaultLink uses operational transformation (OT) to handle concurrent edits and maintain consistency across clients. + +## Operational Transformation + +Operational transformation is a technique for managing concurrent edits to the same document. It transforms operations (edits) so they can be applied in different orders while preserving user intent. + +### Why OT? + +Traditional conflict resolution approaches: + +- **Last write wins**: Loses data, frustrating for users +- **Manual merging**: Interrupts workflow, requires user intervention +- **Version branching**: Complex, not suitable for real-time sync + +Operational transformation: + +- **Automatic**: No user intervention required +- **Preserves all edits**: No data loss +- **Real-time**: Changes appear immediately +- **Intuitive**: Behaviour matches user expectations + +## The reconcile-text Library + +VaultLink uses the [`reconcile-text`](https://crates.io/crates/reconcile-text) Rust library for operational transformation on text documents. + +### Why reconcile-text over CRDTs? + +VaultLink faces a **differential synchronisation** challenge: users edit Obsidian vaults with various editors (Obsidian desktop, Obsidian mobile, Vim, VS Code, or any text editor), often while offline. This means we only observe the **final state** of each document after editing, not the individual keystrokes or operations that produced it. + +**The fundamental problem**: + +- **CRDTs and traditional OT** require capturing every individual operation (each character insertion, deletion, cursor movement) +- **VaultLink's reality**: Users edit files with arbitrary tools, sync happens after the fact +- **What we know**: Parent version and two modified versions +- **What we don't know**: The sequence of operations that created those modifications + +**Why reconcile-text wins for this use case**: + +1. **Works with end states only**: reconcile-text performs conflict-free 3-way merging given just parent, left, and right versions—no operation history needed + +2. **Editor-agnostic**: Users can edit with any tool without requiring VaultLink-specific plugins or operation tracking + +3. **Offline-first**: Edits made while disconnected are merged cleanly when sync resumes, because we're diffing final states rather than replaying operations + +4. **No conflict markers**: Unlike Git merge, produces clean merged output without `<<<<<<<` markers that interrupt note-taking flow + +5. **Human text forgiveness**: For knowledge bases and documentation, a slightly imperfect merge (e.g., minor word order issues) is vastly preferable to manual conflict resolution + +6. **Simpler infrastructure**: No need for complex operation capture, transformation logs, or tombstone management that CRDTs require + +**The trade-off**: + +CRDTs excel when you control the entire editing infrastructure and can capture every operation. reconcile-text excels when you're synchronising independently-edited files—exactly VaultLink's scenario. The merge quality depends on Myers' diff algorithm rather than operation history, which is the correct trade-off for differential sync. + +For note-taking workflows where users value editor freedom and offline editing, this approach provides superior user experience compared to either CRDTs (which would require operation tracking) or Git-style merging (which requires manual conflict resolution). + +[Learn more about reconcile-text →](https://schmelczer.dev/reconcile) + +### How It Works + +Given three versions (parent, left, right), reconcile-text produces a merged result. + +**How reconcile-text works**: + +1. **Tokenisation**: Split text into words (using `BuiltinTokenizer::Word`) +2. **Three-way diff**: Compare parent→left and parent→right changes +3. **Merge**: Combine non-conflicting changes, prefer content preservation for conflicts +4. **Result**: Merged text with both edits applied + +**Example**: + +``` +Parent: "The quick brown fox" +User A: "The quick red fox" (changes "brown" → "red") +User B: "The very quick brown fox" (inserts "very ") + +Merged: "The very quick red fox" (both changes applied) +``` + +**Merge conditions**: Only `.md` and `.txt` files with valid UTF-8 get merged. Binary files or other extensions use last-write-wins. + +### Operation Types + +The algorithm handles these operations: + +- **Insert**: Add text at position +- **Delete**: Remove text from position +- **Retain**: Keep existing text unchanged + +### Transformation Process + +1. **Client A** makes edit and sends to server +2. **Client B** makes concurrent edit and sends to server +3. **Server** receives both edits +4. **Server** transforms operations to account for concurrent changes +5. **Server** applies merged result to database +6. **Server** sends transformed operations to both clients +7. **Clients** apply transformed operations locally + +## Sync State Management + +VaultLink maintains sync state to track which changes have been applied. + +### Version Vectors + +Each document has a version tracked by: + +- **Server version**: Incremented on each change +- **Client cursors**: Track which version each client has seen + +This enables: + +- Efficient syncing (only send changes since last sync) +- Conflict detection (concurrent edits to same version) +- Ordering of operations + +### Cursor Management + +Clients maintain a cursor position: + +```rust +struct Cursor { + vault_id: String, + client_id: String, + last_version: u64, + last_updated: DateTime, +} +``` + +On sync: + +1. Client sends cursor (last seen version) +2. Server returns all changes since that version +3. Client applies changes and updates cursor + +## Conflict Resolution Flow + +### Scenario: Concurrent Edits + +Two users edit the same paragraph simultaneously. + +**Initial state**: + +``` +Version 10: "The quick brown fox jumps over the lazy dog." +``` + +**User A's edit** (version 11): + +``` +"The quick brown fox jumps over the very lazy dog." +``` + +_Inserts "very " at position 40_ + +**User B's edit** (also from version 10): + +``` +"The quick red fox jumps over the lazy dog." +``` + +_Replaces "brown" with "red" at position 10_ + +### Server Processing + +1. **Receive User A's operation**: + - Base: version 10 + - Operation: Insert("very ", position=40) + - Apply to database → version 11 + +2. **Receive User B's operation**: + - Base: version 10 + - Operation: Replace("brown"→"red", position=10) + - **Conflict detected**: Base is version 10, but current is version 11 + +3. **Transform User B's operation**: + - Transform against User A's operation + - Adjust positions/content as needed + - Apply transformed operation → version 12 + +4. **Broadcast updates**: + - Send User A's operation to User B + - Send transformed User B's operation to User A + +### Final Result + +``` +Version 12: "The quick red fox jumps over the very lazy dog." +``` + +Both edits are preserved in the final document. + +## Edge Cases + +### 1. Delete vs Insert Conflict + +**Scenario**: User A deletes a paragraph while User B edits it. + +**Resolution**: + +- OT algorithm prioritizes preservation of content +- Insert operation is transformed to account for deletion +- Typically results in inserted content appearing nearby + +**Example**: + +``` +Base: "Line 1\nLine 2\nLine 3" + +User A: Delete Line 2 → "Line 1\nLine 3" +User B: Edit Line 2 → "Line 1\nLine 2 modified\nLine 3" + +Result: "Line 1\nLine 2 modified\nLine 3" +``` + +(Insert takes precedence, preserving user content) + +### 2. Overlapping Edits + +**Scenario**: Two users edit overlapping regions. + +**Resolution**: + +- OT splits operations into non-overlapping segments +- Applies each segment independently +- Merges results + +### 3. Delete vs Delete + +**Scenario**: Two users delete overlapping text. + +**Resolution**: + +- Deletes are merged +- Final result has the union of deleted ranges removed + +### 4. Network Partitions + +**Scenario**: Client loses connection, makes edits offline, reconnects. + +**Resolution**: + +1. Client queues edits locally +2. On reconnect, sends all queued operations +3. Server applies OT against all operations that happened during partition +4. Client receives transformed operations and applies + +## Performance Characteristics + +### Time Complexity + +- **Single operation**: O(1) for most operations +- **Transformation**: O(n) where n is operation size +- **Conflict resolution**: O(m × n) where m is number of concurrent operations + +### Space Complexity + +- **Version history**: Grows with number of changes +- **Cursors**: O(clients × vaults) +- **Active operations**: Minimal (processed in real-time) + +### Optimisation + +VaultLink optimises for: + +- Small, frequent edits (typical typing patterns) +- Text documents (not binary files) +- Real-time processing (no batching delay) + +## Limitations + +### Binary and Non-Mergeable Files + +Only **`.md`** and **`.txt`** files get automatic merging. Everything else uses last-write-wins. + +**Binary detection**: + +- Files with NUL bytes (`0x00`) +- Files failing UTF-8 validation + +Even `.md` files are treated as binary if they fail UTF-8 checks. + +**Last-write-wins behaviour**: + +``` +User A uploads image.png → Server version 1 +User B uploads image.png → Server version 2 (A's upload lost) +``` + +**Workaround**: Avoid concurrent edits to non-text files. [See all limitations →](/guide/limitations) + +### Large Documents + +Very large documents (> 1MB) may have: + +- Higher transformation costs +- Slower sync times +- Increased memory usage + +**Workaround**: Split large documents or increase timeout settings. + +### Complex Formatting + +Markdown with complex structures may occasionally produce unexpected results: + +- Nested lists +- Tables +- Code blocks + +**Workaround**: Manual cleanup if needed, or minimize concurrent edits to complex structures. + +## Consistency Guarantees + +### Strong Consistency + +VaultLink provides **strong eventual consistency**: + +- All clients eventually converge to the same state +- Operations applied in causal order +- No data loss under normal operation + +### Ordering Guarantees + +- Operations from the same client are applied in order +- Concurrent operations may be applied in any order +- Final result is independent of operation order (commutative) + +### Durability + +- Operations are written to SQLite before acknowledgment +- SQLite ACID guarantees protect against data loss +- Clients retry failed uploads + +## Comparison with Other Approaches + +### Git-style Merging + +| Aspect | Git Merge | VaultLink OT | +| -------------------------- | ------------ | ----------------------- | +| Real-time | No | Yes | +| Manual conflict resolution | Yes | No | +| Branching | Yes | No | +| Automatic merge | Limited | Always | +| Use case | Code changes | Collaborative documents | + +### CRDTs (Conflict-free Replicated Data Types) + +| Aspect | CRDTs | VaultLink (reconcile-text) | +| ----------------------------- | ------------------------------------ | ------------------------------------------------- | +| **Operation tracking** | Required (every keystroke) | Not required (end states only) | +| **Editor freedom** | Limited (must use CRDT-aware editor) | Unlimited (any text editor works) | +| **Offline editing** | Requires operation log | Works with file comparison | +| **Server required** | No | Yes | +| **Memory overhead** | Higher (tombstones, metadata) | Lower (versions only) | +| **Infrastructure complexity** | Higher | Lower | +| **Best for** | Controlled editing environments | Independent file editing (Obsidian, Vim, VS Code) | + +**Key insight**: CRDTs are superior when you can capture every operation. reconcile-text is superior when users edit files independently with arbitrary tools—exactly VaultLink's scenario. + +### Last Write Wins + +| Aspect | LWW | VaultLink OT | +| --------------- | ---- | ------------ | +| Data loss | Yes | No | +| Simplicity | High | Medium | +| User experience | Poor | Excellent | +| Performance | Best | Good | + +## Algorithm Details + +### Transformation Rules + +When transforming operation `A` against operation `B`: + +1. **Insert vs Insert**: + - If positions equal: Order by client ID + - If different positions: Adjust positions + +2. **Insert vs Delete**: + - If insert in deleted range: Shift insert position + - If insert after delete: Adjust position by deleted length + +3. **Delete vs Delete**: + - If ranges overlap: Merge delete ranges + - If ranges disjoint: Adjust positions + +4. **Retain vs Any**: + - Retain operations don't conflict + - Simply adjust positions + +### Transformation Example + +```rust +// Pseudo-code for transformation +fn transform(op_a: Operation, op_b: Operation) -> (Operation, Operation) { + match (op_a, op_b) { + (Insert(pos_a, text_a), Insert(pos_b, text_b)) => { + if pos_a < pos_b { + (op_a, Insert(pos_b + text_a.len(), text_b)) + } else if pos_a > pos_b { + (Insert(pos_a + text_b.len(), text_a), op_b) + } else { + // Same position, use client ID to break tie + if client_id_a < client_id_b { + (op_a, Insert(pos_b + text_a.len(), text_b)) + } else { + (Insert(pos_a + text_b.len(), text_a), op_b) + } + } + } + // ... other cases + } +} +``` + +## Best Practices + +### For Smooth Collaboration + +1. **Small edits**: Make small, focused changes for easier merging +2. **Coordinate major changes**: Discuss large refactors with team +3. **Monitor sync status**: Ensure changes are uploaded before signing off +4. **Test conflict resolution**: Verify behaviour matches expectations + +### For Developers + +1. **Text files preferred**: OT works best on text +2. **Limit file sizes**: Keep documents reasonably sized +3. **Binary files**: Use versioning or avoid concurrent edits +4. **Testing**: Test concurrent edit scenarios thoroughly + +## Further Reading + +- [reconcile-text library](https://crates.io/crates/reconcile-text) +- [Operational Transformation FAQ](https://en.wikipedia.org/wiki/Operational_transformation) +- [Data flow architecture →](/architecture/data-flow) diff --git a/docs/config/advanced.md b/docs/config/advanced.md new file mode 100644 index 00000000..5275be93 --- /dev/null +++ b/docs/config/advanced.md @@ -0,0 +1,603 @@ +# Advanced Configuration + +Advanced topics for optimising and customising your VaultLink deployment. + +## Database Optimisation + +### SQLite Tuning + +While VaultLink handles most SQLite configuration automatically, you can optimise for specific workloads. + +#### WAL Mode + +VaultLink uses Write-Ahead Logging (WAL) mode by default for better concurrency. + +**Benefits**: + +- Readers don't block writers +- Writers don't block readers +- Better performance for concurrent access + +**Maintenance**: + +```bash +# Checkpoint WAL to main database (run periodically) +sqlite3 databases/vault.db "PRAGMA wal_checkpoint(TRUNCATE);" +``` + +#### Database Size Management + +Over time, databases can grow with version history: + +```bash +# Check database size +du -h databases/*.db + +# Vacuum to reclaim space (offline only) +sqlite3 databases/vault.db "VACUUM;" + +# Analyse for query optimisation +sqlite3 databases/vault.db "ANALYZE;" +``` + +**Schedule maintenance**: + +```bash +#!/bin/bash +# monthly-maintenance.sh + +for db in databases/*.db; do + echo "Optimising $db" + sqlite3 "$db" "PRAGMA optimize;" + sqlite3 "$db" "PRAGMA wal_checkpoint(TRUNCATE);" +done +``` + +### Version History Cleanup + +VaultLink stores **all versions indefinitely** by default. Database grows with every change. + +**Database schema**: Each version stored in `documents` table with `vault_update_id` (sequential). + +Manual cleanup (keep last 100 versions per document): + +```bash +#!/bin/bash +# prune-old-versions.sh + +for db in databases/*.db; do + sqlite3 "$db" < /dev/null; then + echo "Health check failed at $(date)" | mail -s "VaultLink Down" admin@example.com + # Optionally restart + # docker restart vaultlink-server + fi + sleep 30 +done +``` + +### Backup Automation + +Automated backup script: + +```bash +#!/bin/bash +# backup-vaultlink.sh + +BACKUP_DIR="/backup/vaultlink" +DATA_DIR="/data" +DATE=$(date +%Y%m%d-%H%M%S) +RETENTION_DAYS=30 + +# Create backup directory +mkdir -p "$BACKUP_DIR/$DATE" + +# Backup databases (with WAL checkpoint) +for db in "$DATA_DIR"/databases/*.db; do + sqlite3 "$db" "PRAGMA wal_checkpoint(TRUNCATE);" + cp "$db" "$BACKUP_DIR/$DATE/" + [ -f "${db}-wal" ] && cp "${db}-wal" "$BACKUP_DIR/$DATE/" + [ -f "${db}-shm" ] && cp "${db}-shm" "$BACKUP_DIR/$DATE/" +done + +# Backup configuration +cp "$DATA_DIR/config.yml" "$BACKUP_DIR/$DATE/" + +# Compress backup +tar -czf "$BACKUP_DIR/vaultlink-$DATE.tar.gz" -C "$BACKUP_DIR" "$DATE" +rm -rf "$BACKUP_DIR/$DATE" + +# Clean old backups +find "$BACKUP_DIR" -name "vaultlink-*.tar.gz" -mtime +$RETENTION_DAYS -delete + +# Upload to remote storage (optional) +# rclone copy "$BACKUP_DIR/vaultlink-$DATE.tar.gz" remote:backups/ +``` + +Schedule with cron: + +```cron +0 2 * * * /opt/vaultlink/backup-vaultlink.sh +``` + +### Restore from Backup + +```bash +#!/bin/bash +# restore-vaultlink.sh + +BACKUP_FILE="$1" +DATA_DIR="/data" + +if [ -z "$BACKUP_FILE" ]; then + echo "Usage: $0 " + exit 1 +fi + +# Stop server +docker stop vaultlink-server + +# Extract backup +tar -xzf "$BACKUP_FILE" -C /tmp/ +BACKUP_DATE=$(basename "$BACKUP_FILE" .tar.gz | cut -d- -f2-) + +# Restore databases +cp /tmp/"$BACKUP_DATE"/databases/*.db "$DATA_DIR/databases/" + +# Restore config (careful!) +# cp /tmp/$BACKUP_DATE/config.yml "$DATA_DIR/" + +# Cleanup +rm -rf /tmp/"$BACKUP_DATE" + +# Start server +docker start vaultlink-server + +echo "Restore complete. Check server logs." +``` + +## Monitoring and Metrics + +### Prometheus Metrics + +While VaultLink doesn't expose metrics natively, monitor Docker: + +```yaml +# docker-compose.yml +services: + vaultlink-server: + image: ghcr.io/schmelczer/vault-link-server:latest + labels: + - "prometheus.io/scrape=true" + - "prometheus.io/port=3000" + + cadvisor: + image: gcr.io/cadvisor/cadvisor:latest + volumes: + - /:/rootfs:ro + - /var/run:/var/run:ro + - /sys:/sys:ro + - /var/lib/docker/:/var/lib/docker:ro + ports: + - 8080:8080 +``` + +### Log Analysis + +Analyze logs for insights: + +```bash +# Most active users +grep "authenticated" logs/*.log | cut -d"'" -f2 | sort | uniq -c | sort -rn + +# Failed authentications by IP +grep "Authentication failed" logs/*.log | grep -oP '\d+\.\d+\.\d+\.\d+' | sort | uniq -c | sort -rn + +# Upload activity +grep "Upload:" logs/*.log | wc -l + +# Average files per vault +grep "Sync complete" logs/*.log | grep -oP '\d+ files' | cut -d' ' -f1 | awk '{sum+=$1; count++} END {print sum/count}' +``` + +### Alerting + +Simple alerting with cron: + +```bash +#!/bin/bash +# alert-errors.sh + +ERROR_THRESHOLD=10 +ERROR_COUNT=$(grep -c "ERROR" logs/latest.log) + +if [ "$ERROR_COUNT" -gt "$ERROR_THRESHOLD" ]; then + echo "VaultLink has $ERROR_COUNT errors in the last hour" | \ + mail -s "VaultLink Alert" admin@example.com +fi +``` + +## Security Hardening + +### Network Isolation + +Run VaultLink in isolated network: + +```yaml +services: + vaultlink-server: + image: ghcr.io/schmelczer/vault-link-server:latest + networks: + - vaultlink-internal + - proxy-external + +networks: + vaultlink-internal: + internal: true + proxy-external: + driver: bridge +``` + +### Read-Only Root Filesystem + +Run with read-only root (mount writable volumes for data): + +```yaml +services: + vaultlink-server: + image: ghcr.io/schmelczer/vault-link-server:latest + read_only: true + volumes: + - ./data:/data + - /tmp +``` + +### Drop Capabilities + +Run with minimal privileges: + +```yaml +services: + vaultlink-server: + image: ghcr.io/schmelczer/vault-link-server:latest + security_opt: + - no-new-privileges:true + cap_drop: + - ALL +``` + +## Migration + +### Moving to New Server + +1. **Backup on old server**: + + ```bash + ./backup-vaultlink.sh + ``` + +2. **Transfer backup**: + + ```bash + scp vaultlink-backup.tar.gz new-server:/tmp/ + ``` + +3. **Restore on new server**: + + ```bash + ./restore-vaultlink.sh /tmp/vaultlink-backup.tar.gz + ``` + +4. **Update DNS/clients** to point to new server + +5. **Verify sync** on all clients + +### Version Upgrades + +```bash +# Pull latest image +docker pull ghcr.io/schmelczer/vault-link-server:latest + +# Backup first +./backup-vaultlink.sh + +# Stop old container +docker stop vaultlink-server +docker rm vaultlink-server + +# Start with new image +docker run -d \ + --name vaultlink-server \ + --restart unless-stopped \ + -p 3000:3000 \ + -v ./data:/data \ + ghcr.io/schmelczer/vault-link-server:latest \ + /app/sync_server /data/config.yml + +# Check logs +docker logs -f vaultlink-server +``` + +## Next Steps + +- [Understand the architecture →](/architecture/) +- [Deploy the server →](/guide/server-setup) +- [Configure clients →](/guide/obsidian-plugin) diff --git a/docs/config/authentication.md b/docs/config/authentication.md new file mode 100644 index 00000000..944e56f2 --- /dev/null +++ b/docs/config/authentication.md @@ -0,0 +1,558 @@ +# Authentication Configuration + +VaultLink uses token-based authentication with per-user vault access control. This guide covers all authentication and authorization options. + +## Overview + +Authentication in VaultLink: + +- **Token-based**: Users authenticate with secure tokens +- **Configured in YAML**: All users defined in `config.yml` +- **Vault-level access**: Control which vaults each user can access +- **No password hashing**: Tokens are treated as secrets + +## Basic Configuration + +```yaml +users: + user_configs: + - name: alice + token: alice-secure-token-here + vault_access: + type: allow_access_to_all +``` + +## User Configuration Fields + +### `name` + +**Type**: String +**Required**: Yes + +Human-readable identifier for the user. Used in logs and auditing. + +```yaml +- name: alice +``` + +**Notes**: + +- Must be unique across all users +- Used for identification only, not authentication +- Appears in server logs +- Can be any string (e.g., email, username) + +### `token` + +**Type**: String +**Required**: Yes + +Authentication token for the user. Must be kept secret. + +```yaml +- token: 1a2b3c4d5e6f7g8h9i0j... +``` + +**Best practices**: + +- Generate with: `openssl rand -hex 32` +- Minimum length: 32 characters +- Use different token per user +- Never commit to version control +- Rotate periodically + +**Example token generation**: + +```bash +# Generate a secure token +openssl rand -hex 32 +# Output: a7f3c9d1e8b2f4a6c3d9e1f7b8a4c2d6e9f1a3b7c5d8e2f4a6b9c3d1e8f7a4b2 +``` + +### `vault_access` + +**Type**: Object +**Required**: Yes + +Defines which vaults the user can access. + +**Three modes**: + +1. `allow_access_to_all`: Access to all vaults +2. `allow_list`: Access to specific vaults only +3. `deny_list`: Access to all vaults except specific ones + +## Access Control Modes + +### Allow Access to All + +Grant access to every vault: + +```yaml +users: + user_configs: + - name: admin + token: admin-token + vault_access: + type: allow_access_to_all +``` + +**Use cases**: + +- Administrator accounts +- Personal single-user deployments +- Development/testing + +### Allow List + +Grant access only to specific vaults: + +```yaml +users: + user_configs: + - name: alice + token: alice-token + vault_access: + type: allow_list + allowed: + - personal + - shared-team + - project-alpha +``` + +**Use cases**: + +- Multi-user deployments +- Restricted access scenarios +- Separation of concerns + +**Notes**: + +- User can only access listed vaults +- Attempting to access other vaults returns authentication error +- Empty list = no access to any vault + +### Deny List + +Grant access to all vaults except specific ones: + +```yaml +users: + user_configs: + - name: bob + token: bob-token + vault_access: + type: deny_list + denied: + - restricted + - admin-only +``` + +**Use cases**: + +- Users with broad access except sensitive vaults +- Simplify configuration when most vaults are accessible + +**Notes**: + +- User can access any vault not in the deny list +- Attempting to access denied vaults returns authentication error + +## Multi-User Scenarios + +### Personal Use (Single User) + +```yaml +users: + user_configs: + - name: me + token: my-super-secret-token + vault_access: + type: allow_access_to_all +``` + +### Small Team (Shared Vaults) + +```yaml +users: + user_configs: + - name: alice + token: alice-token + vault_access: + type: allow_list + allowed: + - personal-alice + - team-shared + - name: bob + token: bob-token + vault_access: + type: allow_list + allowed: + - personal-bob + - team-shared + - name: charlie + token: charlie-token + vault_access: + type: allow_list + allowed: + - personal-charlie + - team-shared +``` + +### Organization (Mixed Access) + +```yaml +users: + user_configs: + - name: admin + token: admin-token + vault_access: + type: allow_access_to_all + + - name: developer + token: dev-token + vault_access: + type: allow_list + allowed: + - engineering-docs + - api-specs + - shared + + - name: designer + token: design-token + vault_access: + type: allow_list + allowed: + - design-docs + - brand-assets + - shared + + - name: readonly + token: readonly-token + vault_access: + type: allow_list + allowed: + - public-wiki +``` + +## Authentication Flow + +### Connection + +1. Client connects via WebSocket +2. Client sends authentication message: + ```json + { + "type": "auth", + "token": "user-token", + "vault": "vault-name" + } + ``` +3. Server validates: + - Token exists in config + - User has access to requested vault +4. Server responds: + - Success: Connection established + - Failure: Connection closed with error + +### Validation + +Server checks: + +1. **Token match**: Token exists in `user_configs` +2. **Vault access**: User has permission for vault +3. **Connection limits**: Not exceeding `max_clients_per_vault` + +### Errors + +**Invalid token**: + +``` +Authentication failed: Invalid token +``` + +**No vault access**: + +``` +Authentication failed: User does not have access to vault 'restricted' +``` + +**Connection limit**: + +``` +Connection rejected: Maximum clients reached for vault +``` + +## Security Best Practices + +### Token Generation + +Generate strong tokens: + +```bash +# 64 character hex token (256 bits) +openssl rand -hex 32 + +# Base64 encoded (256 bits) +openssl rand -base64 32 + +# UUID v4 +uuidgen +``` + +### Token Storage + +**In config file**: + +```yaml +users: + user_configs: + - name: alice + token: !ENV ALICE_TOKEN # Read from environment variable +``` + +**Load from environment**: + +```bash +export ALICE_TOKEN="$(openssl rand -hex 32)" +./sync_server config.yml +``` + +### Token Rotation + +Periodically change tokens: + +1. Generate new token +2. Update `config.yml` +3. Restart server +4. Update clients with new token + +### Token Revocation + +To revoke access: + +1. Remove user from `config.yml` +2. Restart server +3. User's connections will be rejected + +For immediate revocation: + +- Remove user from config +- Restart server +- Existing connections are terminated + +## Access Patterns + +### Read-Only Users + +VaultLink doesn't distinguish read-only vs read-write. Implement via client: + +```yaml +# Server: Grant access +users: + user_configs: + - name: readonly + token: readonly-token + vault_access: + type: allow_list + allowed: + - public + +# Client: Use CLI in read-only mode (mount vault read-only) +docker run -v /vault:/vault:ro ... +``` + +### Temporary Access + +Grant temporary access: + +1. Add user to config +2. Set reminder to remove later +3. Remove user when no longer needed +4. Restart server + +For automation: + +```bash +# Add user with expiry comment +echo " - name: temp-user # EXPIRES: 2024-12-31" >> config.yml +echo " token: temp-token" >> config.yml +``` + +### Shared Tokens (Not Recommended) + +Multiple users sharing a token: + +- All appear as same user in logs +- Can't revoke individual access +- Security risk if one person leaves + +**Instead**: Create separate users with same vault access. + +## Monitoring + +### Server Logs + +Authentication events are logged: + +``` +2024-01-01 12:00:00 INFO User 'alice' authenticated for vault 'personal' +2024-01-01 12:00:05 WARN Authentication failed: Invalid token from 192.168.1.100 +2024-01-01 12:00:10 WARN User 'bob' denied access to vault 'restricted' +``` + +### Audit Trail + +Monitor authentication: + +```bash +# View authentication logs +grep "authenticated" logs/*.log + +# View failed authentications +grep "Authentication failed" logs/*.log + +# View access denials +grep "denied access" logs/*.log +``` + +## Advanced Scenarios + +### Multiple Servers + +Same user across multiple server instances: + +```yaml +# Server 1 config.yml +users: + user_configs: + - name: alice + token: alice-global-token + vault_access: + type: allow_list + allowed: + - vault-1 + - vault-2 + +# Server 2 config.yml +users: + user_configs: + - name: alice + token: alice-global-token # Same token + vault_access: + type: allow_list + allowed: + - vault-3 + - vault-4 +``` + +### Service Accounts + +Tokens for automated systems: + +```yaml +users: + user_configs: + - name: backup-service + token: backup-service-token + vault_access: + type: allow_access_to_all + + - name: ci-pipeline + token: ci-token + vault_access: + type: allow_list + allowed: + - documentation + + - name: monitoring + token: monitoring-token + vault_access: + type: allow_list + allowed: + - metrics +``` + +### Dynamic Vault Access + +VaultLink doesn't support runtime user management. To change access: + +1. Update `config.yml` +2. Restart server +3. Users reconnect automatically + +For frequent changes, consider: + +- Over-provision access (deny list) +- Use external authentication proxy +- Script config updates + reload + +## Troubleshooting + +### Can't connect + +**Check token**: + +```bash +# Verify token in config matches client +grep "token:" config.yml +``` + +**Check vault name**: + +```bash +# Ensure vault is in allowed list +grep -A 5 "name: alice" config.yml +``` + +**Check server logs**: + +```bash +tail -f logs/*.log | grep -i auth +``` + +### Access denied + +**Verify vault access**: + +```yaml +# Check user's vault_access configuration +users: + user_configs: + - name: alice + vault_access: + type: allow_list + allowed: + - vault-name # Must match exactly +``` + +**Case sensitivity**: + +- Vault names are case-sensitive +- `Vault` ≠ `vault` +- Ensure exact match in config and client + +### Token not working + +**Check for typos**: + +- Extra spaces +- Hidden characters +- Wrong quotes in YAML + +**Regenerate token**: + +```bash +# Generate new token +openssl rand -hex 32 + +# Update config +# Restart server +# Update client +``` + +## Next Steps + +- [Server configuration reference →](/config/server) +- [Advanced configuration →](/config/advanced) +- [Deploy the server →](/guide/server-setup) diff --git a/docs/config/server.md b/docs/config/server.md new file mode 100644 index 00000000..26eb894a --- /dev/null +++ b/docs/config/server.md @@ -0,0 +1,489 @@ +# Server Configuration + +Complete reference for configuring the VaultLink sync server via `config.yml`. + +## Configuration File Format + +The server is configured using a YAML file passed as a command-line argument: + +```bash +/app/sync_server /path/to/config.yml +``` + +## Complete Example + +```yaml +database: + databases_directory_path: databases + max_connections_per_vault: 12 + cursor_timeout_seconds: 60 + +server: + host: 0.0.0.0 + port: 3000 + max_body_size_mb: 512 + max_clients_per_vault: 256 + response_timeout_seconds: 60 + +users: + user_configs: + - name: admin + token: your-secure-random-token + vault_access: + type: allow_access_to_all + - name: alice + token: alice-token + vault_access: + type: allow_list + allowed: + - personal + - shared + - name: bob + token: bob-token + vault_access: + type: deny_list + denied: + - restricted + +logging: + log_directory: logs + log_rotation: 7days +``` + +## Database Section + +### `databases_directory_path` + +**Type**: String +**Required**: Yes +**Default**: None + +Directory where SQLite database files are stored. One database file per vault. + +```yaml +database: + databases_directory_path: /data/databases +``` + +The directory structure: + +``` +databases/ +├── vault-1.db +├── vault-2.db +└── personal.db +``` + +**Notes**: + +- Path is relative to working directory or absolute +- Directory must be writable by the server process +- Ensure adequate disk space for vault data +- Back up this directory regularly + +### `max_connections_per_vault` + +**Type**: Integer +**Required**: Yes +**Default**: None +**Recommended**: 12 + +Maximum concurrent database connections per vault. + +```yaml +database: + max_connections_per_vault: 12 +``` + +**Tuning**: + +- Higher values: Better performance under load +- Lower values: Less memory usage +- Typical range: 8-20 +- Consider: Number of concurrent users × average operations per user + +### `cursor_timeout_seconds` + +**Type**: Integer +**Required**: Yes +**Default**: None +**Recommended**: 60 + +How long to keep database cursors alive for inactive clients. + +```yaml +database: + cursor_timeout_seconds: 60 +``` + +**Notes**: + +- Cursors track client sync state +- Timeout too short: Clients may need to re-sync frequently +- Timeout too long: More memory usage +- Typical range: 30-300 seconds + +## Server Section + +### `host` + +**Type**: String +**Required**: Yes +**Default**: None + +Network interface to bind the server to. + +```yaml +server: + host: 0.0.0.0 # All interfaces + # OR + host: 127.0.0.1 # Localhost only + # OR + host: 192.168.1.100 # Specific interface +``` + +**Common values**: + +- `0.0.0.0`: Listen on all network interfaces (production) +- `127.0.0.1`: Listen on localhost only (development/testing) +- Specific IP: Listen on specific interface + +### `port` + +**Type**: Integer +**Required**: Yes +**Default**: None +**Recommended**: 3000 + +TCP port to listen on. + +```yaml +server: + port: 3000 +``` + +**Notes**: + +- Must be available (not in use) +- Privileged ports (< 1024) require root +- Common ports: 3000, 8080, 8888 +- Configure firewall to allow this port + +### `max_body_size_mb` + +**Type**: Integer +**Required**: Yes +**Default**: None +**Recommended**: 512 + +Maximum size of HTTP request body in megabytes. + +```yaml +server: + max_body_size_mb: 512 +``` + +**Usage**: + +- Limits file upload size +- Prevents memory exhaustion attacks +- Must be larger than largest expected file +- Consider client `max_file_size_mb` settings + +**Tuning**: + +- Small vaults (mostly text): 100 MB +- Medium vaults (some images): 512 MB +- Large vaults (many images/PDFs): 1024+ MB + +### `max_clients_per_vault` + +**Type**: Integer +**Required**: Yes +**Default**: None +**Recommended**: 256 + +Maximum concurrent clients per vault. + +```yaml +server: + max_clients_per_vault: 256 +``` + +**Notes**: + +- Limits concurrent WebSocket connections +- Prevents resource exhaustion +- Consider expected number of users +- Each client uses memory and file descriptors + +**Scaling**: + +- Personal use: 10-50 +- Small team: 50-100 +- Large team: 100-500 + +### `response_timeout_seconds` + +**Type**: Integer +**Required**: Yes +**Default**: None +**Recommended**: 60 + +Maximum time to wait for client responses. + +```yaml +server: + response_timeout_seconds: 60 +``` + +**Usage**: + +- Timeout for HTTP requests +- Timeout for WebSocket operations +- Clients disconnected if unresponsive + +**Tuning**: + +- Fast networks: 30 seconds +- Slow networks: 90-120 seconds +- Large file uploads: Increase proportionally + +## Users Section + +See [Authentication Configuration →](/config/authentication) for detailed user configuration. + +## Logging Section + +### `log_directory` + +**Type**: String +**Required**: Yes +**Default**: None + +Directory where log files are written. + +```yaml +logging: + log_directory: /data/logs + # OR + log_directory: logs # Relative to working directory +``` + +**Notes**: + +- Path is relative to working directory or absolute +- Directory must be writable +- Logs are rotated based on `log_rotation` +- Monitor disk usage + +### `log_rotation` + +**Type**: String +**Required**: Yes +**Default**: None + +How often to rotate log files. + +```yaml +logging: + log_rotation: 7days + # OR + log_rotation: 24hours + # OR + log_rotation: 30days +``` + +**Format**: `` + +**Units**: + +- `hours`: Hours (e.g., `12hours`, `24hours`) +- `days`: Days (e.g., `7days`, `30days`) + +**Recommendations**: + +- Development: `24hours` or `7days` +- Production: `7days` or `30days` +- High traffic: `24hours` (logs can be large) + +## Environment-Specific Configurations + +### Development + +```yaml +database: + databases_directory_path: ./databases + max_connections_per_vault: 8 + cursor_timeout_seconds: 30 + +server: + host: 127.0.0.1 + port: 3000 + max_body_size_mb: 100 + max_clients_per_vault: 10 + response_timeout_seconds: 30 + +users: + user_configs: + - name: dev + token: dev-token + vault_access: + type: allow_access_to_all + +logging: + log_directory: logs + log_rotation: 24hours +``` + +### Production + +```yaml +database: + databases_directory_path: /data/databases + max_connections_per_vault: 16 + cursor_timeout_seconds: 120 + +server: + host: 0.0.0.0 + port: 3000 + max_body_size_mb: 512 + max_clients_per_vault: 256 + response_timeout_seconds: 90 + +users: + user_configs: + - name: admin + token: + vault_access: + type: allow_access_to_all + # Additional users... + +logging: + log_directory: /data/logs + log_rotation: 7days +``` + +## Validation + +The server validates configuration on startup: + +```bash +# Start server +./sync_server config.yml + +# Check for errors in logs +tail -f logs/latest.log +``` + +**Common errors**: + +- Missing required fields +- Invalid YAML syntax +- Invalid values (negative numbers, etc.) +- Directory not writable + +## Performance Tuning + +### High Concurrency + +For many concurrent users: + +```yaml +database: + max_connections_per_vault: 20 # Increase + +server: + max_clients_per_vault: 500 # Increase + response_timeout_seconds: 120 # Increase for slow clients +``` + +### Large Files + +For vaults with large files: + +```yaml +server: + max_body_size_mb: 1024 # Allow larger uploads + response_timeout_seconds: 180 # More time for uploads +``` + +### Resource-Constrained Systems + +For limited CPU/memory: + +```yaml +database: + max_connections_per_vault: 6 # Reduce + +server: + max_clients_per_vault: 50 # Reduce + max_body_size_mb: 256 # Reduce +``` + +## Security Considerations + +### Token Security + +- Use strong random tokens: `openssl rand -hex 32` +- Never commit tokens to version control +- Rotate tokens periodically +- Use different tokens per user + +### Network Security + +- Bind to `127.0.0.1` if using reverse proxy on same host +- Use firewall to restrict access +- Enable SSL/TLS via reverse proxy + +### Resource Limits + +- Set `max_clients_per_vault` to prevent DoS +- Set `max_body_size_mb` to prevent memory exhaustion +- Configure `response_timeout_seconds` to prevent hanging connections + +## Troubleshooting + +### Server won't start + +**Check YAML syntax**: + +```bash +# Use a YAML validator +python -c 'import yaml, sys; yaml.safe_load(open("config.yml"))' +``` + +**Check file paths**: + +```bash +# Ensure directories exist and are writable +mkdir -p databases logs +chmod 755 databases logs +``` + +**Check port availability**: + +```bash +# Verify port is not in use +lsof -i :3000 +``` + +### High memory usage + +- Reduce `max_connections_per_vault` +- Reduce `max_clients_per_vault` +- Reduce `max_body_size_mb` +- Check for large vaults or many concurrent users + +### Slow performance + +- Increase `max_connections_per_vault` +- Increase database connection pool +- Use SSD for database storage +- Monitor database size (vacuum if needed) + +## Next Steps + +- [Configure authentication →](/config/authentication) +- [Advanced configuration options →](/config/advanced) +- [Deploy the server →](/guide/server-setup) diff --git a/docs/guide/alternatives.md b/docs/guide/alternatives.md new file mode 100644 index 00000000..7f314127 --- /dev/null +++ b/docs/guide/alternatives.md @@ -0,0 +1,324 @@ +# Comparison with Alternatives + +VaultLink is one of several solutions for synchronising Obsidian vaults. This page compares VaultLink with popular alternatives to help you choose the right tool. + +## Key Differentiator: Editor Agnostic + +**VaultLink is not tied to Obsidian.** While it includes an Obsidian plugin for convenience, VaultLink synchronises plain text files and works with any editor: + +- Edit with **Obsidian desktop** on your laptop +- Edit with **Vim** on your server +- Edit with **VS Code** on your workstation +- Edit with **Obsidian mobile** on your phone +- Use the **CLI client** for automated workflows + +All changes merge automatically without conflict markers, regardless of which editor you use. This is possible because VaultLink uses [reconcile-text](/architecture/sync-algorithm#why-reconcile-text-over-crdts) for differential synchronisation rather than requiring operation-level tracking. + +## VaultLink's Core Strengths + +Before diving into comparisons: + +1. **Fully self-hosted**: Server and all components are open source +2. **Collaborative editing**: Real-time sync with operational transformation +3. **Automatic conflict resolution**: No manual intervention or paid features required +4. **Cursor tracking**: See where other users are editing +5. **Extensively tested**: Comprehensive test suite for server and client +6. **Editor freedom**: Use any text editor, not just Obsidian +7. **Production-ready**: Docker images, health checks, monitoring + +## Obsidian Sync Alternatives + +### Self-hosted LiveSync + +**Downloads**: ~300,000 +**Repository**: https://github.com/vrtmrz/obsidian-livesync + +**Overview**: CouchDB/IBM Cloudant-based sync with end-to-end encryption. + +| Aspect | Self-hosted LiveSync | VaultLink | +| ------------------------- | --------------------------- | -------------------------------------- | +| **Self-hosted** | Yes (CouchDB required) | Yes (single binary or Docker) | +| **Conflict resolution** | Manual or automatic (basic) | Automatic (operational transformation) | +| **Collaborative editing** | No | Yes (real-time with cursors) | +| **Editor support** | Obsidian only | Any text editor | +| **Infrastructure** | CouchDB database | SQLite (bundled) | +| **Deployment complexity** | Medium (external DB) | Low (single container) | +| **End-to-end encryption** | Yes | No (transport encryption only) | +| **Out-of-band edits** | Limited support | Full support (edit with any tool) | + +**When to use LiveSync**: + +- Need end-to-end encryption +- Already running CouchDB +- Only use Obsidian (no external editors) + +**When to use VaultLink**: + +- Want collaborative editing with multiple users +- Edit files with various tools (Vim, VS Code, etc.) +- Need simpler deployment (no external database) +- Want operational transformation for better merges + +--- + +### Remotely Save + +**Downloads**: ~1.1M +**Repository**: https://github.com/remotely-save/remotely-save + +**Overview**: Sync to cloud storage providers (S3, Dropbox, OneDrive, WebDAV). + +| Aspect | Remotely Save | VaultLink | +| ------------------------- | ---------------------------- | ------------------------ | +| **Self-hosted** | Partial (uses cloud storage) | Fully self-hosted | +| **Conflict resolution** | Paid Pro feature | Free and automatic | +| **Collaborative editing** | No | Yes | +| **Editor support** | Obsidian only | Any text editor | +| **Storage backend** | Cloud providers | Self-hosted SQLite | +| **Cost** | Free (basic) / Paid (Pro) | Free (open source) | +| **Code quality** | No tests, complex codebase | Comprehensive test suite | +| **Real-time sync** | No (periodic polling) | Yes (WebSocket) | + +**When to use Remotely Save**: + +- Already use cloud storage (S3, Dropbox) +- Don't need real-time sync +- Single-user scenario + +**When to use VaultLink**: + +- Want full control over data +- Need automatic conflict resolution without paying +- Want real-time collaborative editing +- Value code quality and testing + +**Note**: Remotely Save's conflict resolution is a paid feature. VaultLink provides superior automatic merging for free. + +--- + +### Relay + +**Downloads**: ~24,000 +**Repository**: https://github.com/No-Instructions/Relay + +**Overview**: CRDT-based sync with proprietary server component. + +| Aspect | Relay | VaultLink | +| -------------------------- | ---------------------------- | ----------------------- | +| **Self-hosted** | No (proprietary server) | Yes (fully open source) | +| **Conflict resolution** | CRDT (automatic) | OT (automatic) | +| **Collaborative editing** | Yes | Yes | +| **Editor support** | Obsidian only | Any text editor | +| **Out-of-band edits** | No (breaks CRDT consistency) | Yes (differential sync) | +| **Server open source** | No | Yes | +| **Infrastructure control** | Limited | Full | +| **Per-file overhead** | High (CRDT metadata) | Low (version history) | + +**When to use Relay**: + +- Want hosted solution (don't self-host) +- Only edit within Obsidian +- Don't need out-of-band editing + +**When to use VaultLink**: + +- Need fully open source solution +- Want to self-host completely +- Edit files outside Obsidian (Vim, VS Code) +- Value infrastructure control + +**Critical limitation**: Relay's CRDT approach requires tracking every operation within Obsidian. Editing files outside Obsidian breaks the CRDT state. VaultLink's differential sync works regardless of how files are edited. + +--- + +### Obsidian Git + +**Downloads**: ~1.4M +**Repository**: https://github.com/denolehov/obsidian-git + +**Overview**: Uses Git for version control and synchronisation. + +| Aspect | Obsidian Git | VaultLink | +| ------------------------- | ----------------------------- | ----------------------- | +| **Self-hosted** | Yes (Git server) | Yes (sync server) | +| **Conflict resolution** | Manual (conflict markers) | Automatic (no markers) | +| **Collaborative editing** | No | Yes (real-time) | +| **Editor support** | Any (it's Git) | Any (differential sync) | +| **Version history** | Full Git history | Document versions | +| **Real-time sync** | No (commit-based) | Yes (instant) | +| **Merge conflicts** | Manual resolution | Automatic | +| **Learning curve** | High (Git knowledge required) | Low | +| **Workflow interruption** | Yes (resolve conflicts) | No | + +**When to use Obsidian Git**: + +- Need full version control (branches, tags, etc.) +- Already familiar with Git workflows +- Want integration with existing Git repos +- Don't mind manual conflict resolution + +**When to use VaultLink**: + +- Want automatic conflict-free merging +- Need real-time collaborative editing +- Don't want workflow interruptions from merge conflicts +- Prefer simpler mental model (sync, not commits) + +**Key difference**: Git requires manual conflict resolution with `<<<<<<<` markers. VaultLink automatically merges all changes using operational transformation, never interrupting your workflow. + +--- + +### Syncthing Integration + +**Downloads**: ~22,600 +**Repository**: https://github.com/LBF38/obsidian-syncthing-integration + +**Overview**: Wrapper around Syncthing for file synchronisation. + +| Aspect | Syncthing Integration | VaultLink | +| ------------------------- | ------------------------------ | ----------------- | +| **Self-hosted** | Yes (Syncthing) | Yes (sync server) | +| **Conflict resolution** | Manual | Automatic | +| **Collaborative editing** | No | Yes | +| **Editor support** | Any | Any | +| **Status** | Unfinished | Production-ready | +| **Conflict files** | Creates `.sync-conflict` files | No conflict files | +| **Real-time sync** | Yes | Yes | +| **Automatic merging** | No | Yes | + +**When to use Syncthing Integration**: + +- Already use Syncthing for other files +- Don't need automatic conflict resolution +- Single-user with multiple devices + +**When to use VaultLink**: + +- Want automatic conflict resolution +- Need collaborative editing +- Want production-ready solution +- Don't want to manage conflict files + +**Status note**: Syncthing Integration is marked as unfinished. VaultLink is production-ready with comprehensive testing. + +--- + +### Remotely Sync + +**Downloads**: ~38,000 +**Repository**: https://github.com/sboesen/remotely-sync + +**Overview**: Similar to Remotely Save, syncs to cloud storage. + +| Aspect | Remotely Sync | VaultLink | +| ----------------------- | ----------------------- | ------------------- | +| **Self-hosted** | Partial (cloud storage) | Fully self-hosted | +| **Conflict resolution** | Limited/Paid | Free and automatic | +| **Code quality** | No tests | Comprehensive tests | +| **Maintenance** | Low activity | Active development | + +**Same concerns as Remotely Save**: No test suite, conflict resolution limitations, cloud storage dependency. + +**When to use VaultLink**: See Remotely Save comparison above. + +--- + +### SyncFTP + +**Downloads**: ~5,000 +**Repository**: https://github.com/alex-donnan/SyncFTP + +**Overview**: Simple FTP-based file synchronisation. + +| Aspect | SyncFTP | VaultLink | +| ------------------------- | ---------------------- | ---------------- | +| **Conflict resolution** | None (last write wins) | Automatic (OT) | +| **Data loss risk** | High (overwrites) | None (merges) | +| **Collaborative editing** | No | Yes | +| **Sophistication** | Minimal | Production-grade | + +**When to use SyncFTP**: Don't use SyncFTP for any scenario where data integrity matters. + +**When to use VaultLink**: Any scenario requiring reliable synchronisation. + +--- + +## Feature Comparison Matrix + +| Feature | VaultLink | LiveSync | Relay | Git | Remotely Save | Syncthing | +| --------------------------------- | --------- | -------- | ----- | --- | ------------- | --------- | +| **Fully open source** | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | +| **Self-hosted** | ✅ | ✅ | ❌ | ✅ | Partial | ✅ | +| **Automatic conflict resolution** | ✅ | Basic | ✅ | ❌ | Paid | ❌ | +| **Real-time sync** | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | +| **Collaborative editing** | ✅ | ❌ | ✅ | ❌ | ❌ | ❌ | +| **Cursor tracking** | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | +| **Editor agnostic** | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | +| **Out-of-band edits** | ✅ | Limited | ❌ | ✅ | ❌ | ✅ | +| **No conflict markers** | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | +| **Comprehensive tests** | ✅ | ❌ | ❌ | N/A | ❌ | N/A | +| **Simple deployment** | ✅ | ❌ | N/A | ❌ | ✅ | ❌ | +| **Low infrastructure** | ✅ | ❌ | N/A | ✅ | ✅ | ✅ | + +--- + +## VaultLink's Unique Position + +VaultLink is the **only** solution that combines: + +1. **Fully open source** self-hosted server +2. **Editor agnostic** operation (not locked to Obsidian) +3. **Automatic conflict-free merging** using operational transformation +4. **Real-time collaborative editing** with cursor tracking +5. **Differential synchronisation** supporting out-of-band edits +6. **Comprehensive test coverage** ensuring reliability +7. **Simple deployment** via Docker or single binary + +## Use Case Recommendations + +### Choose VaultLink when you: + +- Edit vaults with multiple editors (Obsidian + Vim + VS Code) +- Need real-time collaboration with teammates +- Want automatic conflict resolution without manual intervention +- Value full control over infrastructure +- Need production-ready reliability with comprehensive testing +- Want to edit files while offline and sync later seamlessly + +### Consider alternatives when you: + +- **LiveSync**: Need end-to-end encryption and only use Obsidian +- **Git**: Need full version control with branches and advanced Git features +- **Remotely Save**: Already committed to cloud storage providers +- **Syncthing**: Already use Syncthing and don't need automatic merging + +## Migration from Other Solutions + +VaultLink works with plain Markdown files, making migration simple: + +1. **From Git**: Clone your repo, point VaultLink to the directory +2. **From cloud sync**: Download files, configure VaultLink client +3. **From LiveSync**: Export vault, import to VaultLink +4. **From Syncthing**: Point VaultLink to synced directory + +All solutions work with the same Markdown files—VaultLink just syncs them better. + +## Beyond Obsidian + +Because VaultLink is editor-agnostic, you can use it for: + +- **Documentation teams**: Sync technical docs edited in VS Code +- **Academic writing**: Collaborate on papers with various Markdown editors +- **Personal knowledge bases**: Use Obsidian on mobile, Vim on servers +- **Automated workflows**: CLI client for backup systems and CI/CD +- **Multi-tool workflows**: Different team members use different editors + +VaultLink doesn't lock you into Obsidian—it's a general-purpose differential sync system that happens to work excellently with Obsidian vaults. + +## Next Steps + +Ready to try VaultLink? + +- [Get started →](/guide/getting-started) +- [Understand the architecture →](/architecture/) +- [See how sync works →](/architecture/sync-algorithm) diff --git a/docs/guide/cli-client.md b/docs/guide/cli-client.md new file mode 100644 index 00000000..eeb11131 --- /dev/null +++ b/docs/guide/cli-client.md @@ -0,0 +1,532 @@ +# CLI Client + +Sync vaults without Obsidian. Works on servers, automation, backups, headless systems. + +## Installation + +### Docker (Recommended) + +Pull the latest image: + +```bash +docker pull ghcr.io/schmelczer/vault-link-cli:latest +``` + +### npm + +Install globally: + +```bash +npm install -g @schmelczer/local-client-cli +``` + +Verify installation: + +```bash +vaultlink --version +``` + +### From Source + +Build from the repository: + +```bash +git clone https://github.com/schmelczer/vault-link.git +cd vault-link/frontend/local-client-cli +npm install +npm run build +node dist/cli.js --help +``` + +## Usage + +### Basic Usage + +```bash +vaultlink \ + --local-path /path/to/vault \ + --remote-uri wss://sync.example.com \ + --token your-auth-token \ + --vault-name default +``` + +### Docker Usage + +```bash +docker run -v /path/to/vault:/vault \ + ghcr.io/schmelczer/vault-link-cli:latest \ + -l /vault \ + -r wss://sync.example.com \ + -t your-auth-token \ + -v default +``` + +### Docker Compose + +Create `docker-compose.yml`: + +```yaml +services: + vaultlink-cli: + image: ghcr.io/schmelczer/vault-link-cli:latest + restart: unless-stopped + volumes: + - ./vault:/vault + command: + - "-l" + - "/vault" + - "-r" + - "wss://sync.example.com" + - "-t" + - "your-token" + - "-v" + - "default" +``` + +Start the client: + +```bash +docker compose up -d +``` + +## Configuration Options + +### Required Arguments + +| Argument | Short | Description | Example | +| -------------- | ----- | ----------------------- | ------------------------ | +| `--local-path` | `-l` | Local directory to sync | `/vault` | +| `--remote-uri` | `-r` | Server WebSocket URI | `wss://sync.example.com` | +| `--token` | `-t` | Authentication token | `abc123...` | +| `--vault-name` | `-v` | Vault name on server | `default` | + +### Optional Arguments + +| Argument | Default | Description | +| ------------------------------- | ------- | -------------------------------------- | +| `--sync-concurrency` | `1` | Concurrent file operations | +| `--max-file-size-mb` | `10` | Max file size in MB | +| `--ignore-pattern` | - | Glob pattern to ignore (repeatable) | +| `--websocket-retry-interval-ms` | `3500` | Reconnection interval | +| `--log-level` | `INFO` | Log level: DEBUG, INFO, WARNING, ERROR | + +### Environment Variables + +Alternative to command-line arguments: + +```bash +export VAULTLINK_LOCAL_PATH="/vault" +export VAULTLINK_REMOTE_URI="wss://sync.example.com" +export VAULTLINK_TOKEN="your-token" +export VAULTLINK_VAULT_NAME="default" + +vaultlink +``` + +## Examples + +### Basic Sync + +Sync a local directory to the server: + +```bash +vaultlink \ + -l ./my-notes \ + -r wss://sync.example.com \ + -t my-secure-token \ + -v personal +``` + +### With Ignore Patterns + +Exclude specific files or directories: + +```bash +vaultlink \ + -l ./vault \ + -r wss://sync.example.com \ + -t token123 \ + -v default \ + --ignore-pattern "*.tmp" \ + --ignore-pattern ".DS_Store" \ + --ignore-pattern "node_modules/**" +``` + +### Debug Logging + +Enable verbose logging: + +```bash +vaultlink \ + -l ./vault \ + -r wss://sync.example.com \ + -t token123 \ + -v default \ + --log-level DEBUG +``` + +### High Concurrency + +Faster initial sync: + +```bash +vaultlink \ + -l ./vault \ + -r wss://sync.example.com \ + -t token123 \ + -v default \ + --sync-concurrency 5 +``` + +### Large Files + +Allow larger file uploads: + +```bash +vaultlink \ + -l ./vault \ + -r wss://sync.example.com \ + -t token123 \ + -v default \ + --max-file-size-mb 50 +``` + +## Docker Deployment + +### Long-Running Sync + +Run as a daemon for continuous synchronisation: + +```bash +docker run -d \ + --name vaultlink-sync \ + --restart unless-stopped \ + -v $(pwd)/vault:/vault \ + ghcr.io/schmelczer/vault-link-cli:latest \ + -l /vault \ + -r wss://sync.example.com \ + -t your-token \ + -v default +``` + +Monitor logs: + +```bash +docker logs -f vaultlink-sync +``` + +### Health Monitoring + +The Docker image includes built-in health checks: + +```bash +# Check health status +docker ps + +# View detailed health info +docker inspect --format='{{json .State.Health}}' vaultlink-sync | jq +``` + +Health check verifies: + +- Health file exists +- Status updated within last 30 seconds +- WebSocket connection is active + +Configure custom health check: + +```yaml +services: + vaultlink-cli: + image: ghcr.io/schmelczer/vault-link-cli:latest + healthcheck: + test: ["CMD", "node", "/app/healthcheck.js"] + interval: 15s + timeout: 5s + retries: 5 + start_period: 20s +``` + +### Read-Only Vault + +Mount vault as read-only to prevent local changes: + +```bash +docker run -d \ + -v $(pwd)/vault:/vault:ro \ + ghcr.io/schmelczer/vault-link-cli:latest \ + -l /vault \ + -r wss://sync.example.com \ + -t token \ + -v default +``` + +::: warning +The CLI needs write access to create `.vaultlink` metadata directory. Mount as read-write or provide a separate writeable directory. +::: + +## How It Works + +### Initial Sync + +On startup: + +1. Creates `.vaultlink/` directory for metadata +2. Scans local filesystem +3. Uploads all local files to server +4. Downloads files from server not present locally +5. Resolves conflicts using operational transformation + +### Real-Time Synchronization + +After initial sync: + +1. Watches filesystem for changes using `fs.watch` +2. Uploads changed files immediately +3. Receives real-time updates from server via WebSocket +4. Handles bidirectional sync automatically + +### Graceful Shutdown + +On SIGINT (Ctrl+C) or SIGTERM: + +1. Completes pending uploads +2. Closes WebSocket connection cleanly +3. Flushes metadata to disk +4. Exits gracefully + +## Use Cases + +### Automated Backups + +Continuously backup vaults to a remote server: + +```bash +docker run -d \ + --name vault-backup \ + -v /important/notes:/vault:ro \ + ghcr.io/schmelczer/vault-link-cli:latest \ + -l /vault -r wss://backup.example.com -t backup-token -v backups +``` + +### CI/CD Documentation + +Sync documentation in automated pipelines: + +```bash +# In your CI pipeline +docker run \ + -v $(pwd)/docs:/vault \ + ghcr.io/schmelczer/vault-link-cli:latest \ + -l /vault -r wss://docs.example.com -t ci-token -v prod-docs +``` + +### Multi-Location Sync + +Sync between different geographic locations: + +```bash +# Location A +vaultlink -l /data/vault -r wss://hub.example.com -t token -v shared + +# Location B +vaultlink -l /backup/vault -r wss://hub.example.com -t token -v shared +``` + +### Development Environment + +Keep documentation in sync across dev environments: + +```bash +# In docker-compose.yml for your dev stack +services: + docs-sync: + image: ghcr.io/schmelczer/vault-link-cli:latest + volumes: + - ./docs:/vault + command: ["-l", "/vault", "-r", "wss://docs-server", "-t", "dev-token", "-v", "dev"] +``` + +## Troubleshooting + +### Client won't connect + +**Check server accessibility**: + +```bash +curl https://sync.example.com/vaults/test/ping +``` + +**Verify WebSocket protocol**: + +- Use `ws://` for HTTP servers +- Use `wss://` for HTTPS servers + +**Check authentication**: + +- Token must match server config +- User must have access to the vault + +### Permission errors + +**Docker volume permissions**: + +```bash +# Ensure directory is writable +chmod 755 /path/to/vault + +# Check Docker user ID +docker run --rm ghcr.io/schmelczer/vault-link-cli:latest id +``` + +**SELinux issues**: + +```bash +# Add :z flag to volume mount +docker run -v /path/to/vault:/vault:z ... +``` + +### Files not syncing + +**Check ignore patterns**: + +- View logs to see which files are skipped +- Ensure patterns don't match unintentionally + +**File size limits**: + +- Check `--max-file-size-mb` setting +- Large files are skipped with a warning + +**Check metadata**: + +```bash +# View sync metadata +cat /path/to/vault/.vaultlink/metadata.json +``` + +### High memory usage + +**Reduce concurrency**: + +```bash +--sync-concurrency 1 +``` + +**Limit file sizes**: + +```bash +--max-file-size-mb 5 +``` + +**Check vault size**: + +- Very large vaults may need more resources +- Consider splitting into multiple vaults + +### Connection keeps dropping + +**Increase retry interval**: + +```bash +--websocket-retry-interval-ms 5000 +``` + +**Check network stability**: + +```bash +# Monitor connection +docker logs -f vaultlink-sync | grep -i websocket +``` + +**Server timeout settings**: + +- Verify reverse proxy WebSocket timeout +- Check server `response_timeout_seconds` + +## Advanced Usage + +### Custom Healthcheck Script + +Create your own health monitoring: + +```bash +#!/bin/bash +HEALTH_FILE="/tmp/vaultlink-health.json" + +if [ ! -f "$HEALTH_FILE" ]; then + exit 1 +fi + +# Check file is recent (within 60 seconds) +if [ $(( $(date +%s) - $(stat -c %Y "$HEALTH_FILE") )) -gt 60 ]; then + exit 1 +fi + +# Check WebSocket is connected +if ! jq -e '.connected == true' "$HEALTH_FILE" > /dev/null; then + exit 1 +fi + +exit 0 +``` + +### Automated Recovery + +Restart on failure with exponential backoff: + +```bash +#!/bin/bash +RETRY_DELAY=5 + +while true; do + vaultlink -l /vault -r wss://server -t token -v default + + echo "Client exited, restarting in ${RETRY_DELAY}s..." + sleep $RETRY_DELAY + + # Exponential backoff up to 5 minutes + RETRY_DELAY=$((RETRY_DELAY * 2)) + if [ $RETRY_DELAY -gt 300 ]; then + RETRY_DELAY=300 + fi +done +``` + +### Integration with systemd + +Create `/etc/systemd/system/vaultlink-cli.service`: + +```ini +[Unit] +Description=VaultLink CLI Sync +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Restart=always +RestartSec=10 +Environment="VAULTLINK_LOCAL_PATH=/data/vault" +Environment="VAULTLINK_REMOTE_URI=wss://sync.example.com" +Environment="VAULTLINK_TOKEN=your-token" +Environment="VAULTLINK_VAULT_NAME=default" +ExecStart=/usr/local/bin/vaultlink + +[Install] +WantedBy=multi-user.target +``` + +Enable and start: + +```bash +sudo systemctl daemon-reload +sudo systemctl enable vaultlink-cli +sudo systemctl start vaultlink-cli +``` + +## Next Steps + +- [Configure server authentication →](/config/authentication) +- [Learn about the sync algorithm →](/architecture/sync-algorithm) +- [Set up Obsidian plugin →](/guide/obsidian-plugin) diff --git a/docs/guide/getting-started.md b/docs/guide/getting-started.md new file mode 100644 index 00000000..02b20ae0 --- /dev/null +++ b/docs/guide/getting-started.md @@ -0,0 +1,125 @@ +# Getting Started + +Set up VaultLink in 5 minutes. Deploy server, connect clients, done. + +## Prerequisites + +- Docker (or Rust toolchain if building from source) +- A server (VPS, home server, or localhost for testing) + +## Step 1: Deploy Server + +Create `config.yml`: + +```yaml +database: + databases_directory_path: databases + max_connections_per_vault: 12 + cursor_timeout_seconds: 60 +server: + host: 0.0.0.0 + port: 3000 + max_body_size_mb: 512 + max_clients_per_vault: 256 + response_timeout_seconds: 60 +users: + user_configs: + - name: admin + token: change-this-to-secure-random-token + vault_access: + type: allow_access_to_all +logging: + log_directory: logs + log_rotation: 7days +``` + +::: tip +Generate secure token: `openssl rand -hex 32` +::: + +Run server: + +```bash +docker run -d \ + --name vaultlink-server \ + --restart unless-stopped \ + -p 3000:3000 \ + -v $(pwd):/data \ + ghcr.io/schmelczer/vault-link-server:latest \ + /app/sync_server /data/config.yml +``` + +Verify: `curl http://localhost:3000/vaults/test/ping` should return server version and auth status + +## Step 2: Connect Client + +### Obsidian Plugin + +1. Settings → Community Plugins → Browse +2. Search "VaultLink", install, enable +3. Configure: + - Server URL: `ws://localhost:3000` (or `wss://your-server.com` for SSL) + - Token: Your token from config.yml + - Vault Name: `default` + +[Full plugin guide →](/guide/obsidian-plugin) + +### CLI Client + +```bash +docker run -d \ + --name vaultlink-cli \ + --restart unless-stopped \ + -v /path/to/vault:/vault \ + ghcr.io/schmelczer/vault-link-cli:latest \ + -l /vault -r ws://localhost:3000 -t your-token -v default +``` + +[Full CLI guide →](/guide/cli-client) + +## Production Setup + +For production: + +1. **SSL/TLS**: Use Nginx/Caddy reverse proxy for `wss://` ([setup guide](/guide/server-setup#ssl-tls-with-reverse-proxy)) +2. **Secure tokens**: Generate with `openssl rand -hex 32`, don't reuse the example +3. **Firewall**: Only expose port 3000 to reverse proxy +4. **Backups**: SQLite databases are in `databases/` directory + +## Multiple Users + +```yaml +users: + user_configs: + - name: alice + token: alice-token + vault_access: + type: allow_list + allowed: + - personal + - shared + - name: bob + token: bob-token + vault_access: + type: allow_list + allowed: + - shared +``` + +[Auth docs →](/config/authentication) + +## Troubleshooting + +**Server won't start**: `docker logs vaultlink-server` + +**Client can't connect**: + +1. Verify server: `curl http://your-server:3000/vaults/test/ping` +2. Check URL: `ws://` for HTTP, `wss://` for HTTPS +3. Verify token matches config.yml + +**Understanding limitations**: [See what VaultLink can and can't do →](/guide/limitations) + +**Files not syncing**: Check client logs, verify vault name matches + +[Server setup →](/guide/server-setup) | [Architecture →](/architecture/) diff --git a/docs/guide/limitations.md b/docs/guide/limitations.md new file mode 100644 index 00000000..1c514939 --- /dev/null +++ b/docs/guide/limitations.md @@ -0,0 +1,192 @@ +# Limitations + +VaultLink works well for most Obsidian vaults, but has some constraints you should know about. + +## File Type Limitations + +### Mergeable Files + +Only **`.md`** and **`.txt`** files get automatic conflict-free merging. + +Other file types (images, PDFs, etc.) use last-write-wins: + +``` +User A updates diagram.png → Server stores version 1 +User B updates diagram.png → Server stores version 2 (overwrites A's changes) +``` + +**Workaround**: Avoid editing the same non-text file simultaneously. + +### Binary Detection + +Files are treated as binary if they: + +- Contain NUL bytes (`0x00`) +- Fail UTF-8 validation + +Binary files within `.md` or `.txt` extensions still get last-write-wins (no merge). + +## Performance Constraints + +### Server Limits (Configurable) + +| Resource | Default | Maximum Tested | +| ------------------------ | ------- | -------------- | +| Clients per vault | 256 | ~256 | +| Database connections | 12 | 20 | +| Max file size | 512 MB | 4096 MB | +| Request timeout | 60s | 180s | +| WebSocket cursor timeout | 60s | 300s | +| Database busy timeout | 3600s | - | + +### Vault Size + +- **Small vaults** (< 1000 files): Excellent performance +- **Medium vaults** (1000-10000 files): Good performance +- **Large vaults** (> 10000 files): Works, but initial sync slower + +No hard file count limit—constrained by disk space and sync time. + +### Resource Usage + +Rough estimates (varies by vault size and activity): + +- **RAM**: ~50-200 MB base + ~1-5 MB per active client +- **CPU**: Low (< 5%) for typical usage, spikes during merges +- **Disk**: Vault size + version history (grows over time) + +## Version History + +### Storage + +- All versions stored indefinitely (no automatic cleanup) +- Each vault is a separate SQLite database +- Deleted files marked as deleted (not purged) + +**Growth**: Version history grows with every change. A 10 MB vault with frequent edits might grow to 100+ MB over months. + +**Cleanup**: Manual only (see [Advanced Configuration](/config/advanced#version-history-cleanup)). + +### Implications + +- Disk usage grows over time +- Database size affects backup time +- No built-in retention policy + +## Merge Quality + +### Text Merging + +VaultLink uses word-level tokenisation for merging: + +```markdown +Parent: "The quick brown fox" +User A: "The quick red fox" +User B: "The very quick brown fox" +Result: "The very quick red fox" ← Both changes preserved +``` + +**Imperfect scenarios**: + +- Complex nested Markdown (tables, code blocks) +- Simultaneous edits to the same sentence +- Large structural changes (moving sections around) + +**Result**: Merged file might need manual cleanup in ~1-5% of concurrent edits. + +## Scalability + +### SQLite Limitations + +- One SQLite database per vault +- Single-server architecture (no built-in clustering) +- Write serialisation through database + +**For high concurrency**: Consider multiple vaults instead of one massive shared vault. + +### Horizontal Scaling + +Not currently supported. Running multiple servers requires manual vault partitioning. + +## Network Requirements + +### Latency + +- Real-time sync typically < 500ms on good connections +- Mobile/slow networks: 1-5s latency possible +- Timeout failures on very slow connections (> 60s) + +### Offline Behaviour + +- Clients queue changes locally +- On reconnect, sync all changes since last connection +- Conflicts resolved automatically (for mergeable files) + +**Limitation**: No offline conflict preview—merged result appears after reconnect. + +## Security + +### No End-to-End Encryption + +- Server sees all file contents +- Transport encryption only (WSS/TLS) +- Trust your server + +**Workaround**: Self-host on infrastructure you control. + +### Authentication + +- Token-based only (no OAuth, SAML, etc.) +- Tokens configured in server config file +- No runtime user management + +## Known Edge Cases + +### Simultaneous Deletes and Edits + +``` +User A deletes note.md +User B edits note.md +Result: Edit wins (file recreated with B's content) +``` + +Operational transformation prioritises content preservation. + +### Large File Uploads + +Files > 100 MB may time out on slow connections. Increase `response_timeout_seconds` or split large files. + +### Mobile Sync + +- Mobile networks may drop WebSocket connections frequently +- Client auto-reconnects, but causes sync delays +- Battery impact from constant reconnections + +## What VaultLink is NOT + +- **Not a backup solution**: Version history helps but isn't a backup (make backups!) +- **Not Git**: No branching, no commit messages, no diffs to review before merge +- **Not encrypted storage**: Server sees everything +- **Not multi-master**: One server, multiple clients (not peer-to-peer) + +## Recommendations + +### Good Use Cases + +- Personal multi-device sync (< 10 devices) +- Small team collaboration (< 20 people) +- Primarily text/Markdown content +- Trusted server environment + +### Poor Use Cases + +- Large teams (> 50 concurrent users per vault) +- Primarily binary files (images, videos, large PDFs) +- Untrusted server (need E2E encryption) +- Highly regulated environments (HIPAA, etc.) + +## Next Steps + +- [Server configuration limits →](/config/server) +- [Advanced tuning →](/config/advanced) +- [Architecture details →](/architecture/) diff --git a/docs/guide/obsidian-plugin.md b/docs/guide/obsidian-plugin.md new file mode 100644 index 00000000..5b63e43d --- /dev/null +++ b/docs/guide/obsidian-plugin.md @@ -0,0 +1,276 @@ +# Obsidian Plugin + +Real-time sync for Obsidian vaults. + +## Installation + +### From Obsidian Community Plugins + +1. Open Obsidian Settings +2. Navigate to **Community Plugins** +3. Click **Browse** and search for "VaultLink" +4. Click **Install** +5. Enable the plugin + +### Manual Installation + +1. Download the latest release from [GitHub Releases](https://github.com/schmelczer/vault-link/releases) +2. Extract `main.js`, `manifest.json`, and `styles.css` +3. Copy to `.obsidian/plugins/vault-link/` in your vault +4. Reload Obsidian +5. Enable VaultLink in Community Plugins settings + +## Configuration + +After installation, configure the plugin in **Settings → VaultLink**. + +### Required Settings + +#### Server URL + +The WebSocket URL of your sync server. + +- **Development/Local**: `ws://localhost:3000` +- **Production (SSL)**: `wss://sync.example.com` + +::: tip +Use `ws://` for unencrypted connections and `wss://` for SSL connections (production). +::: + +#### Authentication Token + +Your authentication token from the server's `config.yml`. + +Generate a secure token: + +```bash +openssl rand -hex 32 +``` + +#### Vault Name + +The name of the vault on the server. Can be any string. + +Multiple Obsidian vaults can sync to the same server vault name (for shared vaults), or use unique names for separate vaults. + +### Optional Settings + +#### Sync Concurrency + +Number of files to sync simultaneously. + +- **Default**: 1 +- **Range**: 1-10 +- Higher values = faster initial sync, more resource usage + +#### Max File Size + +Maximum file size to sync (in MB). + +- **Default**: 10 +- Files larger than this are skipped + +#### Ignore Patterns + +Glob patterns for files to exclude from sync. + +Examples: + +- `*.tmp` - Ignore temporary files +- `.trash/**` - Ignore trash folder +- `private/**` - Ignore private directory + +#### WebSocket Retry Interval + +Milliseconds between reconnection attempts when disconnected. + +- **Default**: 3500ms +- Increase for flaky networks to avoid connection spam + +## Usage + +### Initial Sync + +When first connecting: + +1. The plugin uploads all local files to the server +2. Downloads any missing files from the server +3. Resolves any conflicts using operational transformation +4. Begins real-time synchronisation + +Initial sync time depends on vault size and `sync_concurrency` setting. + +### Real-Time Sync + +Once connected: + +- **File changes**: Automatically synced when saved +- **File creation**: New files immediately uploaded +- **File deletion**: Deletions propagated to other clients +- **File renames**: Tracked and synchronised + +The plugin watches your vault filesystem and syncs changes in real-time via WebSocket. + +### Status Indicators + +The plugin provides visual feedback: + +- **Connected**: Green status in settings +- **Syncing**: Progress indicator during uploads +- **Disconnected**: Red status, automatic reconnection attempts +- **Error**: Error message in settings and console + +Check the Obsidian console (Ctrl+Shift+I / Cmd+Option+I) for detailed logs. + +## Features + +### Automatic Conflict Resolution + +When multiple users edit the same file simultaneously, operational transformation merges changes automatically: + +- All edits are preserved +- No manual conflict resolution required +- Changes appear in real-time as others type + +### Mobile Support + +VaultLink works on Obsidian mobile (iOS and Android): + +- Same configuration as desktop +- Real-time sync across all devices +- Handle network changes gracefully + +::: warning +Ensure your sync server is accessible from mobile networks (use WSS with a public domain or VPN). +::: + +### Offline Support + +The plugin handles offline scenarios: + +- Continue working when disconnected +- Changes queue locally +- Automatic sync when connection restored +- Conflict resolution if others edited the same files + +## Collaboration Workflows + +### Personal Multi-Device Sync + +Sync the same vault across devices: + +1. Configure each Obsidian instance with the same vault name +2. Use the same authentication token +3. All devices stay in sync automatically + +### Team Shared Vault + +Multiple users collaborating: + +1. Each user has their own token (configured in server `config.yml`) +2. All users connect to the same vault name +3. Real-time collaborative editing with automatic conflict resolution + +### Selective Sharing + +Share specific folders while keeping others private: + +1. Use different vault names for shared vs. private content +2. Configure access control on the server per vault +3. Use ignore patterns to exclude sensitive directories + +## Troubleshooting + +### Plugin won't connect + +1. **Verify server is running**: + + ```bash + curl http://your-server:3000/vaults/test/ping + ``` + + Should return `pong` + +2. **Check URL format**: + - Local: `ws://localhost:3000` + - Remote (SSL): `wss://sync.example.com` + - Don't include `/vault/name` in the URL + +3. **Verify token**: + - Must match server config exactly + - No extra spaces or quotes + - Check server logs for authentication errors + +4. **Check firewall**: + - Ensure port is accessible from your network + - For mobile, server must be publicly accessible or use VPN + +### Files not syncing + +1. **Check ignore patterns**: File may match an exclusion pattern +2. **File size**: Check if file exceeds `max_file_size_mb` +3. **Permissions**: Ensure vault directory is readable/writable +4. **Console errors**: Open dev tools (Ctrl+Shift+I) and check console + +### Slow initial sync + +1. **Increase concurrency**: Set `sync_concurrency` higher (e.g., 5) +2. **Network speed**: Check internet connection +3. **Server resources**: Ensure server isn't overloaded +4. **Large files**: Consider increasing timeout settings + +### Conflicts not resolving + +Operational transformation should handle conflicts automatically. If issues persist: + +1. Check console for sync errors +2. Verify both clients are connected +3. Check server logs for processing errors +4. Ensure files are text-based (binary files may not merge well) + +### High CPU/Memory usage + +1. **Reduce concurrency**: Lower `sync_concurrency` +2. **Add ignore patterns**: Exclude unnecessary files +3. **File watchers**: Large vaults may trigger many filesystem events +4. **Check for sync loops**: Ensure no circular dependencies + +## Advanced Configuration + +### Multiple Vaults + +To sync multiple Obsidian vaults to different server vaults: + +1. Each Obsidian vault has its own VaultLink plugin configuration +2. Use different vault names for each +3. Can use the same or different tokens (depending on access control) + +### Custom Sync Patterns + +Combine ignore patterns for fine-grained control: + +``` +# Ignore patterns +*.tmp +*.bak +.DS_Store +.trash/** +private/** +drafts/**/*.draft.md +``` + +### Development/Testing + +For plugin development: + +1. Clone the repository +2. `cd frontend && npm install` +3. `npm run dev` to build in watch mode +4. Plugin rebuilds automatically on changes +5. Reload Obsidian to test changes + +## Next Steps + +- [Learn about the sync algorithm →](/architecture/sync-algorithm) +- [Configure the server →](/config/server) +- [Set up the CLI client →](/guide/cli-client) diff --git a/docs/guide/server-setup.md b/docs/guide/server-setup.md new file mode 100644 index 00000000..7754da54 --- /dev/null +++ b/docs/guide/server-setup.md @@ -0,0 +1,379 @@ +# Server Setup + +Deploy VaultLink server via Docker, binary, or build from source. + +## Deployment Options + +### Docker (Recommended) + +Easiest deployment path, includes health checks. + +#### Basic Docker Deployment + +```bash +# Pull the latest image +docker pull ghcr.io/schmelczer/vault-link-server:latest + +# Create data directory +mkdir -p ~/vaultlink-data + +# Create config.yml (see Configuration section below) + +# Run the container +docker run -d \ + --name vaultlink-server \ + --restart unless-stopped \ + -p 3000:3000 \ + -v ~/vaultlink-data:/data \ + ghcr.io/schmelczer/vault-link-server:latest \ + /app/sync_server /data/config.yml +``` + +#### Docker Compose + +Create `docker-compose.yml`: + +```yaml +services: + vaultlink-server: + image: ghcr.io/schmelczer/vault-link-server:latest + container_name: vaultlink-server + restart: unless-stopped + ports: + - "3000:3000" + volumes: + - ./data:/data + command: ["/app/sync_server", "/data/config.yml"] + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:3000/vaults/fake/ping"] + interval: 30s + timeout: 5s + retries: 3 + start_period: 10s +``` + +Start the server: + +```bash +docker compose up -d +``` + +### Binary Installation + +Download pre-built binaries from [GitHub Releases](https://github.com/schmelczer/vault-link/releases): + +```bash +# Download the binary for your platform +wget https://github.com/schmelczer/vault-link/releases/latest/download/sync_server-linux-x86_64 + +# Make executable +chmod +x sync_server-linux-x86_64 + +# Run the server +./sync_server-linux-x86_64 config.yml +``` + +### Build from Source + +Requirements: Rust 1.89.0+, SQLite development headers, SQLx CLI + +```bash +# Clone the repository +git clone https://github.com/schmelczer/vault-link.git +cd vault-link/sync-server + +# Install SQLx CLI +cargo install sqlx-cli + +# Set up the database +sqlx database create --database-url sqlite://db.sqlite3 +sqlx migrate run --source src/app_state/database/migrations --database-url sqlite://db.sqlite3 +cargo sqlx prepare --workspace + +# Build in release mode +cargo build --release + +# Run the server +./target/release/sync_server config.yml +``` + +## Configuration + +Create a `config.yml` file with your server configuration: + +```yaml +database: + databases_directory_path: databases + max_connections_per_vault: 12 + cursor_timeout_seconds: 60 + +server: + host: 0.0.0.0 + port: 3000 + max_body_size_mb: 512 + max_clients_per_vault: 256 + response_timeout_seconds: 60 + +users: + user_configs: + - name: admin + token: your-secure-random-token-here + vault_access: + type: allow_access_to_all + +logging: + log_directory: logs + log_rotation: 7days +``` + +### Configuration Fields + +#### Database + +- `databases_directory_path`: Directory for SQLite databases (one per vault) +- `max_connections_per_vault`: Maximum concurrent database connections +- `cursor_timeout_seconds`: How long to keep database cursors alive + +#### Server + +- `host`: Bind address (use `0.0.0.0` for all interfaces) +- `port`: Port to listen on (default: 3000) +- `max_body_size_mb`: Maximum upload size +- `max_clients_per_vault`: Concurrent client limit per vault +- `response_timeout_seconds`: Request timeout + +#### Users + +See [Authentication Configuration →](/config/authentication) for detailed user setup. + +#### Logging + +- `log_directory`: Where to store log files +- `log_rotation`: How often to rotate logs (e.g., `7days`, `24hours`) + +## Production Deployment + +### SSL/TLS with Reverse Proxy + +VaultLink doesn't handle SSL directly. Use a reverse proxy like Nginx or Caddy. + +#### Nginx Configuration + +```nginx +upstream vaultlink { + server localhost:3000; +} + +server { + listen 443 ssl http2; + server_name sync.example.com; + + ssl_certificate /path/to/cert.pem; + ssl_certificate_key /path/to/key.pem; + + location / { + proxy_pass http://vaultlink; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + # WebSocket specific + proxy_read_timeout 3600s; + proxy_send_timeout 3600s; + } +} +``` + +Reload Nginx: + +```bash +sudo nginx -t +sudo systemctl reload nginx +``` + +#### Caddy Configuration + +Caddy handles SSL automatically: + +```caddy +sync.example.com { + reverse_proxy localhost:3000 +} +``` + +Start Caddy: + +```bash +caddy run --config Caddyfile +``` + +### Systemd Service + +Create `/etc/systemd/system/vaultlink.service`: + +```ini +[Unit] +Description=VaultLink Sync Server +After=network.target + +[Service] +Type=simple +User=vaultlink +WorkingDirectory=/opt/vaultlink +ExecStart=/opt/vaultlink/sync_server /opt/vaultlink/config.yml +Restart=always +RestartSec=10 + +[Install] +WantedBy=multi-user.target +``` + +Enable and start: + +```bash +sudo systemctl daemon-reload +sudo systemctl enable vaultlink +sudo systemctl start vaultlink +sudo systemctl status vaultlink +``` + +### Security Best Practices + +1. **Use strong tokens**: Generate with `openssl rand -hex 32` +2. **Enable firewall**: Only expose port 3000 to reverse proxy +3. **Regular updates**: Keep Docker images and binaries updated +4. **Backup databases**: SQLite files in `databases_directory_path` +5. **Monitor logs**: Check log directory for errors and anomalies +6. **Limit access**: Use vault-specific access controls per user + +### Backup Strategy + +The SQLite databases contain all vault data and history: + +```bash +# Backup script +#!/bin/bash +BACKUP_DIR="/backup/vaultlink/$(date +%Y%m%d)" +DATA_DIR="/data/databases" + +mkdir -p "$BACKUP_DIR" +cp -r "$DATA_DIR" "$BACKUP_DIR/" + +# Keep 30 days of backups +find /backup/vaultlink -type d -mtime +30 -exec rm -rf {} + +``` + +Run daily via cron: + +```cron +0 2 * * * /opt/vaultlink/backup.sh +``` + +### Monitoring + +#### Health Checks + +The server exposes a ping endpoint: + +```bash +curl http://localhost:3000/vaults/test/ping +# Returns: {"server_version":"0.10.1","is_authenticated":false} +``` + +Replace `test` with any vault name. The endpoint returns: + +- `server_version`: Current server version +- `is_authenticated`: Whether the request included a valid token + +Docker health check is built-in and checks this endpoint every 30 seconds. + +#### Prometheus Metrics + +For advanced monitoring, collect Docker stats or implement custom metrics. + +#### Log Monitoring + +Logs are written to the configured `log_directory`. Monitor for: + +- Connection failures +- Authentication errors +- Database errors +- WebSocket disconnections + +Example log watching: + +```bash +tail -f /data/logs/*.log | grep -i error +``` + +## Scaling + +### Horizontal Scaling + +VaultLink currently uses SQLite, which limits horizontal scaling. For multiple servers: + +1. Run separate instances for different vaults +2. Use load balancer with sticky sessions (same vault → same server) +3. Consider database architecture for your scale needs + +### Vertical Scaling + +Increase resources for the server: + +- More CPU for handling concurrent connections +- More RAM for database caching +- Faster storage (SSD) for database operations + +Tune configuration: + +- Increase `max_clients_per_vault` for more concurrent users +- Increase `max_connections_per_vault` for database performance +- Adjust `max_body_size_mb` based on typical file sizes + +## Troubleshooting + +### Server won't start + +```bash +# Check Docker logs +docker logs vaultlink-server + +# Common issues: +# - Port already in use: Change port mapping +# - Config syntax error: Validate YAML +# - Permission error: Check volume permissions +``` + +### High memory usage + +- Reduce `max_connections_per_vault` +- Reduce `max_clients_per_vault` +- Check for large vaults (may need database optimisation) + +### Database corruption + +```bash +# Verify database integrity +sqlite3 databases/your-vault.db "PRAGMA integrity_check;" + +# If corrupted, restore from backup +cp /backup/databases/your-vault.db /data/databases/ +``` + +### WebSocket connection drops + +- Check reverse proxy timeout settings +- Verify firewall isn't closing connections +- Review client retry intervals +- Check server logs for errors + +## Next Steps + +- [Configure authentication and access control →](/config/authentication) +- [Set up Obsidian plugin →](/guide/obsidian-plugin) +- [Deploy CLI client →](/guide/cli-client) +- [Understand the architecture →](/architecture/) diff --git a/docs/guide/what-is-vaultlink.md b/docs/guide/what-is-vaultlink.md new file mode 100644 index 00000000..070b312c --- /dev/null +++ b/docs/guide/what-is-vaultlink.md @@ -0,0 +1,71 @@ +# What is VaultLink? + +Self-hosted sync for Obsidian vaults with automatic conflict-free merging. Edit with any tool, collaborate in real-time, no conflict markers. + +## The Problem + +Syncing Obsidian vaults across devices or sharing with teammates sucks: + +- **Commercial services**: Lock-in, subscriptions, third-party access to your data +- **Git**: Manual conflict resolution with `<<<<<<<` markers interrupting your workflow +- **Cloud storage**: Last-write-wins data loss or manual conflict resolution +- **CRDT solutions**: Only work if you edit inside Obsidian (break if you use Vim, VS Code, etc.) + +## VaultLink's Solution + +Differential synchronisation with operational transformation for Markdown and text files. + +Edit `.md` and `.txt` files with Obsidian, Vim, VS Code, or any editor. VaultLink compares versions and automatically merges all changes. No operation tracking required, no conflict markers. + +**Note**: Binary files (images, PDFs, etc.) use last-write-wins. [See limitations →](/guide/limitations) + +## How It Works + +1. **Server**: Rust WebSocket server with SQLite stores document versions +2. **Clients**: Obsidian plugin or CLI client watches filesystem changes +3. **Sync**: Changes upload to server, server broadcasts to other clients +4. **Merge**: [reconcile-text](https://schmelczer.dev/reconcile) automatically merges concurrent edits + +No CRDT infrastructure. No operation logs. Just file comparison and smart merging. + +## Key Advantages + +**Editor agnostic**: Edit files with any tool. Other solutions break when you edit outside their ecosystem. + +**Self-hosted**: Your data, your server. No third parties, no subscriptions, no surprises. + +**Automatic merging**: Operational transformation handles conflicts without interrupting your workflow. + +**Production-ready**: Comprehensive tests, E2E tests, battle-tested. Many alternatives have zero tests. + +**Collaborative**: Real-time sync with cursor tracking. See where teammates are editing. + +## Not Tied to Obsidian + +VaultLink syncs Markdown files. Use it for: + +- Obsidian vaults (Obsidian desktop + mobile + CLI) +- Technical documentation (VS Code, your-editor, CLI) +- Academic writing (multiple Markdown editors) +- Automated workflows (CLI client for backups/CI/CD) + +The Obsidian plugin is just a convenience wrapper around the sync client. + +## Quick Comparison + +| Feature | VaultLink | Git | Cloud Sync | CRDT Solutions | +| ------------------- | --------- | --- | ---------- | -------------- | +| Self-hosted | ✅ | ✅ | ❌ | Varies | +| Any editor | ✅ | ✅ | ✅ | ❌ | +| No conflict markers | ✅ | ❌ | ❌ | ✅ | +| Real-time | ✅ | ❌ | ❌ | ✅ | +| No subscriptions | ✅ | ✅ | ❌ | Varies | +| Comprehensive tests | ✅ | N/A | N/A | ❌ | + +[Detailed comparison with alternatives →](/guide/alternatives) + +## Next Steps + +- [Get started →](/guide/getting-started) (5 minute setup) +- [See the architecture →](/architecture/) (understand how it works) +- [Compare alternatives →](/guide/alternatives) (why VaultLink vs others) diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 00000000..6a7d610d --- /dev/null +++ b/docs/index.md @@ -0,0 +1,55 @@ +--- +layout: home + +hero: + name: VaultLink + text: Self-Hosted Obsidian Sync + tagline: Edit with any tool. Automatic conflict-free merging. Your infrastructure. + image: + src: /logo.svg + alt: VaultLink + actions: + - theme: brand + text: Get Started + link: /guide/getting-started + - theme: alt + text: Why VaultLink? + link: /guide/what-is-vaultlink + +features: + - title: Edit Anywhere + details: Use Obsidian, Vim, VS Code, or any editor. VaultLink syncs files, not keystrokes—edit however you want + - title: Your Data, Your Server + details: Fully self-hosted. No third parties, no subscriptions, no data mining. Single Docker container or binary + - title: No Conflict Markers + details: Automatic merge using operational transformation. Never see conflict markers in your notes again + - title: Real-Time Collaboration + details: See teammate cursors, merge edits instantly. Rust-powered WebSocket server with SQLite + - title: Open Source Everything + details: MIT licensed. Server, clients, and sync algorithm are all open source. No proprietary components + - title: Battle-Tested + details: Comprehensive test suite. E2E tests. Used in production. Unlike alternatives with zero tests +--- + +## Why Self-Host? + +**You own your knowledge base.** Commercial sync services can disappear, change pricing, or lock you out. VaultLink runs on your infrastructure—VPS, home server, or localhost. + +**Edit with any tool.** Other solutions require CRDT-aware editors or break when you edit outside Obsidian. VaultLink uses differential sync: edit files however you want, sync handles the rest. + +**No conflict markers.** Git forces manual merging. Other tools use last-write-wins. VaultLink's operational transformation automatically merges Markdown and text files without conflict markers or workflow interruption. [See what's supported →](/guide/limitations) + +[See how VaultLink compares to alternatives →](/guide/alternatives) + +## Quick Start + +Deploy server (single command): + +```bash +docker run -d -p 3000:3000 -v $(pwd)/data:/data \ + ghcr.io/schmelczer/vault-link-server:latest +``` + +Then install the [Obsidian plugin](/guide/obsidian-plugin) or [CLI client](/guide/cli-client). + +[Full setup guide →](/guide/getting-started) diff --git a/docs/package.json b/docs/package.json new file mode 100644 index 00000000..6904b5e5 --- /dev/null +++ b/docs/package.json @@ -0,0 +1,25 @@ +{ + "name": "docs", + "version": "1.0.0", + "description": "", + "main": "index.js", + "scripts": { + "dev": "vitepress dev", + "build": "vitepress build", + "preview": "vitepress preview", + "format": "prettier --write \"**/*.md\" \"**/*.mts\"", + "format:check": "prettier --check \"**/*.md\" \"**/*.mts\"", + "spell": "cspell \"**/*.md\" \"**/*.mts\"", + "spell:check": "cspell \"**/*.md\" \"**/*.mts\"" + }, + "keywords": [], + "author": "", + "license": "ISC", + "devDependencies": { + "@cspell/dict-en-gb": "^5.0.19", + "cspell": "^9.3.2", + "prettier": "^3.6.2", + "vitepress": "^1.6.4", + "vue": "^3.5.24" + } +} diff --git a/docs/public/logo.svg b/docs/public/logo.svg new file mode 100644 index 00000000..cccc6fd8 --- /dev/null +++ b/docs/public/logo.svg @@ -0,0 +1,47 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/frontend/eslint.config.mjs b/frontend/eslint.config.mjs index 8e13be78..b2ed7a35 100644 --- a/frontend/eslint.config.mjs +++ b/frontend/eslint.config.mjs @@ -37,6 +37,24 @@ export default [ "@typescript-eslint/no-magic-numbers": "off", "@typescript-eslint/prefer-readonly-parameter-types": "off", "@typescript-eslint/naming-convention": "off", + "no-restricted-properties": [ + "error", + { + object: "Promise", + property: "all", + message: "Use `awaitAll` instead of Promise.all to always await all promises." + }, + { + object: "Promise", + property: "allSettled", + message: "Use `awaitAll` instead of Promise.allSettled to always await all promises and throw on errors." + }, + { + object: "String", + property: "replace", + message: "Use replaceAll instead of replace to replace all occurrences of a substring." + } + ], "unused-imports/no-unused-vars": [ "warn", { diff --git a/frontend/local-client-cli/src/cli.ts b/frontend/local-client-cli/src/cli.ts index 2a4cef98..36449d8d 100644 --- a/frontend/local-client-cli/src/cli.ts +++ b/frontend/local-client-cli/src/cli.ts @@ -87,19 +87,20 @@ async function main(): Promise { ]; const settings: SyncSettings = { + ...DEFAULT_SETTINGS, remoteUri: args.remoteUri, token: args.token, vaultName: args.vaultName, syncConcurrency: args.syncConcurrency ?? DEFAULT_SETTINGS.syncConcurrency, maxFileSizeMB: args.maxFileSizeMB ?? DEFAULT_SETTINGS.maxFileSizeMB, - diffCacheSizeMB: DEFAULT_SETTINGS.diffCacheSizeMB, ignorePatterns, webSocketRetryIntervalMs: args.webSocketRetryIntervalMs ?? DEFAULT_SETTINGS.webSocketRetryIntervalMs, isSyncEnabled: true, - enableTelemetry: args.enableTelemetry ?? false + enableTelemetry: + args.enableTelemetry ?? DEFAULT_SETTINGS.enableTelemetry }; const client = await SyncClient.create({ @@ -187,7 +188,8 @@ async function main(): Promise { ); fileWatcher.stop(); - await client.waitAndStop(); + await client.waitUntilFinished(); + await client.destroy(); console.log(colorize("Shutdown complete", "green")); process.exit(0); }; @@ -226,7 +228,7 @@ async function main(): Promise { ); fileWatcher.stop(); - await client.waitAndStop(); + await client.destroy(); process.exit(1); } } diff --git a/frontend/local-client-cli/src/node-filesystem.ts b/frontend/local-client-cli/src/node-filesystem.ts index 252385c9..f40143c8 100644 --- a/frontend/local-client-cli/src/node-filesystem.ts +++ b/frontend/local-client-cli/src/node-filesystem.ts @@ -1,8 +1,11 @@ import * as fs from "fs/promises"; import type { Dirent } from "fs"; import * as path from "path"; -import type { FileSystemOperations, RelativePath } from "sync-client"; -import type { TextWithCursors } from "reconcile-text"; +import type { + FileSystemOperations, + RelativePath, + TextWithCursors +} from "sync-client"; export class NodeFileSystemOperations implements FileSystemOperations { public constructor(private readonly basePath: string) {} diff --git a/frontend/obsidian-plugin/src/obsidian-file-system.ts b/frontend/obsidian-plugin/src/obsidian-file-system.ts index 44407890..bc8265fd 100644 --- a/frontend/obsidian-plugin/src/obsidian-file-system.ts +++ b/frontend/obsidian-plugin/src/obsidian-file-system.ts @@ -1,12 +1,12 @@ import type { Stat, Vault, Workspace } from "obsidian"; import { MarkdownView, normalizePath } from "obsidian"; +import type { CursorPosition, TextWithCursors } from "sync-client"; import { utils, type FileSystemOperations, type RelativePath } from "sync-client"; import { getSelectionsFromEditor } from "./views/cursors/get-selections-from-editor"; -import type { TextWithCursors, CursorPosition } from "reconcile-text"; export class ObsidianFileSystemOperations implements FileSystemOperations { public constructor( diff --git a/frontend/obsidian-plugin/src/vault-link-plugin.ts b/frontend/obsidian-plugin/src/vault-link-plugin.ts index fc16aae2..54e302f8 100644 --- a/frontend/obsidian-plugin/src/vault-link-plugin.ts +++ b/frontend/obsidian-plugin/src/vault-link-plugin.ts @@ -5,7 +5,7 @@ import type { TAbstractFile, WorkspaceLeaf } from "obsidian"; -import { Platform, Plugin, TFile } from "obsidian"; +import { Notice, Platform, Plugin, TFile } from "obsidian"; import "../manifest.json"; import { HistoryView } from "./views/history/history-view"; import { StatusBar } from "./views/status-bar/status-bar"; @@ -30,124 +30,59 @@ import { LocalCursorUpdateListener } from "./views/cursors/local-cursor-update-l import { renderCursorsInFileExplorer } from "./views/cursors/file-explorer"; const MIN_WAIT_BETWEEN_UPDATES_IN_MS = 250; +const IS_DEBUG_BUILD = process.env.NODE_ENV === "development"; export default class VaultLinkPlugin extends Plugin { - private readonly disposables: (() => unknown)[] = []; - - private settingsTab: SyncSettingsTab | undefined; - private client!: SyncClient; private readonly rateLimitedUpdatesPerFile = new Map< string, () => Promise >(); + private readonly syncClient: SyncClient | undefined; + private settingsTab: SyncSettingsTab | undefined; + public async onload(): Promise { - DEFAULT_SETTINGS.ignorePatterns.push( - ".obsidian/**", - ".git/**", - ".trash/**" - ); - - const isDebugBuild = process.env.NODE_ENV === "development"; - const debugOptions = isDebugBuild - ? { - fetch: debugging.slowFetchFactory(1), - webSocket: debugging.slowWebSocketFactory(1, new Logger()) - } - : {}; - - this.client = await SyncClient.create({ - fs: new ObsidianFileSystemOperations( - this.app.vault, - this.app.workspace - ), - persistence: { - load: this.loadData.bind(this), - save: this.saveData.bind(this) - }, - nativeLineEndings: Platform.isWin ? "\r\n" : "\n", - ...debugOptions - }); - - if (isDebugBuild) { - debugging.logToConsole(this.client); - } - - const statusDescription = new StatusDescription(this.client); - - this.settingsTab = new SyncSettingsTab({ - app: this.app, - plugin: this, - syncClient: this.client, - statusDescription - }); - this.addSettingTab(this.settingsTab); - - new StatusBar(this, this.client); - - this.registerView( - HistoryView.TYPE, - (leaf) => new HistoryView(this.client, leaf) - ); - - this.registerView( - LogsView.TYPE, - (leaf) => new LogsView(this.client, leaf) - ); - - this.registerEditorExtension([remoteCursorsTheme, remoteCursorsPlugin]); - - this.client.addRemoteCursorsUpdateListener((cursors) => { - RemoteCursorsPluginValue.setCursors(cursors, this.app); - renderCursorsInFileExplorer(cursors, this.app); - }); - - const cursorListener = new LocalCursorUpdateListener( - this.client, - this.app.workspace - ); - this.disposables.push(() => { - cursorListener.dispose(); - }); - - this.app.workspace.updateOptions(); - - this.addRibbonIcon( - HistoryView.ICON, - "Open VaultLink events", - async (_: MouseEvent) => this.activateView(HistoryView.TYPE) - ); - - this.addRibbonIcon( - LogsView.ICON, - "Open VaultLink logs", - async (_: MouseEvent) => this.activateView(LogsView.TYPE) - ); - this.app.workspace.onLayoutReady(async () => { - this.registerEditorEvents(); - await this.client.start(); + // eslint-disable-next-line + if ((globalThis as any).VAULT_LINK_RUNNING_INSTANCE) { + new Notice( + "Another instance of VaultLink is already running. Please disable the duplicate instance." + ); + throw new Error("VaultLink instance already running"); + } + // eslint-disable-next-line + (globalThis as any).VAULT_LINK_RUNNING_INSTANCE = this; - const editorStatusDisplayManager = new EditorStatusDisplayManager( - this, - this.app.workspace, - this.client - ); - this.disposables.push(() => { - editorStatusDisplayManager.stop(); + const client = await this.createSyncClient(); + + this.registerObsidianExtensions(client); + + this.registerEditorEvents(client); + + this.register(async () => { + await client.waitUntilFinished(); + await client.destroy(); }); + + await client.start(); }); } - public onunload(): void { - this.client.waitAndStop().catch((err: unknown) => { - this.client.logger.error( - `Error while stopping the sync client: ${err}` + public onUserEnable(): void { + new Notice( + "VaultLink has been enabled, check out the docs for tips on getting started!" + ); + void this.activateView(HistoryView.TYPE).catch((e: unknown) => { + this.syncClient?.logger.error( + `Failed to open history view on enable: ${e}` ); }); - this.disposables.forEach((disposable) => { - disposable(); + void this.activateView(LogsView.TYPE).catch((e: unknown) => { + this.syncClient?.logger.error( + `Failed to open logs view on enable: ${e}` + ); }); + this.openSettings(); } public openSettings(): void { @@ -180,7 +115,112 @@ export default class VaultLinkPlugin extends Plugin { } } - private registerEditorEvents(): void { + private async createSyncClient(): Promise { + DEFAULT_SETTINGS.ignorePatterns.push( + ".obsidian/**", + ".git/**", + ".trash/**", + "**/.DS_Store" + ); + + const client = await SyncClient.create({ + fs: new ObsidianFileSystemOperations( + this.app.vault, + this.app.workspace + ), + persistence: { + load: this.loadData.bind(this), + save: this.saveData.bind(this) + }, + nativeLineEndings: Platform.isWin ? "\r\n" : "\n", + ...(IS_DEBUG_BUILD + ? { + fetch: debugging.slowFetchFactory(1), + webSocket: debugging.slowWebSocketFactory( + 1, + new Logger() + ) + } + : {}) + }); + + if (IS_DEBUG_BUILD) { + debugging.logToConsole(client); + } + + return client; + } + + private registerObsidianExtensions(client: SyncClient): void { + const statusDescription = new StatusDescription(client); + + this.settingsTab = new SyncSettingsTab({ + app: this.app, + plugin: this, + syncClient: client, + statusDescription + }); + this.addSettingTab(this.settingsTab); + + new StatusBar(this, client); + + this.registerView(HistoryView.TYPE, (leaf) => { + const view = new HistoryView(client, leaf); + this.register(async () => view.onClose()); + return view; + }); + + this.registerView(LogsView.TYPE, (leaf) => new LogsView(client, leaf)); + + this.registerEditorExtension([remoteCursorsTheme, remoteCursorsPlugin]); + + client.addRemoteCursorsUpdateListener((cursors) => { + RemoteCursorsPluginValue.setCursors(cursors, this.app); + renderCursorsInFileExplorer(cursors, this.app); + }); + + const cursorListener = new LocalCursorUpdateListener( + client, + this.app.workspace + ); + this.register(() => { + cursorListener.dispose(); + }); + + this.app.workspace.updateOptions(); + + this.addRibbonIcons(); + + const editorStatusDisplayManager = new EditorStatusDisplayManager( + this, + this.app.workspace, + client + ); + this.register(() => { + editorStatusDisplayManager.dispose(); + }); + + this.register(() => { + // eslint-disable-next-line + (globalThis as any).VAULT_LINK_RUNNING_INSTANCE = null; + }); + } + + private addRibbonIcons(): void { + this.addRibbonIcon( + HistoryView.ICON, + "Open VaultLink events", + async (_: MouseEvent) => this.activateView(HistoryView.TYPE) + ); + + this.addRibbonIcon( + LogsView.ICON, + "Open VaultLink logs", + async (_: MouseEvent) => this.activateView(LogsView.TYPE) + ); + } + + private registerEditorEvents(client: SyncClient): void { [ this.app.workspace.on( "editor-change", @@ -190,28 +230,28 @@ export default class VaultLinkPlugin extends Plugin { ) => { const { file } = info; if (file) { - await this.rateLimitedUpdate(file.path); + await this.rateLimitedUpdate(file.path, client); } } ), this.app.vault.on("create", async (file: TAbstractFile) => { if (file instanceof TFile) { - await this.client.syncLocallyCreatedFile(file.path); + await client.syncLocallyCreatedFile(file.path); } }), this.app.vault.on("modify", async (file: TAbstractFile) => { if (file instanceof TFile) { - await this.rateLimitedUpdate(file.path); + await this.rateLimitedUpdate(file.path, client); } }), this.app.vault.on("delete", async (file: TAbstractFile) => { - await this.client.syncLocallyDeletedFile(file.path); + await client.syncLocallyDeletedFile(file.path); }), this.app.vault.on( "rename", async (file: TAbstractFile, oldPath: string) => { if (file instanceof TFile) { - await this.client.syncLocallyUpdatedFile({ + await client.syncLocallyUpdatedFile({ oldPath, relativePath: file.path }); @@ -223,13 +263,16 @@ export default class VaultLinkPlugin extends Plugin { }); } - private async rateLimitedUpdate(path: string): Promise { + private async rateLimitedUpdate( + path: string, + client: SyncClient + ): Promise { if (!this.rateLimitedUpdatesPerFile.has(path)) { this.rateLimitedUpdatesPerFile.set( path, rateLimit( async () => - this.client.syncLocallyUpdatedFile({ + client.syncLocallyUpdatedFile({ relativePath: path }), MIN_WAIT_BETWEEN_UPDATES_IN_MS diff --git a/frontend/obsidian-plugin/src/views/editor-status-display-manager/editor-status-display-manager.ts b/frontend/obsidian-plugin/src/views/editor-status-display-manager/editor-status-display-manager.ts index 5075b847..0725c1ea 100644 --- a/frontend/obsidian-plugin/src/views/editor-status-display-manager/editor-status-display-manager.ts +++ b/frontend/obsidian-plugin/src/views/editor-status-display-manager/editor-status-display-manager.ts @@ -22,7 +22,7 @@ export class EditorStatusDisplayManager { }, EditorStatusDisplayManager.UPDATE_INTERVAL_IN_MS); } - public stop(): void { + public dispose(): void { clearInterval(this.intervalId); } diff --git a/frontend/obsidian-plugin/src/views/history/history-view.ts b/frontend/obsidian-plugin/src/views/history/history-view.ts index 631fde72..1094e575 100644 --- a/frontend/obsidian-plugin/src/views/history/history-view.ts +++ b/frontend/obsidian-plugin/src/views/history/history-view.ts @@ -108,6 +108,7 @@ export class HistoryView extends ItemView { this.historyContainer = container.createDiv({ cls: "logs-container" }); await this.updateView(); + this.clearTimer(); this.timer = setInterval( () => void this.updateView().catch((error: unknown) => { @@ -120,8 +121,13 @@ export class HistoryView extends ItemView { } public async onClose(): Promise { + this.clearTimer(); + } + + private clearTimer(): void { if (this.timer) { clearInterval(this.timer); + this.timer = null; } } diff --git a/frontend/obsidian-plugin/src/views/logs/logs-view.scss b/frontend/obsidian-plugin/src/views/logs/logs-view.scss index 82ed1037..2bffe693 100644 --- a/frontend/obsidian-plugin/src/views/logs/logs-view.scss +++ b/frontend/obsidian-plugin/src/views/logs/logs-view.scss @@ -14,8 +14,22 @@ margin: 0; } - select { - cursor: pointer; + .logs-controls { + display: flex; + align-items: center; + gap: var(--size-4-2); + + button { + display: flex; + align-items: center; + gap: var(--size-2-1); + padding: var(--size-2-2) var(--size-4-2); + cursor: pointer; + } + + select { + cursor: pointer; + } } } diff --git a/frontend/obsidian-plugin/src/views/logs/logs-view.ts b/frontend/obsidian-plugin/src/views/logs/logs-view.ts index 19cf4701..395cfe09 100644 --- a/frontend/obsidian-plugin/src/views/logs/logs-view.ts +++ b/frontend/obsidian-plugin/src/views/logs/logs-view.ts @@ -1,7 +1,7 @@ import "./logs-view.scss"; import type { WorkspaceLeaf } from "obsidian"; -import { ItemView } from "obsidian"; +import { ItemView, Notice, setIcon } from "obsidian"; import type { LogLine } from "sync-client"; import { LogLevel, type SyncClient } from "sync-client"; @@ -78,7 +78,20 @@ export class LogsView extends ItemView { text: "VaultLink logs" }); - verbositySection.createEl("select", {}, (dropdown) => { + const controls = verbositySection.createDiv({ + cls: "logs-controls" + }); + + const copyButton = controls.createEl("button", { + text: "Copy logs", + cls: "clickable-icon" + }); + setIcon(copyButton, "clipboard-copy"); + copyButton.addEventListener("click", () => { + this.copyLogsToClipboard(); + }); + + controls.createEl("select", {}, (dropdown) => { logLevels.forEach(({ label, value }) => dropdown.createEl("option", { text: label, value }) ); @@ -102,6 +115,35 @@ export class LogsView extends ItemView { this.updateView(); } + private copyLogsToClipboard(): void { + const logs = this.client.logger.getMessages(this.minLogLevel); + + if (logs.length === 0) { + new Notice("No logs to copy"); + return; + } + + const formattedLogs = logs + .map((logLine) => { + const timestamp = logLine.timestamp.toLocaleString(); + const level = logLine.level.toUpperCase(); + return `[${timestamp}] ${level}: ${logLine.message}`; + }) + .join("\n"); + + navigator.clipboard + .writeText(formattedLogs) + .then(() => { + new Notice(`Copied ${logs.length} log entries to clipboard`); + }) + .catch((error: unknown) => { + this.client.logger.error( + `Failed to copy logs to clipboard: ${error}` + ); + new Notice("Failed to copy logs to clipboard"); + }); + } + private updateView(): void { const container = this.logsContainer; if (container === undefined) { diff --git a/frontend/obsidian-plugin/src/views/settings/settings-tab.scss b/frontend/obsidian-plugin/src/views/settings/settings-tab.scss index dcc3e806..0aabbadc 100644 --- a/frontend/obsidian-plugin/src/views/settings/settings-tab.scss +++ b/frontend/obsidian-plugin/src/views/settings/settings-tab.scss @@ -13,45 +13,122 @@ } } -.vault-link-settings { - h2 { - display: flex; - align-items: center; - font-size: var(--h2-size); +.vault-link-settings-container { + position: relative; - .version { - @include number-card; - margin: var(--size-2-2) 0 0 var(--size-4-2); - background-color: var(--color-base-30); - color: var(--color-base-70); - font-size: var(--font-ui-smaller); + .vault-link-settings { + h2 { + display: flex; + align-items: center; + font-size: var(--h2-size); + + .version { + @include number-card; + margin: var(--size-2-2) 0 0 var(--size-4-2); + background-color: var(--color-base-30); + color: var(--color-base-70); + font-size: var(--font-ui-smaller); + } + } + + .button-container { + display: flex; + gap: var(--size-4-2); + } + + h3 { + font-size: var(--font-ui-large); + margin-top: var(--heading-spacing); + } + + button, + input[type="range"], + .checkbox-container, + .slider::-webkit-slider-thumb { + cursor: pointer; + } + + input[type="text"], + textarea { + width: 250px; + } + + textarea { + resize: none; + height: 75px; + } + + .applying-changes-overlay { + position: absolute; + top: 50%; + left: 50%; + transform: translateY(-50%) translateX(-50%); + z-index: 10; + backdrop-filter: blur(10px); + + .spinner-container { + background-color: rgba(var(--background-primary), 0.5); + border: 1px solid var(--background-modifier-border); + border-radius: var(--radius-m); + padding: var(--size-4-8); + display: flex; + flex-direction: column; + align-items: center; + gap: var(--size-4-3); + box-shadow: 0 8px 24px rgba(0, 0, 0, 0.3); + min-width: 200px; + } + + .spinner { + width: 48px; + height: 48px; + border: 4px solid var(--background-modifier-border); + border-top-color: var(--interactive-accent); + border-radius: 50%; + animation: spin 0.8s linear infinite; + } + + .spinner-text { + color: var(--text-normal); + font-size: var(--font-ui-medium); + font-weight: 500; + } + + .spinner-warning { + color: var(--text-muted); + font-size: var(--font-ui-small); + text-align: center; + margin-top: var(--size-2-2); + } + } + + @keyframes spin { + from { + transform: rotate(0deg); + } + + to { + transform: rotate(360deg); + } + } + + &.applying-changes { + .setting-item-control { + pointer-events: none; + opacity: 0.5; + } + + button:not(.applying-changes-overlay button) { + pointer-events: none; + opacity: 0.5; + } + + input, + textarea, + select { + pointer-events: none; + opacity: 0.5; + } } } - - .button-container { - display: flex; - gap: var(--size-4-2); - } - - h3 { - font-size: var(--font-ui-large); - margin-top: var(--heading-spacing); - } - - button, - input[type="range"], - .checkbox-container, - .slider::-webkit-slider-thumb { - cursor: pointer; - } - - input[type="text"], - textarea { - width: 250px; - } - - textarea { - resize: none; - height: 75px; - } -} +} \ No newline at end of file diff --git a/frontend/obsidian-plugin/src/views/settings/settings-tab.ts b/frontend/obsidian-plugin/src/views/settings/settings-tab.ts index e4c16e6e..1ff78a4b 100644 --- a/frontend/obsidian-plugin/src/views/settings/settings-tab.ts +++ b/frontend/obsidian-plugin/src/views/settings/settings-tab.ts @@ -13,6 +13,9 @@ export class SyncSettingsTab extends PluginSettingTab { private editedToken: string; private editedVaultName: string; + private _isApplyingChanges = false; + private syncEnabledOverride: boolean | undefined = undefined; + private readonly plugin: VaultLinkPlugin; private readonly syncClient: SyncClient; private readonly statusDescription: StatusDescription; @@ -64,11 +67,28 @@ export class SyncSettingsTab extends PluginSettingTab { ); } + private get isApplyingChanges(): boolean { + return this._isApplyingChanges; + } + + private set isApplyingChanges(value: boolean) { + this._isApplyingChanges = value; + this.display(); + } + public display(): void { const { containerEl } = this; containerEl.empty(); containerEl.addClass("vault-link-settings"); + containerEl.parentElement?.addClass("vault-link-settings-container"); + if (this.isApplyingChanges) { + containerEl.addClass("applying-changes"); + } else { + containerEl.removeClass("applying-changes"); + } + + this.renderApplyingChanges(containerEl); this.renderSettingsHeader(containerEl); this.renderConnectionSettings(containerEl); this.renderSyncSettings(containerEl); @@ -80,6 +100,32 @@ export class SyncSettingsTab extends PluginSettingTab { this.setStatusDescriptionSubscription(); } + private renderApplyingChanges(containerEl: HTMLElement): void { + if (this.isApplyingChanges) { + const overlay = containerEl.createDiv({ + cls: "applying-changes-overlay" + }); + + const spinnerContainer = overlay.createDiv({ + cls: "spinner-container" + }); + + spinnerContainer.createDiv({ + cls: "spinner" + }); + + spinnerContainer.createDiv({ + text: "Applying changes...", + cls: "spinner-text" + }); + + spinnerContainer.createDiv({ + text: "You can exit, but changes won't be saved", + cls: "spinner-warning" + }); + } + } + private renderSettingsHeader(containerEl: HTMLElement): void { containerEl.createEl("h2", { text: "VaultLink" }).createSpan({ text: this.plugin.manifest.version, @@ -197,23 +243,40 @@ export class SyncSettingsTab extends PluginSettingTab { new Setting(containerEl).addButton((button) => button .setButtonText("Apply & test connection") - .onClick(async () => { - if (this.areThereUnsavedChanges()) { - await this.syncClient.setSettings({ - vaultName: this.editedVaultName, - remoteUri: this.editedServerUri, - token: this.editedToken - }); - new Notice("Checking connection to the server..."); - new Notice( - ( - await this.syncClient.checkConnection() - ).serverMessage - ); - await this.statusDescription.updateConnectionState(); - } else { - new Notice("No changes to apply"); - } + .setDisabled(this.isApplyingChanges) + .setTooltip( + this.isApplyingChanges + ? "Waiting for applying changes to finish..." + : "Apply the changes made to the connection settings and test the connection to the server." + ) + .onClick(() => { + // don't show loader within the button + void (async (): Promise => { + if (this.areThereUnsavedChanges()) { + new Notice("Applying changes to the server..."); + + this.isApplyingChanges = true; + try { + await this.syncClient.setSettings({ + vaultName: this.editedVaultName, + remoteUri: this.editedServerUri, + token: this.editedToken + }); + } finally { + this.isApplyingChanges = false; + } + + new Notice("Checking connection to the server..."); + new Notice( + ( + await this.syncClient.checkConnection() + ).serverMessage + ); + await this.statusDescription.updateConnectionState(); + } else { + new Notice("No changes to apply"); + } + })(); }) ); } @@ -239,9 +302,31 @@ export class SyncSettingsTab extends PluginSettingTab { ) .addToggle((toggle) => toggle - .setValue(this.syncClient.getSettings().isSyncEnabled) - .onChange(async (value) => - this.syncClient.setSetting("isSyncEnabled", value) + .setValue( + this.syncEnabledOverride ?? + this.syncClient.getSettings().isSyncEnabled + ) + .setDisabled(this.isApplyingChanges) + .setTooltip( + this.isApplyingChanges + ? "Waiting for applying changes to finish..." + : "Enable or disable syncing." + ) + .onChange( + (value) => + void (async (): Promise => { + this.syncEnabledOverride = value; + this.isApplyingChanges = true; + try { + await this.syncClient.setSetting( + "isSyncEnabled", + value + ); + } finally { + this.syncEnabledOverride = undefined; + this.isApplyingChanges = false; + } + })() ) ); @@ -321,12 +406,29 @@ export class SyncSettingsTab extends PluginSettingTab { "Delete the local metadata database while leaving the local and remote files intact." ) .addButton((button) => - button.setButtonText("Reset sync state").onClick(async () => { - await this.syncClient.reset(); - new Notice( - "Sync state has been reset, you will need to resync" - ); - }) + button + .setDisabled(this.isApplyingChanges) + .setTooltip( + this.isApplyingChanges + ? "Waiting for applying changes to finish..." + : "Reset sync state" + ) + .setButtonText("Reset sync state") + .onClick( + () => + void (async (): Promise => { + this.isApplyingChanges = true; + try { + await this.syncClient.reset(); + } finally { + this.isApplyingChanges = false; + } + + new Notice( + "Sync state has been reset, you will need to resync" + ); + })() + ) ); } @@ -348,6 +450,76 @@ export class SyncSettingsTab extends PluginSettingTab { this.syncClient.setSetting("enableTelemetry", value) ) ); + + containerEl.createEl("h3", { text: "Advanced" }); + + new Setting(containerEl) + .setName("Network retry interval (ms)") + .setDesc( + "The time to wait between retrying failed network requests, in milliseconds." + ) + .addText((input) => + input + .setValue( + this.syncClient + .getSettings() + .networkRetryIntervalMs.toString() + ) + .onChange(async (value) => { + if (value === "") { + return; + } + let parsedValue = Number.parseInt(value, 10); + if (Number.isNaN(parsedValue) || parsedValue < 0) { + parsedValue = + this.syncClient.getSettings() + .networkRetryIntervalMs; + } + + if (value !== parsedValue.toString()) { + input.setValue(parsedValue.toString()); + } + + return this.syncClient.setSetting( + "networkRetryIntervalMs", + parsedValue + ); + }) + ); + + new Setting(containerEl) + .setName("Minimum save interval (ms)") + .setDesc( + "The minimum time between saving settings and database to disk, in milliseconds. Lower values save more frequently but may impact performance." + ) + .addText((input) => + input + .setValue( + this.syncClient + .getSettings() + .minimumSaveIntervalMs.toString() + ) + .onChange(async (value) => { + if (value === "") { + return; + } + let parsedValue = Number.parseInt(value, 10); + if (Number.isNaN(parsedValue) || parsedValue < 0) { + parsedValue = + this.syncClient.getSettings() + .minimumSaveIntervalMs; + } + + if (value !== parsedValue.toString()) { + input.setValue(parsedValue.toString()); + } + + return this.syncClient.setSetting( + "minimumSaveIntervalMs", + parsedValue + ); + }) + ); } private setStatusDescriptionSubscription( diff --git a/frontend/package-lock.json b/frontend/package-lock.json index f60d140b..6242aec3 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1565,6 +1565,7 @@ }, "node_modules/balanced-match": { "version": "1.0.2", + "dev": true, "license": "MIT" }, "node_modules/big.js": { @@ -1649,6 +1650,7 @@ }, "node_modules/byte-base64": { "version": "1.1.0", + "dev": true, "license": "MIT" }, "node_modules/call-bind-apply-helpers": { @@ -2251,6 +2253,7 @@ }, "node_modules/eventemitter3": { "version": "5.0.1", + "dev": true, "license": "MIT" }, "node_modules/events": { @@ -3146,6 +3149,7 @@ }, "node_modules/p-queue": { "version": "8.1.0", + "dev": true, "license": "MIT", "dependencies": { "eventemitter3": "^5.0.1", @@ -3160,6 +3164,7 @@ }, "node_modules/p-timeout": { "version": "6.1.4", + "dev": true, "license": "MIT", "engines": { "node": ">=14.16" @@ -3484,6 +3489,7 @@ "version": "0.7.1", "resolved": "https://registry.npmjs.org/reconcile-text/-/reconcile-text-0.7.1.tgz", "integrity": "sha512-khedcYvAKs7ELKh5Z8mz2vyomMY5TqznV1dB+k/7qUAX9cheMNN5/EPJVQYZepOMunYbnQitvhFJX3kD4IMcNw==", + "dev": true, "license": "MIT" }, "node_modules/regex-parser": { @@ -4303,6 +4309,7 @@ "version": "13.0.0", "resolved": "https://registry.npmjs.org/uuid/-/uuid-13.0.0.tgz", "integrity": "sha512-XQegIaBTVUjSHliKqcnFqYypAd4S+WCYt5NIeRs6w/UAry7z8Y9j5ZwRRL4kzq9U3sD6v+85er9FvkEaBpji2w==", + "dev": true, "funding": [ "https://github.com/sponsors/broofa", "https://github.com/sponsors/ctavan" @@ -4664,20 +4671,18 @@ }, "sync-client": { "version": "0.10.1", - "dependencies": { + "devDependencies": { + "@sentry/browser": "^10.8.0", + "@types/node": "^24.8.1", "byte-base64": "^1.1.0", "minimatch": "^10.0.1", "p-queue": "^8.1.0", "reconcile-text": "^0.7.1", - "uuid": "^13.0.0" - }, - "devDependencies": { - "@sentry/browser": "^10.8.0", - "@types/node": "^24.8.1", "ts-loader": "^9.5.2", "tslib": "2.8.1", "tsx": "^4.20.5", "typescript": "5.8.3", + "uuid": "^13.0.0", "webpack": "^5.99.9", "webpack-cli": "^6.0.1", "webpack-merge": "^6.0.1", @@ -4688,6 +4693,7 @@ "version": "2.0.2", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", + "dev": true, "license": "MIT", "dependencies": { "balanced-match": "^1.0.0" @@ -4695,6 +4701,7 @@ }, "sync-client/node_modules/minimatch": { "version": "10.0.1", + "dev": true, "license": "ISC", "dependencies": { "brace-expansion": "^2.0.1" diff --git a/frontend/sync-client/package.json b/frontend/sync-client/package.json index 0c7c8266..f6234b80 100644 --- a/frontend/sync-client/package.json +++ b/frontend/sync-client/package.json @@ -12,14 +12,12 @@ "build": "webpack --mode production", "test": "tsx --test src/**/*.test.ts" }, - "dependencies": { + "devDependencies": { "byte-base64": "^1.1.0", "minimatch": "^10.0.1", "p-queue": "^8.1.0", "reconcile-text": "^0.7.1", - "uuid": "^13.0.0" - }, - "devDependencies": { + "uuid": "^13.0.0", "@types/node": "^24.8.1", "ts-loader": "^9.5.2", "tslib": "2.8.1", diff --git a/frontend/sync-client/src/consts.ts b/frontend/sync-client/src/consts.ts new file mode 100644 index 00000000..b90c48c3 --- /dev/null +++ b/frontend/sync-client/src/consts.ts @@ -0,0 +1,6 @@ +export const TIMEOUT_FOR_MERGING_HISTORY_ENTRIES_IN_SECONDS = 60; +export const DIFF_CACHE_SIZE_MB = 2; +export const MAX_LOG_MESSAGE_COUNT = 100000; +export const MAX_HISTORY_ENTRY_COUNT = 5000; +export const SUPPORTED_API_VERSION = 1; +export const WEBSOCKET_DISCONNECT_TIMEOUT_IN_S = 10; diff --git a/frontend/sync-client/src/file-operations/file-not-found-error.ts b/frontend/sync-client/src/file-operations/file-not-found-error.ts index 63af7dab..8725e81e 100644 --- a/frontend/sync-client/src/file-operations/file-not-found-error.ts +++ b/frontend/sync-client/src/file-operations/file-not-found-error.ts @@ -1,5 +1,8 @@ export class FileNotFoundError extends Error { - public constructor(message: string) { + public constructor( + message: string, + public readonly filePath: string + ) { super(message); this.name = "FileNotFoundError"; } diff --git a/frontend/sync-client/src/file-operations/file-operations.test.ts b/frontend/sync-client/src/file-operations/file-operations.test.ts index 675fdce1..353312a3 100644 --- a/frontend/sync-client/src/file-operations/file-operations.test.ts +++ b/frontend/sync-client/src/file-operations/file-operations.test.ts @@ -9,6 +9,17 @@ import { Logger } from "../tracing/logger"; import { assertSetContainsExactly } from "../utils/assert-set-contains-exactly"; import type { FileSystemOperations } from "./filesystem-operations"; import type { TextWithCursors } from "reconcile-text"; +import type { ServerConfig, ServerConfigData } from "../services/server-config"; + +class MockServerConfig implements Pick { + public getConfig(): ServerConfigData { + return { + mergeableFileExtensions: ["md", "txt"], + supportedApiVersion: 1, + isAuthenticated: true + }; + } +} class MockDatabase implements Partial { public getLatestDocumentByRelativePath( @@ -79,7 +90,8 @@ describe("File operations", () => { const fileOperations = new FileOperations( new Logger(), new MockDatabase() as Database, // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion - fileSystemOperations + fileSystemOperations, + new MockServerConfig() as ServerConfig // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion ); await fileOperations.create("a", new Uint8Array()); @@ -108,7 +120,8 @@ describe("File operations", () => { const fileOperations = new FileOperations( new Logger(), new MockDatabase() as Database, // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion - fileSystemOperations + fileSystemOperations, + new MockServerConfig() as ServerConfig // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion ); await fileOperations.create("b.md", new Uint8Array()); @@ -147,7 +160,8 @@ describe("File operations", () => { const fileOperations = new FileOperations( new Logger(), new MockDatabase() as Database, // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion - fileSystemOperations + fileSystemOperations, + new MockServerConfig() as ServerConfig // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion ); await fileOperations.create("a/b.c/d", new Uint8Array()); @@ -159,4 +173,63 @@ describe("File operations", () => { "a/b.c/e (1)" ); }); + + it("should continue deconfliction from existing number in filename", async () => { + const fileSystemOperations = new FakeFileSystemOperations(); + const fileOperations = new FileOperations( + new Logger(), + new MockDatabase() as Database, // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion + fileSystemOperations, + new MockServerConfig() as ServerConfig // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion + ); + + await fileOperations.create("document (5).md", new Uint8Array()); + await fileOperations.create("other.md", new Uint8Array()); + + await fileOperations.move("other.md", "document (5).md"); + assertSetContainsExactly( + fileSystemOperations.names, + "document (5).md", + "document (6).md" + ); + + await fileOperations.create("another.md", new Uint8Array()); + await fileOperations.move("another.md", "document (5).md"); + assertSetContainsExactly( + fileSystemOperations.names, + "document (5).md", + "document (6).md", + "document (7).md" + ); + }); + + it("should handle dotfiles correctly", async () => { + const fileSystemOperations = new FakeFileSystemOperations(); + const fileOperations = new FileOperations( + new Logger(), + new MockDatabase() as Database, // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion + fileSystemOperations, + new MockServerConfig() as ServerConfig // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion + ); + + await fileOperations.create(".gitignore", new Uint8Array()); + await fileOperations.create("temp", new Uint8Array()); + await fileOperations.move("temp", ".gitignore"); + assertSetContainsExactly( + fileSystemOperations.names, + ".gitignore", + ".gitignore (1)" + ); + + await fileOperations.create(".config.json", new Uint8Array()); + await fileOperations.create("temp2", new Uint8Array()); + await fileOperations.move("temp2", ".config.json"); + assertSetContainsExactly( + fileSystemOperations.names, + ".gitignore", + ".gitignore (1)", + ".config.json", + ".config (1).json" + ); + }); }); diff --git a/frontend/sync-client/src/file-operations/file-operations.ts b/frontend/sync-client/src/file-operations/file-operations.ts index e85c7fda..6bfdc305 100644 --- a/frontend/sync-client/src/file-operations/file-operations.ts +++ b/frontend/sync-client/src/file-operations/file-operations.ts @@ -6,15 +6,17 @@ import type { TextWithCursors } from "reconcile-text"; import { reconcile } from "reconcile-text"; import { isFileTypeMergable } from "../utils/is-file-type-mergable"; import { isBinary } from "../utils/is-binary"; +import type { ServerConfig } from "../services/server-config"; export class FileOperations { - private static readonly PARENTHESES_REGEX = / \((\d+)\)$/; + private static readonly PARENTHESES_REGEX = / \((?\d+)\)$/; private readonly fs: SafeFileSystemOperations; public constructor( private readonly logger: Logger, private readonly database: Database, fs: FileSystemOperations, + private readonly serverConfig: ServerConfig, private readonly nativeLineEndings = "\n" ) { this.fs = new SafeFileSystemOperations(fs, logger); @@ -25,7 +27,7 @@ export class FileOperations { ): [RelativePath, RelativePath] { const pathParts = path.split("/"); const fileName = pathParts.pop(); - if (fileName == "" || fileName == null) { + if (fileName == null || fileName === "") { throw new Error(`Path '${path}' cannot be empty`); } @@ -59,12 +61,16 @@ export class FileOperations { public async ensureClearPath(path: RelativePath): Promise { if (await this.fs.exists(path)) { const deconflictedPath = await this.deconflictPath(path); - this.logger.debug( - `Didn't expect ${path} to exist, deconflicting by moving it to '${deconflictedPath}'` - ); + try { + this.logger.debug( + `Didn't expect ${path} to exist, deconflicting by moving it to '${deconflictedPath}'` + ); - this.database.move(path, deconflictedPath); - await this.fs.rename(path, deconflictedPath); + this.database.move(path, deconflictedPath); + await this.fs.rename(path, deconflictedPath, true); + } finally { + this.fs.unlock(deconflictedPath); + } } else { await this.createParentDirectories(path); } @@ -89,7 +95,10 @@ export class FileOperations { } if ( - !isFileTypeMergable(path) || + !isFileTypeMergable( + path, + this.serverConfig.getConfig().mergeableFileExtensions + ) || isBinary(expectedContent) || isBinary(newContent) ) { @@ -114,14 +123,14 @@ export class FileOperations { `Performing a 3-way merge for ${path} with the expected content` ); - text = text.replace(this.nativeLineEndings, "\n"); + text = text.replaceAll(this.nativeLineEndings, "\n"); const merged = reconcile( expectedText, { text, cursors }, newText ); - const resultText = merged.text.replace( + const resultText = merged.text.replaceAll( "\n", this.nativeLineEndings ); @@ -166,6 +175,10 @@ export class FileOperations { await this.deletingEmptyParentDirectoriesOfDeletedFile(oldPath); } + public reset(): void { + this.fs.reset(); + } + private async deletingEmptyParentDirectoriesOfDeletedFile( path: RelativePath ): Promise { @@ -197,7 +210,7 @@ export class FileOperations { const decoder = new TextDecoder("utf-8"); let text = decoder.decode(content); - text = text.replace(this.nativeLineEndings, "\n"); + text = text.replaceAll(this.nativeLineEndings, "\n"); return new TextEncoder().encode(text); } @@ -208,7 +221,7 @@ export class FileOperations { const decoder = new TextDecoder("utf-8"); let text = decoder.decode(content); - text = text.replace("\n", this.nativeLineEndings); + text = text.replaceAll("\n", this.nativeLineEndings); return new TextEncoder().encode(text); } @@ -225,6 +238,13 @@ export class FileOperations { } } + /** + * Deconflicts the given path by appending (1), (2), etc. before the file extension until a non-existent path is found. + * The returned path has a lock acquired on it; it must be released by the caller when no longer needed. + * + * @param path The starting path to deconflict + * @returns a non-existent path with a lock acquired on it + */ private async deconflictPath(path: RelativePath): Promise { // eslint-disable-next-line prefer-const let [directory, fileName] = FileOperations.getParentDirAndFile(path); @@ -234,20 +254,38 @@ export class FileOperations { } const nameParts = fileName.split("."); + // Handle dotfiles: ".gitignore" should have no extension, ".config.json" should have ".json" + const isDotfile = fileName.startsWith(".") && nameParts[0] === ""; const extension = - nameParts.length > 1 ? "." + nameParts[nameParts.length - 1] : ""; + nameParts.length > 1 && !(isDotfile && nameParts.length === 2) + ? "." + nameParts[nameParts.length - 1] + : ""; let stem = extension ? nameParts.slice(0, -1).join(".") : fileName; let currentCount = Number.parseInt( - FileOperations.PARENTHESES_REGEX.exec(stem)?.groups?.[0] ?? "0" + FileOperations.PARENTHESES_REGEX.exec(stem)?.groups?.count ?? "0" ); stem = stem.replace(FileOperations.PARENTHESES_REGEX, ""); let newName = path; - do { + + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition + while (true) { currentCount++; newName = `${directory}${stem} (${currentCount})${extension}`; - } while (await this.fs.exists(newName)); - return newName; + // Avoid multiple deconflictPath calls returning the same path + if (this.fs.tryLock(newName)) { + const newDocument = + this.database.getLatestDocumentByRelativePath(newName); + if ( + newDocument?.isDeleted === false || // the document might have been confirmed by the server at a new path but haven't yet moved there locally + (await this.fs.exists(newName, true)) + ) { + this.fs.unlock(newName); + } else { + return newName; + } + } + } } } diff --git a/frontend/sync-client/src/file-operations/safe-filesystem-operations.ts b/frontend/sync-client/src/file-operations/safe-filesystem-operations.ts index 2c865c9f..33984be4 100644 --- a/frontend/sync-client/src/file-operations/safe-filesystem-operations.ts +++ b/frontend/sync-client/src/file-operations/safe-filesystem-operations.ts @@ -1,7 +1,7 @@ import type { RelativePath } from "../persistence/database"; import type { FileSystemOperations } from "./filesystem-operations"; import type { Logger } from "../tracing/logger"; -import { Locks } from "../utils/locks"; +import { Locks } from "../utils/data-structures/locks"; import { FileNotFoundError } from "./file-not-found-error"; import type { TextWithCursors } from "reconcile-text"; @@ -73,9 +73,16 @@ export class SafeFileSystemOperations implements FileSystemOperations { ); } - public async exists(path: RelativePath): Promise { + public async exists( + path: RelativePath, + skipLock = false + ): Promise { this.logger.debug(`Checking if file '${path}' exists`); - return this.locks.withLock(path, async () => this.fs.exists(path)); + if (skipLock) { + return this.fs.exists(path); + } else { + return this.locks.withLock(path, async () => this.fs.exists(path)); + } } public async createDirectory(path: RelativePath): Promise { @@ -92,19 +99,41 @@ export class SafeFileSystemOperations implements FileSystemOperations { public async rename( oldPath: RelativePath, - newPath: RelativePath + newPath: RelativePath, + skipLock = false ): Promise { this.logger.debug(`Renaming file '${oldPath}' to '${newPath}'`); return this.safeOperation( oldPath, - async () => - this.locks.withLock([oldPath, newPath], async () => - this.fs.rename(oldPath, newPath) - ), + async () => { + if (skipLock) { + return this.fs.rename(oldPath, newPath); + } else { + return this.locks.withLock([oldPath, newPath], async () => + this.fs.rename(oldPath, newPath) + ); + } + }, "rename" ); } + public tryLock(path: RelativePath): boolean { + return this.locks.tryLock(path); + } + + public async waitForLock(path: RelativePath): Promise { + return this.locks.waitForLock(path); + } + + public unlock(path: RelativePath): void { + this.locks.unlock(path); + } + + public reset(): void { + this.locks.reset(); + } + /** * Decorate an operation to ensure that the file exists before running it. * If the operation fails, it will check if the file still exists and throw @@ -117,7 +146,8 @@ export class SafeFileSystemOperations implements FileSystemOperations { ): Promise { if (!(await this.fs.exists(path))) { throw new FileNotFoundError( - `File '${path}' not found before trying to ${operationName}` + `File not found before trying to ${operationName}`, + path ); } @@ -131,7 +161,8 @@ export class SafeFileSystemOperations implements FileSystemOperations { throw error; } else { throw new FileNotFoundError( - `File '${path}' not found when trying to ${operationName}` + `File not found when trying to ${operationName}`, + path ); } } diff --git a/frontend/sync-client/src/index.ts b/frontend/sync-client/src/index.ts index a73f63dd..f09d339c 100644 --- a/frontend/sync-client/src/index.ts +++ b/frontend/sync-client/src/index.ts @@ -1,6 +1,6 @@ -import { logToConsole } from "./debugging/log-to-console"; -import { slowFetchFactory } from "./debugging/slow-fetch-factory"; -import { slowWebSocketFactory } from "./debugging/slow-web-socket-factory"; +import { logToConsole } from "./utils/debugging/log-to-console"; +import { slowFetchFactory } from "./utils/debugging/slow-fetch-factory"; +import { slowWebSocketFactory } from "./utils/debugging/slow-web-socket-factory"; import { getRandomColor } from "./utils/get-random-color"; import { lineAndColumnToPosition } from "./utils/line-and-column-to-position"; import { positionToLineAndColumn } from "./utils/position-to-line-and-column"; @@ -25,9 +25,12 @@ export type { PersistenceProvider } from "./persistence/persistence"; export type { CursorSpan } from "./services/types/CursorSpan"; export type { ClientCursors } from "./services/types/ClientCursors"; export type { NetworkConnectionStatus } from "./types/network-connection-status"; +export type { ServerVersionMismatchError } from "./services/server-version-mismatch-error"; +export type { AuthenticationError } from "./services/authentication-error"; export type { MaybeOutdatedClientCursors } from "./types/maybe-outdated-client-cursors"; export { DocumentSyncStatus } from "./types/document-sync-status"; export { SyncClient } from "./sync-client"; +export type { TextWithCursors, CursorPosition } from "reconcile-text"; export const debugging = { slowFetchFactory, diff --git a/frontend/sync-client/src/persistence/database.ts b/frontend/sync-client/src/persistence/database.ts index 9425c629..d42651ae 100644 --- a/frontend/sync-client/src/persistence/database.ts +++ b/frontend/sync-client/src/persistence/database.ts @@ -1,6 +1,7 @@ import type { Logger } from "../tracing/logger"; import { EMPTY_HASH } from "../utils/hash"; -import { CoveredValues } from "../utils/min-covered"; +import { CoveredValues } from "../utils/data-structures/min-covered"; +import { awaitAll } from "../utils/await-all"; export type VaultUpdateId = number; export type DocumentId = string; @@ -74,6 +75,10 @@ export class Database { Math.max(0, lastSeenUpdateId ?? 0) // the first updateId will be 1 which is the first integer after -1 ); + this.documents.forEach((doc) => { + this.lastSeenUpdateIds.add(doc.metadata?.parentVersionId); + }); + this.hasInitialSyncCompleted = initialState.hasInitialSyncCompleted ?? false; this.logger.debug( @@ -132,7 +137,7 @@ export class Database { toUpdate.metadata = metadata; - this.save(); + this.saveInTheBackground(); } public removeDocumentPromise(promise: Promise): void { @@ -152,7 +157,7 @@ export class Database { public removeDocument(find: DocumentRecord): void { this.documents = this.documents.filter((document) => document !== find); - this.save(); + this.saveInTheBackground(); } public getLatestDocumentByRelativePath( @@ -183,7 +188,7 @@ export class Database { const currentPromises = entry.updates; entry.updates = [...currentPromises, promise]; - await Promise.all(currentPromises); + await awaitAll(currentPromises); return entry; } @@ -193,6 +198,9 @@ export class Database { relativePath: RelativePath, promise: Promise ): DocumentRecord { + this.logger.debug( + `Creating new pending document: ${relativePath} (${documentId})` + ); const previousEntry = this.getLatestDocumentByRelativePath(relativePath); @@ -209,7 +217,7 @@ export class Database { }; this.documents.push(entry); - this.save(); + this.saveInTheBackground(); return entry; } @@ -233,7 +241,7 @@ export class Database { }; this.documents.push(entry); - this.save(); + this.saveInTheBackground(); return entry; } @@ -270,7 +278,7 @@ export class Database { oldDocument.parallelVersion = newDocument !== undefined ? newDocument.parallelVersion + 1 : 0; - this.save(); + this.saveInTheBackground(); } public delete(relativePath: RelativePath): void { @@ -289,7 +297,7 @@ export class Database { public setHasInitialSyncCompleted(value: boolean): void { this.hasInitialSyncCompleted = value; - this.save(); + this.saveInTheBackground(); } public getLastSeenUpdateId(): VaultUpdateId { @@ -300,13 +308,13 @@ export class Database { const previousMin = this.lastSeenUpdateIds.min; this.lastSeenUpdateIds.add(value); if (previousMin !== this.lastSeenUpdateIds.min) { - this.save(); + this.saveInTheBackground(); } } public setLastSeenUpdateId(value: number): void { this.lastSeenUpdateIds.min = value; - this.save(); + this.saveInTheBackground(); } public reset(): void { @@ -315,12 +323,11 @@ export class Database { 0 // the first updateId will be 1 which is the first integer after -1 ); this.hasInitialSyncCompleted = false; - this.save(); + this.saveInTheBackground(); } - private save(): void { - this.ensureConsistency(); - void this.saveData({ + public async save(): Promise { + return this.saveData({ documents: this.resolvedDocuments.map( ({ relativePath, documentId, metadata }) => ({ documentId, @@ -331,8 +338,6 @@ export class Database { ), lastSeenUpdateId: this.lastSeenUpdateIds.min, hasInitialSyncCompleted: this.hasInitialSyncCompleted - }).catch((error: unknown) => { - this.logger.error(`Error saving data: ${error}`); }); } @@ -357,4 +362,11 @@ export class Database { ); } } + + private saveInTheBackground(): void { + this.ensureConsistency(); + void this.save().catch((error: unknown) => { + this.logger.error(`Error saving data: ${error}`); + }); + } } diff --git a/frontend/sync-client/src/persistence/settings.ts b/frontend/sync-client/src/persistence/settings.ts index 87821728..81044a38 100644 --- a/frontend/sync-client/src/persistence/settings.ts +++ b/frontend/sync-client/src/persistence/settings.ts @@ -1,4 +1,6 @@ import type { Logger } from "../tracing/logger"; +import { awaitAll } from "../utils/await-all"; +import { Lock } from "../utils/data-structures/locks"; export interface SyncSettings { remoteUri: string; @@ -11,6 +13,8 @@ export interface SyncSettings { webSocketRetryIntervalMs: number; diffCacheSizeMB: number; enableTelemetry: boolean; + networkRetryIntervalMs: number; + minimumSaveIntervalMs: number; } export const DEFAULT_SETTINGS: SyncSettings = { @@ -23,11 +27,14 @@ export const DEFAULT_SETTINGS: SyncSettings = { ignorePatterns: [], webSocketRetryIntervalMs: 3500, diffCacheSizeMB: 4, - enableTelemetry: false + enableTelemetry: false, + networkRetryIntervalMs: 1000, + minimumSaveIntervalMs: 1000 }; export class Settings { private settings: SyncSettings; + private readonly lock: Lock = new Lock(); private readonly onSettingsChangeHandlers: (( newSettings: SyncSettings, @@ -54,32 +61,52 @@ export class Settings { } public addOnSettingsChangeListener( - handler: (settings: SyncSettings, oldSettings: SyncSettings) => unknown + listener: (settings: SyncSettings, oldSettings: SyncSettings) => unknown ): void { - this.onSettingsChangeHandlers.push(handler); + this.onSettingsChangeHandlers.push(listener); + } + + public removeOnSettingsChangeListener( + listener: (settings: SyncSettings, oldSettings: SyncSettings) => unknown + ): void { + const index = this.onSettingsChangeHandlers.indexOf(listener); + if (index !== -1) { + this.onSettingsChangeHandlers.splice(index, 1); + } } public async setSetting( key: T, value: SyncSettings[T] ): Promise { - this.logger.debug(`Setting '${key}' to '${value}'`); await this.setSettings({ [key]: value }); } public async setSettings(value: Partial): Promise { - const oldSettings = this.settings; - this.settings = { - ...this.settings, - ...value - }; + await this.lock.withLock(async () => { + this.logger.debug( + `Updating settings with: ${JSON.stringify(value)}` + ); + const oldSettings = this.settings; + this.settings = { + ...this.settings, + ...value + }; - this.onSettingsChangeHandlers.forEach((handler) => { - handler(this.settings, oldSettings); + await awaitAll( + this.onSettingsChangeHandlers + .map((handler) => { + return handler(this.settings, oldSettings); + }) + .filter((result): result is Promise => { + return result instanceof Promise; + }) + ); + + await this.save(); }); - await this.save(); } private async save(): Promise { diff --git a/frontend/sync-client/src/services/authentication-error.ts b/frontend/sync-client/src/services/authentication-error.ts new file mode 100644 index 00000000..9daa1937 --- /dev/null +++ b/frontend/sync-client/src/services/authentication-error.ts @@ -0,0 +1,6 @@ +export class AuthenticationError extends Error { + public constructor(message: string) { + super(message); + this.name = "AuthenticationError"; + } +} diff --git a/frontend/sync-client/src/services/connection-status.ts b/frontend/sync-client/src/services/connection-status.ts deleted file mode 100644 index 18f53a0d..00000000 --- a/frontend/sync-client/src/services/connection-status.ts +++ /dev/null @@ -1,98 +0,0 @@ -import type { Settings } from "../persistence/settings"; -import type { Logger } from "../tracing/logger"; -import { createPromise } from "../utils/create-promise"; -import { SyncResetError } from "./sync-reset-error"; - -export class ConnectionStatus { - private static readonly UNTIL_RESOLUTION = Symbol(); - private canFetch: boolean; - private until: Promise; - private resolveUntil: (result: symbol) => unknown; - private rejectUntil: (reason: unknown) => unknown; - - public constructor( - settings: Settings, - private readonly logger: Logger - ) { - this.canFetch = settings.getSettings().isSyncEnabled; - - [this.until, this.resolveUntil, this.rejectUntil] = - createPromise(); - - settings.addOnSettingsChangeListener((newSettings, oldSettings) => { - if (oldSettings.isSyncEnabled != newSettings.isSyncEnabled) { - this.canFetch = newSettings.isSyncEnabled; - this.resolveUntil(ConnectionStatus.UNTIL_RESOLUTION); - [this.until, this.resolveUntil, this.rejectUntil] = - createPromise(); - } - }); - } - - private static getUrlFromInput(input: RequestInfo | URL): string { - if (input instanceof URL) { - return input.href; - } - if (typeof input === "string") { - return input; - } - return input.url; - } - - public startReset(): void { - this.rejectUntil(new SyncResetError()); - } - - public finishReset(): void { - [this.until, this.resolveUntil, this.rejectUntil] = createPromise(); - } - - public getFetchImplementation( - logger: Logger, - fetch: typeof globalThis.fetch = globalThis.fetch - ): typeof globalThis.fetch { - return async ( - input: RequestInfo | URL, - init?: RequestInit - ): Promise => { - while (!this.canFetch) { - await this.until; - } - - try { - // https://github.com/jonbern/fetch-retry/blob/8684ef4e688375f623bd76f13add76dbc1d67cfb/index.js#L67C1-L70C21 - const _input = - typeof Request !== "undefined" && input instanceof Request - ? input.clone() - : input; - - const fetchPromise = fetch(_input, init); - - // We only want to catch rejections from `this.until` - let result: symbol | Response | undefined = undefined; - do { - result = await Promise.race([this.until, fetchPromise]); - } while (result === ConnectionStatus.UNTIL_RESOLUTION); - - const fetchResult: Response = result as Response; // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion - - if (!fetchResult.ok) { - this.logger.warn( - `Fetch for ${ConnectionStatus.getUrlFromInput( - input - )}, got status ${fetchResult.status}` - ); - } - - return fetchResult; - } catch (error) { - logger.warn( - `Fetch for ${ConnectionStatus.getUrlFromInput( - input - )}, got error: ${error}` - ); - throw error; - } - }; - } -} diff --git a/frontend/sync-client/src/services/fetch-controller.test.ts b/frontend/sync-client/src/services/fetch-controller.test.ts new file mode 100644 index 00000000..4ff57c55 --- /dev/null +++ b/frontend/sync-client/src/services/fetch-controller.test.ts @@ -0,0 +1,177 @@ +import type { Mock } from "node:test"; +import { describe, it, mock, beforeEach, afterEach } from "node:test"; +import assert from "node:assert"; +import { FetchController } from "./fetch-controller"; +import { Logger } from "../tracing/logger"; +import { SyncResetError } from "./sync-reset-error"; +import { sleep } from "../utils/sleep"; + +describe("FetchController", () => { + const createMockFetch = ( + shouldSleep: boolean + ): Mock<() => Promise> => + mock.fn(async () => { + if (shouldSleep) { + await sleep(30); + } + return Promise.resolve(new Response("OK", { status: 200 })); + }); + + beforeEach(() => { + mock.timers.enable({ apis: ["setTimeout"] }); + }); + + afterEach(() => { + mock.timers.reset(); + }); + + it("should allow fetch when canFetch is true", async () => { + const logger = new Logger(); + const controller = new FetchController(true, logger); + const mockFetch = createMockFetch(false); + const controlledFetch = controller.getControlledFetchImplementation( + logger, + mockFetch + ); + + const response = await controlledFetch("http://example.com"); + + assert.strictEqual(await response.text(), "OK"); + assert.strictEqual(mockFetch.mock.calls.length, 1); + }); + + it("should block fetch until canFetch becomes true", async () => { + const logger = new Logger(); + const controller = new FetchController(false, logger); + const mockFetch = createMockFetch(true); + const controlledFetch = controller.getControlledFetchImplementation( + logger, + mockFetch + ); + + const fetchPromise = controlledFetch("http://example.com"); + assert.strictEqual(mockFetch.mock.calls.length, 0); + + controller.canFetch = true; + await Promise.resolve(); + mock.timers.tick(30); + + const response = await fetchPromise; + assert.strictEqual(await response.text(), "OK"); + assert.strictEqual(mockFetch.mock.calls.length, 1); + }); + + it("should reject during reset", async () => { + const logger = new Logger(); + const controller = new FetchController(true, logger); + const mockFetch = createMockFetch(true); + const controlledFetch = controller.getControlledFetchImplementation( + logger, + mockFetch + ); + + const firstRequest = controlledFetch("http://example.com"); + assert.strictEqual(mockFetch.mock.calls.length, 1); + + controller.startReset(); + + const secondRequest = controlledFetch("http://example.com"); + + await assert.rejects( + firstRequest, + (error: unknown) => error instanceof SyncResetError + ); + await assert.rejects( + secondRequest, + (error: unknown) => error instanceof SyncResetError + ); + assert.strictEqual(mockFetch.mock.calls.length, 1); + }); + + it("should allow fetch after reset finishes", async () => { + const logger = new Logger(); + const controller = new FetchController(true, logger); + const mockFetch = createMockFetch(false); + const controlledFetch = controller.getControlledFetchImplementation( + logger, + mockFetch + ); + + controller.startReset(); + controller.finishReset(); + + const response = await controlledFetch("http://example.com"); + assert.strictEqual(await response.text(), "OK"); + }); + + it("should defer canFetch changes during reset", async () => { + const logger = new Logger(); + const controller = new FetchController(false, logger); + const mockFetch = createMockFetch(true); + const controlledFetch = controller.getControlledFetchImplementation( + logger, + mockFetch + ); + + controller.startReset(); + controller.canFetch = true; + + await assert.rejects( + async () => controlledFetch("http://example.com"), + (error: unknown) => error instanceof SyncResetError + ); + + controller.finishReset(); + + const fetchPromise = controlledFetch("http://example.com"); + mock.timers.tick(30); + + const response = await fetchPromise; + assert.strictEqual(await response.text(), "OK"); + }); + + it("should handle different input types", async () => { + const logger = new Logger(); + const controller = new FetchController(true, logger); + const mockFetch = createMockFetch(false); + const controlledFetch = controller.getControlledFetchImplementation( + logger, + mockFetch + ); + + await controlledFetch("http://example.com"); + await controlledFetch(new URL("http://example.com")); + await controlledFetch( + new Request("http://example.com", { method: "POST" }) + ); + + assert.strictEqual(mockFetch.mock.calls.length, 3); + }); + + it("should handle fetch errors", async () => { + const logger = new Logger(); + const controller = new FetchController(true, logger); + const mockFetch = mock.fn(async () => { + throw new Error("Network error"); + }); + const controlledFetch = controller.getControlledFetchImplementation( + logger, + mockFetch + ); + + await assert.rejects( + async () => controlledFetch("http://example.com"), + (error: unknown) => + error instanceof Error && error.message === "Network error" + ); + }); + + it("should not create unhandled rejection on reset with no waiting fetches", async () => { + const logger = new Logger(); + const controller = new FetchController(true, logger); + + controller.startReset(); + mock.timers.tick(10); + controller.finishReset(); + }); +}); diff --git a/frontend/sync-client/src/services/fetch-controller.ts b/frontend/sync-client/src/services/fetch-controller.ts new file mode 100644 index 00000000..1e93c853 --- /dev/null +++ b/frontend/sync-client/src/services/fetch-controller.ts @@ -0,0 +1,149 @@ +import type { Logger } from "../tracing/logger"; +import { createPromise } from "../utils/create-promise"; +import { SyncResetError } from "./sync-reset-error"; + +/** + * Offers a resettable fetch implementation that waits until syncing is enabled + * and aborts outstanding requests when a reset is started. + */ +export class FetchController { + private static readonly UNTIL_RESOLUTION = Symbol(); + + private isResetting = false; + + // Promise resolves on the next state change: sync enabled/disabled or reset started/ended + private until: Promise; + private resolveUntil: (result: symbol) => unknown; + private rejectUntil: (reason: unknown) => unknown; + + public constructor( + private _canFetch: boolean, + private readonly logger: Logger + ) { + [this.until, this.resolveUntil, this.rejectUntil] = + createPromise(); + } + + /** + * Whether the fetch implementation can immediately send requests once outside of a reset. + */ + public get canFetch(): boolean { + return this._canFetch; + } + + /** + * Allow or disallow fetching. The changes only take effect if not resetting. + * When called during a reset, its effect is deferred until the reset is finished. + * + * @param canFetch Whether fetching is enabled + */ + public set canFetch(canFetch: boolean) { + this._canFetch = canFetch; + + if (!this.isResetting) { + const previousResolve = this.resolveUntil; + [this.until, this.resolveUntil, this.rejectUntil] = + createPromise(); + previousResolve(FetchController.UNTIL_RESOLUTION); + } + } + + private static getUrlFromInput(input: RequestInfo | URL): string { + if (input instanceof URL) { + return input.href; + } + if (typeof input === "string") { + return input; + } + return input.url; + } + + /** + * Starts a reset, causing all ongoing and future fetches to be rejected + * with a SyncResetError until finishReset is called. + */ + public startReset(): void { + this.isResetting = true; + this.rejectUntil(new SyncResetError()); + // Catch unhandled rejection if no fetches are waiting + this.until.catch(() => { + // Intentionally ignore - this rejection is handled by waiting fetches + }); + } + + /** + * Finishes a reset, allowing fetches to proceed or wait again depending on + * the current sync settings. + */ + public finishReset(): void { + if (!this.isResetting) { + return; + } + + this.isResetting = false; + [this.until, this.resolveUntil, this.rejectUntil] = createPromise(); + } + + /** + * + * |------------------|---------------|-----------------------------------------------------| + * | | Sync enabled | Sync disabled | + * |------------------|-------------- |-----------------------------------------------------| + * | During reset | Rejects with SyncResetError without sending request | + * |------------------|-------------- |-----------------------------------------------------| + * | Outside of reset | Same as fetch | Blocks until sync is enabled and then same as fetch | + * |------------------|---------------|-----------------------------------------------------| + * + * @param logger for errors + * @param fetch to wrap + * @returns a wrapped fetch implementation affected by the FetchController state + */ + public getControlledFetchImplementation( + logger: Logger, + fetch: typeof globalThis.fetch = globalThis.fetch + ): typeof globalThis.fetch { + return async ( + input: RequestInfo | URL, + init?: RequestInit + ): Promise => { + while (!this.canFetch || this.isResetting) { + await this.until; + } + + try { + // https://github.com/jonbern/fetch-retry/blob/8684ef4e688375f623bd76f13add76dbc1d67cfb/index.js#L67C1-L70C21 + const _input = + typeof Request !== "undefined" && input instanceof Request + ? input.clone() + : input; + + const fetchPromise = fetch(_input, init); + + // We only want to catch rejections from `this.until` + let result: symbol | Response | undefined = undefined; + do { + result = await Promise.race([this.until, fetchPromise]); + } while (result === FetchController.UNTIL_RESOLUTION); + + const fetchResult: Response = result as Response; // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion + + if (!fetchResult.ok) { + this.logger.warn( + `Fetch for ${FetchController.getUrlFromInput( + input + )}, got status ${fetchResult.status}` + ); + } + + return fetchResult; + } catch (error) { + logger.warn( + `Fetch for ${FetchController.getUrlFromInput( + input + )}, got error: ${error}` + ); + throw error; + } + }; + } +} diff --git a/frontend/sync-client/src/services/server-config.ts b/frontend/sync-client/src/services/server-config.ts new file mode 100644 index 00000000..b3107d10 --- /dev/null +++ b/frontend/sync-client/src/services/server-config.ts @@ -0,0 +1,87 @@ +import { SUPPORTED_API_VERSION } from "../consts"; +import { AuthenticationError } from "./authentication-error"; +import { ServerVersionMismatchError } from "./server-version-mismatch-error"; +import type { SyncService } from "./sync-service"; +import type { PingResponse } from "./types/PingResponse"; + +export interface ServerConfigData { + mergeableFileExtensions: string[]; + supportedApiVersion: number; + isAuthenticated: boolean; +} + +export class ServerConfig { + private response: Promise | undefined; + private config: ServerConfigData | undefined; + + public constructor(private readonly syncService: SyncService) {} + + public async initialize(): Promise { + this.response = this.syncService.ping(); + this.config = await this.response; + + if (this.config.supportedApiVersion !== SUPPORTED_API_VERSION) { + const shouldUpgradeClient = + this.config.supportedApiVersion > SUPPORTED_API_VERSION; + throw new ServerVersionMismatchError( + `Unsupported API version: ${this.config.supportedApiVersion}. Consider upgrading the ${ + shouldUpgradeClient ? "client" : "sync-server" + } to ensure compatibility.` + ); + } + + if (!this.config.isAuthenticated) { + throw new AuthenticationError( + "Failed to authenticate with the sync-server." + ); + } + } + + public async checkConnection(forceUpdate = false): Promise<{ + isSuccessful: boolean; + message: string; + }> { + try { + let { response } = this; + if (!response && !forceUpdate) { + throw new Error("ServerConfig not initialized"); + } else if (forceUpdate) { + response = this.response = this.syncService.ping(); + } + + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + const result: PingResponse = (await response)!; // it must be defined, otherwise we would have thrown above + this.config = result; + + if (result.isAuthenticated) { + return { + isSuccessful: true, + message: `Successfully connected to server (version: ${result.serverVersion}) and authenticated` + }; + } + + return { + isSuccessful: false, + message: `Successfully connected to server (version: ${result.serverVersion}) but failed to authenticate` + }; + } catch (e) { + return { + isSuccessful: false, + message: `Failed to connect to server: ${e}` + }; + } + } + + public getConfig(): ServerConfigData { + if (!this.config) { + throw new Error("ServerConfig not initialized"); + } + + return this.config; + } + + public reset(): void { + this.response = undefined; + this.config = undefined; + } +} diff --git a/frontend/sync-client/src/services/server-version-mismatch-error.ts b/frontend/sync-client/src/services/server-version-mismatch-error.ts new file mode 100644 index 00000000..0f37fc6f --- /dev/null +++ b/frontend/sync-client/src/services/server-version-mismatch-error.ts @@ -0,0 +1,6 @@ +export class ServerVersionMismatchError extends Error { + public constructor(message: string) { + super(message); + this.name = "ServerVersionMismatchError"; + } +} diff --git a/frontend/sync-client/src/services/sync-reset-error.ts b/frontend/sync-client/src/services/sync-reset-error.ts index 5e27dfb6..3fd8a86c 100644 --- a/frontend/sync-client/src/services/sync-reset-error.ts +++ b/frontend/sync-client/src/services/sync-reset-error.ts @@ -1,6 +1,6 @@ export class SyncResetError extends Error { public constructor() { - super("Sync was reset"); + super("SyncClient has been reset, cleaning up"); this.name = "SyncResetError"; } } diff --git a/frontend/sync-client/src/services/sync-service.ts b/frontend/sync-client/src/services/sync-service.ts index 5bbf01e6..ba047b5e 100644 --- a/frontend/sync-client/src/services/sync-service.ts +++ b/frontend/sync-client/src/services/sync-service.ts @@ -6,7 +6,7 @@ import type { import type { Logger } from "../tracing/logger"; import type { Settings } from "../persistence/settings"; -import type { ConnectionStatus } from "./connection-status"; +import type { FetchController } from "./fetch-controller"; import { sleep } from "../utils/sleep"; import { SyncResetError } from "./sync-reset-error"; import type { SerializedError } from "./types/SerializedError"; @@ -18,19 +18,13 @@ import type { PingResponse } from "./types/PingResponse"; import type { DeleteDocumentVersion } from "./types/DeleteDocumentVersion"; import type { UpdateTextDocumentVersion } from "./types/UpdateTextDocumentVersion"; -export interface CheckConnectionResult { - isSuccessful: boolean; - message: string; -} - export class SyncService { - private static readonly NETWORK_RETRY_INTERVAL_MS = 1000; private readonly client: typeof globalThis.fetch; private readonly pingClient: typeof globalThis.fetch; public constructor( private readonly deviceId: string, - private readonly connectionStatus: ConnectionStatus, + private readonly fetchController: FetchController, private readonly settings: Settings, private readonly logger: Logger, fetchImplementation: typeof globalThis.fetch = globalThis.fetch @@ -39,7 +33,7 @@ export class SyncService { const unboundFetch: typeof globalThis.fetch = async (...args) => fetchImplementation(...args); - this.client = this.connectionStatus.getFetchImplementation( + this.client = this.fetchController.getControlledFetchImplementation( this.logger, unboundFetch ); @@ -65,7 +59,7 @@ export class SyncService { relativePath: RelativePath; contentBytes: Uint8Array; }): Promise { - return this.withRetries(async () => { + return this.retryForever(async () => { const formData = new FormData(); if (documentId !== undefined) { formData.append("document_id", documentId); @@ -114,7 +108,7 @@ export class SyncService { relativePath: RelativePath; content: (number | string)[]; }): Promise { - return this.withRetries(async () => { + return this.retryForever(async () => { this.logger.debug( `Updating text document ${documentId} with parent version ${parentVersionId} and relative path ${relativePath}` ); @@ -166,7 +160,7 @@ export class SyncService { relativePath: RelativePath; contentBytes: Uint8Array; }): Promise { - return this.withRetries(async () => { + return this.retryForever(async () => { this.logger.debug( `Updating binary document ${documentId} with parent version ${parentVersionId} and relative path ${relativePath}` ); @@ -215,7 +209,7 @@ export class SyncService { documentId: DocumentId; relativePath: RelativePath; }): Promise { - return this.withRetries(async () => { + return this.retryForever(async () => { const request: DeleteDocumentVersion = { relativePath }; @@ -252,7 +246,7 @@ export class SyncService { }: { documentId: DocumentId; }): Promise { - return this.withRetries(async () => { + return this.retryForever(async () => { const response = await this.client( this.getUrl(`/documents/${documentId}`), { @@ -280,7 +274,7 @@ export class SyncService { public async getAll( since?: VaultUpdateId ): Promise { - return this.withRetries(async () => { + return this.retryForever(async () => { const url = new URL(this.getUrl("/documents")); if (since !== undefined) { url.searchParams.append("since", since.toString()); @@ -308,43 +302,30 @@ export class SyncService { }); } - public async checkConnection(): Promise { - try { - const response = await this.pingClient(this.getUrl("/ping"), { - headers: this.getDefaultHeaders() - }); - const result: PingResponse | SerializedError = - (await response.json()) as PingResponse | SerializedError; // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion + public async ping(): Promise { + const response = await this.pingClient(this.getUrl("/ping"), { + headers: this.getDefaultHeaders() + }); + const result: PingResponse | SerializedError = + (await response.json()) as PingResponse | SerializedError; // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion - if ("errorType" in result) { - throw new Error( - `Failed to ping server: ${SyncService.formatError(result)}` - ); - } - - if (result.isAuthenticated) { - return { - isSuccessful: true, - message: `Successfully connected to server (version: ${result.serverVersion}) and authenticated` - }; - } - - return { - isSuccessful: false, - message: `Successfully connected to server (version: ${result.serverVersion}) but failed to authenticate` - }; - } catch (e) { - return { - isSuccessful: false, - message: `Failed to connect to server: ${e}` - }; + if ("errorType" in result) { + throw new Error( + `Failed to ping server: ${SyncService.formatError(result)}` + ); } + + this.logger.debug( + `Pinged server, got response: ${JSON.stringify(result)}` + ); + + return result; } private getUrl(path: string): string { const { vaultName, remoteUri } = this.settings.getSettings(); - const safeRemoteUri = remoteUri.replace(/\/+$/, ""); - return `${safeRemoteUri}/vaults/${vaultName}${path}`; + const remoteUriWithoutTrailingSlash = remoteUri.replace(/\/+$/, ""); + return `${remoteUriWithoutTrailingSlash}/vaults/${vaultName}${path}`; } private getDefaultHeaders( @@ -362,7 +343,7 @@ export class SyncService { return headers; } - private async withRetries(fn: () => Promise): Promise { + private async retryForever(fn: () => Promise): Promise { // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition while (true) { try { @@ -373,10 +354,12 @@ export class SyncService { throw e; } + const retryInterval = + this.settings.getSettings().networkRetryIntervalMs; this.logger.error( - `Failed network call (${e}), retrying in ${SyncService.NETWORK_RETRY_INTERVAL_MS}ms` + `Failed network call (${e}), retrying in ${retryInterval}ms` ); - await sleep(SyncService.NETWORK_RETRY_INTERVAL_MS); + await sleep(retryInterval); } } } diff --git a/frontend/sync-client/src/services/types/PingResponse.ts b/frontend/sync-client/src/services/types/PingResponse.ts index b0d993f2..cc7370e7 100644 --- a/frontend/sync-client/src/services/types/PingResponse.ts +++ b/frontend/sync-client/src/services/types/PingResponse.ts @@ -13,4 +13,13 @@ export interface PingResponse { * header. */ isAuthenticated: boolean; + /** + * List of file extensions that are allowed to be merged. + */ + mergeableFileExtensions: string[]; + /** + * API version ensuring backwards & forwards compatibility between the client + * and server. + */ + supportedApiVersion: number; } diff --git a/frontend/sync-client/src/services/websocket-manager.test.ts b/frontend/sync-client/src/services/websocket-manager.test.ts new file mode 100644 index 00000000..13aca939 --- /dev/null +++ b/frontend/sync-client/src/services/websocket-manager.test.ts @@ -0,0 +1,298 @@ +/* eslint-disable @typescript-eslint/no-unsafe-type-assertion */ +import { describe, it, beforeEach } from "node:test"; +import assert from "node:assert"; +import { WebSocketManager } from "./websocket-manager"; +import type { Logger } from "../tracing/logger"; +import type { Settings } from "../persistence/settings"; +// eslint-disable-next-line @typescript-eslint/no-require-imports +const WebSocket = require("ws") as typeof globalThis.WebSocket; + +class MockCloseEvent extends Event { + public code: number; + public reason: string; + + public constructor( + type: string, + options: { code: number; reason: string } + ) { + super(type); + this.code = options.code; + this.reason = options.reason; + } +} + +class MockMessageEvent extends Event { + public data: string; + + public constructor(type: string, options: { data: string }) { + super(type); + this.data = options.data; + } +} + +class MockWebSocket { + public readyState: number = WebSocket.CONNECTING; + public onopen: ((event: Event) => void) | null = null; + public onclose: ((event: MockCloseEvent) => void) | null = null; + public onmessage: ((event: MockMessageEvent) => void) | null = null; + public onerror: ((event: Event) => void) | null = null; + + public sentMessages: string[] = []; + + public constructor(public url: string) { + setTimeout(() => { + if (this.readyState === WebSocket.CONNECTING) { + this.readyState = WebSocket.OPEN; + this.onopen?.(new Event("open")); + } + }, 0); + } + + public send(data: string): void { + if (this.readyState !== WebSocket.OPEN) { + throw new Error("WebSocket is not open"); + } + this.sentMessages.push(data); + } + + public close(code?: number, reason?: string): void { + this.readyState = WebSocket.CLOSED; + this.onclose?.( + new MockCloseEvent("close", { + code: code ?? 1000, + reason: reason ?? "" + }) + ); + } + + public simulateMessage(data: unknown): void { + this.onmessage?.( + new MockMessageEvent("message", { data: JSON.stringify(data) }) + ); + } +} + +type MockFn unknown> = T & { + calls: Parameters[]; +}; + +function createMockFn unknown>( + implementation?: T +): MockFn { + const calls: Parameters[] = []; + const mockFn = ((...args: Parameters) => { + calls.push(args); + return implementation?.(...args); + }) as unknown as MockFn; + mockFn.calls = calls; + return mockFn; +} + +describe("WebSocketManager", () => { + let mockLogger: Logger = undefined as unknown as Logger; + let mockSettings: Settings = undefined as unknown as Settings; + let deviceId = "test-device-123"; + + beforeEach(() => { + deviceId = "test-device-123"; + const noop = (): void => { + // Intentionally empty for mock + }; + mockLogger = { + info: createMockFn(noop), + warn: createMockFn(noop), + error: createMockFn(noop), + debug: createMockFn(noop) + } as unknown as Logger; + + mockSettings = { + getSettings: () => ({ + remoteUri: "https://example.com", + vaultName: "test-vault", + webSocketRetryIntervalMs: 1000 + }) + } as unknown as Settings; + }); + + it("cleans up promises after message handling", async () => { + const manager = new WebSocketManager( + deviceId, + mockLogger, + mockSettings, + MockWebSocket as unknown as typeof WebSocket + ); + + manager.addRemoteVaultUpdateListener(async () => { + await new Promise((resolve) => setTimeout(resolve, 10)); + }); + manager.start(); + await new Promise((resolve) => setTimeout(resolve, 50)); + + const { outstandingPromises } = manager as unknown as { + outstandingPromises: Promise[]; + }; + const mockWs = (manager as unknown as { webSocket: MockWebSocket }) + .webSocket; + + mockWs.simulateMessage({ type: "vaultUpdate", updates: [] }); + mockWs.simulateMessage({ type: "vaultUpdate", updates: [] }); + mockWs.simulateMessage({ type: "vaultUpdate", updates: [] }); + + await new Promise((resolve) => setTimeout(resolve, 100)); + + assert.strictEqual(outstandingPromises.length, 0); + await manager.stop(); + }); + + it("cleans up cursor position promises", async () => { + const manager = new WebSocketManager( + deviceId, + mockLogger, + mockSettings, + MockWebSocket as unknown as typeof WebSocket + ); + + manager.addRemoteCursorsUpdateListener(async () => { + await new Promise((resolve) => setTimeout(resolve, 10)); + }); + manager.start(); + await new Promise((resolve) => setTimeout(resolve, 50)); + + const { outstandingPromises } = manager as unknown as { + outstandingPromises: Promise[]; + }; + const mockWs = (manager as unknown as { webSocket: MockWebSocket }) + .webSocket; + + mockWs.simulateMessage({ + type: "cursorPositions", + clients: [{ deviceId: "other-device", cursors: [] }] + }); + + await new Promise((resolve) => setTimeout(resolve, 100)); + assert.strictEqual(outstandingPromises.length, 0); + await manager.stop(); + }); + + it("logs handshake send errors", async () => { + const manager = new WebSocketManager( + deviceId, + mockLogger, + mockSettings, + MockWebSocket as unknown as typeof WebSocket + ); + + manager.start(); + await new Promise((resolve) => setTimeout(resolve, 50)); + + const mockWs = (manager as unknown as { webSocket: MockWebSocket }) + .webSocket; + mockWs.send = (): void => { + throw new Error("Buffer full"); + }; + + assert.throws(() => { + manager.sendHandshakeMessage({ + type: "handshake", + token: "test", + deviceId: "test", + lastSeenVaultUpdateId: null + }); + }); + + await manager.stop(); + }); + + it("completes stop with timeout protection", async () => { + const manager = new WebSocketManager( + deviceId, + mockLogger, + mockSettings, + MockWebSocket as unknown as typeof WebSocket + ); + + manager.start(); + await new Promise((resolve) => setTimeout(resolve, 50)); + + await manager.stop(); + assert.ok(true); + }); + + it("clears old handlers on reconnection", async () => { + const manager = new WebSocketManager( + deviceId, + mockLogger, + mockSettings, + MockWebSocket as unknown as typeof WebSocket + ); + + let statusChangeCount = 0; + manager.addWebSocketStatusChangeListener(() => { + statusChangeCount++; + }); + + manager.start(); + await new Promise((resolve) => setTimeout(resolve, 50)); + + const firstWs = (manager as unknown as { webSocket: MockWebSocket }) + .webSocket; + + statusChangeCount = 0; + + ( + manager as unknown as { initializeWebSocket: () => void } + ).initializeWebSocket(); + await new Promise((resolve) => setTimeout(resolve, 50)); + + statusChangeCount = 0; + + // Old handler should be cleared + firstWs.onclose?.( + new MockCloseEvent("close", { code: 1000, reason: "test" }) + ); + + assert.strictEqual(statusChangeCount, 0); + await manager.stop(); + }); + + it("tracks message handling promises", async () => { + const manager = new WebSocketManager( + deviceId, + mockLogger, + mockSettings, + MockWebSocket as unknown as typeof WebSocket + ); + + // eslint-disable-next-line @typescript-eslint/init-declarations + let resolveListener: () => void; + const listenerPromise = new Promise((resolve) => { + resolveListener = resolve; + }); + + manager.addRemoteVaultUpdateListener(async () => { + await listenerPromise; + }); + + manager.start(); + await new Promise((resolve) => setTimeout(resolve, 50)); + + const mockWs = (manager as unknown as { webSocket: MockWebSocket }) + .webSocket; + mockWs.simulateMessage({ type: "vaultUpdate", updates: [] }); + + await new Promise((resolve) => setTimeout(resolve, 10)); + + const { outstandingPromises } = manager as unknown as { + outstandingPromises: Promise[]; + }; + + assert.ok(outstandingPromises.length > 0); + + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + resolveListener!(); + await new Promise((resolve) => setTimeout(resolve, 50)); + + assert.strictEqual(outstandingPromises.length, 0); + await manager.stop(); + }); +}); diff --git a/frontend/sync-client/src/services/websocket-manager.ts b/frontend/sync-client/src/services/websocket-manager.ts index a30774f4..015a778e 100644 --- a/frontend/sync-client/src/services/websocket-manager.ts +++ b/frontend/sync-client/src/services/websocket-manager.ts @@ -1,31 +1,40 @@ -import type { Database } from "../persistence/database"; import type { Logger } from "../tracing/logger"; -import type { Settings, SyncSettings } from "../persistence/settings"; +import type { Settings } from "../persistence/settings"; import type { WebSocketServerMessage } from "./types/WebSocketServerMessage"; -import type { Syncer } from "../sync-operations/syncer"; import type { WebSocketClientMessage } from "./types/WebSocketClientMessage"; import type { CursorPositionFromClient } from "./types/CursorPositionFromClient"; import type { ClientCursors } from "./types/ClientCursors"; +import { createPromise } from "../utils/create-promise"; +import type { WebSocketVaultUpdate } from "./types/WebSocketVaultUpdate"; +import { awaitAll } from "../utils/await-all"; +import { WEBSOCKET_DISCONNECT_TIMEOUT_IN_S } from "../consts"; export class WebSocketManager { - private readonly webSocketStatusChangeListeners: (() => unknown)[] = []; - private readonly remoteCursorsUpdateListeners: (( - cursors: ClientCursors[] + private readonly webSocketStatusChangeListeners: (( + isConnected: boolean ) => unknown)[] = []; - private webSocket: WebSocket | undefined; + private readonly remoteVaultUpdateListeners: (( + update: WebSocketVaultUpdate + ) => Promise)[] = []; + + private readonly remoteCursorsUpdateListeners: (( + cursors: ClientCursors[] + ) => Promise)[] = []; private isStopped = true; - private _isFirstSyncCompleted = false; + private resolveDisconnectingPromise: null | (() => unknown) = null; + private reconnectTimeoutId: ReturnType | undefined; + private readonly outstandingPromises: Promise[] = []; + + private webSocket: WebSocket | undefined; private readonly webSocketFactoryImplementation: typeof globalThis.WebSocket; public constructor( private readonly deviceId: string, private readonly logger: Logger, - private readonly database: Database, private readonly settings: Settings, - private readonly syncer: Syncer, webSocketImplementation?: typeof globalThis.WebSocket ) { if (webSocketImplementation) { @@ -41,16 +50,6 @@ export class WebSocketManager { this.webSocketFactoryImplementation = WebSocket; } } - - settings.addOnSettingsChangeListener((newSettings, oldSettings) => { - if ( - newSettings.remoteUri !== oldSettings.remoteUri || - newSettings.vaultName !== oldSettings.vaultName || - newSettings.token !== oldSettings.token - ) { - this.initializeWebSocket(newSettings); - } - }); } public get isWebSocketConnected(): boolean { @@ -60,96 +59,205 @@ export class WebSocketManager { ); } - public get isFirstSyncCompleted(): boolean { - return this._isFirstSyncCompleted; - } - - public addWebSocketStatusChangeListener(listener: () => unknown): void { + public addWebSocketStatusChangeListener( + listener: (isConnected: boolean) => unknown + ): void { this.webSocketStatusChangeListeners.push(listener); } public addRemoteCursorsUpdateListener( - listener: (cursors: ClientCursors[]) => unknown + listener: (cursors: ClientCursors[]) => Promise ): void { this.remoteCursorsUpdateListeners.push(listener); } - public start(): void { - this.isStopped = false; - this._isFirstSyncCompleted = false; - this.initializeWebSocket(this.settings.getSettings()); + public addRemoteVaultUpdateListener( + listener: (update: WebSocketVaultUpdate) => Promise + ): void { + this.remoteVaultUpdateListeners.push(listener); } - public stop(): void { + public start(): void { + this.isStopped = false; + this.initializeWebSocket(); + } + + public async stop(): Promise { + const [promise, resolve] = createPromise(); + this.resolveDisconnectingPromise = resolve; + this.isStopped = true; + + if (this.reconnectTimeoutId !== undefined) { + clearTimeout(this.reconnectTimeoutId); + this.reconnectTimeoutId = undefined; + } + this.webSocket?.close(1000, "WebSocketManager has been stopped"); + + // eslint-disable-next-line @typescript-eslint/init-declarations + let timeoutId: ReturnType | undefined; + const timeoutPromise = new Promise((_, reject) => { + timeoutId = setTimeout(() => { + reject( + new Error( + `Timeout waiting for WebSocket to close after ${WEBSOCKET_DISCONNECT_TIMEOUT_IN_S} seconds` + ) + ); + }, WEBSOCKET_DISCONNECT_TIMEOUT_IN_S * 1000); + }); + + try { + while (this.isWebSocketConnected) { + await Promise.race([promise, timeoutPromise]); + } + } catch (error) { + this.logger.error( + `Error while waiting for WebSocket to close: ${String(error)}` + ); + // Force cleanup even if close didn't work + this.resolveDisconnectingPromise(); + this.resolveDisconnectingPromise = null; + } finally { + // Clear timeout to prevent unhandled rejection + if (timeoutId !== undefined) { + clearTimeout(timeoutId); + } + } + + await this.waitUntilFinished(); + } + + public async waitUntilFinished(): Promise { + await awaitAll(this.outstandingPromises); + } + + public sendHandshakeMessage( + message: WebSocketClientMessage & { type: "handshake" } + ): void { + const { webSocket } = this; + if (!webSocket) { + throw new Error( + "WebSocket is not connected, cannot send handshake message" + ); + } + + try { + webSocket.send(JSON.stringify(message)); + } catch (error) { + this.logger.error( + `Failed to send handshake message: ${String(error)}` + ); + throw error; + } } public updateLocalCursors(cursorPositions: CursorPositionFromClient): void { - if (!this.isWebSocketConnected) { + if (!this.isWebSocketConnected || !this.webSocket) { + // A missing cursor update is fine, we can just skip it if needed this.logger.warn( "WebSocket is not connected, cannot send cursor positions" ); return; } + const message: WebSocketClientMessage = { type: "cursorPositions", ...cursorPositions }; - this.webSocket?.send(JSON.stringify(message)); - this.logger.debug( - `Sent cursor positions: ${JSON.stringify(cursorPositions)}` - ); - } - - private initializeWebSocket(settings: SyncSettings): void { - if (this.isStopped) { - return; - } try { - this.webSocket?.close(); - } catch (e) { - this.logger.warn(`Failed to close WebSocket: ${e}`); + this.webSocket.send(JSON.stringify(message)); + this.logger.debug( + `Sent cursor positions: ${JSON.stringify(cursorPositions)}` + ); + } catch (error) { + this.logger.warn( + `Failed to send cursor positions: ${String(error)}` + ); + } + } + + private initializeWebSocket(): void { + // Clean up old WebSocket handlers to prevent race conditions + if (this.webSocket) { + try { + // Remove handlers to prevent them from firing after new connection + this.webSocket.onopen = null; + this.webSocket.onclose = null; + this.webSocket.onmessage = null; + this.webSocket.onerror = null; + this.webSocket.close(); + } catch (e) { + this.logger.error( + `Failed to close previous WebSocket connection: ${e}` + ); + } } - const wsUri = new URL(settings.remoteUri); + const wsUri = new URL(this.settings.getSettings().remoteUri); wsUri.protocol = wsUri.protocol === "https" ? "wss" : "ws"; - wsUri.pathname = `/vaults/${settings.vaultName}/ws`; + wsUri.pathname = `/vaults/${this.settings.getSettings().vaultName}/ws`; this.logger.info(`Connecting to WebSocket at ${wsUri.toString()}`); this.webSocket = new this.webSocketFactoryImplementation(wsUri); - // The JS WebSocket API doesn't support setting headers, so we have to send the token as a message this.webSocket.onopen = (): void => { this.logger.info("WebSocket connection opened"); - this.webSocketStatusChangeListeners.forEach((l) => l()); - - const message: WebSocketClientMessage = { - type: "handshake", - deviceId: this.deviceId, - token: settings.token, - lastSeenVaultUpdateId: this.database.getLastSeenUpdateId() - }; - this.webSocket?.send(JSON.stringify(message)); + this.webSocketStatusChangeListeners.forEach((listener) => + listener(true) + ); }; - this.webSocket.onmessage = async (event): Promise => { - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion - const message = JSON.parse(event.data) as WebSocketServerMessage; - return this.handleWebSocketMessage(message); + this.webSocket.onmessage = (event): void => { + try { + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion + const message = JSON.parse( + event.data + ) as WebSocketServerMessage; + + // Track the message handling promise + const messageHandlingPromise = this.handleWebSocketMessage( + message + ) + .catch((error: unknown) => { + this.logger.error( + `Error handling WebSocket message: ${String(error)}` + ); + }) + .finally(() => { + const index = this.outstandingPromises.indexOf( + messageHandlingPromise + ); + if (index !== -1) { + void this.outstandingPromises.splice(index, 1); // ignore the returned promise + } + }); + + void this.outstandingPromises.push(messageHandlingPromise); // ignore the returned promise + } catch (error) { + this.logger.error( + `Error parsing WebSocket message: ${String(error)}` + ); + } }; this.webSocket.onclose = (event): void => { this.logger.warn( `WebSocket closed with code ${event.code} (${event.reason == "" ? "unknown reason" : event.reason})` ); - this.webSocketStatusChangeListeners.forEach((l) => l()); + this.webSocketStatusChangeListeners.forEach((listener) => + listener(false) + ); - if (!this.isStopped) { - setTimeout(() => { - this.initializeWebSocket(this.settings.getSettings()); + if (this.isStopped) { + this.resolveDisconnectingPromise?.(); + this.resolveDisconnectingPromise = null; + } else { + this.reconnectTimeoutId = setTimeout(() => { + this.reconnectTimeoutId = undefined; + this.initializeWebSocket(); }, this.settings.getSettings().webSocketRetryIntervalMs); } }; @@ -159,37 +267,31 @@ export class WebSocketManager { message: WebSocketServerMessage ): Promise { if (message.type === "vaultUpdate") { - try { - await Promise.all( - message.documents.map(async (document) => - this.syncer.syncRemotelyUpdatedFile(document) - ) - ); + await awaitAll( + this.remoteVaultUpdateListeners.map(async (listener) => { + await listener(message).catch((error: unknown) => { + this.logger.error( + `Error in vault update listener: ${String(error)}` + ); + }); + }) + ); - if (message.isInitialSync && message.documents.length > 0) { - this.database.setLastSeenUpdateId( - message.documents - .map((document) => document.vaultUpdateId) - .reduce((a, b) => Math.max(a, b)) - ); - } - - this._isFirstSyncCompleted = true; - } catch (e) { - this.logger.error(`Failed to sync remotely updated file: ${e}`); - } // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition } else if (message.type === "cursorPositions") { this.logger.debug( `Received cursor positions for ${JSON.stringify(message.clients)}` ); - this.remoteCursorsUpdateListeners.forEach((listener) => { - listener( - message.clients.filter( - (client) => client.deviceId !== this.deviceId - ) - ); - }); + + await awaitAll( + this.remoteCursorsUpdateListeners.map(async (listener) => { + await listener(message.clients).catch((error: unknown) => { + this.logger.error( + `Error in cursor positions listener: ${String(error)}` + ); + }); + }) + ); } else { this.logger.warn( `Received unknown message type: ${JSON.stringify(message)}` diff --git a/frontend/sync-client/src/sync-client.ts b/frontend/sync-client/src/sync-client.ts index 9547af65..b76da9d9 100644 --- a/frontend/sync-client/src/sync-client.ts +++ b/frontend/sync-client/src/sync-client.ts @@ -1,16 +1,17 @@ import type { PersistenceProvider } from "./persistence/persistence"; import type { HistoryEntry, HistoryStats } from "./tracing/sync-history"; import { SyncHistory } from "./tracing/sync-history"; -import { Logger } from "./tracing/logger"; +import { Logger, LogLevel, LogLine } from "./tracing/logger"; import type { RelativePath, StoredDatabase } from "./persistence/database"; import { Database } from "./persistence/database"; +import * as Sentry from "@sentry/browser"; import type { SyncSettings } from "./persistence/settings"; -import { Settings } from "./persistence/settings"; +import { DEFAULT_SETTINGS, Settings } from "./persistence/settings"; import { SyncService } from "./services/sync-service"; import { Syncer } from "./sync-operations/syncer"; import type { FileSystemOperations } from "./file-operations/filesystem-operations"; import { FileOperations } from "./file-operations/file-operations"; -import { ConnectionStatus } from "./services/connection-status"; +import { FetchController } from "./services/fetch-controller"; import { UnrestrictedSyncer } from "./sync-operations/unrestricted-syncer"; import { rateLimit } from "./utils/rate-limit"; import type { NetworkConnectionStatus } from "./types/network-connection-status"; @@ -21,13 +22,16 @@ import { CursorTracker } from "./sync-operations/cursor-tracker"; import type { CursorSpan } from "./services/types/CursorSpan"; import type { MaybeOutdatedClientCursors } from "./types/maybe-outdated-client-cursors"; import { FileChangeNotifier } from "./sync-operations/file-change-notifier"; -import { FixedSizeDocumentCache } from "./utils/fix-sized-cache"; +import { FixedSizeDocumentCache } from "./utils/data-structures/fix-sized-cache"; import { setUpTelemetry } from "./utils/set-up-telemetry"; +import { DIFF_CACHE_SIZE_MB } from "./consts"; +import { ServerConfig } from "./services/server-config"; export class SyncClient { - private static readonly MINIMUM_SAVE_INTERVAL_MS = 1000; private hasStartedOfflineSync = false; private hasFinishedOfflineSync = false; + private hasStarted = false; + private hasBeenDestroyed = false; private unloadTelemetry?: () => void; private constructor( @@ -35,56 +39,21 @@ export class SyncClient { private readonly settings: Settings, private readonly database: Database, private readonly syncer: Syncer, - private readonly syncService: SyncService, private readonly webSocketManager: WebSocketManager, - private readonly _logger: Logger, - private readonly connectionStatus: ConnectionStatus, + public readonly logger: Logger, + private readonly fetchController: FetchController, private readonly cursorTracker: CursorTracker, private readonly fileChangeNotifier: FileChangeNotifier, - private readonly contentCache: FixedSizeDocumentCache - ) { - if (settings.getSettings().enableTelemetry) { - this.unloadTelemetry = setUpTelemetry(); - } - - this.settings.addOnSettingsChangeListener( - async (newSettings, oldSettings) => { - if (newSettings.vaultName !== oldSettings.vaultName) { - await this.reset(); - } - - if (newSettings.isSyncEnabled !== oldSettings.isSyncEnabled) { - if (newSettings.isSyncEnabled) { - await this.start(); - } else { - this.stop(); - } - } - - if ( - newSettings.diffCacheSizeMB !== oldSettings.diffCacheSizeMB - ) { - this.contentCache.resize( - newSettings.diffCacheSizeMB * 1024 * 1024 - ); - } - - if ( - newSettings.enableTelemetry !== oldSettings.enableTelemetry - ) { - if (newSettings.enableTelemetry) { - this.unloadTelemetry = setUpTelemetry(); - } else { - this.unloadTelemetry?.(); - } - } - } - ); - } - - public get logger(): Logger { - return this._logger; - } + private readonly contentCache: FixedSizeDocumentCache, + private readonly fileOperations: FileOperations, + private readonly serverConfig: ServerConfig, + private readonly persistence: PersistenceProvider< + Partial<{ + settings: Partial; + database: Partial; + }> + > + ) {} public get documentCount(): number { return this.database.length; @@ -93,7 +62,6 @@ export class SyncClient { public get isWebSocketConnected(): boolean { return this.webSocketManager.isWebSocketConnected; } - public static async create({ fs, persistence, @@ -116,7 +84,7 @@ export class SyncClient { const deviceId = createClientId(); - logger.info(`Initialising SyncClient with client id ${deviceId}`); + logger.info(`Creating SyncClient with client id ${deviceId}`); const history = new SyncHistory(logger); @@ -125,9 +93,20 @@ export class SyncClient { database: undefined }; + const settings = new Settings( + logger, + state.settings, + async (data): Promise => { + state = { ...state, settings: data }; + // we're not rate-limiting settings saves as (1) we need to initialise the settings to know the rate limit + // and (2) settings changes are infrequent enough that rate-limiting is not necessary + await persistence.save(state); + } + ); + const rateLimitedSave = rateLimit( persistence.save, - SyncClient.MINIMUM_SAVE_INTERVAL_MS + () => settings.getSettings().minimumSaveIntervalMs ); const database = new Database( @@ -139,32 +118,37 @@ export class SyncClient { } ); - const settings = new Settings( - logger, - state.settings, - async (data): Promise => { - state = { ...state, settings: data }; - await rateLimitedSave(state); - } + const fetchController = new FetchController( + settings.getSettings().isSyncEnabled, + logger ); + settings.addOnSettingsChangeListener((newSettings, oldSettings) => { + if (oldSettings.isSyncEnabled != newSettings.isSyncEnabled) { + fetchController.canFetch = newSettings.isSyncEnabled; + } + }); - const connectionStatus = new ConnectionStatus(settings, logger); const syncService = new SyncService( deviceId, - connectionStatus, + fetchController, settings, logger, fetch ); + const serverConfig = new ServerConfig(syncService); + const fileOperations = new FileOperations( logger, database, fs, + serverConfig, nativeLineEndings ); - const contentCache = new FixedSizeDocumentCache(1024 * 1024 * 2); // 2 MB cache + const contentCache = new FixedSizeDocumentCache( + 1024 * 1024 * DIFF_CACHE_SIZE_MB + ); const unrestrictedSyncer = new UnrestrictedSyncer( logger, database, @@ -172,25 +156,26 @@ export class SyncClient { syncService, fileOperations, history, - contentCache - ); - - const syncer = new Syncer( - logger, - database, - settings, - syncService, - fileOperations, - unrestrictedSyncer + contentCache, + serverConfig ); const webSocketManager = new WebSocketManager( + deviceId, + logger, + settings, + webSocket + ); + + const syncer = new Syncer( deviceId, logger, database, settings, - syncer, - webSocket + syncService, + webSocketManager, + fileOperations, + unrestrictedSyncer ); const fileChangeNotifier = new FileChangeNotifier(); @@ -205,22 +190,78 @@ export class SyncClient { settings, database, syncer, - syncService, webSocketManager, logger, - connectionStatus, + fetchController, cursorTracker, fileChangeNotifier, - contentCache + contentCache, + fileOperations, + serverConfig, + persistence ); - logger.info("SyncClient initialised"); + logger.info("SyncClient created successfully"); return client; } + public async start(): Promise { + this.checkIfDestroyed("start"); + + if (this.hasStarted) { + throw new Error("SyncClient has already been started"); + } + this.hasStarted = true; + + if ( + !this.unloadTelemetry && + this.settings.getSettings().enableTelemetry + ) { + this.unloadTelemetry = setUpTelemetry(); + } + + this.logger.addOnMessageListener((log): void => { + if (log.level === LogLevel.ERROR && Sentry.isInitialized()) { + Sentry.captureMessage(log.message); + } + }); + + this.settings.addOnSettingsChangeListener( + this.onSettingsChange.bind(this) + ); + + if (this.settings.getSettings().isSyncEnabled) { + this.logger.info("Starting SyncClient"); + await this.startSyncing(); + this.logger.info("SyncClient has successfully started"); + } + } + + /** + * Reload settings from disk overriding current in-memory settings. + * Missing values will be filled in from DEFAULT_SETTINGS rather than + * retaining current in-memory settings. + */ + public async reloadSettings(): Promise { + this.checkIfDestroyed("reloadSettings"); + + const state = (await this.persistence.load()) ?? { + settings: undefined + }; + + const settings = { + ...DEFAULT_SETTINGS, + ...(state.settings ?? {}) + }; + + await this.setSettings(settings); + } + public async checkConnection(): Promise { - const server = await this.syncService.checkConnection(); + this.checkIfDestroyed("checkConnection"); + + const server = await this.serverConfig.checkConnection(true); return { isSuccessful: server.isSuccessful, serverMessage: server.message, @@ -235,42 +276,34 @@ export class SyncClient { public addSyncHistoryUpdateListener( listener: (stats: HistoryStats) => unknown ): void { + this.checkIfDestroyed("addSyncHistoryUpdateListener"); + this.history.addSyncHistoryUpdateListener(listener); } - public async start(): Promise { - if (!this.hasStartedOfflineSync) { - await this.syncer.scheduleSyncForOfflineChanges(); - this.hasStartedOfflineSync = true; - } - - this.hasFinishedOfflineSync = true; - this.webSocketManager.start(); - } - - public stop(): void { - this.hasFinishedOfflineSync = false; - this.webSocketManager.stop(); - } - - public async waitAndStop(): Promise { - this.stop(); - await this.syncer.waitUntilFinished(); - } - - /// Wait for the in-flight operations to finish, reset all tracking, - /// and the local database but retain the settings. - /// The SyncClient can be used again after calling this method. + /** + * Wait for the in-flight operations to finish, reset all tracking, + * and the local database but retain the settings. + * The SyncClient can be used again after calling this method. + */ public async reset(): Promise { - this.stop(); - this.connectionStatus.startReset(); - this.contentCache.clear(); - await this.syncer.reset(); - this.history.reset(); + this.checkIfDestroyed("reset"); + + this.logger.info( + "Stopping SyncClient to apply changed connection settings" + ); + await this.pause(); + + // clear all local state + this.logger.info("Resetting SyncClient's local state"); this.database.reset(); - this._logger.reset(); - this.connectionStatus.finishReset(); - await this.start(); + await this.database.save(); // ensure the new database reads as empty + this.resetInMemoryState(); + this.hasStartedOfflineSync = false; + this.hasFinishedOfflineSync = false; + this.serverConfig.reset(); + + await this.startSyncing(); } public getSettings(): SyncSettings { @@ -281,32 +314,44 @@ export class SyncClient { key: T, value: SyncSettings[T] ): Promise { + this.checkIfDestroyed("setSetting"); + await this.settings.setSetting(key, value); } public async setSettings(value: Partial): Promise { + this.checkIfDestroyed("setSettings"); + await this.settings.setSettings(value); } public addOnSettingsChangeListener( - handler: (settings: SyncSettings, oldSettings: SyncSettings) => unknown + listener: (settings: SyncSettings, oldSettings: SyncSettings) => unknown ): void { - this.settings.addOnSettingsChangeListener(handler); + this.checkIfDestroyed("addOnSettingsChangeListener"); + + this.settings.addOnSettingsChangeListener(listener); } public addRemainingSyncOperationsListener( listener: (remainingOperations: number) => unknown ): void { + this.checkIfDestroyed("addRemainingSyncOperationsListener"); + this.syncer.addRemainingOperationsListener(listener); } public addWebSocketStatusChangeListener(listener: () => unknown): void { + this.checkIfDestroyed("addWebSocketStatusChangeListener"); + this.webSocketManager.addWebSocketStatusChangeListener(listener); } public async syncLocallyCreatedFile( relativePath: RelativePath ): Promise { + this.checkIfDestroyed("syncLocallyCreatedFile"); + this.fileChangeNotifier.notifyOfFileChange(relativePath); return this.syncer.syncLocallyCreatedFile(relativePath); } @@ -314,6 +359,8 @@ export class SyncClient { public async syncLocallyDeletedFile( relativePath: RelativePath ): Promise { + this.checkIfDestroyed("syncLocallyDeletedFile"); + this.fileChangeNotifier.notifyOfFileChange(relativePath); return this.syncer.syncLocallyDeletedFile(relativePath); } @@ -325,6 +372,8 @@ export class SyncClient { oldPath?: RelativePath; relativePath: RelativePath; }): Promise { + this.checkIfDestroyed("syncLocallyUpdatedFile"); + this.fileChangeNotifier.notifyOfFileChange(relativePath); return this.syncer.syncLocallyUpdatedFile({ oldPath, @@ -335,14 +384,13 @@ export class SyncClient { public getDocumentSyncingStatus( relativePath: RelativePath ): DocumentSyncStatus { + this.checkIfDestroyed("getDocumentSyncingStatus"); + if (!this.settings.getSettings().isSyncEnabled) { return DocumentSyncStatus.SYNCING_IS_DISABLED; } - if ( - !this.webSocketManager.isFirstSyncCompleted || - !this.hasFinishedOfflineSync - ) { + if (!this.syncer.isFirstSyncComplete || !this.hasFinishedOfflineSync) { return DocumentSyncStatus.SYNCING; } @@ -359,12 +407,114 @@ export class SyncClient { public async updateLocalCursors( documentToCursors: Record ): Promise { + this.checkIfDestroyed("updateLocalCursors"); + await this.cursorTracker.sendLocalCursorsToServer(documentToCursors); } public addRemoteCursorsUpdateListener( listener: (cursors: MaybeOutdatedClientCursors[]) => unknown ): void { + this.checkIfDestroyed("addRemoteCursorsUpdateListener"); + this.cursorTracker.addRemoteCursorsUpdateListener(listener); } + + public async waitUntilFinished(): Promise { + this.checkIfDestroyed("waitUntilIdle"); + await this.syncer.waitUntilFinished(); + await this.webSocketManager.waitUntilFinished(); + await this.database.save(); // flush all changes to disk + } + + /** + * Completely destroy the SyncClient, cancelling all in-progress operations. + * After calling this method, the SyncClient cannot be used again. + */ + public async destroy(): Promise { + this.checkIfDestroyed("destroy"); + + // cancel everything that's in progress + await this.pause(); + + this.hasBeenDestroyed = true; + + this.resetInMemoryState(); + + this.logger.info("SyncClient has been successfully disposed"); + + this.unloadTelemetry?.(); + } + + private async startSyncing(): Promise { + this.checkIfDestroyed("startSyncing"); + this.fetchController.finishReset(); + + await this.serverConfig.initialize(); + this.webSocketManager.start(); + + if (!this.hasStartedOfflineSync) { + this.hasStartedOfflineSync = true; + await this.syncer.scheduleSyncForOfflineChanges(); + } + + this.hasFinishedOfflineSync = true; + } + + private async pause(): Promise { + this.fetchController.startReset(); + await this.webSocketManager.stop(); + await this.waitUntilFinished(); + } + + private resetInMemoryState(): void { + this.history.reset(); + this.contentCache.reset(); + // don't reset the logger + this.cursorTracker.reset(); + this.syncer.reset(); + this.fileOperations.reset(); + } + + private async onSettingsChange( + newSettings: SyncSettings, + oldSettings: SyncSettings + ): Promise { + this.checkIfDestroyed("onSettingsChange"); + + if ( + newSettings.vaultName !== oldSettings.vaultName || + newSettings.remoteUri !== oldSettings.remoteUri + ) { + await this.reset(); + } + + if (newSettings.isSyncEnabled !== oldSettings.isSyncEnabled) { + if (newSettings.isSyncEnabled) { + await this.startSyncing(); + } else { + await this.pause(); + } + } + + if (newSettings.diffCacheSizeMB !== oldSettings.diffCacheSizeMB) { + this.contentCache.resize(newSettings.diffCacheSizeMB * 1024 * 1024); + } + + if (newSettings.enableTelemetry !== oldSettings.enableTelemetry) { + if (newSettings.enableTelemetry) { + this.unloadTelemetry = setUpTelemetry(); + } else { + this.unloadTelemetry?.(); + } + } + } + + private checkIfDestroyed(origin: string): void { + if (this.hasBeenDestroyed) { + throw new Error( + `SyncClient has been destroyed and can no longer be used; called from ${origin}` + ); + } + } } diff --git a/frontend/sync-client/src/sync-operations/cursor-tracker.ts b/frontend/sync-client/src/sync-operations/cursor-tracker.ts index 17f166c4..d4cf3c53 100644 --- a/frontend/sync-client/src/sync-operations/cursor-tracker.ts +++ b/frontend/sync-client/src/sync-operations/cursor-tracker.ts @@ -8,7 +8,7 @@ import type { MaybeOutdatedClientCursors } from "../types/maybe-outdated-client- import { DocumentUpToDateness } from "../types/document-up-to-dateness"; import { hash } from "../utils/hash"; import type { FileChangeNotifier } from "./file-change-notifier"; -import { Lock } from "../utils/locks"; +import { Lock } from "../utils/data-structures/locks"; // Cursor positions are updated separately from documents. However, a given cursor position is only // valid within a certain version of the document it belongs to. This class tracks previous and the latest @@ -157,6 +157,13 @@ export class CursorTracker { }); } + public reset(): void { + this.knownRemoteCursors = []; + this.lastLocalCursorState = []; + this.lastLocalCursorStateWithoutDirtyDocuments = []; + this.updateLock.reset(); + } + private getRelevantAndPruneKnownClientCursors(): MaybeOutdatedClientCursors[] { const result: MaybeOutdatedClientCursors[] = []; const included = new Set(); @@ -167,14 +174,14 @@ export class CursorTracker { continue; } - if (clientCursors.upToDateness == DocumentUpToDateness.Later) { + if (clientCursors.upToDateness === DocumentUpToDateness.Later) { continue; } result.push({ ...clientCursors, isOutdated: - clientCursors.upToDateness == DocumentUpToDateness.Prior + clientCursors.upToDateness === DocumentUpToDateness.Prior }); included.add(clientCursors.deviceId); diff --git a/frontend/sync-client/src/sync-operations/file-change-notifier.ts b/frontend/sync-client/src/sync-operations/file-change-notifier.ts index 8a7af66c..2c099b6f 100644 --- a/frontend/sync-client/src/sync-operations/file-change-notifier.ts +++ b/frontend/sync-client/src/sync-operations/file-change-notifier.ts @@ -9,6 +9,15 @@ export class FileChangeNotifier { this.listeners.push(listener); } + public removeFileChangeListener( + listener: (filePath: RelativePath) => unknown + ): void { + const index = this.listeners.indexOf(listener); + if (index !== -1) { + this.listeners.splice(index, 1); + } + } + public notifyOfFileChange(filePath: RelativePath): void { this.listeners.forEach((listener) => listener(filePath)); } diff --git a/frontend/sync-client/src/sync-operations/syncer.ts b/frontend/sync-client/src/sync-operations/syncer.ts index a4badd9a..12008b59 100644 --- a/frontend/sync-client/src/sync-operations/syncer.ts +++ b/frontend/sync-client/src/sync-operations/syncer.ts @@ -15,24 +15,32 @@ import { findMatchingFile } from "../utils/find-matching-file"; import type { UnrestrictedSyncer } from "./unrestricted-syncer"; import { createPromise } from "../utils/create-promise"; import { SyncResetError } from "../services/sync-reset-error"; -import { Locks } from "../utils/locks"; +import { Locks } from "../utils/data-structures/locks"; import type { DocumentVersionWithoutContent } from "../services/types/DocumentVersionWithoutContent"; -import type { FixedSizeDocumentCache } from "../utils/fix-sized-cache"; +import type { WebSocketVaultUpdate } from "../services/types/WebSocketVaultUpdate"; +import type { WebSocketManager } from "../services/websocket-manager"; +import type { WebSocketClientMessage } from "../services/types/WebSocketClientMessage"; +import { awaitAll } from "../utils/await-all"; export class Syncer { private readonly remoteDocumentsLock: Locks; private readonly remainingOperationsListeners: (( remainingOperations: number ) => unknown)[] = []; + + // FIFO to limit the number of concurrent sync operations private readonly syncQueue: PQueue; + private _isFirstSyncComplete = false; private runningScheduleSyncForOfflineChanges: Promise | undefined; public constructor( + private readonly deviceId: string, private readonly logger: Logger, private readonly database: Database, - settings: Settings, + private readonly settings: Settings, private readonly syncService: SyncService, + private readonly webSocketManager: WebSocketManager, private readonly operations: FileOperations, private readonly internalSyncer: UnrestrictedSyncer ) { @@ -53,6 +61,22 @@ export class Syncer { listener(this.syncQueue.size); }); }); + + this.webSocketManager.addWebSocketStatusChangeListener( + (isConnected) => { + if (isConnected) { + // The JS WebSocket API doesn't support setting headers, so we have to send the token as a message + this.sendHandshakeMessage(); + } + } + ); + this.webSocketManager.addRemoteVaultUpdateListener( + this.syncRemotelyUpdatedFile.bind(this) + ); + } + + public get isFirstSyncComplete(): boolean { + return this._isFirstSyncComplete; } public addRemainingOperationsListener( @@ -83,10 +107,6 @@ export class Syncer { promise ); - this.logger.debug( - `Creating new pending document ${relativePath} with id ${id}` - ); - try { await this.syncQueue.add(async () => this.internalSyncer.unrestrictedSyncLocallyCreatedFile(document) @@ -246,14 +266,53 @@ export class Syncer { public async waitUntilFinished(): Promise { await this.runningScheduleSyncForOfflineChanges; - return this.syncQueue.onEmpty(); - } - - public async reset(): Promise { - await this.waitUntilFinished(); + await this.syncQueue.onEmpty(); } public async syncRemotelyUpdatedFile( + message: WebSocketVaultUpdate + ): Promise { + try { + const handlerPromise = awaitAll( + message.documents.map(async (document) => + this.internalSyncRemotelyUpdatedFile(document) + ) + ); + + await handlerPromise; + + if (message.isInitialSync && message.documents.length > 0) { + this.database.setLastSeenUpdateId( + message.documents + .map((document) => document.vaultUpdateId) + .reduce((a, b) => Math.max(a, b)) + ); + } + + this._isFirstSyncComplete = true; + } catch (e) { + this.logger.error(`Failed to sync remotely updated file: ${e}`); + } + } + + public reset(): void { + this._isFirstSyncComplete = false; + this.syncQueue.clear(); + this.remoteDocumentsLock.reset(); + this.runningScheduleSyncForOfflineChanges = undefined; + } + + private sendHandshakeMessage(): void { + const message: WebSocketClientMessage = { + type: "handshake", + deviceId: this.deviceId, + token: this.settings.getSettings().token, + lastSeenVaultUpdateId: this.database.getLastSeenUpdateId() + }; + this.webSocketManager.sendHandshakeMessage(message); + } + + private async internalSyncRemotelyUpdatedFile( remoteVersion: DocumentVersionWithoutContent ): Promise { let document = this.database.getDocumentByDocumentId( @@ -337,6 +396,9 @@ export class Syncer { await this.createFakeDocumentsFromRemoteState(); const allLocalFiles = await this.operations.listFilesRecursively(); + this.logger.info( + `Scheduling sync for ${allLocalFiles.length} local files` + ); let locallyPossiblyDeletedFiles: DocumentRecord[] = []; @@ -349,7 +411,7 @@ export class Syncer { } } - const updates = Promise.all( + await awaitAll( allLocalFiles.map(async (relativePath) => { if ( this.database.getLatestDocumentByRelativePath(relativePath) @@ -407,7 +469,9 @@ export class Syncer { }) ); - const deletes = Promise.all( + // this has to happen strictly after the previous awaitAll, as that one + // might have removed some of the documents from the list + await awaitAll( locallyPossiblyDeletedFiles.map(async ({ relativePath }) => { this.logger.debug( `Document ${relativePath} has been deleted locally, scheduling sync to delete it` @@ -417,8 +481,6 @@ export class Syncer { return this.syncLocallyDeletedFile(relativePath); }) ); - - await Promise.all([updates, deletes]); } /** @@ -431,7 +493,7 @@ export class Syncer { return; } - const [allLocalFiles, remote] = await Promise.all([ + const [allLocalFiles, remote] = await awaitAll([ this.operations.listFilesRecursively(), this.syncQueue.add(async () => this.syncService.getAll()) ]); diff --git a/frontend/sync-client/src/sync-operations/unrestricted-syncer.ts b/frontend/sync-client/src/sync-operations/unrestricted-syncer.ts index f9f6e2c1..ebbb076f 100644 --- a/frontend/sync-client/src/sync-operations/unrestricted-syncer.ts +++ b/frontend/sync-client/src/sync-operations/unrestricted-syncer.ts @@ -18,7 +18,8 @@ import type { } from "../tracing/sync-history"; import { SyncStatus, SyncType } from "../tracing/sync-history"; import { EMPTY_HASH, hash } from "../utils/hash"; -import { deserialize } from "../utils/deserialize"; + +import { base64ToBytes } from "byte-base64"; import type { Settings } from "../persistence/settings"; import type { FileOperations } from "../file-operations/file-operations"; import { createPromise } from "../utils/create-promise"; @@ -28,9 +29,10 @@ import { globsToRegexes } from "../utils/globs-to-regexes"; import type { DocumentVersion } from "../services/types/DocumentVersion"; import type { DocumentUpdateResponse } from "../services/types/DocumentUpdateResponse"; import type { DocumentVersionWithoutContent } from "../services/types/DocumentVersionWithoutContent"; -import type { FixedSizeDocumentCache } from "../utils/fix-sized-cache"; +import type { FixedSizeDocumentCache } from "../utils/data-structures/fix-sized-cache"; import { isFileTypeMergable } from "../utils/is-file-type-mergable"; import { isBinary } from "../utils/is-binary"; +import type { ServerConfig } from "../services/server-config"; export class UnrestrictedSyncer { private ignorePatterns: RegExp[]; @@ -42,7 +44,8 @@ export class UnrestrictedSyncer { private readonly syncService: SyncService, private readonly operations: FileOperations, private readonly history: SyncHistory, - private readonly contentCache: FixedSizeDocumentCache + private readonly contentCache: FixedSizeDocumentCache, + private readonly serverConfig: ServerConfig ) { this.ignorePatterns = globsToRegexes( this.settings.getSettings().ignorePatterns, @@ -66,24 +69,35 @@ export class UnrestrictedSyncer { }; return this.executeSync(updateDetails, async () => { + const originalRelativePath = document.relativePath; if (document.isDeleted) { this.logger.debug( - `Document ${document.relativePath} has been already deleted, no need to create it` + `Document ${originalRelativePath} has been already deleted, no need to create it` ); return; } - const contentBytes = await this.operations.read( - document.relativePath - ); // this can throw FileNotFoundError + const contentBytes = + await this.operations.read(originalRelativePath); // this can throw FileNotFoundError const contentHash = hash(contentBytes); const response = await this.syncService.create({ documentId: document.documentId, - relativePath: document.relativePath, + relativePath: originalRelativePath, contentBytes }); + // In case a document with the same name (but different ID) had existed remotely that we haven't known about + if (response.relativePath != originalRelativePath) { + this.logger.debug( + `Document ${originalRelativePath} has been created remotely at a different path: ${response.relativePath}, moving it locally` + ); + await this.operations.move( + document.relativePath, + response.relativePath + ); // this can throw FileNotFoundError + } + this.database.updateDocumentMetadata( { parentVersionId: response.vaultUpdateId, @@ -92,6 +106,7 @@ export class UnrestrictedSyncer { }, document ); + this.database.addSeenUpdateId(response.vaultUpdateId); this.updateCache( response.vaultUpdateId, @@ -189,7 +204,10 @@ export class UnrestrictedSyncer { if (areThereLocalChanges) { const isText = !isBinary(contentBytes) && - isFileTypeMergable(document.relativePath); + isFileTypeMergable( + document.relativePath, + this.serverConfig.getConfig().mergeableFileExtensions + ); const cachedVersion = this.contentCache.get( document.metadata.parentVersionId ); @@ -249,33 +267,7 @@ export class UnrestrictedSyncer { } if (response.isDeleted) { - this.history.addHistoryEntry({ - status: SyncStatus.SUCCESS, - details: { - type: SyncType.DELETE, - relativePath: document.relativePath - }, - message: - "File has been deleted remotely, so we deleted it locally", - author: response.userId, - timestamp: new Date(response.updatedDate) - }); - - this.database.delete(document.relativePath); - this.database.updateDocumentMetadata( - { - parentVersionId: response.vaultUpdateId, - hash: EMPTY_HASH, - remoteRelativePath: response.relativePath - }, - document - ); - - await this.operations.delete(document.relativePath); - - this.database.addSeenUpdateId(response.vaultUpdateId); - - return; + return this.applyRemoteDeleteLocally(document, response); } let actualPath = document.relativePath; @@ -292,7 +284,7 @@ export class UnrestrictedSyncer { } if (!("type" in response) || response.type === "MergingUpdate") { - const responseBytes = deserialize(response.contentBase64); + const responseBytes = base64ToBytes(response.contentBase64); contentHash = hash(responseBytes); this.database.updateDocumentMetadata( @@ -439,7 +431,7 @@ export class UnrestrictedSyncer { return; } - const contentBytes = deserialize(content); + const contentBytes = base64ToBytes(content); await this.operations.ensureClearPath(remoteVersion.relativePath); @@ -562,8 +554,44 @@ export class UnrestrictedSyncer { contentBytes: Uint8Array, filePath: RelativePath ): void { - if (isFileTypeMergable(filePath) && !isBinary(contentBytes)) { + if ( + isFileTypeMergable( + filePath, + this.serverConfig.getConfig().mergeableFileExtensions + ) && + !isBinary(contentBytes) + ) { this.contentCache.put(updateId, contentBytes); } } + + private async applyRemoteDeleteLocally( + document: DocumentRecord, + response: DocumentVersion | DocumentUpdateResponse + ): Promise { + this.history.addHistoryEntry({ + status: SyncStatus.SUCCESS, + details: { + type: SyncType.DELETE, + relativePath: document.relativePath + }, + message: "File has been deleted remotely, so we deleted it locally", + author: response.userId, + timestamp: new Date(response.updatedDate) + }); + + this.database.delete(document.relativePath); + this.database.updateDocumentMetadata( + { + parentVersionId: response.vaultUpdateId, + hash: EMPTY_HASH, + remoteRelativePath: response.relativePath + }, + document + ); + + await this.operations.delete(document.relativePath); + + this.database.addSeenUpdateId(response.vaultUpdateId); + } } diff --git a/frontend/sync-client/src/tracing/logger.ts b/frontend/sync-client/src/tracing/logger.ts index cf39e4de..96b93b0d 100644 --- a/frontend/sync-client/src/tracing/logger.ts +++ b/frontend/sync-client/src/tracing/logger.ts @@ -1,3 +1,5 @@ +import { MAX_LOG_MESSAGE_COUNT } from "../consts"; + export enum LogLevel { DEBUG = "DEBUG", INFO = "INFO", @@ -21,7 +23,6 @@ export class LogLine { } export class Logger { - private static readonly MAX_MESSAGES = 100000; private readonly messages: LogLine[] = []; private readonly onMessageListeners: ((message: LogLine) => unknown)[] = []; @@ -59,6 +60,15 @@ export class Logger { this.onMessageListeners.push(listener); } + public removeOnMessageListener( + listener: (message: LogLine) => unknown + ): void { + const index = this.onMessageListeners.indexOf(listener); + if (index !== -1) { + this.onMessageListeners.splice(index, 1); + } + } + public reset(): void { this.messages.length = 0; this.debug("Logger has been reset"); @@ -68,7 +78,7 @@ export class Logger { const logLine = new LogLine(level, message); this.messages.push(logLine); - while (this.messages.length > Logger.MAX_MESSAGES) { + while (this.messages.length > MAX_LOG_MESSAGE_COUNT) { this.messages.shift(); } diff --git a/frontend/sync-client/src/tracing/sync-history.ts b/frontend/sync-client/src/tracing/sync-history.ts index 92904ce6..0fb1a754 100644 --- a/frontend/sync-client/src/tracing/sync-history.ts +++ b/frontend/sync-client/src/tracing/sync-history.ts @@ -1,3 +1,7 @@ +import { + MAX_HISTORY_ENTRY_COUNT, + TIMEOUT_FOR_MERGING_HISTORY_ENTRIES_IN_SECONDS +} from "../consts"; import type { RelativePath } from "../persistence/database"; import type { Logger } from "./logger"; @@ -64,9 +68,6 @@ export interface HistoryStats { } export class SyncHistory { - private static readonly MAX_ENTRIES = 5000; - private static readonly TIMEOUT_FOR_MERGING_ENTRIES_IN_SECONDS = 60; - private _entries: HistoryEntry[] = []; private readonly syncHistoryUpdateListeners: (( @@ -104,7 +105,7 @@ export class SyncHistory { // Insert the entry at the beginning this._entries.unshift(historyEntry); - if (this._entries.length > SyncHistory.MAX_ENTRIES) { + if (this._entries.length > MAX_HISTORY_ENTRY_COUNT) { this._entries.pop(); } @@ -118,6 +119,15 @@ export class SyncHistory { listener({ ...this.status }); } + public removeSyncHistoryUpdateListener( + listener: (stats: HistoryStats) => unknown + ): void { + const index = this.syncHistoryUpdateListeners.indexOf(listener); + if (index !== -1) { + this.syncHistoryUpdateListeners.splice(index, 1); + } + } + public reset(): void { this._entries.length = 0; this.status = { @@ -145,7 +155,7 @@ export class SyncHistory { candidate !== undefined && (this._entries[0] === candidate || candidate.timestamp.getTime() + - SyncHistory.TIMEOUT_FOR_MERGING_ENTRIES_IN_SECONDS * 1000 > + TIMEOUT_FOR_MERGING_HISTORY_ENTRIES_IN_SECONDS * 1000 > entry.timestamp.getTime()) ) { return candidate; @@ -164,7 +174,7 @@ export class SyncHistory { this.logger.error(`Cannot sync file: ${message}`); break; case SyncStatus.SKIPPED: - this.logger.error(`Skipping file: ${message}`); + this.logger.warn(`Skipping file: ${message}`); break; } diff --git a/frontend/sync-client/src/utils/await-all.test.ts b/frontend/sync-client/src/utils/await-all.test.ts new file mode 100644 index 00000000..bbce9423 --- /dev/null +++ b/frontend/sync-client/src/utils/await-all.test.ts @@ -0,0 +1,56 @@ +import { test } from "node:test"; +import assert from "node:assert"; +import { awaitAll } from "./await-all"; + +void test("awaitAll resolves promises of the same type", async () => { + const promises = [ + Promise.resolve(1), + Promise.resolve(2), + Promise.resolve(3) + ]; + + const results = await awaitAll(promises); + assert.deepStrictEqual(results, [1, 2, 3]); +}); + +void test("awaitAll resolves promises of different types", async () => { + const promises = [ + Promise.resolve("hello"), + Promise.resolve(42), + Promise.resolve(true) + ] as const; + + const results = await awaitAll(promises); + + // Type assertions to verify type inference + const str: string = results[0]; + const num: number = results[1]; + const bool: boolean = results[2]; + + assert.strictEqual(str, "hello"); + assert.strictEqual(num, 42); + assert.strictEqual(bool, true); +}); + +void test("awaitAll throws on first rejection", async () => { + const error = new Error("Test error"); + const promises = [ + Promise.resolve(1), + Promise.reject(error), + Promise.resolve(3) + ]; + + await assert.rejects(async () => { + await awaitAll(promises); + }, error); +}); + +void test("awaitAll works with async functions", async () => { + const asyncString = async (): Promise => "async"; + const asyncNumber = async (): Promise => 123; + + const results = await awaitAll([asyncString(), asyncNumber()]); + + assert.strictEqual(results[0], "async"); + assert.strictEqual(results[1], 123); +}); diff --git a/frontend/sync-client/src/utils/await-all.ts b/frontend/sync-client/src/utils/await-all.ts new file mode 100644 index 00000000..b8d50250 --- /dev/null +++ b/frontend/sync-client/src/utils/await-all.ts @@ -0,0 +1,25 @@ +type PromiseTuple = readonly [ + ...{ [K in keyof T]: Promise } +]; + +type ResolvedTuple = { + [K in keyof T]: T[K]; +}; + +export const awaitAll = async ( + promises: PromiseTuple +): Promise> => { + // eslint-disable-next-line no-restricted-properties + const result = await Promise.allSettled(promises); + for (const res of result) { + if (res.status === "rejected") { + throw res.reason; + } + } + + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion + return result.map( + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion + (res) => (res as PromiseFulfilledResult).value + ) as ResolvedTuple; +}; diff --git a/frontend/sync-client/src/utils/fix-sized-cache.test.ts b/frontend/sync-client/src/utils/data-structures/fix-sized-cache.test.ts similarity index 99% rename from frontend/sync-client/src/utils/fix-sized-cache.test.ts rename to frontend/sync-client/src/utils/data-structures/fix-sized-cache.test.ts index 4a24aafb..a118815b 100644 --- a/frontend/sync-client/src/utils/fix-sized-cache.test.ts +++ b/frontend/sync-client/src/utils/data-structures/fix-sized-cache.test.ts @@ -89,7 +89,7 @@ describe("fixedSizeDocumentCache", () => { assert.equal(cache.get(1), doc1); assert.equal(cache.get(2), doc2); - cache.clear(); + cache.reset(); assert.equal(cache.get(1), undefined); assert.equal(cache.get(2), undefined); diff --git a/frontend/sync-client/src/utils/fix-sized-cache.ts b/frontend/sync-client/src/utils/data-structures/fix-sized-cache.ts similarity index 96% rename from frontend/sync-client/src/utils/fix-sized-cache.ts rename to frontend/sync-client/src/utils/data-structures/fix-sized-cache.ts index cf0ba47e..1541d72f 100644 --- a/frontend/sync-client/src/utils/fix-sized-cache.ts +++ b/frontend/sync-client/src/utils/data-structures/fix-sized-cache.ts @@ -1,6 +1,6 @@ // Implements an in-memory fixed-size cache for document contents, -import type { VaultUpdateId } from "../persistence/database"; +import type { VaultUpdateId } from "../../persistence/database"; // Doubly-linked list node for O(1) LRU operations class LRUNode { @@ -57,7 +57,7 @@ export class FixedSizeDocumentCache { this.fitBelowMaxSize(); } - public clear(): void { + public reset(): void { this.cache.clear(); this.head = null; this.tail = null; diff --git a/frontend/sync-client/src/utils/locks.test.ts b/frontend/sync-client/src/utils/data-structures/locks.test.ts similarity index 84% rename from frontend/sync-client/src/utils/locks.test.ts rename to frontend/sync-client/src/utils/data-structures/locks.test.ts index 5626becc..a13bb274 100644 --- a/frontend/sync-client/src/utils/locks.test.ts +++ b/frontend/sync-client/src/utils/data-structures/locks.test.ts @@ -1,8 +1,10 @@ import { describe, it, beforeEach } from "node:test"; import assert from "node:assert"; -import { Logger } from "../tracing/logger"; -import type { RelativePath } from "../persistence/database"; +import { Logger } from "../../tracing/logger"; +import type { RelativePath } from "../../persistence/database"; import { Locks } from "./locks"; +import { awaitAll } from "../await-all"; +import { sleep } from "../sleep"; describe("withLock", () => { const testPath: RelativePath = "test/document/path"; @@ -31,7 +33,7 @@ describe("withLock", () => { let executionCount = 0; const result = await locks.withLock(testPath, async () => { executionCount++; - await new Promise((resolve) => setTimeout(resolve, 10)); + await sleep(10); return "async-success"; }); @@ -56,19 +58,19 @@ describe("withLock", () => { // Start two concurrent operations with keys in different orders const promise1 = locks.withLock([testPath2, testPath], async () => { executionOrder.push("operation1-start"); - await new Promise((resolve) => setTimeout(resolve, 50)); + await sleep(50); executionOrder.push("operation1-end"); return "result1"; }); const promise2 = locks.withLock([testPath, testPath2], async () => { executionOrder.push("operation2-start"); - await new Promise((resolve) => setTimeout(resolve, 50)); + await sleep(50); executionOrder.push("operation2-end"); return "result2"; }); - const [result1, result2] = await Promise.all([promise1, promise2]); + const [result1, result2] = await awaitAll([promise1, promise2]); assert.strictEqual(result1, "result1"); assert.strictEqual(result2, "result2"); @@ -86,19 +88,19 @@ describe("withLock", () => { const promise1 = locks.withLock(testPath, async () => { executionOrder.push("operation1-start"); - await new Promise((resolve) => setTimeout(resolve, 50)); + await sleep(50); executionOrder.push("operation1-end"); return "result1"; }); const promise2 = locks.withLock(testPath, async () => { executionOrder.push("operation2-start"); - await new Promise((resolve) => setTimeout(resolve, 30)); + await sleep(30); executionOrder.push("operation2-end"); return "result2"; }); - const [result1, result2] = await Promise.all([promise1, promise2]); + const [result1, result2] = await awaitAll([promise1, promise2]); assert.strictEqual(result1, "result1"); assert.strictEqual(result2, "result2"); @@ -115,19 +117,20 @@ describe("withLock", () => { const promise1 = locks.withLock(testPath, async () => { executionOrder.push("operation1-start"); - await new Promise((resolve) => setTimeout(resolve, 50)); + await sleep(50); + executionOrder.push("operation1-end"); return "result1"; }); const promise2 = locks.withLock(testPath2, async () => { executionOrder.push("operation2-start"); - await new Promise((resolve) => setTimeout(resolve, 30)); + await sleep(30); executionOrder.push("operation2-end"); return "result2"; }); - const [result1, result2] = await Promise.all([promise1, promise2]); + const [result1, result2] = await awaitAll([promise1, promise2]); assert.strictEqual(result1, "result1"); assert.strictEqual(result2, "result2"); @@ -159,7 +162,8 @@ describe("withLock", () => { await assert.rejects( locks.withLock(testPath, async () => { - await new Promise((resolve) => setTimeout(resolve, 10)); + await sleep(10); + throw error; }), { message: "async test error" } @@ -184,30 +188,30 @@ describe("withLock", () => { // Start first operation that holds the lock const firstPromise = locks.withLock(testPath, async () => { executionOrder.push("first-start"); - await new Promise((resolve) => setTimeout(resolve, 100)); + await sleep(100); executionOrder.push("first-end"); return "first"; }); // Small delay to ensure first operation starts - await new Promise((resolve) => setTimeout(resolve, 10)); + await sleep(10); // Queue second and third operations const secondPromise = locks.withLock(testPath, async () => { executionOrder.push("second-start"); - await new Promise((resolve) => setTimeout(resolve, 30)); + await sleep(50); executionOrder.push("second-end"); return "second"; }); const thirdPromise = locks.withLock(testPath, async () => { executionOrder.push("third-start"); - await new Promise((resolve) => setTimeout(resolve, 20)); + await sleep(20); executionOrder.push("third-end"); return "third"; }); - const [first, second, third] = await Promise.all([ + const [first, second, third] = await awaitAll([ firstPromise, secondPromise, thirdPromise diff --git a/frontend/sync-client/src/utils/locks.ts b/frontend/sync-client/src/utils/data-structures/locks.ts similarity index 83% rename from frontend/sync-client/src/utils/locks.ts rename to frontend/sync-client/src/utils/data-structures/locks.ts index e09da236..fccccf8c 100644 --- a/frontend/sync-client/src/utils/locks.ts +++ b/frontend/sync-client/src/utils/data-structures/locks.ts @@ -1,4 +1,5 @@ -import type { Logger } from "../tracing/logger"; +import type { Logger } from "../../tracing/logger"; +import { awaitAll } from "../await-all"; /** * Manages exclusive locks on items to prevent concurrent modifications. @@ -49,19 +50,27 @@ export class Locks { fn: () => R | Promise ): Promise { const keys = Array.isArray(keyOrKeys) ? keyOrKeys : [keyOrKeys]; - keys.sort((a, b) => String(a).localeCompare(String(b))); // Ensure consistent order to prevent deadlocks - await Promise.all(keys.map(async (key) => this.waitForLock(key))); + // Deduplicate keys to prevent deadlock from acquiring same lock twice + const uniqueKeys = Array.from(new Set(keys)); + uniqueKeys.sort((a, b) => String(a).localeCompare(String(b))); // Ensure consistent order to prevent deadlocks + + await awaitAll(uniqueKeys.map(async (key) => this.waitForLock(key))); try { return await fn(); } finally { - keys.forEach((key) => { + uniqueKeys.forEach((key) => { this.unlock(key); }); } } + public reset(): void { + this.locked.clear(); + this.waiters.clear(); + } + /** * Attempts to acquire a lock immediately without waiting. * Must call `unlock()` if successful. @@ -69,7 +78,7 @@ export class Locks { * @param key The key to lock * @returns `true` if lock acquired, `false` if already locked */ - private tryLock(key: T): boolean { + public tryLock(key: T): boolean { if (this.locked.has(key)) { return false; } @@ -86,7 +95,7 @@ export class Locks { * @param key The key to wait for and lock * @returns Promise that resolves when lock is acquired */ - private async waitForLock(key: T): Promise { + public async waitForLock(key: T): Promise { if (this.tryLock(key)) { return Promise.resolve(); } @@ -112,9 +121,9 @@ export class Locks { * @param key The key to unlock * @throws {Error} If key is not currently locked */ - private unlock(key: T): void { + public unlock(key: T): void { if (!this.locked.has(key)) { - throw new Error(`Key '${key}' is not locked, cannot unlock`); + return; } // Remove first waiter to ensure FIFO order @@ -139,4 +148,8 @@ export class Lock { public async withLock(fn: () => R | Promise): Promise { return this.locks.withLock(true, fn); } + + public reset(): void { + this.locks.reset(); + } } diff --git a/frontend/sync-client/src/utils/min-covered.test.ts b/frontend/sync-client/src/utils/data-structures/min-covered.test.ts similarity index 73% rename from frontend/sync-client/src/utils/min-covered.test.ts rename to frontend/sync-client/src/utils/data-structures/min-covered.test.ts index 82f792c3..1bbd1425 100644 --- a/frontend/sync-client/src/utils/min-covered.test.ts +++ b/frontend/sync-client/src/utils/data-structures/min-covered.test.ts @@ -48,15 +48,29 @@ describe("CoveredValues", () => { assert.strictEqual(covered.min, 6); }); - it("should handle force setting min value", () => { + it("should auto-advance when setting min value", () => { const covered = new CoveredValues(5); covered.add(7); covered.add(8); covered.add(9); assert.strictEqual(covered.min, 5); + // Setting min to 6 should auto-advance through 7, 8, 9 covered.min = 6; - assert.strictEqual(covered.min, 6); + assert.strictEqual(covered.min, 9); covered.add(10); assert.strictEqual(covered.min, 10); }); + + it("should handle setting min value with no consecutive values", () => { + const covered = new CoveredValues(5); + covered.add(10); + covered.add(15); + assert.strictEqual(covered.min, 5); + // Setting min to 8 should not auto-advance (no consecutive values) + covered.min = 8; + assert.strictEqual(covered.min, 8); + // Add 9 to trigger auto-advance to 10 + covered.add(9); + assert.strictEqual(covered.min, 10); + }); }); diff --git a/frontend/sync-client/src/utils/min-covered.ts b/frontend/sync-client/src/utils/data-structures/min-covered.ts similarity index 81% rename from frontend/sync-client/src/utils/min-covered.ts rename to frontend/sync-client/src/utils/data-structures/min-covered.ts index c453ef88..be480597 100644 --- a/frontend/sync-client/src/utils/min-covered.ts +++ b/frontend/sync-client/src/utils/data-structures/min-covered.ts @@ -24,11 +24,12 @@ export class CoveredValues { public set min(value: number) { this.minValue = Math.max(value, this.minValue); - this.seenValues = this.seenValues.filter((v) => v > value); + this.seenValues = this.seenValues.filter((v) => v > this.minValue); + this.advanceMinWhilePossible(); } - public add(value: number): void { - if (value < this.minValue) { + public add(value: number | undefined): void { + if (value === undefined || value < this.minValue) { return; } @@ -45,6 +46,10 @@ export class CoveredValues { this.seenValues.splice(i, 0, value); } + this.advanceMinWhilePossible(); + } + + private advanceMinWhilePossible(): void { while ( this.seenValues.length > 0 && this.seenValues[0] === this.minValue + 1 diff --git a/frontend/sync-client/src/debugging/log-to-console.ts b/frontend/sync-client/src/utils/debugging/log-to-console.ts similarity index 76% rename from frontend/sync-client/src/debugging/log-to-console.ts rename to frontend/sync-client/src/utils/debugging/log-to-console.ts index ace58db0..2d1a12e8 100644 --- a/frontend/sync-client/src/debugging/log-to-console.ts +++ b/frontend/sync-client/src/utils/debugging/log-to-console.ts @@ -1,6 +1,6 @@ -import type { SyncClient } from "../sync-client"; -import type { LogLine } from "../tracing/logger"; -import { LogLevel } from "../tracing/logger"; +import type { SyncClient } from "../../sync-client"; +import type { LogLine } from "../../tracing/logger"; +import { LogLevel } from "../../tracing/logger"; export function logToConsole(client: SyncClient): void { client.logger.addOnMessageListener((logLine: LogLine) => { diff --git a/frontend/sync-client/src/debugging/slow-fetch-factory.ts b/frontend/sync-client/src/utils/debugging/slow-fetch-factory.ts similarity index 56% rename from frontend/sync-client/src/debugging/slow-fetch-factory.ts rename to frontend/sync-client/src/utils/debugging/slow-fetch-factory.ts index cd07dd1a..4c2ddedb 100644 --- a/frontend/sync-client/src/debugging/slow-fetch-factory.ts +++ b/frontend/sync-client/src/utils/debugging/slow-fetch-factory.ts @@ -1,4 +1,4 @@ -import { sleep } from "../utils/sleep"; +import { sleep } from "../sleep"; export const slowFetchFactory = (jitterScaleInSeconds: number) => @@ -7,10 +7,14 @@ export const slowFetchFactory = init?: RequestInit ): Promise => { if (jitterScaleInSeconds > 0) { - await sleep(Math.random() * jitterScaleInSeconds * 1000); + await sleep(((Math.random() * jitterScaleInSeconds) / 2) * 1000); } const response = await fetch(input, init); + if (jitterScaleInSeconds > 0) { + await sleep(((Math.random() * jitterScaleInSeconds) / 2) * 1000); + } + return response; }; diff --git a/frontend/sync-client/src/debugging/slow-web-socket-factory.ts b/frontend/sync-client/src/utils/debugging/slow-web-socket-factory.ts similarity index 75% rename from frontend/sync-client/src/debugging/slow-web-socket-factory.ts rename to frontend/sync-client/src/utils/debugging/slow-web-socket-factory.ts index 51a27a5f..e52ff76b 100644 --- a/frontend/sync-client/src/debugging/slow-web-socket-factory.ts +++ b/frontend/sync-client/src/utils/debugging/slow-web-socket-factory.ts @@ -1,29 +1,29 @@ -import { sleep } from "../utils/sleep"; -import { Locks } from "../utils/locks"; -import type { Logger } from "../tracing/logger"; +import { sleep } from "../sleep"; +import { Locks } from "../data-structures/locks"; +import type { Logger } from "../../tracing/logger"; export function slowWebSocketFactory( jitterScaleInSeconds: number, logger: Logger ): typeof WebSocket { - // eslint-disable-next-line + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion return class FlakyWebSocket extends WebSocket { private static readonly RECEIVE_KEY = "websocket-receive"; private static readonly SEND_KEY = "websocket-send"; private readonly locks = new Locks(logger); - public set onopen(callback: (event: Event) => void) { + public set onopen(callback: ((event: Event) => void) | null) { super.onopen = async (event: Event): Promise => { if (jitterScaleInSeconds > 0) { await sleep(Math.random() * jitterScaleInSeconds * 1000); } - callback(event); + callback?.(event); }; } - public set onmessage(callback: (event: MessageEvent) => void) { + public set onmessage(callback: ((event: MessageEvent) => void) | null) { super.onmessage = async (event: MessageEvent): Promise => { await this.locks.withLock( FlakyWebSocket.RECEIVE_KEY, @@ -34,27 +34,27 @@ export function slowWebSocketFactory( ); } - callback(event); + callback?.(event); } ); }; } - public set onclose(callback: (event: CloseEvent) => void) { + public set onclose(callback: ((event: CloseEvent) => void) | null) { super.onclose = async (event: CloseEvent): Promise => { if (jitterScaleInSeconds > 0) { await sleep(Math.random() * jitterScaleInSeconds * 1000); } - callback(event); + callback?.(event); }; } - public set onerror(callback: (event: Event) => void) { + public set onerror(callback: ((event: Event) => void) | null) { super.onerror = async (event: Event): Promise => { if (jitterScaleInSeconds > 0) { await sleep(Math.random() * jitterScaleInSeconds * 1000); } - callback(event); + callback?.(event); }; } diff --git a/frontend/sync-client/src/utils/deserialize.ts b/frontend/sync-client/src/utils/deserialize.ts deleted file mode 100644 index 4255479f..00000000 --- a/frontend/sync-client/src/utils/deserialize.ts +++ /dev/null @@ -1,5 +0,0 @@ -import { base64ToBytes } from "byte-base64"; - -export function deserialize(data: string): Uint8Array { - return base64ToBytes(data); -} diff --git a/frontend/sync-client/src/utils/is-equal-bytes.test.ts b/frontend/sync-client/src/utils/is-equal-bytes.test.ts deleted file mode 100644 index a887309f..00000000 --- a/frontend/sync-client/src/utils/is-equal-bytes.test.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { describe, it } from "node:test"; -import assert from "node:assert"; -import { isEqualBytes } from "./is-equal-bytes"; - -describe("isEqualBytes", () => { - it("should return true for equal byte arrays", () => { - const bytes1 = new Uint8Array([1, 2, 3, 4]); - const bytes2 = new Uint8Array([1, 2, 3, 4]); - assert.strictEqual(isEqualBytes(bytes1, bytes2), true); - }); - - it("should return false for byte arrays of different lengths", () => { - const bytes1 = new Uint8Array([1, 2, 3, 4]); - const bytes2 = new Uint8Array([1, 2, 3]); - assert.strictEqual(isEqualBytes(bytes1, bytes2), false); - }); - - it("should return true for empty byte arrays", () => { - const bytes1 = new Uint8Array([]); - const bytes2 = new Uint8Array([]); - assert.strictEqual(isEqualBytes(bytes1, bytes2), true); - }); - - it("should return false for byte arrays with same length but different content", () => { - const bytes1 = new Uint8Array([1, 2, 3, 4]); - const bytes2 = new Uint8Array([4, 3, 2, 1]); - assert.strictEqual(isEqualBytes(bytes1, bytes2), false); - }); -}); diff --git a/frontend/sync-client/src/utils/is-equal-bytes.ts b/frontend/sync-client/src/utils/is-equal-bytes.ts deleted file mode 100644 index d0688d44..00000000 --- a/frontend/sync-client/src/utils/is-equal-bytes.ts +++ /dev/null @@ -1,13 +0,0 @@ -export function isEqualBytes(bytes1: Uint8Array, bytes2: Uint8Array): boolean { - if (bytes1.length !== bytes2.length) { - return false; - } - - for (let i = 0; i < bytes1.length; i++) { - if (bytes1[i] !== bytes2[i]) { - return false; - } - } - - return true; -} diff --git a/frontend/sync-client/src/utils/is-file-type-mergable.test.ts b/frontend/sync-client/src/utils/is-file-type-mergable.test.ts index 3f3fffbb..a2268d19 100644 --- a/frontend/sync-client/src/utils/is-file-type-mergable.test.ts +++ b/frontend/sync-client/src/utils/is-file-type-mergable.test.ts @@ -2,41 +2,72 @@ import { describe, it } from "node:test"; import assert from "node:assert"; import { isFileTypeMergable } from "./is-file-type-mergable"; +const mergableExtensions = ["md", "txt"]; describe("isFileTypeMergable", () => { it("should return true for .md files", () => { - assert.strictEqual(isFileTypeMergable(".md"), true); - assert.strictEqual(isFileTypeMergable("hi.md"), true); + assert.strictEqual(isFileTypeMergable(".md", mergableExtensions), true); assert.strictEqual( - isFileTypeMergable("my/path/to/my/document.md"), + isFileTypeMergable("hi.md", mergableExtensions), + true + ); + assert.strictEqual( + isFileTypeMergable("my/path/to/my/document.md", mergableExtensions), true ); }); it("should return true for .txt files", () => { - assert.strictEqual(isFileTypeMergable(".txt"), true); - assert.strictEqual(isFileTypeMergable("hi.txt"), true); assert.strictEqual( - isFileTypeMergable("my/path/to/my/document.txt"), + isFileTypeMergable(".txt", mergableExtensions), + true + ); + assert.strictEqual( + isFileTypeMergable("hi.txt", mergableExtensions), + true + ); + assert.strictEqual( + isFileTypeMergable( + "my/path/to/my/document.txt", + mergableExtensions + ), true ); }); it("should be case insensitive", () => { - assert.strictEqual(isFileTypeMergable("hi.MD"), true); assert.strictEqual( - isFileTypeMergable("my/path/to/my/DOCUMENT.MD"), + isFileTypeMergable("hi.MD", mergableExtensions), true ); - assert.strictEqual(isFileTypeMergable("hi.TXT"), true); assert.strictEqual( - isFileTypeMergable("my/path/to/my/DOCUMENT.TXT"), + isFileTypeMergable("my/path/to/my/DOCUMENT.MD", mergableExtensions), + true + ); + assert.strictEqual( + isFileTypeMergable("hi.TXT", mergableExtensions), + true + ); + assert.strictEqual( + isFileTypeMergable( + "my/path/to/my/DOCUMENT.TXT", + mergableExtensions + ), true ); }); it("should return false for non-mergable file types", () => { - assert.strictEqual(isFileTypeMergable(".json"), false); - assert.strictEqual(isFileTypeMergable("HELLO.JSON"), false); - assert.strictEqual(isFileTypeMergable("my/config.yml"), false); + assert.strictEqual( + isFileTypeMergable(".json", mergableExtensions), + false + ); + assert.strictEqual( + isFileTypeMergable("HELLO.JSON", mergableExtensions), + false + ); + assert.strictEqual( + isFileTypeMergable("my/config.yml", mergableExtensions), + false + ); }); }); diff --git a/frontend/sync-client/src/utils/is-file-type-mergable.ts b/frontend/sync-client/src/utils/is-file-type-mergable.ts index 3b149285..4eec2733 100644 --- a/frontend/sync-client/src/utils/is-file-type-mergable.ts +++ b/frontend/sync-client/src/utils/is-file-type-mergable.ts @@ -1,6 +1,9 @@ -export function isFileTypeMergable(pathOrFileName: string): boolean { +export function isFileTypeMergable( + pathOrFileName: string, + mergeableExtensions: string[] +): boolean { const parts = pathOrFileName.split("."); const fileExtension = parts.at(-1) ?? ""; - return ["md", "txt"].includes(fileExtension.toLowerCase()); + return mergeableExtensions.includes(fileExtension.toLowerCase()); } diff --git a/frontend/sync-client/src/utils/line-and-column-to-position.ts b/frontend/sync-client/src/utils/line-and-column-to-position.ts index 670d8cac..2ee6b2a4 100644 --- a/frontend/sync-client/src/utils/line-and-column-to-position.ts +++ b/frontend/sync-client/src/utils/line-and-column-to-position.ts @@ -13,7 +13,7 @@ export function lineAndColumnToPosition( line: number, column: number ): number { - const lines = text.replace("\r", "").split("\n"); + const lines = text.replaceAll("\r", "").split("\n"); if (line >= lines.length) { throw new Error(`Line number ${line} is out of range.`); diff --git a/frontend/sync-client/src/utils/position-to-line-and-column.test.ts b/frontend/sync-client/src/utils/position-to-line-and-column.test.ts index bc21b983..2341b7c5 100644 --- a/frontend/sync-client/src/utils/position-to-line-and-column.test.ts +++ b/frontend/sync-client/src/utils/position-to-line-and-column.test.ts @@ -43,6 +43,28 @@ describe("positionToLineAndColumn", () => { }); }); + test("with multiple carriage returns", () => { + // Test that all \r characters are removed, not just the first one + const text = "line1\r\nline2\r\nline3\r\n"; + + assert.deepStrictEqual(positionToLineAndColumn(text, 0), { + line: 0, + column: 0 + }); + + // Position 6 = start of 'line2' after all \r removed + assert.deepStrictEqual(positionToLineAndColumn(text, 6), { + line: 1, + column: 0 + }); + + // Position 12 = start of 'line3' after all \r removed + assert.deepStrictEqual(positionToLineAndColumn(text, 12), { + line: 2, + column: 0 + }); + }); + test("handles empty input", () => { assert.deepStrictEqual(positionToLineAndColumn("", 0), { line: 0, diff --git a/frontend/sync-client/src/utils/position-to-line-and-column.ts b/frontend/sync-client/src/utils/position-to-line-and-column.ts index 3df61ded..116b9f15 100644 --- a/frontend/sync-client/src/utils/position-to-line-and-column.ts +++ b/frontend/sync-client/src/utils/position-to-line-and-column.ts @@ -14,13 +14,10 @@ export function positionToLineAndColumn( throw new Error("Position cannot be negative"); } - text = text.replace("\r", ""); + text = text.replaceAll("\r", ""); - if ( - position > - text.length + 1 - // +1 to account for the cursor being after last character - ) { + if (position > text.length) { + // position == text.length accounts for the cursor being after last character throw new Error( `Position ${position} exceeds text length ${text.length}` ); diff --git a/frontend/sync-client/src/utils/rate-limit.ts b/frontend/sync-client/src/utils/rate-limit.ts index 4de89ae8..2c6d018b 100644 --- a/frontend/sync-client/src/utils/rate-limit.ts +++ b/frontend/sync-client/src/utils/rate-limit.ts @@ -10,7 +10,8 @@ import { sleep } from "./sleep"; * * @template T - Type of the function to be rate limited * @param {T} fn - The asynchronous function to rate limit - * @param {number} minIntervalMs - The minimum interval in milliseconds between function calls + * @param {number | (() => number)} minIntervalMs - Minimum interval in milliseconds between calls, + * or a function that returns the minimum interval * @returns {(...args: Parameters) => ReturnType | Promise} A decorated function that respects the rate limit. * Returns the original function's return type when executed, or undefined if the call was superseded by a newer one. */ @@ -21,7 +22,7 @@ export function rateLimit< ) => Promise >( fn: T, - minIntervalMs: number + minIntervalMs: number | (() => number) ): (...args: Parameters) => Promise { let newArgs: Parameters | undefined = undefined; let running: Promise | undefined = undefined; @@ -46,7 +47,11 @@ export function rateLimit< const [promise, resolve] = createPromise(); running = promise; - sleep(minIntervalMs) + sleep( + typeof minIntervalMs === "function" + ? minIntervalMs() + : minIntervalMs + ) .then(resolve) .catch(() => { // sleep cannot fail diff --git a/frontend/test-client/src/agent/mock-agent.ts b/frontend/test-client/src/agent/mock-agent.ts index a6ced45d..42d9490d 100644 --- a/frontend/test-client/src/agent/mock-agent.ts +++ b/frontend/test-client/src/agent/mock-agent.ts @@ -18,6 +18,7 @@ export class MockAgent extends MockClient { initialSettings: Partial, public readonly name: string, private readonly doDeletes: boolean, + private readonly doResets: boolean, useSlowFileEvents: boolean, private readonly jitterScaleInSeconds: number ) { @@ -107,28 +108,41 @@ export class MockAgent extends MockClient { } } - this.pendingActions.push( - (async (): Promise => { - try { - return await choose(options)(); - } catch (error) { - this.client.logger.error( - `Failed to perform an action: ${error}` - ); - this.client.logger.info(JSON.stringify(this.data, null, 2)); - this.client.logger.info( - JSON.stringify(this.localFiles, null, 2) - ); - throw error; - } - })() - ); + if (Math.random() < 0.015 && this.doResets) { + // we can't just queue this up as once it's destroyed, no more method calls can go to SyncClient + await this.resetClient(); + } else { + this.pendingActions.push( + (async (): Promise => { + try { + return await choose(options)(); + } catch (error) { + this.client.logger.error( + `Failed to perform an action: ${error}` + ); + this.client.logger.info( + JSON.stringify(this.data, null, 2) + ); + this.client.logger.info( + JSON.stringify(this.localFiles, null, 2) + ); + throw error; + } + })() + ); + } } public async finish(): Promise { await this.client.setSetting("isSyncEnabled", true); + // eslint-disable-next-line no-restricted-properties await Promise.all(this.pendingActions); - await this.client.waitAndStop(); + await this.client.waitUntilFinished(); + } + + public async destroy(): Promise { + await this.client.waitUntilFinished(); + await this.client.destroy(); } public assertFileSystemsAreConsistent(otherAgent: MockAgent): void { @@ -228,6 +242,12 @@ export class MockAgent extends MockClient { } } + private async resetClient(): Promise { + this.client.logger.info(`Resetting client ${this.name}`); + await this.client.destroy(); + await this.init(); + } + private async createFileAction(): Promise { const file = this.getFileName(); diff --git a/frontend/test-client/src/agent/mock-client.ts b/frontend/test-client/src/agent/mock-client.ts index 2b384c24..3121db29 100644 --- a/frontend/test-client/src/agent/mock-client.ts +++ b/frontend/test-client/src/agent/mock-client.ts @@ -1,4 +1,4 @@ -import type { StoredDatabase } from "sync-client"; +import type { StoredDatabase, TextWithCursors } from "sync-client"; import { assert } from "../utils/assert"; import { type RelativePath, @@ -6,7 +6,6 @@ import { type SyncSettings, SyncClient } from "sync-client"; -import type { TextWithCursors } from "reconcile-text"; export class MockClient implements FileSystemOperations { protected readonly localFiles = new Map(); diff --git a/frontend/test-client/src/cli.ts b/frontend/test-client/src/cli.ts index 4a3aab4f..531cf102 100644 --- a/frontend/test-client/src/cli.ts +++ b/frontend/test-client/src/cli.ts @@ -14,6 +14,7 @@ async function runTest({ concurrency, iterations, doDeletes, + doResets, useSlowFileEvents, jitterScaleInSeconds }: { @@ -21,12 +22,13 @@ async function runTest({ concurrency: number; iterations: number; doDeletes: boolean; + doResets: boolean; useSlowFileEvents: boolean; jitterScaleInSeconds: number; }): Promise { slowFileEvents = useSlowFileEvents; - const settings = `with ${agentCount} agents, concurrency ${concurrency}, iterations ${iterations}, doDeletes ${doDeletes}, jitterScaleInSeconds ${jitterScaleInSeconds}, useSlowFileEvents ${useSlowFileEvents}`; + const settings = `with ${agentCount} agents, concurrency ${concurrency}, iterations ${iterations}, doDeletes ${doDeletes}, doResets ${doResets}, jitterScaleInSeconds ${jitterScaleInSeconds}, useSlowFileEvents ${useSlowFileEvents}`; console.info(`Running test ${settings}`); const vaultName = uuidv4(); @@ -46,6 +48,7 @@ async function runTest({ initialSettings, `agent-${i}`, doDeletes, + doResets, useSlowFileEvents, jitterScaleInSeconds ) @@ -53,10 +56,12 @@ async function runTest({ } try { + // eslint-disable-next-line no-restricted-properties await Promise.all(clients.map(async (client) => client.init())); for (let i = 0; i < iterations; i++) { console.info(`Iteration ${i + 1}/${iterations}`); + // eslint-disable-next-line no-restricted-properties await Promise.all(clients.map(async (client) => client.act())); await sleep(100); } @@ -77,7 +82,7 @@ async function runTest({ // then we need a second pass to ensure that all agents pull the same state. for (const client of clients) { try { - await client.finish(); + await client.destroy(); } catch (err) { if (!slowFileEvents) { throw err; @@ -112,6 +117,16 @@ async function runTest({ async function runTests(): Promise { for (let i = 0; i < TEST_ITERATIONS; i++) { + await runTest({ + agentCount: 2, + concurrency: 16, + iterations: 100, + doDeletes: true, + doResets: true, + useSlowFileEvents: true, + jitterScaleInSeconds: 0.75 + }); + for (const useSlowFileEvents of [false, true]) { for (const concurrency of [ 16, @@ -123,6 +138,7 @@ async function runTests(): Promise { concurrency, iterations: 100, doDeletes, + doResets: false, useSlowFileEvents, jitterScaleInSeconds: 0.75 }); diff --git a/scripts/check.sh b/scripts/check.sh index eccc5714..4f69dfb2 100755 --- a/scripts/check.sh +++ b/scripts/check.sh @@ -21,6 +21,7 @@ else cargo fmt --all -- --check fi +cargo install cargo-machete cargo machete --with-metadata echo "Running checks in frontend" @@ -41,6 +42,7 @@ cd .. if [[ "$FIX_MODE" == true ]]; then $0 +else + echo "Success" fi -echo "Success" diff --git a/sync-server/config-e2e.yml b/sync-server/config-e2e.yml index 0b8491ee..58410948 100644 --- a/sync-server/config-e2e.yml +++ b/sync-server/config-e2e.yml @@ -8,6 +8,9 @@ server: max_body_size_mb: 512 max_clients_per_vault: 256 response_timeout_seconds: 60 + mergeable_file_extensions: + - md + - txt users: user_configs: - name: admin diff --git a/sync-server/src/app_state/database.rs b/sync-server/src/app_state/database.rs index 346fea38..d64bd560 100644 --- a/sync-server/src/app_state/database.rs +++ b/sync-server/src/app_state/database.rs @@ -2,6 +2,7 @@ use core::time::Duration; use std::{collections::HashMap, sync::Arc}; use anyhow::{Context as _, Result}; +use log::info; use models::{ DocumentId, DocumentVersionWithoutContent, StoredDocumentVersion, VaultId, VaultUpdateId, }; @@ -10,6 +11,7 @@ use sqlx::{sqlite::SqliteConnectOptions, types::chrono::Utc}; pub mod models; use sqlx::{Pool, Sqlite, sqlite::SqlitePoolOptions}; use tokio::sync::Mutex; +use tokio::time::Instant; use uuid::fmt::Hyphenated; use super::websocket::{ @@ -18,11 +20,26 @@ use super::websocket::{ }; use crate::config::database_config::DatabaseConfig; +#[derive(Clone)] +struct PoolWithTimestamp { + pool: Pool, + last_accessed: Instant, +} + +impl std::fmt::Debug for PoolWithTimestamp { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("PoolWithTimestamp") + .field("pool", &"Pool") + .field("last_accessed", &self.last_accessed) + .finish() + } +} + #[derive(Clone, Debug)] pub struct Database { config: DatabaseConfig, broadcasts: Broadcasts, - connection_pools: Arc>>>, + connection_pools: Arc>>, } pub type Transaction<'a> = sqlx::Transaction<'a, Sqlite>; @@ -33,7 +50,7 @@ impl Database { .await .with_context(|| { format!( - "Failed to create databases directory: {}", + "Failed to create databases directory at `{}`", config.databases_directory_path.to_string_lossy() ) })?; @@ -52,17 +69,26 @@ impl Database { .trim_end_matches(".sqlite") .to_owned(); + let pool = Self::create_vault_database(config, &vault).await?; connection_pools.insert( vault.clone(), - Self::create_vault_database(config, &vault).await?, + PoolWithTimestamp { + pool, + last_accessed: Instant::now(), + }, ); } - Ok(Self { + let database = Self { config: config.clone(), connection_pools: Arc::new(Mutex::new(connection_pools)), broadcasts: broadcasts.clone(), - }) + }; + + // Start background task to cleanup idle connection pools + database.start_idle_pool_cleanup(); + + Ok(database) } async fn create_vault_database( @@ -84,7 +110,7 @@ impl Database { .test_before_acquire(true) .connect_with(connection_options) .await - .with_context(|| format!("Cannot open database at {}", file_name.display()))?; + .with_context(|| format!("Cannot open database at `{}`", file_name.display()))?; Self::run_migrations(&pool).await?; @@ -100,16 +126,26 @@ impl Database { async fn get_connection_pool(&self, vault: &VaultId) -> Result> { let mut pools = self.connection_pools.lock().await; + if !pools.contains_key(vault) { let pool = Self::create_vault_database(&self.config, vault).await?; - pools.insert(vault.clone(), pool); + pools.insert( + vault.clone(), + PoolWithTimestamp { + pool, + last_accessed: Instant::now(), + }, + ); } - let pool = pools - .get(vault) + let pool_with_timestamp = pools + .get_mut(vault) .expect("Pool was just inserted or already exists"); - Ok(pool.clone()) + // Update last accessed time + pool_with_timestamp.last_accessed = Instant::now(); + + Ok(pool_with_timestamp.pool.clone()) } /// Attempting to write from this transaction might result in a @@ -218,7 +254,7 @@ impl Database { .await } .with_context(|| { - format!("Cannot fetch latest documents since vault_update_id {vault_update_id}") + format!("Cannot fetch latest documents since vault_update_id `{vault_update_id}`") }) .map(|rows| { rows.into_iter() @@ -434,4 +470,42 @@ impl Database { Ok(()) } + + /// Cleanup idle connection pools that haven't been accessed in more than 5 minutes + async fn cleanup_idle_pools(&self) { + let mut pools = self.connection_pools.lock().await; + let now = Instant::now(); + let idle_timeout = Duration::from_secs(5 * 60); // 5 minutes + + // Collect vaults to remove + let vaults_to_remove: Vec = pools + .iter() + .filter(|(_, pool_with_timestamp)| { + now.duration_since(pool_with_timestamp.last_accessed) > idle_timeout + }) + .map(|(vault_id, _)| vault_id.clone()) + .collect(); + + // Close and remove idle pools + for vault_id in &vaults_to_remove { + if let Some(pool_with_timestamp) = pools.remove(vault_id) { + info!("Closing idle database connection pool for vault `{vault_id}`"); + pool_with_timestamp.pool.close().await; + } + } + } + + /// Start a background task that periodically cleans up idle connection pools + fn start_idle_pool_cleanup(&self) { + let database = self.clone(); + tokio::spawn(async move { + let mut interval = tokio::time::interval(Duration::from_secs(60)); // Check every minute + interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + + loop { + interval.tick().await; + database.cleanup_idle_pools().await; + } + }); + } } diff --git a/sync-server/src/app_state/websocket/broadcasts.rs b/sync-server/src/app_state/websocket/broadcasts.rs index cef6ee6a..60ae0219 100644 --- a/sync-server/src/app_state/websocket/broadcasts.rs +++ b/sync-server/src/app_state/websocket/broadcasts.rs @@ -1,6 +1,7 @@ use std::{collections::HashMap, sync::Arc}; use anyhow::Context; +use log::{debug, warn}; use tokio::sync::{Mutex, broadcast}; use super::models::WebSocketServerMessageWithOrigin; @@ -32,13 +33,18 @@ impl Broadcasts { } /// Notify all clients (who are subscribed to the vault) about an update. - /// We only log failures. + /// We only log failures and don't propagate them. pub async fn send_document_update( &self, vault: VaultId, document: WebSocketServerMessageWithOrigin, ) { - let tx = self.get_or_create(vault).await; + let tx = self.get_or_create(vault.clone()).await; + + if tx.receiver_count() == 0 { + debug!("Skipping broadcast, no clients connected for vault `{vault}`"); + return; + } let result = tx .send(document) @@ -46,7 +52,7 @@ impl Broadcasts { .map_err(server_error); if result.is_err() { - log::debug!("Failed to send message: {result:?}"); + warn!("Failed to send message: {result:?}"); } } diff --git a/sync-server/src/config.rs b/sync-server/src/config.rs index 2e1a6e39..6a003d2e 100644 --- a/sync-server/src/config.rs +++ b/sync-server/src/config.rs @@ -30,7 +30,7 @@ impl Config { pub async fn read_or_create(path: &Path) -> Result { let config = if path.exists() { info!( - "Loading configuration from '{}'", + "Loading configuration from `{}`", path.canonicalize().unwrap().display() ); Self::load_from_file(path).await? @@ -40,7 +40,7 @@ impl Config { config.write(path).await?; info!( - "Updated configuration at '{}'", + "Updated configuration at `{}`", path.canonicalize().unwrap().display() ); @@ -50,14 +50,12 @@ impl Config { pub async fn load_from_file(path: &Path) -> Result { let contents = fs::read_to_string(path).await.with_context(|| { format!( - "Cannot load configuration from disk from {}", + "Cannot load configuration from disk from `{}`", path.display() ) })?; - let config = serde_yaml::from_str(&contents).context("Failed to parse configuration")?; - - Ok(config) + serde_yaml::from_str(&contents).context("Failed to parse configuration") } pub async fn write(&self, path: &Path) -> Result<()> { diff --git a/sync-server/src/config/logging_config.rs b/sync-server/src/config/logging_config.rs index 95ab9350..79d4fa1e 100644 --- a/sync-server/src/config/logging_config.rs +++ b/sync-server/src/config/logging_config.rs @@ -24,7 +24,7 @@ impl Default for LoggingConfig { } fn default_log_directory() -> String { - debug!("Using default log directory: {DEFAULT_LOG_DIRECTORY}"); + debug!("Using default log directory: `{DEFAULT_LOG_DIRECTORY}`"); DEFAULT_LOG_DIRECTORY.to_owned() } diff --git a/sync-server/src/config/server_config.rs b/sync-server/src/config/server_config.rs index ce922fb9..fc6034ed 100644 --- a/sync-server/src/config/server_config.rs +++ b/sync-server/src/config/server_config.rs @@ -2,8 +2,8 @@ use log::debug; use serde::{Deserialize, Serialize}; use crate::consts::{ - DEFAULT_HOST, DEFAULT_MAX_BODY_SIZE_MB, DEFAULT_MAX_CLIENTS_PER_VAULT, DEFAULT_PORT, - DEFAULT_RESPONSE_TIMEOUT_SECONDS, + DEFAULT_HOST, DEFAULT_MAX_BODY_SIZE_MB, DEFAULT_MAX_CLIENTS_PER_VAULT, + DEFAULT_MERGEABLE_FILE_EXTENSIONS, DEFAULT_PORT, DEFAULT_RESPONSE_TIMEOUT_SECONDS, }; #[derive(Debug, Deserialize, Serialize, Clone, Default)] @@ -22,6 +22,9 @@ pub struct ServerConfig { #[serde(default = "default_response_timeout_seconds")] pub response_timeout_seconds: u64, + + #[serde(default = "default_mergeable_file_extensions")] + pub mergeable_file_extensions: Vec, } fn default_host() -> String { @@ -35,7 +38,7 @@ fn default_port() -> u16 { } fn default_max_body_size_mb() -> usize { - debug!("Using default max body size (MB): {DEFAULT_MAX_BODY_SIZE_MB}"); + debug!("Using default max body size {DEFAULT_MAX_BODY_SIZE_MB} MB"); DEFAULT_MAX_BODY_SIZE_MB } @@ -45,6 +48,14 @@ fn default_max_clients_per_vault() -> usize { } fn default_response_timeout_seconds() -> u64 { - debug!("Using default response timeout (seconds): {DEFAULT_RESPONSE_TIMEOUT_SECONDS}"); + debug!("Using default response timeout: {DEFAULT_RESPONSE_TIMEOUT_SECONDS} seconds"); DEFAULT_RESPONSE_TIMEOUT_SECONDS } + +fn default_mergeable_file_extensions() -> Vec { + debug!("Using default mergeable file extensions: {DEFAULT_MERGEABLE_FILE_EXTENSIONS:?}"); + DEFAULT_MERGEABLE_FILE_EXTENSIONS + .iter() + .map(|s| (*s).to_owned()) + .collect() +} diff --git a/sync-server/src/config/user_config.rs b/sync-server/src/config/user_config.rs index ed7ecc23..cdfed838 100644 --- a/sync-server/src/config/user_config.rs +++ b/sync-server/src/config/user_config.rs @@ -20,7 +20,7 @@ where for user in &users { if let Some(existing_name) = user_token_map.get_by_right(&user.token) { return Err(D::Error::custom(format!( - "Duplicate user token found: '{}' for users '{}' and '{}'. User tokens must be \ + "Duplicate user token found: `{}` for users `{}` and `{}`. User tokens must be \ unique.", user.token, existing_name, user.name ))); @@ -28,7 +28,7 @@ where if user_token_map.contains_left(&user.name) { return Err(D::Error::custom(format!( - "Duplicate user name found: '{}'. User names must be unique.", + "Duplicate user name found: `{}`. User names must be unique.", user.name ))); } diff --git a/sync-server/src/consts.rs b/sync-server/src/consts.rs index d973ca4a..3c672520 100644 --- a/sync-server/src/consts.rs +++ b/sync-server/src/consts.rs @@ -14,3 +14,7 @@ pub const DEFAULT_MAX_CLIENTS_PER_VAULT: usize = 256; pub const DEFAULT_LOG_DIRECTORY: &str = "logs"; pub const DEFAULT_LOG_ROTATION_INTERVAL: Duration = Duration::from_secs(60 * 60 * 24); // 1 day + +pub const DEFAULT_MERGEABLE_FILE_EXTENSIONS: &[&str] = &["md", "txt"]; + +pub const SUPPORTED_API_VERSION: u32 = 1; diff --git a/sync-server/src/server/auth.rs b/sync-server/src/server/auth.rs index d27c16e3..e56f4acc 100644 --- a/sync-server/src/server/auth.rs +++ b/sync-server/src/server/auth.rs @@ -52,14 +52,14 @@ pub fn auth(state: &AppState, token: &str, vault_id: &VaultId) -> Result allowed.contains(vault_id), } { info!( - "User '{}' is authenticated and is authorised to access to vault '{vault_id}'", + "User `{}` is authenticated and is authorised to access to vault `{vault_id}`", user.name ); Ok(user) } else { info!( - "User '{}' is authenticated but is not authorised to access vault '{vault_id}'", + "User `{}` is authenticated but is not authorised to access vault `{vault_id}`", user.name ); diff --git a/sync-server/src/server/create_document.rs b/sync-server/src/server/create_document.rs index 0f698538..859c0db4 100644 --- a/sync-server/src/server/create_document.rs +++ b/sync-server/src/server/create_document.rs @@ -4,6 +4,7 @@ use axum::{ }; use axum_extra::TypedHeader; use axum_typed_multipart::TypedMultipart; +use log::{debug, info}; use serde::Deserialize; use super::{device_id_header::DeviceIdHeader, requests::CreateDocumentVersion}; @@ -14,7 +15,10 @@ use crate::{ }, config::user_config::User, errors::{SyncServerError, client_error, server_error}, - utils::{normalize::normalize, sanitize_path::sanitize_path}, + utils::{ + find_first_available_path::find_first_available_path, normalize::normalize, + sanitize_path::sanitize_path, + }, }; #[derive(Deserialize)] @@ -34,6 +38,8 @@ pub async fn create_document( State(state): State, TypedMultipart(request): TypedMultipart, ) -> Result, SyncServerError> { + debug!("Creating document in vault `{vault_id}`"); + let mut transaction = state .database .create_write_transaction(&vault_id) @@ -50,7 +56,7 @@ pub async fn create_document( if existing_version.is_some() { return Err(client_error(anyhow::anyhow!( - "Document with the same ID already exists" + "Document with the same ID `{document_id}` already exists" ))); } @@ -66,11 +72,25 @@ pub async fn create_document( .map_err(server_error)?; let sanitized_relative_path = sanitize_path(&request.relative_path); + let deduped_path = find_first_available_path( + &vault_id, + &sanitized_relative_path, + &state.database, + &mut transaction, + ) + .await + .map_err(server_error)?; + + if deduped_path != sanitized_relative_path { + info!( + "Document already exists at new location: `{sanitized_relative_path}` when trying to create it in vault `{vault_id}`, deconflicting by creating at `{deduped_path}`" + ); + } let new_version = StoredDocumentVersion { vault_update_id: last_update_id + 1, document_id, - relative_path: sanitized_relative_path, + relative_path: deduped_path, content: request.content.contents.to_vec(), updated_date: chrono::Utc::now(), is_deleted: false, diff --git a/sync-server/src/server/delete_document.rs b/sync-server/src/server/delete_document.rs index f7080417..e126d6b5 100644 --- a/sync-server/src/server/delete_document.rs +++ b/sync-server/src/server/delete_document.rs @@ -1,8 +1,10 @@ +use anyhow::Context; use axum::{ Extension, Json, extract::{Path, State}, }; use axum_extra::TypedHeader; +use log::{debug, info}; use serde::Deserialize; use super::{device_id_header::DeviceIdHeader, requests::DeleteDocumentVersion}; @@ -37,6 +39,8 @@ pub async fn delete_document( State(state): State, Json(request): Json, ) -> Result, SyncServerError> { + debug!("Deleting document `{document_id}` in vault `{vault_id}`"); + let mut transaction = state .database .create_write_transaction(&vault_id) @@ -49,12 +53,26 @@ pub async fn delete_document( .await .map_err(server_error)?; - let latest_content = state + let latest_version = state .database .get_latest_document(&vault_id, &document_id, Some(&mut transaction)) .await - .map_err(server_error)? - .map_or_else(Vec::new, |version| version.content); // in case the document has never existed before deleting it + .map_err(server_error)?; + + if let Some(latest_version) = &latest_version + && latest_version.is_deleted + { + transaction + .rollback() + .await + .context("Failed to roll back transaction") + .map_err(server_error)?; + + info!("Document `{document_id}` has already been deleted",); + return Ok(Json(latest_version.clone().into())); + } + + let latest_content = latest_version.map_or_else(Vec::new, |version| version.content); // in case the document has never existed before deleting it let new_version = StoredDocumentVersion { vault_update_id: last_update_id + 1, diff --git a/sync-server/src/server/fetch_document_version.rs b/sync-server/src/server/fetch_document_version.rs index 5b571a7b..67e72ca4 100644 --- a/sync-server/src/server/fetch_document_version.rs +++ b/sync-server/src/server/fetch_document_version.rs @@ -3,6 +3,7 @@ use axum::{ Json, extract::{Path, State}, }; +use log::debug; use serde::Deserialize; use crate::{ @@ -32,6 +33,10 @@ pub async fn fetch_document_version( }): Path, State(state): State, ) -> Result, SyncServerError> { + debug!( + "Fetching document version `{vault_update_id}` for document `{document_id}` in vault `{vault_id}`" + ); + let result = state .database .get_document_version(&vault_id, vault_update_id, None) diff --git a/sync-server/src/server/fetch_document_version_content.rs b/sync-server/src/server/fetch_document_version_content.rs index a419b7bf..a74e88ec 100644 --- a/sync-server/src/server/fetch_document_version_content.rs +++ b/sync-server/src/server/fetch_document_version_content.rs @@ -3,6 +3,7 @@ use axum::{ body::Bytes, extract::{Path, State}, }; +use log::debug; use serde::Deserialize; use crate::{ @@ -32,6 +33,10 @@ pub async fn fetch_document_version_content( }): Path, State(state): State, ) -> Result { + debug!( + "Fetching document version `{vault_update_id}` for document `{document_id}` in vault `{vault_id}`" + ); + let result = state .database .get_document_version(&vault_id, vault_update_id, None) diff --git a/sync-server/src/server/fetch_latest_document_version.rs b/sync-server/src/server/fetch_latest_document_version.rs index 07f07860..a9973606 100644 --- a/sync-server/src/server/fetch_latest_document_version.rs +++ b/sync-server/src/server/fetch_latest_document_version.rs @@ -3,6 +3,7 @@ use axum::{ Json, extract::{Path, State}, }; +use log::debug; use serde::Deserialize; use crate::{ @@ -30,6 +31,8 @@ pub async fn fetch_latest_document_version( }): Path, State(state): State, ) -> Result, SyncServerError> { + debug!("Fetching latest document version for document `{document_id}` in vault `{vault_id}`"); + let latest_version = state .database .get_latest_document(&vault_id, &document_id, None) diff --git a/sync-server/src/server/fetch_latest_documents.rs b/sync-server/src/server/fetch_latest_documents.rs index 6101f55c..209374ce 100644 --- a/sync-server/src/server/fetch_latest_documents.rs +++ b/sync-server/src/server/fetch_latest_documents.rs @@ -2,6 +2,7 @@ use axum::{ Json, extract::{Path, Query, State}, }; +use log::debug; use serde::Deserialize; use super::responses::FetchLatestDocumentsResponse; @@ -31,6 +32,8 @@ pub async fn fetch_latest_documents( Query(QueryParams { since_update_id }): Query, State(state): State, ) -> Result, SyncServerError> { + debug!("Fetching latest documents in vault `{vault_id}` since update ID `{since_update_id:?}`"); + let documents = if let Some(since_update_id) = since_update_id { state .database diff --git a/sync-server/src/server/ping.rs b/sync-server/src/server/ping.rs index 620ef0d4..31aa8acd 100644 --- a/sync-server/src/server/ping.rs +++ b/sync-server/src/server/ping.rs @@ -6,11 +6,13 @@ use axum_extra::{ TypedHeader, headers::{Authorization, authorization::Bearer}, }; +use log::debug; use serde::Deserialize; use super::{auth::auth, responses::PingResponse}; use crate::{ app_state::{AppState, database::models::VaultId}, + consts::SUPPORTED_API_VERSION, errors::SyncServerError, utils::normalize::normalize, }; @@ -27,11 +29,15 @@ pub async fn ping( Path(PingPathParams { vault_id }): Path, State(state): State, ) -> Result, SyncServerError> { + debug!("Pinging vault `{vault_id}`"); + let is_authenticated = maybe_auth_header .is_some_and(|auth_header| auth(&state, auth_header.token(), &vault_id).is_ok()); Ok(Json(PingResponse { server_version: env!("CARGO_PKG_VERSION").to_owned(), is_authenticated, + mergeable_file_extensions: state.config.server.mergeable_file_extensions.clone(), + supported_api_version: SUPPORTED_API_VERSION, })) } diff --git a/sync-server/src/server/responses.rs b/sync-server/src/server/responses.rs index 5cfaa5d5..a8b3fcd7 100644 --- a/sync-server/src/server/responses.rs +++ b/sync-server/src/server/responses.rs @@ -16,6 +16,13 @@ pub struct PingResponse { /// Whether the client is authenticated based on the sent Authorization /// header. pub is_authenticated: bool, + + /// List of file extensions that are allowed to be merged. + pub mergeable_file_extensions: Vec, + + /// API version ensuring backwards & forwards compatibility between the client + /// and server. + pub supported_api_version: u32, } /// Response to a fetch latest documents request. diff --git a/sync-server/src/server/update_document.rs b/sync-server/src/server/update_document.rs index cb81361b..9da37832 100644 --- a/sync-server/src/server/update_document.rs +++ b/sync-server/src/server/update_document.rs @@ -5,7 +5,7 @@ use axum::{ }; use axum_extra::TypedHeader; use axum_typed_multipart::TypedMultipart; -use log::info; +use log::{debug, info}; use reconcile_text::{BuiltinTokenizer, EditedText, reconcile}; use serde::Deserialize; @@ -22,7 +22,7 @@ use crate::{ errors::{SyncServerError, not_found_error, server_error}, server::requests::UpdateBinaryDocumentVersion, utils::{ - dedup_paths::dedup_paths, is_binary::is_binary, + find_first_available_path::find_first_available_path, is_binary::is_binary, is_file_type_mergable::is_file_type_mergable, normalize::normalize, sanitize_path::sanitize_path, }, @@ -129,6 +129,8 @@ async fn update_document( relative_path: &str, content: Vec, ) -> Result, SyncServerError> { + debug!("Updating document `{document_id}` in vault `{vault_id}`"); + let sanitized_relative_path = sanitize_path(relative_path); let mut transaction = state @@ -164,6 +166,7 @@ async fn update_document( .context("Failed to roll back transaction") .map_err(server_error)?; + info!("Document `{document_id}` has been deleted, ignoring update to it",); return Ok(Json(DocumentUpdateResponse::FastForwardUpdate( latest_version.into(), ))); @@ -173,7 +176,9 @@ async fn update_document( // version if content == latest_version.content && sanitized_relative_path == latest_version.relative_path { - info!("Document content is the same as the latest version, skipping update"); + info!( + "Document content is the same as the latest version for `{document_id}`, skipping update" + ); transaction .rollback() .await @@ -185,12 +190,15 @@ async fn update_document( ))); } - let are_all_participants_mergable = is_file_type_mergable(&sanitized_relative_path) - && !is_binary(&parent_document.content) + let are_all_participants_mergable = is_file_type_mergable( + &sanitized_relative_path, + &state.config.server.mergeable_file_extensions, + ) && !is_binary(&parent_document.content) && !is_binary(&latest_version.content) && !is_binary(&content); let merged_content = if are_all_participants_mergable { + info!("Merging changes for document `{document_id}` in vault `{vault_id}`"); reconcile( str::from_utf8(&parent_document.content) .expect("parent must be valid UTF-8 because it's not binary"), @@ -215,21 +223,22 @@ async fn update_document( let new_relative_path = if parent_document.relative_path == latest_version.relative_path && latest_version.relative_path != sanitized_relative_path { - let mut new_relative_path = String::default(); - for candidate in dedup_paths(&sanitized_relative_path) { - if state - .database - .get_latest_document_by_path(&vault_id, &candidate, Some(&mut transaction)) - .await - .map_err(server_error)? - .is_none() - { - new_relative_path = candidate; - break; - } + let new_path = find_first_available_path( + &vault_id, + &sanitized_relative_path, + &state.database, + &mut transaction, + ) + .await + .map_err(server_error)?; + + if new_path != sanitized_relative_path { + info!( + "Document already exists at new location: `{sanitized_relative_path}` when trying to update it in vault `{vault_id}`, deconflicting by creating at `{new_path}`" + ); } - new_relative_path + new_path } else { latest_version.relative_path.clone() }; diff --git a/sync-server/src/server/websocket.rs b/sync-server/src/server/websocket.rs index 5e94b277..bb10b49f 100644 --- a/sync-server/src/server/websocket.rs +++ b/sync-server/src/server/websocket.rs @@ -43,12 +43,12 @@ pub async fn websocket_handler( } async fn websocket_wrapped(state: AppState, stream: WebSocket, vault_id: VaultId) { - info!("WebSocket connection opened on vault '{vault_id}'"); + info!("WebSocket connection opened on vault `{vault_id}`"); let result = websocket(state, stream, vault_id.clone()).await; if let Err(err) = result { - debug!("WebSocket connection error on vault '{vault_id}': {err}"); + debug!("WebSocket connection error on vault `{vault_id}`: {err}"); } } @@ -71,7 +71,7 @@ async fn websocket( )?; info!( - "WebSocket handshake successful for vault '{vault_id}' for '{}'", + "WebSocket handshake successful for vault `{vault_id}` for `{}`", authed_handshake.handshake.device_id ); @@ -184,7 +184,7 @@ async fn websocket( if result.is_err() { info!( - "WebSocket disconnected on vault '{vault_id}' for '{}'", + "WebSocket disconnected on vault `{vault_id}` for `{}`", authed_handshake.handshake.device_id ); } diff --git a/sync-server/src/utils.rs b/sync-server/src/utils.rs index 7345880d..460a1466 100644 --- a/sync-server/src/utils.rs +++ b/sync-server/src/utils.rs @@ -1,4 +1,5 @@ pub mod dedup_paths; +pub mod find_first_available_path; pub mod is_binary; pub mod is_file_type_mergable; pub mod normalize; diff --git a/sync-server/src/utils/dedup_paths.rs b/sync-server/src/utils/dedup_paths.rs index c35ad33b..bc687f6a 100644 --- a/sync-server/src/utils/dedup_paths.rs +++ b/sync-server/src/utils/dedup_paths.rs @@ -9,16 +9,24 @@ pub fn dedup_paths(path: &str) -> impl Iterator { directory.push('/'); } - let name_parts = file_name.rsplitn(2, '.').collect::>(); - let mut reverse_parts = name_parts.into_iter().rev(); - let (stem, extension) = match (reverse_parts.next(), reverse_parts.next()) { - (Some(stem), maybe_extension) => ( - stem.to_owned(), - maybe_extension - .map(|ext| format!(".{ext}")) - .unwrap_or_default(), - ), - _ => unreachable!("Path must have at least one part"), + // Handle dotfiles: ".gitignore" should have no extension, ".config.json" should split as ".config" + ".json" + let is_simple_dotfile = file_name.starts_with('.') && file_name.matches('.').count() == 1; + + let (stem, extension) = if is_simple_dotfile { + (file_name.clone(), String::new()) + } else { + // Regular file or dotfile with extension + let name_parts = file_name.rsplitn(2, '.').collect::>(); + let mut reverse_parts = name_parts.into_iter().rev(); + match (reverse_parts.next(), reverse_parts.next()) { + (Some(stem), maybe_extension) => ( + stem.to_owned(), + maybe_extension + .map(|ext| format!(".{ext}")) + .unwrap_or_default(), + ), + _ => unreachable!("Path must have at least one part"), + } }; let regex = Regex::new(r" \((\d+)\)$").unwrap(); @@ -85,4 +93,58 @@ mod test { Some("my/path.with.dots/file (6)".to_owned()) ); } + + #[test] + fn test_regex_capturing_group() { + // Single digit in parentheses + let mut deduped = dedup_paths("document (5).md"); + assert_eq!(deduped.next(), Some("document (5).md".to_owned())); + assert_eq!(deduped.next(), Some("document (6).md".to_owned())); + assert_eq!(deduped.next(), Some("document (7).md".to_owned())); + + // Multi-digit number + let mut deduped = dedup_paths("report (123).pdf"); + assert_eq!(deduped.next(), Some("report (123).pdf".to_owned())); + assert_eq!(deduped.next(), Some("report (124).pdf".to_owned())); + assert_eq!(deduped.next(), Some("report (125).pdf".to_owned())); + + // Number without extension + let mut deduped = dedup_paths("folder (99)"); + assert_eq!(deduped.next(), Some("folder (99)".to_owned())); + assert_eq!(deduped.next(), Some("folder (100)".to_owned())); + assert_eq!(deduped.next(), Some("folder (101)".to_owned())); + } + + #[test] + fn test_dedup_dotfiles() { + // Simple dotfile (no extension) + let mut deduped = dedup_paths(".gitignore"); + assert_eq!(deduped.next(), Some(".gitignore".to_owned())); + assert_eq!(deduped.next(), Some(".gitignore (1)".to_owned())); + assert_eq!(deduped.next(), Some(".gitignore (2)".to_owned())); + + // Dotfile with extension + let mut deduped = dedup_paths(".config.json"); + assert_eq!(deduped.next(), Some(".config.json".to_owned())); + assert_eq!(deduped.next(), Some(".config (1).json".to_owned())); + assert_eq!(deduped.next(), Some(".config (2).json".to_owned())); + + // Dotfile with number + let mut deduped = dedup_paths(".gitignore (5)"); + assert_eq!(deduped.next(), Some(".gitignore (5)".to_owned())); + assert_eq!(deduped.next(), Some(".gitignore (6)".to_owned())); + assert_eq!(deduped.next(), Some(".gitignore (7)".to_owned())); + + // Dotfile with extension and number + let mut deduped = dedup_paths(".config (3).json"); + assert_eq!(deduped.next(), Some(".config (3).json".to_owned())); + assert_eq!(deduped.next(), Some(".config (4).json".to_owned())); + assert_eq!(deduped.next(), Some(".config (5).json".to_owned())); + + // Dotfile in subdirectory + let mut deduped = dedup_paths("my/path/.gitignore"); + assert_eq!(deduped.next(), Some("my/path/.gitignore".to_owned())); + assert_eq!(deduped.next(), Some("my/path/.gitignore (1)".to_owned())); + assert_eq!(deduped.next(), Some("my/path/.gitignore (2)".to_owned())); + } } diff --git a/sync-server/src/utils/find_first_available_path.rs b/sync-server/src/utils/find_first_available_path.rs new file mode 100644 index 00000000..7629d8f1 --- /dev/null +++ b/sync-server/src/utils/find_first_available_path.rs @@ -0,0 +1,26 @@ +use crate::app_state::database::models::VaultId; +use crate::{app_state::database::Transaction, utils::dedup_paths::dedup_paths}; +use anyhow::Result; +use log::{debug, info}; + +pub async fn find_first_available_path( + vault_id: &VaultId, + sanitized_relative_path: &str, + database: &crate::app_state::database::Database, + transaction: &mut Transaction<'_>, +) -> Result { + info!("Finding first available path for `{sanitized_relative_path}` in vault `{vault_id}`"); + for candidate in dedup_paths(sanitized_relative_path) { + debug!("Checking candidate path for deconflicting names: `{candidate}`"); + if database + .get_latest_document_by_path(vault_id, &candidate, Some(transaction)) + .await? + .is_none() + { + info!("Selected available path: `{candidate}`"); + return Ok(candidate); + } + } + + unreachable!("dedup_paths produces infinite paths"); +} diff --git a/sync-server/src/utils/is_file_type_mergable.rs b/sync-server/src/utils/is_file_type_mergable.rs index fba4b323..7aabb393 100644 --- a/sync-server/src/utils/is_file_type_mergable.rs +++ b/sync-server/src/utils/is_file_type_mergable.rs @@ -1,7 +1,10 @@ -pub fn is_file_type_mergable(path_or_file_name: &str) -> bool { +pub fn is_file_type_mergable(path_or_file_name: &str, mergeable_extensions: &[String]) -> bool { let file_extension = path_or_file_name.split('.').next_back().unwrap_or_default(); + let file_extension_lower = file_extension.to_lowercase(); - matches!(file_extension.to_lowercase().as_str(), "md" | "txt") + mergeable_extensions + .iter() + .any(|ext| ext.to_lowercase() == file_extension_lower) } #[cfg(test)] @@ -10,14 +13,22 @@ mod tests { #[test] fn test_is_file_type_mergable() { - assert!(is_file_type_mergable(".md")); - assert!(is_file_type_mergable("hi.md")); - assert!(is_file_type_mergable("my/path/to/my/document.md")); - assert!(is_file_type_mergable("hi.MD")); - assert!(is_file_type_mergable("my/path/to/my/DOCUMENT.MD")); + let mergeable = vec!["md".to_owned(), "txt".to_owned()]; - assert!(!is_file_type_mergable(".json")); - assert!(!is_file_type_mergable("HELLO.JSON")); - assert!(!is_file_type_mergable("my/config.yml")); + assert!(is_file_type_mergable(".md", &mergeable)); + assert!(is_file_type_mergable("hi.md", &mergeable)); + assert!(is_file_type_mergable( + "my/path/to/my/document.md", + &mergeable + )); + assert!(is_file_type_mergable("hi.MD", &mergeable)); + assert!(is_file_type_mergable( + "my/path/to/my/DOCUMENT.MD", + &mergeable + )); + + assert!(!is_file_type_mergable(".json", &mergeable)); + assert!(!is_file_type_mergable("HELLO.JSON", &mergeable)); + assert!(!is_file_type_mergable("my/config.yml", &mergeable)); } }