From d034ad5cb35fe4594ea4183981653595de94d85d Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Mon, 6 Apr 2026 13:01:47 +0100 Subject: [PATCH] WIP --- .gitignore | 9 +- CLAUDE.md | 562 ++++++- frontend/history-ui/index.html | 13 + frontend/history-ui/package.json | 16 + frontend/history-ui/src/App.svelte | 78 + frontend/history-ui/src/app.css | 101 ++ .../src/components/ActivityFeed.svelte | 346 +++++ .../src/components/ConfirmDialog.svelte | 167 ++ .../src/components/Dashboard.svelte | 511 ++++++ .../history-ui/src/components/DiffView.svelte | 288 ++++ .../src/components/DocumentDetail.svelte | 712 +++++++++ .../history-ui/src/components/FileTree.svelte | 124 ++ .../history-ui/src/components/Header.svelte | 144 ++ .../history-ui/src/components/Login.svelte | 176 +++ .../src/components/TimeSlider.svelte | 191 +++ .../src/components/ToastContainer.svelte | 80 + .../src/components/VaultPicker.svelte | 198 +++ frontend/history-ui/src/lib/api.ts | 121 ++ frontend/history-ui/src/lib/stores.svelte.ts | 305 ++++ .../src/lib/types/ListVaultsResponse.ts | 7 + .../history-ui/src/lib/types/VaultInfo.ts | 6 + frontend/history-ui/src/main.ts | 7 + frontend/history-ui/svelte.config.js | 5 + frontend/history-ui/tsconfig.json | 16 + frontend/history-ui/vite.config.ts | 15 + frontend/package-lock.json | 29 +- .../file-operations/file-operations.test.ts | 34 +- .../src/file-operations/file-operations.ts | 45 +- .../sync-client/src/persistence/database.ts | 303 +--- .../sync-client/src/services/sync-service.ts | 37 +- .../src/services/types/ListVaultsResponse.ts | 7 + .../src/services/types/VaultInfo.ts | 6 + .../src/services/websocket-manager.ts | 7 +- frontend/sync-client/src/sync-client.ts | 80 +- .../src/sync-operations/cursor-tracker.ts | 37 +- .../sync-operations/sync-event-queue.test.ts | 465 +++++- .../src/sync-operations/sync-event-queue.ts | 343 +++- .../sync-client/src/sync-operations/syncer.ts | 1375 +++++++++++++---- .../sync-client/src/sync-operations/types.ts | 42 + .../sync-operations/unrestricted-syncer.ts | 612 -------- package-lock.json | 6 + scripts/e2e.sh | 4 +- sync-server/build.rs | 13 +- .../20260314000000_add_idempotency_key.sql | 2 + sync-server/src/app_state/websocket/utils.rs | 2 +- sync-server/src/server.rs | 21 +- .../src/server/fetch_document_versions.rs | 42 + sync-server/src/server/fetch_vault_history.rs | 70 + sync-server/src/server/index.rs | 80 +- .../src/server/restore_document_version.rs | 147 ++ 50 files changed, 6515 insertions(+), 1492 deletions(-) create mode 100644 frontend/history-ui/index.html create mode 100644 frontend/history-ui/package.json create mode 100644 frontend/history-ui/src/App.svelte create mode 100644 frontend/history-ui/src/app.css create mode 100644 frontend/history-ui/src/components/ActivityFeed.svelte create mode 100644 frontend/history-ui/src/components/ConfirmDialog.svelte create mode 100644 frontend/history-ui/src/components/Dashboard.svelte create mode 100644 frontend/history-ui/src/components/DiffView.svelte create mode 100644 frontend/history-ui/src/components/DocumentDetail.svelte create mode 100644 frontend/history-ui/src/components/FileTree.svelte create mode 100644 frontend/history-ui/src/components/Header.svelte create mode 100644 frontend/history-ui/src/components/Login.svelte create mode 100644 frontend/history-ui/src/components/TimeSlider.svelte create mode 100644 frontend/history-ui/src/components/ToastContainer.svelte create mode 100644 frontend/history-ui/src/components/VaultPicker.svelte create mode 100644 frontend/history-ui/src/lib/api.ts create mode 100644 frontend/history-ui/src/lib/stores.svelte.ts create mode 100644 frontend/history-ui/src/lib/types/ListVaultsResponse.ts create mode 100644 frontend/history-ui/src/lib/types/VaultInfo.ts create mode 100644 frontend/history-ui/src/main.ts create mode 100644 frontend/history-ui/svelte.config.js create mode 100644 frontend/history-ui/tsconfig.json create mode 100644 frontend/history-ui/vite.config.ts create mode 100644 frontend/sync-client/src/services/types/ListVaultsResponse.ts create mode 100644 frontend/sync-client/src/services/types/VaultInfo.ts create mode 100644 frontend/sync-client/src/sync-operations/types.ts delete mode 100644 frontend/sync-client/src/sync-operations/unrestricted-syncer.ts create mode 100644 package-lock.json create mode 100644 sync-server/src/app_state/database/migrations/20260314000000_add_idempotency_key.sql create mode 100644 sync-server/src/server/fetch_document_versions.rs create mode 100644 sync-server/src/server/fetch_vault_history.rs create mode 100644 sync-server/src/server/restore_document_version.rs diff --git a/.gitignore b/.gitignore index a1c1ac4f..967b2b65 100644 --- a/.gitignore +++ b/.gitignore @@ -7,15 +7,18 @@ node_modules # Frontend build folders frontend/*/dist -sync-server/db.sqlite3* -sync-server/databases - # Rust build folders sync-server/target sync-server/artifacts sync-server/bindings/*.ts +# build folders +sync-server/db.sqlite3* +**/databases + *.log *.sqlx target + +.task diff --git a/CLAUDE.md b/CLAUDE.md index c77b091b..09bc48dc 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -4,7 +4,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co ## Project Overview -VaultLink is a self-hosted Obsidian plugin for real-time collaborative file syncing. The project consists of a Rust-based sync server and a TypeScript frontend with three main components: an Obsidian plugin, a sync client library, and a test client. +VaultLink is a self-hosted Obsidian plugin for real-time collaborative file syncing. The project consists of a Rust-based sync server and a TypeScript frontend with four main components: an Obsidian plugin, a sync client library, a test client, and a standalone CLI client. ## Architecture @@ -13,21 +13,104 @@ VaultLink is a self-hosted Obsidian plugin for real-time collaborative file sync - **sync-server/**: Rust-based WebSocket server with SQLite database for document versioning and real-time synchronization - **frontend/sync-client/**: TypeScript library providing core sync functionality, WebSocket management, and file operations - **frontend/obsidian-plugin/**: Obsidian plugin that integrates the sync client with Obsidian's API -- **frontend/test-client/**: CLI testing tool for the sync functionality +- **frontend/test-client/**: CLI testing tool for simulating multiple concurrent users +- **frontend/local-client-cli/**: Standalone CLI for VaultLink sync client +- **frontend/history-ui/**: Svelte 5 web UI for browsing vault history, viewing diffs, and restoring versions ### Key Technologies - **Backend**: Rust with Axum framework, SQLite with SQLx, WebSockets for real-time sync -- **Frontend**: TypeScript, Webpack for bundling, Jest for testing +- **Frontend**: TypeScript, Webpack for bundling, Node.js native test runner +- **History UI**: Svelte 5 with runes, Vite for bundling, embedded in server binary via `rust-embed` - **Sync Algorithm**: Uses reconcile-text library for operational transformation +### Architectural Patterns + +**Server Architecture:** + +- `AppState`: Central state container holding `Database`, `Cursors`, and `Broadcasts` +- `Database`: SQLite-backed document versioning with SQLx for compile-time query verification +- `Broadcasts`: WebSocket broadcast system for real-time updates to connected clients +- `Cursors`: Tracks user cursor positions across documents with background cleanup task + +**Client Architecture (Serial Event Queue Model):** + +- `SyncClient`: Main entry point, orchestrates all sync operations +- `SyncService`: HTTP API client for CRUD operations on documents +- `WebSocketManager`: Manages WebSocket connection and real-time updates +- `Syncer`: Coordinates file synchronisation via a serial drain loop over a `SyncEventQueue` +- `SyncEventQueue`: Intent queue that coalesces events and tracks path→documentId mappings +- `CursorTracker`: Manages local and remote cursor positions +- `Database`: Client-side document metadata cache (persisted via `PersistenceProvider`) +- `FileOperations`: Abstraction layer for filesystem operations (3-way merge on write) + +**Dual-Bundle Strategy:** +The sync-client builds two separate bundles: + +- `sync-client.web.js`: Browser-compatible UMD bundle (excludes `ws` package) +- `sync-client.node.js`: Node.js CommonJS bundle with WebSocket support + +**History UI Architecture:** + +The history UI (`frontend/history-ui/`) is a standalone Svelte 5 SPA that provides read-only vault history browsing. It communicates with the server via the same REST API used by sync clients, plus three additional endpoints: + +- `GET /vaults/:vault_id/documents/:document_id/versions` — all versions of a document (without content) +- `GET /vaults/:vault_id/history?limit=&before_update_id=` — paginated vault-wide version history (cursor-based) +- `POST /vaults/:vault_id/documents/:document_id/restore` — restore a document to a historical version (creates a new version with old content) + +Server-side implementation: +- Database methods: `get_document_versions()` and `get_vault_history()` in `database.rs`, plus a `VaultHistoryRow` helper struct for `sqlx::query_as!` +- Handlers: `fetch_document_versions.rs`, `fetch_vault_history.rs`, `restore_document_version.rs` +- Response type: `VaultHistoryResponse { versions, hasMore }` in `responses.rs` +- SPA serving: `rust-embed` embeds `frontend/history-ui/dist/` into the binary; `index.rs` serves the SPA at `/` and assets at `/assets/*` + +Client-side component hierarchy: +- `App.svelte` — session restore, routing +- `Login.svelte` — vault name + token auth via `/ping` +- `Dashboard.svelte` — main layout: file tree sidebar, activity feed, time-travel slider +- `DocumentDetail.svelte` — version timeline, content preview, diff view, restore +- `DiffView.svelte` — unified diff with LCS algorithm +- `FileTree.svelte` — recursive tree built from flat `relativePath` values +- `ActivityFeed.svelte` — git-log-style feed with action pills (created/updated/renamed/deleted/restored) +- `TimeSlider.svelte` — scrubs through `vaultUpdateId` range, reconstructs vault state at any point + +State is managed with Svelte 5 runes (`$state`, `$derived`, `$effect`) in `lib/stores.svelte.ts`. Auth is stored in `sessionStorage`. The API client (`lib/api.ts`) sets `Authorization: Bearer` and `device-id: history-ui` headers on all requests. + ## Development Commands +### Initial Setup + +**Node.js (requires version 25):** + +```bash +curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh | bash +nvm install 25 +nvm use 25 +nvm alias default 25 # Optional: set as system default +``` + +**Rust:** + +```bash +curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh +curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh +cargo install sqlx-cli cargo-machete cargo-edit cargo-insta +``` + +**Frontend:** + +```bash +cd frontend +npm install +``` + ### Server Development + ```bash cd sync-server cargo run config-e2e.yml # Start development server -cargo test --verbose # Run Rust tests +cargo test --verbose # Run all Rust tests +cargo test # Run specific test cargo clippy --all-targets --all-features # Lint Rust code cargo clippy --all-targets --all-features --fix --allow-dirty --allow-staged # Auto-fix clippy warnings cargo fmt --all -- --check # Check Rust formatting @@ -36,75 +119,474 @@ cargo machete --with-metadata # Detect unused dependencies ``` ### Frontend Development + ```bash cd frontend npm run dev # Start development mode (watches sync-client and obsidian-plugin) npm run build # Build all workspaces -npm run test # Run all tests -npm run lint # Lint and format TypeScript code +npm run build -w sync-client # Build specific workspace +npm run test # Run all tests across all workspaces +npm run test -w sync-client # Run tests for specific workspace +npm run lint # Lint and format TypeScript code with ESLint + Prettier ``` -### Database Setup (Development) +### History UI Development + +```bash +cd frontend +npm run dev -w history-ui # Start Vite dev server (localhost:5173, proxies API to localhost:3000) +npm run build -w history-ui # Build for production (output: frontend/history-ui/dist/) +``` + +The history UI is a Svelte 5 SPA embedded in the server binary via `rust-embed`. The build flow is: + +1. `npm run build -w history-ui` produces `frontend/history-ui/dist/` +2. The Rust server embeds these files at compile time (`sync-server/src/server/index.rs`) +3. The server serves `index.html` at `GET /` and static assets at `GET /assets/*` +4. If the dist directory doesn't exist at Rust compile time, `build.rs` creates a placeholder + +During development, run the Vite dev server separately and use its proxy to forward API calls to the running sync server. + +### Database Operations + ```bash cd sync-server +# Create/reset database for development +rm -rf db.sqlite* sqlx database create --database-url sqlite://db.sqlite3 sqlx migrate run --source src/app_state/database/migrations --database-url sqlite://db.sqlite3 cargo sqlx prepare --workspace + +# Add new migration +sqlx migrate add --source src/app_state/database/migrations +sqlx migrate run --source src/app_state/database/migrations --database-url sqlite://db.sqlite3 ``` -### Initial Setup -```bash -# Install required cargo tools -cargo install sqlx-cli cargo-machete cargo-edit -``` +### Project Scripts -### Scripts -- `scripts/check.sh`: Full CI check (builds, lints, tests both server and frontend) +- `scripts/check.sh`: Full CI check (builds, lints, tests both server and frontend). **Run before pushing.** - `scripts/check.sh --fix`: Same as above but auto-fixes linting and formatting issues -- `scripts/e2e.sh`: End-to-end testing +- `scripts/e2e.sh`: End-to-end testing (e.g., `scripts/e2e.sh 8` for 8 concurrent clients) - `scripts/clean-up.sh`: Clean logs and database files -- `scripts/bump-version.sh patch`: Publish new version -- `scripts/update-api-types.sh`: Update TypeScript bindings from Rust types +- `scripts/bump-version.sh patch`: Publish new version (options: patch, minor, major) +- `scripts/update-api-types.sh`: Update TypeScript bindings from Rust types (uses ts-rs) ## Code Structure ### Workspace Configuration -The frontend uses npm workspaces with four packages: -- `sync-client`: Core synchronization logic + +The frontend uses npm workspaces with five packages: + +- `sync-client`: Core synchronization logic (builds dual bundles for web and Node.js) - `obsidian-plugin`: Obsidian-specific integration -- `test-client`: Testing utilities +- `test-client`: Testing utilities for E2E tests - `local-client-cli`: Standalone CLI for VaultLink sync client +- `history-ui`: Svelte 5 SPA for vault history browsing (built with Vite, embedded in server binary) -### Type Generation -Rust structs generate TypeScript types via ts-rs crate, stored in `sync-server/bindings/` and used by frontend packages. +### Type Generation and API Updates -### Key Files -- `sync-server/src/`: Rust server implementation with WebSocket handlers -- `frontend/sync-client/src/sync-client.ts`: Main sync client entry point -- `frontend/obsidian-plugin/src/vault-link-plugin.ts`: Main Obsidian plugin class -- `frontend/sync-client/src/services/sync-service.ts`: Core synchronization logic +Rust structs generate TypeScript types via ts-rs crate: + +1. Rust structs annotated with `#[derive(TS)]` export to `sync-server/bindings/` +2. Run `scripts/update-api-types.sh` to copy bindings to `frontend/sync-client/src/services/types/` +3. Frontend imports these types for type-safe API communication + +### Important Implementation Details + +**SQLx Compile-Time Verification:** + +- SQLx verifies SQL queries at compile time against the database schema +- Run `cargo sqlx prepare --workspace` after schema changes to update `.sqlx/` directory +- CI builds require prepared query metadata to avoid needing a live database ## Testing ### Running Tests -- Server: `cargo test --verbose` -- Frontend: `npm run test` (runs Jest across all workspaces) -- E2E: `scripts/e2e.sh` + +**Server:** + +```bash +cargo test --verbose # All tests +cargo test # Specific test +``` + +**Frontend:** + +```bash +npm run test # All workspaces +npm run test -w sync-client # Specific workspace +``` + +**E2E:** + +```bash +scripts/e2e.sh 8 # 8 concurrent clients +scripts/clean-up.sh # Clean up after tests +``` ### Test Structure -- Rust: Unit tests alongside source files -- TypeScript: `.test.ts` files using Jest -- E2E: Uses test-client to simulate multiple concurrent users -## Code Style +- **Rust**: Unit tests alongside source files, uses `cargo-insta` for snapshot testing +- **TypeScript**: `.test.ts` files using Node.js native test runner (not Jest) +- **E2E**: Uses `test-client` to simulate multiple concurrent users with random operations +- **Deterministic**: Step-by-step sync scenario tests in `frontend/deterministic-tests/` + +### Deterministic Tests (`frontend/deterministic-tests/`) + +Controlled, step-by-step sync scenario tests that exercise specific edge cases. Each test defines a sequence of operations (create, update, rename, delete, enable/disable sync, pause/resume server) and asserts convergence across multiple agents. + +**Running:** + +```bash +cd frontend/deterministic-tests +npx webpack --config webpack.config.js # Build (required after changes) +node dist/cli.js # Run all tests +node dist/cli.js --filter=write-write # Run tests matching a name/key +``` + +Requires the server binary at `sync-server/target/release/sync_server` and `sync-server/config-e2e.yml`. The harness starts/stops servers automatically. + +**Architecture:** + +- `DeterministicAgent` extends `InMemoryFileSystem` — wraps a real `SyncClient` with an in-memory filesystem +- `TestRunner` executes `TestStep[]` sequentially, manages agent lifecycle +- `ServerControl` manages server processes (start/stop/SIGSTOP/SIGCONT) +- Tests that use `pause-server`/`resume-server` get dedicated server instances; regular tests share one +- Each test gets a unique vault name (UUID) for isolation + +**Writing Tests — Step Types:** + +```typescript +{ type: "create", client: 0, path: "A.md", content: "hello" } +{ type: "update", client: 0, path: "A.md", content: "updated" } +{ type: "rename", client: 0, oldPath: "A.md", newPath: "B.md" } +{ type: "delete", client: 0, path: "A.md" } +{ type: "enable-sync", client: 0 } // Connects WS, triggers reconciliation +{ type: "disable-sync", client: 0 } // Disconnects WS +{ type: "sync", client: 0 } // Wait for specific client to settle +{ type: "sync" } // Wait for ALL clients to settle +{ type: "barrier" } // Wait for convergence + check consistency +{ type: "pause-server" } // SIGSTOP the server process +{ type: "resume-server" } // SIGCONT + wait for readiness +{ type: "assert-consistent", verify?: (state: AssertableState) => void } +``` + +**Critical Rules When Writing Tests:** + +1. **Agents start with sync DISABLED.** Do not `disable-sync` on an agent that hasn't been `enable-sync`'d — it's already off. + +2. **Do not put `{ type: "sync" }` before `{ type: "barrier" }`.** The barrier already calls `waitAllAgentsSettled()` (2 rounds of `waitForSync` on all agents). Adding a `sync` before it is pure redundancy. Use targeted `{ type: "sync", client: N }` only when you need a specific client to finish before another client acts. + +3. **`enable-sync` blocks until WebSocket connects.** If the server is paused (SIGSTOP), `enable-sync` will hang for 10 seconds then fail. Never `enable-sync` while the server is paused. Tests that need to stall in-flight requests should enable sync FIRST, then pause the server. + +4. **File operations while sync is disabled are queued.** When `createFile` is called on the agent, `enqueueSync(syncLocallyCreatedFile)` fires immediately but the fetch is disabled. The `scheduleSyncForOfflineChanges` reconciliation scans the filesystem and re-enqueues all pending changes on the next `enable-sync`. + +5. **`barrier` retries for up to 60 seconds.** It calls `waitAllAgentsSettled`, checks consistency, and if clients disagree, sleeps 500ms and retries. Tests that need more settling time should add targeted `sync` steps before the barrier (e.g., `{ type: "sync", client: 0 }` to ensure client 0's operations complete first). + +6. **No comments in test files.** The test name/description and step types are self-documenting. Keep test files comment-free. + +7. **Keep tests minimal.** Each test should reproduce exactly one edge case with the fewest steps possible. Don't add `assert-consistent` after `barrier` unless it has a `verify` callback (barrier already checks consistency). Always use inline arrow functions for `verify` callbacks rather than separate named functions. + +8. **Treat sync as a black box in test names/descriptions.** Don't reference internal implementation details (VFS, coalescing, idempotency keys, reconciliation, parentVersionId, etc.). Describe the observable scenario and expected outcome from the user's perspective. + +**Test Patterns for Common Edge Cases:** + +*Two clients create at same path (offline):* +```typescript +steps: [ + { type: "create", client: 0, path: "A.md", content: "hello" }, + { type: "create", client: 1, path: "A.md", content: "world" }, + { type: "enable-sync", client: 0 }, + { type: "enable-sync", client: 1 }, + { type: "barrier" }, + { type: "assert-consistent", verify: verifyMergedContent } +] +``` + +*Client edits while other client is offline:* +```typescript +steps: [ + { type: "create", client: 0, path: "A.md", content: "original" }, + { type: "enable-sync", client: 0 }, + { type: "enable-sync", client: 1 }, + { type: "barrier" }, + // Client 1 goes offline, client 0 edits + { type: "disable-sync", client: 1 }, + { type: "update", client: 0, path: "A.md", content: "edited" }, + { type: "sync", client: 0 }, + // Client 1 reconnects + { type: "enable-sync", client: 1 }, + { type: "barrier" }, + { type: "assert-consistent" } +] +``` + +*Testing behavior during server pause (stalled HTTP requests):* +```typescript +steps: [ + // Setup FIRST — both clients must be online before pausing + { type: "enable-sync", client: 0 }, + { type: "enable-sync", client: 1 }, + { type: "barrier" }, + // NOW pause — in-flight requests from subsequent operations will stall + { type: "pause-server" }, + { type: "create", client: 0, path: "A.md", content: "hello" }, + { type: "resume-server" }, + { type: "barrier" }, + { type: "assert-consistent" } +] +``` + +**Verify Functions and `AssertableState`:** + +The `verify` callback on `assert-consistent` receives an `AssertableState` object (defined in `utils/assertable-state.ts`) with chainable assertion methods: + +```typescript +state.assertFileCount(2) // exact file count +state.assertFileExists("A.md") // file must exist +state.assertFileNotExists("old.md") // file must not exist +state.assertContent("A.md", "hello") // exact content match +state.assertContains("A.md", "hello", "world") // all substrings present +state.assertContainsAny("A.md", "hello", "world") // at least one substring +state.assertAnyFileContains("content-a") // substring in any file +state.assertSubstringCount("A.md", "hello", 1) // occurrence count +state.assertContentInAtMostOneFile("original") // no duplicate content +state.ifFileExists("A.md", (s) => ...) // conditional assertion +state.getContent("A.md") // raw content access +``` + +All methods return `this` for chaining. The object also exposes `files` and `clientFiles` for custom logic. + +For conflict-resolution tests where the outcome is genuinely ambiguous (delete vs update, rename ordering), use `ifFileExists`. For merges where both sides MUST be preserved, use `assertContains`. When the empty-parent merge (invariant #15) is involved, word boundaries may be garbled — check for fragments, not exact substrings. + +```typescript +function verify(state: AssertableState): void { + state.ifFileExists("A.md", (s) => s.assertContent("A.md", "expected content")); +} + +function verify(state: AssertableState): void { + state.assertContains("A.md", "edit from 0", "edit from 1"); +} +``` + +**Adding a New Test:** + +1. Create `frontend/deterministic-tests/src/tests/your-test-name.test.ts` +2. Export a `TestDefinition` with `clients` and `steps` (the test name is derived from the registry key) +3. Import and register in `test-registry.ts` +4. Build with `npx webpack --config webpack.config.js` +5. Run with `node dist/cli.js --filter=your-test-name` + +**Known Limitations:** + +- Cannot test VFS.move failures — the in-memory filesystem never fails +- Cannot `enable-sync` while the server is paused — the WebSocket connection will time out +- The empty-parent 3-way merge (used for smart creates) can produce garbled word boundaries — check for fragments, not exact substrings +- The test harness can hang during shared server cleanup when transitioning to server-pause tests + +## Code Style and Formatting ### Rust -- Uses extensive Clippy lints (see Cargo.toml) -- Follows pedantic linting rules + +- Extensive Clippy lints (see `Cargo.toml`) +- Pedantic linting rules enabled - Forbids unsafe code -- Uses cargo fmt with default settings +- Uses `rustfmt.toml` for formatting configuration (4 spaces, Unix line endings) +- Run `cargo fmt --all` to format ### TypeScript -- Prettier configuration: 4-space tabs, trailing commas removed, LF line endings -- ESLint with unused imports plugin -- Consistent across all three frontend packages + +- **Prettier**: 4-space indentation, no trailing commas, LF line endings +- **YAML/Markdown override**: 2-space indentation (via prettier config) +- **ESLint**: Strict rules with unused imports detection +- Configuration in `frontend/package.json` +- Run `npm run lint` to format and fix issues + +### Svelte (History UI) + +- Uses Svelte 5 runes syntax (`$state`, `$derived`, `$effect`, `$props`) +- Vite as bundler with `@sveltejs/vite-plugin-svelte` +- Excluded from the main ESLint config (Svelte files need different linting); `history-ui/**` is in the eslint ignores list +- CSS is component-scoped via Svelte's ` diff --git a/frontend/history-ui/src/app.css b/frontend/history-ui/src/app.css new file mode 100644 index 00000000..ff3e6a9c --- /dev/null +++ b/frontend/history-ui/src/app.css @@ -0,0 +1,101 @@ +:root { + --bg: #0d1117; + --bg-secondary: #161b22; + --bg-tertiary: #21262d; + --bg-hover: #30363d; + --border: #30363d; + --border-light: #21262d; + --text: #e6edf3; + --text-muted: #8b949e; + --text-subtle: #6e7681; + --accent: #58a6ff; + --accent-hover: #79c0ff; + --green: #3fb950; + --green-bg: rgba(63, 185, 80, 0.15); + --red: #f85149; + --red-bg: rgba(248, 81, 73, 0.15); + --orange: #d29922; + --orange-bg: rgba(210, 153, 34, 0.15); + --purple: #bc8cff; + --purple-bg: rgba(188, 140, 255, 0.15); + --blue: #58a6ff; + --blue-bg: rgba(88, 166, 255, 0.15); + --mono: "SF Mono", "Fira Code", "Fira Mono", Menlo, Consolas, monospace; + --sans: -apple-system, BlinkMacSystemFont, "Segoe UI", Noto Sans, Helvetica, Arial, sans-serif; + --radius: 6px; + --radius-sm: 4px; + --shadow: 0 1px 3px rgba(0, 0, 0, 0.3); +} + +* { + margin: 0; + padding: 0; + box-sizing: border-box; +} + +html, body, #app { + height: 100%; + width: 100%; + overflow: hidden; +} + +body { + font-family: var(--sans); + font-size: 14px; + line-height: 1.5; + color: var(--text); + background: var(--bg); + -webkit-font-smoothing: antialiased; +} + +button { + font-family: inherit; + font-size: inherit; + cursor: pointer; + border: none; + background: none; + color: inherit; +} + +input { + font-family: inherit; + font-size: inherit; + color: inherit; + background: var(--bg); + border: 1px solid var(--border); + border-radius: var(--radius); + padding: 8px 12px; + outline: none; + transition: border-color 0.15s; +} + +input:focus { + border-color: var(--accent); +} + +a { + color: var(--accent); + text-decoration: none; +} + +a:hover { + text-decoration: underline; +} + +::-webkit-scrollbar { + width: 8px; + height: 8px; +} + +::-webkit-scrollbar-track { + background: transparent; +} + +::-webkit-scrollbar-thumb { + background: var(--bg-tertiary); + border-radius: 4px; +} + +::-webkit-scrollbar-thumb:hover { + background: var(--bg-hover); +} diff --git a/frontend/history-ui/src/components/ActivityFeed.svelte b/frontend/history-ui/src/components/ActivityFeed.svelte new file mode 100644 index 00000000..c1c82c29 --- /dev/null +++ b/frontend/history-ui/src/components/ActivityFeed.svelte @@ -0,0 +1,346 @@ + + +
+ {#if loading && versions.length === 0} +
Loading activity...
+ {:else if versions.length === 0} +
+ No activity yet. Documents will appear here as sync clients + make changes. +
+ {:else} + {#each grouped as group} +
+
{group.date}
+
+ {#each group.items as event} +
+ + +
+ {/each} +
+
+ {/each} + + {#if hasMore} +
+ +
+ {/if} + {/if} +
+ + diff --git a/frontend/history-ui/src/components/ConfirmDialog.svelte b/frontend/history-ui/src/components/ConfirmDialog.svelte new file mode 100644 index 00000000..e91f790a --- /dev/null +++ b/frontend/history-ui/src/components/ConfirmDialog.svelte @@ -0,0 +1,167 @@ + + + + + + + + diff --git a/frontend/history-ui/src/components/Dashboard.svelte b/frontend/history-ui/src/components/Dashboard.svelte new file mode 100644 index 00000000..807b4f8d --- /dev/null +++ b/frontend/history-ui/src/components/Dashboard.svelte @@ -0,0 +1,511 @@ + + +
+
+ +
+ + + + +
+ {#if maxUpdateId > 0} +
+ { + timeSliderValue = v; + }} + /> +
+ {/if} + + {#if selectedDocumentId} + nav.goHome()} + onRestore={handleRefresh} + /> + {:else} +
+ + +
+ + {#if activeTab === "activity"} + { + timeSliderValue = id >= maxUpdateId ? null : id; + }} + /> + {:else} +
+ {#each latestDocuments + .filter((d) => showDeleted || !d.isDeleted) + .sort((a, b) => b.vaultUpdateId - a.vaultUpdateId) as doc} + + {/each} +
+ {/if} + {/if} +
+
+
+ + diff --git a/frontend/history-ui/src/components/DiffView.svelte b/frontend/history-ui/src/components/DiffView.svelte new file mode 100644 index 00000000..be97952c --- /dev/null +++ b/frontend/history-ui/src/components/DiffView.svelte @@ -0,0 +1,288 @@ + + +
+
+ {oldLabel} + → + {newLabel} + + +{stats.added} + -{stats.removed} + +
+
+ {#each diffLines as line} +
+ + {line.oldLineNo ?? ""} + + + {line.newLineNo ?? ""} + + + {#if line.type === "add"}+{:else if line.type === "remove"}-{:else} {/if} + + {line.content} +
+ {/each} +
+
+ + diff --git a/frontend/history-ui/src/components/DocumentDetail.svelte b/frontend/history-ui/src/components/DocumentDetail.svelte new file mode 100644 index 00000000..556a5e8d --- /dev/null +++ b/frontend/history-ui/src/components/DocumentDetail.svelte @@ -0,0 +1,712 @@ + + +
+ +
+ +
+
+ + {currentPath} + + {#if isDeleted} + Deleted + {:else} + Active + {/if} +
+
+ + {documentId.substring(0, 8)}... + + {#if latest} + · + {versions.length} version{versions.length !== 1 ? "s" : ""} + · + Last by {latest.userId} + {/if} +
+
+
+ + {#if loading} +
Loading versions...
+ {:else} + +
+
+ {#if selectedVersion} +
+ + +
+ + Viewing v#{selectedVersion.vaultUpdateId} + · + {relativeTime(selectedVersion.updatedDate)} + +
+ +
+ {#if loadingContent} +
Loading content...
+ {:else if activeTab === "diff" && diffOldContent !== null && diffNewContent !== null} + + {:else if activeTab === "preview"} + {#if isTextFile(selectedVersion.relativePath) || fileExtension(selectedVersion.relativePath) === ""} + 
{loadedContent ?? ""}
+ {:else if isImageFile(selectedVersion.relativePath) && loadedContentBytes} +
+ {selectedVersion.relativePath} +
+ {:else} +
+
📦
+
Binary file
+
+ {formatBytes(selectedVersion.contentSize)} +
+
+ {/if} + {/if} +
+ {/if} +
+ + +
+
Version History
+
+ {#each [...versionEvents].reverse() as event, i} + {@const v = event.version} + {@const isSelected = selectedVersion?.vaultUpdateId === v.vaultUpdateId} +
+ + {#if event.previousPath} +
+ {event.previousPath} → {v.relativePath} +
+ {/if} +
+ {#if i < versionEvents.length - 1} + + {/if} + {#if v !== latest} + + {/if} +
+
+ {/each} +
+
+
+ {/if} +
+ +{#if showRestoreDialog && restoreTarget} + { + showRestoreDialog = false; + restoreTarget = null; + }} + /> +{/if} + + diff --git a/frontend/history-ui/src/components/FileTree.svelte b/frontend/history-ui/src/components/FileTree.svelte new file mode 100644 index 00000000..ec72cbf9 --- /dev/null +++ b/frontend/history-ui/src/components/FileTree.svelte @@ -0,0 +1,124 @@ + + +{#if node.isFolder && depth === 0} + {#each node.children as child} + + {/each} +{:else if node.isFolder} +
+ + {#if isExpanded(node.path)} + {#each node.children as child} + + {/each} + {/if} +
+{:else} + +{/if} + + diff --git a/frontend/history-ui/src/components/Header.svelte b/frontend/history-ui/src/components/Header.svelte new file mode 100644 index 00000000..8e635224 --- /dev/null +++ b/frontend/history-ui/src/components/Header.svelte @@ -0,0 +1,144 @@ + + +
+
+ + + + + + VaultLink + / + {vaultId} +
+ +
+ v{serverVersion} + + {#if auth.availableVaults.length > 1} + + {/if} + +
+
+ + diff --git a/frontend/history-ui/src/components/Login.svelte b/frontend/history-ui/src/components/Login.svelte new file mode 100644 index 00000000..8d331966 --- /dev/null +++ b/frontend/history-ui/src/components/Login.svelte @@ -0,0 +1,176 @@ + + + + + diff --git a/frontend/history-ui/src/components/TimeSlider.svelte b/frontend/history-ui/src/components/TimeSlider.svelte new file mode 100644 index 00000000..79e9e5de --- /dev/null +++ b/frontend/history-ui/src/components/TimeSlider.svelte @@ -0,0 +1,191 @@ + + +
+
+ + + + + Time Travel +
+ +
+ +
+ +
+ {#if isNow} + Now + {:else if currentVersion} + + #{value} + · + {relativeTime(currentVersion.updatedDate)} + + {:else} + #{value} + {/if} +
+ + {#if !isNow} + + {/if} +
+ + diff --git a/frontend/history-ui/src/components/ToastContainer.svelte b/frontend/history-ui/src/components/ToastContainer.svelte new file mode 100644 index 00000000..39ab1705 --- /dev/null +++ b/frontend/history-ui/src/components/ToastContainer.svelte @@ -0,0 +1,80 @@ + + +{#if toasts.items.length > 0} +
+ {#each toasts.items as toast (toast.id)} +
+ {toast.message} + +
+ {/each} +
+{/if} + + diff --git a/frontend/history-ui/src/components/VaultPicker.svelte b/frontend/history-ui/src/components/VaultPicker.svelte new file mode 100644 index 00000000..1bd64165 --- /dev/null +++ b/frontend/history-ui/src/components/VaultPicker.svelte @@ -0,0 +1,198 @@ + + +
+
+
+ +
+ + {#if auth.availableVaults.length === 0} +
+

No vaults found

+

+ Vaults are created when a sync client first connects. +

+
+ {:else} +
    + {#each auth.availableVaults as vault} +
  • + +
    • + {/each} +
+ {/if} +
+
+ + diff --git a/frontend/history-ui/src/lib/api.ts b/frontend/history-ui/src/lib/api.ts new file mode 100644 index 00000000..6d52a0f7 --- /dev/null +++ b/frontend/history-ui/src/lib/api.ts @@ -0,0 +1,121 @@ +import type { + DocumentVersion, + DocumentVersionWithoutContent, + FetchLatestDocumentsResponse, + ListVaultsResponse, + PingResponse, + VaultHistoryResponse +} from "./types"; + +async function fetchJsonWithToken( + path: string, + token: string, + init?: RequestInit +): Promise { + const response = await fetch(path, { + ...init, + headers: { + Authorization: `Bearer ${token}`, + "device-id": "history-ui", + ...init?.headers + } + }); + if (!response.ok) { + const body = await response.text(); + throw new Error(`HTTP ${response.status}: ${body}`); + } + return response.json() as Promise; +} + +export async function listVaults( + token: string +): Promise { + return fetchJsonWithToken("/vaults", token); +} + +export class ApiClient { + constructor( + private vaultId: string, + private token: string + ) {} + + private get baseUrl(): string { + return `/vaults/${encodeURIComponent(this.vaultId)}`; + } + + private async fetchJson( + path: string, + init?: RequestInit + ): Promise { + return fetchJsonWithToken(path, this.token, init); + } + + async ping(): Promise { + return this.fetchJson(`${this.baseUrl}/ping`); + } + + async fetchLatestDocuments(): Promise { + return this.fetchJson(`${this.baseUrl}/documents`); + } + + async fetchDocumentVersions( + documentId: string + ): Promise { + return this.fetchJson( + `${this.baseUrl}/documents/${documentId}/versions` + ); + } + + async fetchDocumentVersion( + documentId: string, + vaultUpdateId: number + ): Promise { + return this.fetchJson( + `${this.baseUrl}/documents/${documentId}/versions/${vaultUpdateId}` + ); + } + + async fetchDocumentVersionContent( + documentId: string, + vaultUpdateId: number + ): Promise { + const response = await fetch( + `${this.baseUrl}/documents/${documentId}/versions/${vaultUpdateId}/content`, + { headers: this.headers() } + ); + if (!response.ok) { + throw new Error(`HTTP ${response.status}`); + } + return response.arrayBuffer(); + } + + async fetchVaultHistory( + limit?: number, + beforeUpdateId?: number + ): Promise { + const params = new URLSearchParams(); + if (limit !== undefined) params.set("limit", String(limit)); + if (beforeUpdateId !== undefined) + params.set("before_update_id", String(beforeUpdateId)); + const qs = params.toString(); + return this.fetchJson( + `${this.baseUrl}/history${qs ? `?${qs}` : ""}` + ); + } + + async restoreVersion( + documentId: string, + vaultUpdateId: number + ): Promise { + return this.fetchJson( + `${this.baseUrl}/documents/${documentId}/restore`, + { + method: "POST", + headers: { + "Content-Type": "application/json" + }, + body: JSON.stringify({ vaultUpdateId }) + } + ); + } +} diff --git a/frontend/history-ui/src/lib/stores.svelte.ts b/frontend/history-ui/src/lib/stores.svelte.ts new file mode 100644 index 00000000..fcba5340 --- /dev/null +++ b/frontend/history-ui/src/lib/stores.svelte.ts @@ -0,0 +1,305 @@ +import { ApiClient } from "./api"; +import type { + DocumentVersionWithoutContent, + VaultInfo, + VersionEvent, + ActionType, + TreeNode +} from "./types"; + +class AuthStore { + token = $state(""); + userName = $state(""); + vaultId = $state(""); + serverVersion = $state(""); + availableVaults = $state([]); + isAuthenticated = $state(false); + api = $state(null); + + authenticate( + token: string, + userName: string, + vaults: VaultInfo[] + ) { + this.token = token; + this.userName = userName; + this.availableVaults = vaults; + sessionStorage.setItem("vaultlink_token", token); + } + + selectVault(vaultId: string) { + this.vaultId = vaultId; + this.isAuthenticated = true; + this.api = new ApiClient(vaultId, this.token); + sessionStorage.setItem("vaultlink_vault", vaultId); + } + + deselectVault() { + this.vaultId = ""; + this.isAuthenticated = false; + this.api = null; + sessionStorage.removeItem("vaultlink_vault"); + } + + logout() { + this.token = ""; + this.userName = ""; + this.vaultId = ""; + this.serverVersion = ""; + this.availableVaults = []; + this.isAuthenticated = false; + this.api = null; + sessionStorage.removeItem("vaultlink_token"); + sessionStorage.removeItem("vaultlink_vault"); + } + + tryRestore(): { token: string; vaultId?: string } | null { + const token = sessionStorage.getItem("vaultlink_token"); + if (!token) return null; + const vaultId = + sessionStorage.getItem("vaultlink_vault") ?? undefined; + return { token, vaultId }; + } +} + +export const auth = new AuthStore(); + +// Navigation +export type View = + | { kind: "dashboard" } + | { kind: "document"; documentId: string }; + +class NavStore { + current = $state({ kind: "dashboard" }); + + goto(view: View) { + this.current = view; + } + + goHome() { + this.current = { kind: "dashboard" }; + } +} + +export const nav = new NavStore(); + +// Toasts +export interface Toast { + id: number; + message: string; + type: "success" | "error" | "info"; +} + +class ToastStore { + items = $state([]); + private nextId = 0; + + add(message: string, type: Toast["type"] = "info") { + const id = this.nextId++; + this.items.push({ id, message, type }); + setTimeout(() => this.dismiss(id), 5000); + } + + dismiss(id: number) { + this.items = this.items.filter((t) => t.id !== id); + } +} + +export const toasts = new ToastStore(); + +// Utilities + +export function inferAction( + version: DocumentVersionWithoutContent, + previousVersion?: DocumentVersionWithoutContent +): ActionType { + if (version.isDeleted) return "deleted"; + if (!previousVersion) return "created"; + if ( + previousVersion.isDeleted && + !version.isDeleted + ) + return "restored"; + if (previousVersion.relativePath !== version.relativePath) + return "renamed"; + return "updated"; +} + +export function enrichVersions( + versions: DocumentVersionWithoutContent[] +): VersionEvent[] { + // versions should be sorted by vaultUpdateId ascending + const sorted = [...versions].sort( + (a, b) => a.vaultUpdateId - b.vaultUpdateId + ); + const byDoc = new Map(); + for (const v of sorted) { + let arr = byDoc.get(v.documentId); + if (!arr) { + arr = []; + byDoc.set(v.documentId, arr); + } + arr.push(v); + } + + return sorted.map((v) => { + const docVersions = byDoc.get(v.documentId)!; + const idx = docVersions.indexOf(v); + const prev = idx > 0 ? docVersions[idx - 1] : undefined; + const action = inferAction(v, prev); + return { + ...v, + action, + previousPath: + action === "renamed" ? prev?.relativePath : undefined + }; + }); +} + +export function buildTree( + documents: DocumentVersionWithoutContent[], + showDeleted: boolean +): TreeNode { + const root: TreeNode = { + name: "", + path: "", + isFolder: true, + children: [] + }; + + const filtered = showDeleted + ? documents + : documents.filter((d) => !d.isDeleted); + + for (const doc of filtered) { + const parts = doc.relativePath.split("/"); + let current = root; + + for (let i = 0; i < parts.length; i++) { + const part = parts[i]; + const isFile = i === parts.length - 1; + const path = parts.slice(0, i + 1).join("/"); + + if (isFile) { + current.children.push({ + name: part, + path, + isFolder: false, + children: [], + document: doc, + isDeleted: doc.isDeleted + }); + } else { + let folder = current.children.find( + (c) => c.isFolder && c.name === part + ); + if (!folder) { + folder = { + name: part, + path, + isFolder: true, + children: [] + }; + current.children.push(folder); + } + current = folder; + } + } + } + + sortTree(root); + return root; +} + +function sortTree(node: TreeNode) { + node.children.sort((a, b) => { + if (a.isFolder !== b.isFolder) return a.isFolder ? -1 : 1; + return a.name.localeCompare(b.name); + }); + for (const child of node.children) { + if (child.isFolder) sortTree(child); + } +} + +export function relativeTime(dateStr: string): string { + const date = new Date(dateStr); + const now = Date.now(); + const diff = now - date.getTime(); + const seconds = Math.floor(diff / 1000); + const minutes = Math.floor(seconds / 60); + const hours = Math.floor(minutes / 60); + const days = Math.floor(hours / 24); + + if (seconds < 60) return "just now"; + if (minutes < 60) return `${minutes}m ago`; + if (hours < 24) return `${hours}h ago`; + if (days < 7) return `${days}d ago`; + return date.toLocaleDateString("en-US", { + month: "short", + day: "numeric", + year: days > 365 ? "numeric" : undefined + }); +} + +export function absoluteTime(dateStr: string): string { + return new Date(dateStr).toLocaleString(); +} + +export function formatBytes(bytes: number): string { + if (bytes === 0) return "0 B"; + const k = 1024; + const sizes = ["B", "KB", "MB", "GB"]; + const i = Math.floor(Math.log(bytes) / Math.log(k)); + return `${parseFloat((bytes / Math.pow(k, i)).toFixed(1))} ${sizes[i]}`; +} + +export function fileExtension(path: string): string { + const dot = path.lastIndexOf("."); + return dot > -1 ? path.substring(dot + 1).toLowerCase() : ""; +} + +export function isTextFile(path: string): boolean { + const textExts = new Set([ + "md", + "txt", + "json", + "yaml", + "yml", + "toml", + "xml", + "html", + "css", + "js", + "ts", + "svelte", + "rs", + "py", + "sh", + "bash", + "zsh", + "csv", + "svg", + "log", + "conf", + "cfg", + "ini", + "env", + "gitignore", + "editorconfig" + ]); + return textExts.has(fileExtension(path)); +} + +export function isImageFile(path: string): boolean { + const imageExts = new Set([ + "png", + "jpg", + "jpeg", + "gif", + "webp", + "svg", + "ico", + "bmp" + ]); + return imageExts.has(fileExtension(path)); +} diff --git a/frontend/history-ui/src/lib/types/ListVaultsResponse.ts b/frontend/history-ui/src/lib/types/ListVaultsResponse.ts new file mode 100644 index 00000000..92b2b3e0 --- /dev/null +++ b/frontend/history-ui/src/lib/types/ListVaultsResponse.ts @@ -0,0 +1,7 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { VaultInfo } from "./VaultInfo"; + +/** + * Response to listing vaults accessible to the authenticated user. + */ +export type ListVaultsResponse = { vaults: Array, hasMore: boolean, userName: string, }; diff --git a/frontend/history-ui/src/lib/types/VaultInfo.ts b/frontend/history-ui/src/lib/types/VaultInfo.ts new file mode 100644 index 00000000..32373346 --- /dev/null +++ b/frontend/history-ui/src/lib/types/VaultInfo.ts @@ -0,0 +1,6 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +/** + * Summary of a single vault returned by the list-vaults endpoint. + */ +export type VaultInfo = { name: string, documentCount: number, createdAt: string | null, }; diff --git a/frontend/history-ui/src/main.ts b/frontend/history-ui/src/main.ts new file mode 100644 index 00000000..c72cabd0 --- /dev/null +++ b/frontend/history-ui/src/main.ts @@ -0,0 +1,7 @@ +import { mount } from "svelte"; +import App from "./App.svelte"; +import "./app.css"; + +const app = mount(App, { target: document.getElementById("app")! }); + +export default app; diff --git a/frontend/history-ui/svelte.config.js b/frontend/history-ui/svelte.config.js new file mode 100644 index 00000000..76a68bfc --- /dev/null +++ b/frontend/history-ui/svelte.config.js @@ -0,0 +1,5 @@ +import { vitePreprocess } from "@sveltejs/vite-plugin-svelte"; + +export default { + preprocess: vitePreprocess() +}; diff --git a/frontend/history-ui/tsconfig.json b/frontend/history-ui/tsconfig.json new file mode 100644 index 00000000..216dc140 --- /dev/null +++ b/frontend/history-ui/tsconfig.json @@ -0,0 +1,16 @@ +{ + "compilerOptions": { + "target": "ESNext", + "module": "ESNext", + "moduleResolution": "bundler", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "resolveJsonModule": true, + "isolatedModules": true, + "verbatimModuleSyntax": true, + "types": ["svelte"] + }, + "include": ["src/**/*", "src/**/*.svelte"] +} diff --git a/frontend/history-ui/vite.config.ts b/frontend/history-ui/vite.config.ts new file mode 100644 index 00000000..18f6be82 --- /dev/null +++ b/frontend/history-ui/vite.config.ts @@ -0,0 +1,15 @@ +import { defineConfig } from "vite"; +import { svelte } from "@sveltejs/vite-plugin-svelte"; + +export default defineConfig({ + plugins: [svelte()], + build: { + outDir: "dist", + emptyOutDir: true + }, + server: { + proxy: { + "/vaults": "http://localhost:3010" + } + } +}); diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 6b8d31f3..f0c60c83 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1423,13 +1423,6 @@ "dev": true, "license": "MIT" }, - "node_modules/@types/murmurhash3js-revisited": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/@types/murmurhash3js-revisited/-/murmurhash3js-revisited-3.0.3.tgz", - "integrity": "sha512-QvlqvYtGBYIDeO8dFdY4djkRubcrc+yTJtBc7n8VZPlJDUS/00A+PssbvERM8f9bYRmcaSEHPZgZojeQj7kzAA==", - "dev": true, - "license": "MIT" - }, "node_modules/@types/node": { "version": "25.0.2", "resolved": "https://registry.npmjs.org/@types/node/-/node-25.0.2.tgz", @@ -3965,15 +3958,6 @@ "dev": true, "license": "MIT" }, - "node_modules/murmurhash3js-revisited": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/murmurhash3js-revisited/-/murmurhash3js-revisited-3.0.0.tgz", - "integrity": "sha512-/sF3ee6zvScXMb1XFJ8gDsSnY+X8PbOyjIuBhtgis10W2Jx4ZjIhikUCIF9c4gpJxVnQIsPAFrSwTCuAjicP6g==", - "license": "MIT", - "engines": { - "node": ">=8.0.0" - } - }, "node_modules/nanoid": { "version": "3.3.11", "dev": true, @@ -5911,17 +5895,13 @@ }, "sync-client": { "version": "0.14.0", - "dependencies": { - "murmurhash3js-revisited": "^3.0.0" - }, "devDependencies": { "@sentry/browser": "^10.30.0", - "@types/murmurhash3js-revisited": "^3.0.3", "@types/node": "^25.0.2", "byte-base64": "^1.1.0", "minimatch": "^10.1.1", "p-queue": "^9.0.1", - "reconcile-text": "^0.11.0", + "reconcile-text": "^0.8.0", "ts-loader": "^9.5.4", "tslib": "2.8.1", "tsx": "^4.21.0", @@ -5946,6 +5926,13 @@ "url": "https://github.com/sponsors/isaacs" } }, + "sync-client/node_modules/reconcile-text": { + "version": "0.8.0", + "resolved": "https://registry.npmjs.org/reconcile-text/-/reconcile-text-0.8.0.tgz", + "integrity": "sha512-evskVha3YgpP2ZelsFxP9t7CuKnwE7TrsH3FdrH2mfKbzjUWiNF7scHXsFbFS921lmFlAOB94DHNAWPvL34Mqg==", + "dev": true, + "license": "MIT" + }, "test-client": { "version": "0.14.0", "bin": { diff --git a/frontend/sync-client/src/file-operations/file-operations.test.ts b/frontend/sync-client/src/file-operations/file-operations.test.ts index 27724ee9..bad1ebb6 100644 --- a/frontend/sync-client/src/file-operations/file-operations.test.ts +++ b/frontend/sync-client/src/file-operations/file-operations.test.ts @@ -1,9 +1,6 @@ import { describe, it } from "node:test"; -import type { - Database, - DocumentRecord, - RelativePath -} from "../persistence/database"; +import type { DocumentId, DocumentRecord, RelativePath } from "../sync-operations/types"; +import type { SyncEventQueue } from "../sync-operations/sync-event-queue"; import { FileOperations } from "./file-operations"; import { Logger } from "../tracing/logger"; import { assertSetContainsExactly } from "../utils/assert-set-contains-exactly"; @@ -21,19 +18,18 @@ class MockServerConfig implements Pick { } } -class MockDatabase implements Partial { - public getLatestDocumentByRelativePath( - _target: RelativePath +class MockQueue implements Pick { + public getDocument( + _path: RelativePath ): DocumentRecord | undefined { - // no-op return undefined; } - public move( - _oldRelativePath: RelativePath, - _newRelativePath: RelativePath - ): void { - // no-op + public moveDocument( + _oldPath: RelativePath, + _newPath: RelativePath + ): DocumentId | undefined { + return undefined; } } @@ -89,7 +85,7 @@ describe("File operations", () => { const fileSystemOperations = new FakeFileSystemOperations(); const fileOperations = new FileOperations( new Logger(), - new MockDatabase() as Database, // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion + new MockQueue() as SyncEventQueue, // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion fileSystemOperations, new MockServerConfig() as ServerConfig // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion ); @@ -119,7 +115,7 @@ describe("File operations", () => { const fileSystemOperations = new FakeFileSystemOperations(); const fileOperations = new FileOperations( new Logger(), - new MockDatabase() as Database, // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion + new MockQueue() as SyncEventQueue, // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion fileSystemOperations, new MockServerConfig() as ServerConfig // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion ); @@ -159,7 +155,7 @@ describe("File operations", () => { const fileSystemOperations = new FakeFileSystemOperations(); const fileOperations = new FileOperations( new Logger(), - new MockDatabase() as Database, // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion + new MockQueue() as SyncEventQueue, // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion fileSystemOperations, new MockServerConfig() as ServerConfig // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion ); @@ -178,7 +174,7 @@ describe("File operations", () => { const fileSystemOperations = new FakeFileSystemOperations(); const fileOperations = new FileOperations( new Logger(), - new MockDatabase() as Database, // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion + new MockQueue() as SyncEventQueue, // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion fileSystemOperations, new MockServerConfig() as ServerConfig // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion ); @@ -207,7 +203,7 @@ describe("File operations", () => { const fileSystemOperations = new FakeFileSystemOperations(); const fileOperations = new FileOperations( new Logger(), - new MockDatabase() as Database, // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion + new MockQueue() as SyncEventQueue, // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion fileSystemOperations, new MockServerConfig() as ServerConfig // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion ); diff --git a/frontend/sync-client/src/file-operations/file-operations.ts b/frontend/sync-client/src/file-operations/file-operations.ts index 863f62af..cc47076b 100644 --- a/frontend/sync-client/src/file-operations/file-operations.ts +++ b/frontend/sync-client/src/file-operations/file-operations.ts @@ -1,6 +1,7 @@ import type { Logger } from "../tracing/logger"; import type { FileSystemOperations } from "./filesystem-operations"; -import type { Database, RelativePath } from "../persistence/database"; +import type { RelativePath } from "../sync-operations/types"; +import type { SyncEventQueue } from "../sync-operations/sync-event-queue"; import { SafeFileSystemOperations } from "./safe-filesystem-operations"; import type { TextWithCursors } from "reconcile-text"; import { reconcile } from "reconcile-text"; @@ -14,7 +15,7 @@ export class FileOperations { public constructor( private readonly logger: Logger, - private readonly database: Database, + private readonly queue: SyncEventQueue, fs: FileSystemOperations, private readonly serverConfig: ServerConfig, private readonly nativeLineEndings = "\n" @@ -58,7 +59,10 @@ export class FileOperations { return this.fs.write(path, this.toNativeLineEndings(newContent)); } - public async ensureClearPath(path: RelativePath): Promise { + // Returns the deconflicted path if a file was moved, undefined otherwise + public async ensureClearPath( + path: RelativePath + ): Promise { if (await this.fs.exists(path)) { const deconflictedPath = await this.deconflictPath(path); try { @@ -66,14 +70,16 @@ export class FileOperations { `Didn't expect ${path} to exist, deconflicting by moving it to '${deconflictedPath}'` ); - this.database.move(path, deconflictedPath); + this.queue.moveDocument(path, deconflictedPath); await this.fs.rename(path, deconflictedPath, true); + return deconflictedPath; } finally { this.fs.unlock(deconflictedPath); } } else { await this.createParentDirectories(path); } + return undefined; } /** @@ -160,21 +166,24 @@ export class FileOperations { return this.fs.exists(path); } + // Returns the deconflicted path if a file at the target was displaced public async move( oldPath: RelativePath, newPath: RelativePath - ): Promise { + ): Promise { if (oldPath === newPath) { - return; + return undefined; } - await this.ensureClearPath(newPath); - this.database.move(oldPath, newPath); + const deconflictedPath = await this.ensureClearPath(newPath); + this.queue.moveDocument(oldPath, newPath); await this.fs.rename(oldPath, newPath); await this.deletingEmptyParentDirectoriesOfDeletedFile(oldPath); + return deconflictedPath; } + public reset(): void { this.fs.reset(); } @@ -274,17 +283,15 @@ export class FileOperations { newName = `${directory}${stem} (${currentCount})${extension}`; // Avoid multiple deconflictPath calls returning the same path - if (this.fs.tryLock(newName)) { - const newDocument = - this.database.getLatestDocumentByRelativePath(newName); - if ( - newDocument?.isDeleted === false || // the document might have been confirmed by the server at a new path but haven't yet moved there locally - (await this.fs.exists(newName, true)) - ) { - this.fs.unlock(newName); - } else { - return newName; - } + await this.fs.waitForLock(newName); + const existingRecord = this.queue.getDocument(newName); + if ( + existingRecord !== undefined || // the document might have been confirmed by the server at a new path but haven't yet moved there locally + (await this.fs.exists(newName, true)) + ) { + this.fs.unlock(newName); + } else { + return newName; } } } diff --git a/frontend/sync-client/src/persistence/database.ts b/frontend/sync-client/src/persistence/database.ts index 2a5e901e..72f15fbd 100644 --- a/frontend/sync-client/src/persistence/database.ts +++ b/frontend/sync-client/src/persistence/database.ts @@ -1,301 +1,2 @@ -import type { Logger } from "../tracing/logger"; -import { EMPTY_HASH } from "../utils/hash"; -import { CoveredValues } from "../utils/data-structures/min-covered"; -import { awaitAll } from "../utils/await-all"; -import { removeFromArray } from "../utils/remove-from-array"; - -export type VaultUpdateId = number; -export type DocumentId = string; -export type RelativePath = string; - -export interface DocumentMetadata { - documentId: DocumentId; - parentVersionId: VaultUpdateId; - hash: string; - remoteRelativePath?: RelativePath; -} - -export interface StoredDocumentMetadata { - relativePath: RelativePath; - documentId: DocumentId; - parentVersionId: VaultUpdateId; - remoteRelativePath?: RelativePath; - hash: string; -} - -export interface StoredDatabase { - documents: StoredDocumentMetadata[]; - lastSeenUpdateId: VaultUpdateId | undefined; -} - -/** - * Represents a document in the database. - * - * It is mutable and its content should always represent the latest - * state of the document on disk based on the update events we have seen. - */ -export interface DocumentRecord { - relativePath: RelativePath; - metadata: DocumentMetadata | undefined; - isDeleted: boolean; - parallelVersion: number; -} - -export class Database { - private documents: DocumentRecord[]; - private lastSeenUpdateIds: CoveredValues; - - public constructor( - private readonly logger: Logger, - initialState: Partial | undefined, - private readonly saveData: (data: StoredDatabase) => Promise - ) { - initialState ??= {}; - - this.documents = - initialState.documents?.map(({ relativePath, ...metadata }) => ({ - relativePath, - metadata, - isDeleted: false, - parallelVersion: 0 - })) ?? []; - - this.ensureConsistency(); - this.logger.debug(`Loaded ${this.documents.length} documents`); - - const { lastSeenUpdateId } = initialState; - this.logger.debug(`Loaded last seen update id: ${lastSeenUpdateId}`); - this.lastSeenUpdateIds = new CoveredValues( - Math.max(0, lastSeenUpdateId ?? 0) // the first updateId will be 1 which is the first integer after -1 - ); - - this.documents.forEach((doc) => { - this.lastSeenUpdateIds.add(doc.metadata?.parentVersionId); - }); - } - - public get length(): number { - return this.documents.length; - } - - public get resolvedDocuments(): DocumentRecord[] { - const paths = new Map(); - this.documents - // eslint-disable-next-line no-restricted-syntax -- Type narrowing, not removing a specific item - .filter(({ metadata }) => metadata !== undefined) - .forEach((record) => - paths.set(record.relativePath, [ - record, - ...(paths.get(record.relativePath) ?? []) - ]) - ); - - return Array.from(paths.values()).map((records) => { - records.sort( - (a, b) => b.parallelVersion - a.parallelVersion // descending - ); - - if ( - records.length > 1 && - records.some((current, i) => - i === 0 - ? false - : records[i - 1].parallelVersion === - current.parallelVersion - ) - ) { - throw new Error( - `Multiple documents with the same parallel version and path at ${records[0].relativePath}` - ); - } - return records[0]; - }); - } - - public updateDocumentMetadata( - metadata: { - documentId: DocumentId; - parentVersionId: VaultUpdateId; - hash: string; - remoteRelativePath: RelativePath; - }, - target: DocumentRecord - ): void { - if (!this.documents.includes(target)) { - throw new Error("Document not found in database"); - } - - this.logger.debug( - `Updating document metadata for ${target.relativePath} from ${JSON.stringify( - target.metadata, - null, - 2 - )} to ${JSON.stringify(metadata, null, 2)}` - ); - - target.metadata = metadata; - - this.saveInTheBackground(); - } - - public getLatestDocumentByRelativePath( - target: RelativePath - ): DocumentRecord | undefined { - const candidates = this.documents.filter( - ({ relativePath }) => relativePath === target - ); - candidates.sort((a, b) => b.parallelVersion - a.parallelVersion); // descending - return candidates[0]; - } - - public createNewPendingDocument( - relativePath: RelativePath - ): DocumentRecord { - this.logger.debug(`Creating new pending document: ${relativePath}`); - const previousEntry = - this.getLatestDocumentByRelativePath(relativePath); - - const entry = { - relativePath, - metadata: undefined, - isDeleted: false, - parallelVersion: - previousEntry?.parallelVersion === undefined - ? 0 - : previousEntry.parallelVersion + 1 - }; - - this.documents.push(entry); - - // no need to save as we only save documents which have metadata - - return entry; - } - - public getDocumentByDocumentId( - target: DocumentId - ): DocumentRecord | undefined { - return this.documents.find( - ({ metadata }) => metadata?.documentId === target - ); - } - - public move( - oldRelativePath: RelativePath, - newRelativePath: RelativePath - ): void { - const oldDocument = - this.getLatestDocumentByRelativePath(oldRelativePath); - - if (oldDocument === undefined) { - return; - } - - const newDocument = - this.getLatestDocumentByRelativePath(newRelativePath); - if (newDocument?.isDeleted === false) { - throw new Error( - `Document already exists at new location: ${newRelativePath}` - ); - } - - oldDocument.relativePath = newRelativePath; - // We might be in a strange state where the target of the move has just got deleted, - // however, its metadata might already have a bunch of updates queued up for - // the document at the new location. We need to keep these updates. - oldDocument.parallelVersion = - newDocument !== undefined ? newDocument.parallelVersion + 1 : 0; - - this.saveInTheBackground(); - } - - public delete(relativePath: RelativePath): void { - const candidate = this.getLatestDocumentByRelativePath(relativePath); - if (candidate === undefined) { - return; - } - candidate.isDeleted = true; - } - - public removeDocument(target: DocumentRecord): void { - removeFromArray(this.documents, target); - this.saveInTheBackground(); - } - - public getLastSeenUpdateId(): VaultUpdateId { - return this.lastSeenUpdateIds.min; - } - - public addSeenUpdateId(value: number): void { - const previousMin = this.lastSeenUpdateIds.min; - this.lastSeenUpdateIds.add(value); - if (previousMin !== this.lastSeenUpdateIds.min) { - this.saveInTheBackground(); - } - } - - public setLastSeenUpdateId(value: number): void { - this.lastSeenUpdateIds.min = value; - this.saveInTheBackground(); - } - - public reset(): void { - this.documents = []; - this.lastSeenUpdateIds = new CoveredValues( - 0 // the first updateId will be 1 which is the first integer after -1 - ); - this.saveInTheBackground(); - } - - public async save(): Promise { - return this.saveData({ - documents: this.resolvedDocuments.map( - ({ relativePath, metadata }) => ({ - relativePath, - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - ...metadata! // `resolvedDocuments` only returns docs with metadata set - }) - ), - lastSeenUpdateId: this.lastSeenUpdateIds.min - }); - } - - private ensureConsistency(): void { - const idToPath = new Map(); - - this.resolvedDocuments.forEach(({ relativePath, metadata }) => { - if (metadata === undefined) { - return; - } - idToPath.set(metadata.documentId, [ - ...(idToPath.get(metadata.documentId) ?? []), - relativePath - ]); - }); - - const duplicates = Array.from(idToPath.entries()) - .filter(([_, paths]) => paths.length > 1) - .map(([id, paths]) => { - let details = ""; - for (const path of paths) { - const doc = this.getLatestDocumentByRelativePath(path); - details += `\n- ${JSON.stringify(doc, null, 2)}`; - } - return `${id} (${paths.join(", ")}): ${details}`; - }); - - if (duplicates.length > 0) { - throw new Error( - "Document IDs are not unique, found duplicates: " + - duplicates.join("; ") - ); - } - } - - private saveInTheBackground(): void { - this.ensureConsistency(); - void this.save().catch((error: unknown) => { - this.logger.error(`Error saving data: ${error}`); - }); - } -} +// This file is intentionally empty +// All document tracking has been moved to sync-event-queue.ts diff --git a/frontend/sync-client/src/services/sync-service.ts b/frontend/sync-client/src/services/sync-service.ts index 647ac8da..ad268814 100644 --- a/frontend/sync-client/src/services/sync-service.ts +++ b/frontend/sync-client/src/services/sync-service.ts @@ -2,13 +2,14 @@ import type { DocumentId, RelativePath, VaultUpdateId -} from "../persistence/database"; +} from "../sync-operations/types"; import type { Logger } from "../tracing/logger"; import type { Settings } from "../persistence/settings"; import type { FetchController } from "./fetch-controller"; import { sleep } from "../utils/sleep"; import { SyncResetError } from "../errors/sync-reset-error"; +import { HttpClientError } from "../errors/http-client-error"; import type { SerializedError } from "./types/SerializedError"; import type { DocumentVersionWithoutContent } from "./types/DocumentVersionWithoutContent"; import type { DocumentUpdateResponse } from "./types/DocumentUpdateResponse"; @@ -139,13 +140,7 @@ export class SyncService { } ); - if (!response.ok) { - throw new Error( - `Failed to update document: ${await SyncService.errorFromResponse( - response - )}` - ); - } + await SyncService.throwIfNotOk(response, "update document"); const result: DocumentUpdateResponse = (await response.json()) as DocumentUpdateResponse; // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion @@ -192,13 +187,7 @@ export class SyncService { } ); - if (!response.ok) { - throw new Error( - `Failed to update document: ${await SyncService.errorFromResponse( - response - )}` - ); - } + await SyncService.throwIfNotOk(response, "update document"); const result: DocumentUpdateResponse = (await response.json()) as DocumentUpdateResponse; // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion @@ -413,8 +402,10 @@ export class SyncService { try { return await fn(); } catch (e) { - // We must not retry errors coming from reset - if (e instanceof SyncResetError) { + if ( + e instanceof SyncResetError || + e instanceof HttpClientError + ) { throw e; } @@ -427,4 +418,16 @@ export class SyncService { } } } + + private static async throwIfNotOk( + response: Response, + operation: string + ): Promise { + if (response.ok) return; + const message = `Failed to ${operation}: ${await SyncService.errorFromResponse(response)}`; + if (response.status >= 400 && response.status < 500) { + throw new HttpClientError(response.status, message); + } + throw new Error(message); + } } diff --git a/frontend/sync-client/src/services/types/ListVaultsResponse.ts b/frontend/sync-client/src/services/types/ListVaultsResponse.ts new file mode 100644 index 00000000..85928d89 --- /dev/null +++ b/frontend/sync-client/src/services/types/ListVaultsResponse.ts @@ -0,0 +1,7 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { VaultInfo } from "./VaultInfo"; + +/** + * Response to listing vaults accessible to the authenticated user. + */ +export interface ListVaultsResponse { vaults: VaultInfo[], hasMore: boolean, userName: string, } diff --git a/frontend/sync-client/src/services/types/VaultInfo.ts b/frontend/sync-client/src/services/types/VaultInfo.ts new file mode 100644 index 00000000..921645f3 --- /dev/null +++ b/frontend/sync-client/src/services/types/VaultInfo.ts @@ -0,0 +1,6 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +/** + * Summary of a single vault returned by the list-vaults endpoint. + */ +export interface VaultInfo { name: string, documentCount: number, createdAt: string | null, } diff --git a/frontend/sync-client/src/services/websocket-manager.ts b/frontend/sync-client/src/services/websocket-manager.ts index e99b8662..5cceec72 100644 --- a/frontend/sync-client/src/services/websocket-manager.ts +++ b/frontend/sync-client/src/services/websocket-manager.ts @@ -4,7 +4,6 @@ import type { WebSocketServerMessage } from "./types/WebSocketServerMessage"; import type { WebSocketClientMessage } from "./types/WebSocketClientMessage"; import type { CursorPositionFromClient } from "./types/CursorPositionFromClient"; import type { ClientCursors } from "./types/ClientCursors"; -import { createPromise } from "../utils/create-promise"; import type { WebSocketVaultUpdate } from "./types/WebSocketVaultUpdate"; import { WEBSOCKET_DISCONNECT_TIMEOUT_IN_SECONDS, @@ -42,6 +41,10 @@ export class WebSocketManager { private readonly webSocketFactoryImplementation: typeof globalThis.WebSocket = WebSocket ) {} + public get hasOutstandingWork(): boolean { + return this.outstandingPromises.length > 0; + } + public get isWebSocketConnected(): boolean { return ( this.webSocket?.readyState === @@ -55,7 +58,7 @@ export class WebSocketManager { } public async stop(): Promise { - const [promise, resolve] = createPromise(); + const { promise, resolve } = Promise.withResolvers(); this.resolveDisconnectingPromise = resolve; this.isStopped = true; diff --git a/frontend/sync-client/src/sync-client.ts b/frontend/sync-client/src/sync-client.ts index db6ff902..1a88c269 100644 --- a/frontend/sync-client/src/sync-client.ts +++ b/frontend/sync-client/src/sync-client.ts @@ -2,8 +2,8 @@ import type { PersistenceProvider } from "./persistence/persistence"; import type { HistoryEntry, HistoryStats } from "./tracing/sync-history"; import { SyncHistory } from "./tracing/sync-history"; import { Logger, LogLevel, LogLine } from "./tracing/logger"; -import type { RelativePath, StoredDatabase } from "./persistence/database"; -import { Database } from "./persistence/database"; +import type { RelativePath, StoredSyncState } from "./sync-operations/types"; +import { SyncEventQueue } from "./sync-operations/sync-event-queue"; import * as Sentry from "@sentry/browser"; import type { SyncSettings } from "./persistence/settings"; import { DEFAULT_SETTINGS, Settings } from "./persistence/settings"; @@ -12,7 +12,6 @@ import { Syncer } from "./sync-operations/syncer"; import type { FileSystemOperations } from "./file-operations/filesystem-operations"; import { FileOperations } from "./file-operations/file-operations"; import { FetchController } from "./services/fetch-controller"; -import { UnrestrictedSyncer } from "./sync-operations/unrestricted-syncer"; import { rateLimit } from "./utils/rate-limit"; import type { NetworkConnectionStatus } from "./types/network-connection-status"; import { DocumentSyncStatus } from "./types/document-sync-status"; @@ -40,7 +39,7 @@ export class SyncClient { public readonly logger: Logger, private readonly history: SyncHistory, private readonly settings: Settings, - private readonly database: Database, + private readonly syncEventQueue: SyncEventQueue, private readonly syncer: Syncer, private readonly webSocketManager: WebSocketManager, private readonly fetchController: FetchController, @@ -52,13 +51,13 @@ export class SyncClient { private readonly persistence: PersistenceProvider< Partial<{ settings: Partial; - database: Partial; + database: Partial; }> > ) { } public get documentCount(): number { - return this.database.length; + return this.syncEventQueue.documentCount; } public get isWebSocketConnected(): boolean { @@ -111,7 +110,7 @@ export class SyncClient { persistence: PersistenceProvider< Partial<{ settings: Partial; - database: Partial; + database: Partial; }> >; fetch?: typeof globalThis.fetch; @@ -136,8 +135,6 @@ export class SyncClient { state.settings, async (data): Promise => { state = { ...state, settings: data }; - // we're not rate-limiting settings saves as (1) we need to initialise the settings to know the rate limit - // and (2) settings changes are infrequent enough that rate-limiting is not necessary await persistence.save(state); } ); @@ -147,7 +144,8 @@ export class SyncClient { () => settings.getSettings().minimumSaveIntervalMs ); - const database = new Database( + const syncEventQueue = new SyncEventQueue( + settings, logger, state.database, async (data): Promise => { @@ -173,7 +171,7 @@ export class SyncClient { const fileOperations = new FileOperations( logger, - database, + syncEventQueue, fs, serverConfig, nativeLineEndings @@ -182,16 +180,6 @@ export class SyncClient { const contentCache = new FixedSizeDocumentCache( 1024 * 1024 * DIFF_CACHE_SIZE_MB ); - const unrestrictedSyncer = new UnrestrictedSyncer( - logger, - database, - settings, - syncService, - fileOperations, - history, - contentCache, - serverConfig - ); const webSocketManager = new WebSocketManager( logger, @@ -202,17 +190,20 @@ export class SyncClient { const syncer = new Syncer( deviceId, logger, - database, settings, webSocketManager, fileOperations, - unrestrictedSyncer + syncService, + history, + contentCache, + serverConfig, + syncEventQueue ); const fileChangeNotifier = new FileChangeNotifier(); const cursorTracker = new CursorTracker( logger, - database, + syncEventQueue, webSocketManager, fileOperations, fileChangeNotifier @@ -221,7 +212,7 @@ export class SyncClient { logger, history, settings, - database, + syncEventQueue, syncer, webSocketManager, fetchController, @@ -319,7 +310,7 @@ export class SyncClient { /** * Wait for the in-flight operations to finish, reset all tracking, - * and the local database but retain the settings. + * and the local state but retain the settings. * The SyncClient can be used again after calling this method. */ public async reset(): Promise { @@ -330,10 +321,9 @@ export class SyncClient { ); await this.pause(); - // clear all local state this.logger.info("Resetting SyncClient's local state"); - this.database.reset(); - await this.database.save(); // ensure the new database reads as empty + this.syncEventQueue.resetState(); + await this.syncEventQueue.save(); this.resetInMemoryState(); this.hasFinishedOfflineSync = false; this.serverConfig.reset(); @@ -362,40 +352,47 @@ export class SyncClient { await this.settings.setSettings(value); } - public async syncLocallyCreatedFile( + public syncLocallyCreatedFile( relativePath: RelativePath - ): Promise { + ): void { this.checkIfDestroyed("syncLocallyCreatedFile"); this.fileChangeNotifier.notifyOfFileChange(relativePath); - return this.syncer.syncLocallyCreatedFile(relativePath); + this.syncer.syncLocallyCreatedFile(relativePath); } - public async syncLocallyDeletedFile( + public syncLocallyDeletedFile( relativePath: RelativePath - ): Promise { + ): void { this.checkIfDestroyed("syncLocallyDeletedFile"); this.fileChangeNotifier.notifyOfFileChange(relativePath); - return this.syncer.syncLocallyDeletedFile(relativePath); + this.syncer.syncLocallyDeletedFile(relativePath); } - public async syncLocallyUpdatedFile({ + public syncLocallyUpdatedFile({ oldPath, relativePath }: { oldPath?: RelativePath; relativePath: RelativePath; - }): Promise { + }): void { this.checkIfDestroyed("syncLocallyUpdatedFile"); this.fileChangeNotifier.notifyOfFileChange(relativePath); - return this.syncer.syncLocallyUpdatedFile({ + this.syncer.syncLocallyUpdatedFile({ oldPath, relativePath }); } + public get hasPendingWork(): boolean { + return ( + this.syncEventQueue.size > 0 || + this.webSocketManager.hasOutstandingWork + ); + } + public getDocumentSyncingStatus( relativePath: RelativePath ): DocumentSyncStatus { @@ -426,7 +423,7 @@ export class SyncClient { this.checkIfDestroyed("waitUntilIdle"); await this.syncer.waitUntilFinished(); await this.webSocketManager.waitUntilFinished(); - await this.database.save(); // flush all changes to disk + await this.syncEventQueue.save(); } /** @@ -436,7 +433,6 @@ export class SyncClient { public async destroy(): Promise { this.checkIfDestroyed("destroy"); - // Prevent concurrent destroy calls if (this.isDestroying) { this.logger.warn( "destroy() called while already destroying, ignoring" @@ -445,14 +441,12 @@ export class SyncClient { } this.isDestroying = true; - // cancel everything that's in progress await this.pause(); this.hasBeenDestroyed = true; this.resetInMemoryState(); - // Clean up event listeners to prevent memory leaks this.eventUnsubscribers.forEach((unsubscribe) => { unsubscribe(); }); @@ -467,7 +461,6 @@ export class SyncClient { this.checkIfDestroyed("startSyncing"); this.fetchController.finishReset(); - // warm the cache await this.serverConfig.getConfig(); await this.syncer.scheduleSyncForOfflineChanges(); @@ -486,7 +479,6 @@ export class SyncClient { private resetInMemoryState(): void { this.history.reset(); this.contentCache.reset(); - // don't reset the logger this.cursorTracker.reset(); this.syncer.reset(); this.fileOperations.reset(); diff --git a/frontend/sync-client/src/sync-operations/cursor-tracker.ts b/frontend/sync-client/src/sync-operations/cursor-tracker.ts index dbce144b..f67f7eb7 100644 --- a/frontend/sync-client/src/sync-operations/cursor-tracker.ts +++ b/frontend/sync-client/src/sync-operations/cursor-tracker.ts @@ -1,5 +1,6 @@ import type { FileOperations } from "../file-operations/file-operations"; -import type { Database, RelativePath } from "../persistence/database"; +import type { RelativePath } from "./types"; +import type { SyncEventQueue } from "./sync-event-queue"; import type { ClientCursors } from "../services/types/ClientCursors"; import type { CursorSpan } from "../services/types/CursorSpan"; import type { DocumentWithCursors } from "../services/types/DocumentWithCursors"; @@ -35,7 +36,7 @@ export class CursorTracker { public constructor( private readonly logger: Logger, - private readonly database: Database, + private readonly queue: SyncEventQueue, private readonly webSocketManager: WebSocketManager, private readonly fileOperations: FileOperations, private readonly fileChangeNotifier: FileChangeNotifier @@ -104,21 +105,16 @@ export class CursorTracker { for (const [relativePath, cursors] of Object.entries( documentToCursors )) { - const record = - this.database.getLatestDocumentByRelativePath(relativePath); + const record = this.queue.getDocument(relativePath); if (!record) { continue; // Let's wait for the file to be created before sending cursors } - if (!record.metadata) { - continue; // this is a new document, no need to sync the cursors - } - documentsWithCursors.push({ relative_path: relativePath, - document_id: record.metadata.documentId, - vault_update_id: record.metadata.parentVersionId, + document_id: record.documentId, + vault_update_id: record.parentVersionId, cursors: cursors.map(({ start, end }) => ({ start: Math.min(start, end), end: Math.max(start, end) @@ -139,10 +135,8 @@ export class CursorTracker { const readContent = await this.fileOperations.read( doc.relative_path ); - const record = this.database.getLatestDocumentByRelativePath( - doc.relative_path - ); - if (record?.metadata?.hash !== (await hash(readContent))) { + const record = this.queue.getDocument(doc.relative_path); + if (record?.hash !== (await hash(readContent))) { doc.vault_update_id = null; } } @@ -227,9 +221,7 @@ export class CursorTracker { private async getDocumentUpToDateness( document: DocumentWithCursors ): Promise { - const record = this.database.getLatestDocumentByRelativePath( - document.relative_path - ); + const record = this.queue.getDocument(document.relative_path); if (!record) { // the document of the cursor must be from the future @@ -237,13 +229,11 @@ export class CursorTracker { } if ( - (record.metadata?.parentVersionId ?? 0) < - (document.vault_update_id ?? 0) + record.parentVersionId < (document.vault_update_id ?? 0) ) { return DocumentUpToDateness.Later; } else if ( - (document.vault_update_id ?? 0) < - (record.metadata?.parentVersionId ?? 0) + (document.vault_update_id ?? 0) < record.parentVersionId ) { // the document of the cursor must be from the past return DocumentUpToDateness.Prior; @@ -253,9 +243,8 @@ export class CursorTracker { document.relative_path ); - return this.database.getLatestDocumentByRelativePath( - document.relative_path - )?.metadata?.hash === (await hash(currentContent)) + const currentRecord = this.queue.getDocument(document.relative_path); + return currentRecord?.hash === (await hash(currentContent)) ? DocumentUpToDateness.UpToDate : DocumentUpToDateness.Prior; } diff --git a/frontend/sync-client/src/sync-operations/sync-event-queue.test.ts b/frontend/sync-client/src/sync-operations/sync-event-queue.test.ts index 7e43b700..d2e32268 100644 --- a/frontend/sync-client/src/sync-operations/sync-event-queue.test.ts +++ b/frontend/sync-client/src/sync-operations/sync-event-queue.test.ts @@ -1,46 +1,443 @@ import { describe, it } from "node:test"; import assert from "node:assert"; -import { SyncEventQueue, type SyncEvent } from "./sync-event-queue"; +import { SyncEventQueue } from "./sync-event-queue"; +import { Settings } from "../persistence/settings"; +import { Logger } from "../tracing/logger"; +import type { DocumentVersionWithoutContent } from "../services/types/DocumentVersionWithoutContent"; +import { SyncEventType } from "./types"; + +function createQueue(ignorePatterns: string[] = []): SyncEventQueue { + const logger = new Logger(); + const settings = new Settings(logger, { ignorePatterns }, async () => {}); + return new SyncEventQueue(settings, logger, undefined, async () => {}); +} + +function fakeRemoteVersion( + documentId: string, + overrides: Partial = {} +): DocumentVersionWithoutContent { + return { + vaultUpdateId: 1, + documentId, + relativePath: `${documentId}.md`, + updatedDate: "2026-01-01", + isDeleted: false, + userId: "user", + deviceId: "device", + contentSize: 100, + ...overrides + }; +} describe("SyncEventQueue", () => { - it("delete collapses interleaved events for one document while leaving the other intact", () => { - const queue = new SyncEventQueue(); - queue.enqueue({ type: "local-content-update", documentId: "A" }); - queue.enqueue({ type: "remote-content-update", documentId: "B" }); - queue.enqueue({ type: "local-content-update", documentId: "A" }); - queue.enqueue({ type: "move", documentId: "A" }); - queue.enqueue({ type: "remote-content-update", documentId: "A" }); - queue.enqueue({ type: "delete", documentId: "A" }); - queue.enqueue({ type: "local-content-update", documentId: "B" }); - - assert.deepStrictEqual(queue.next(), { type: "delete", documentId: "A" }); - assert.deepStrictEqual(queue.next(), { - type: "local-content-update", - documentId: "B" + it("sync-local followed by delete for the same document returns only the delete", () => { + const queue = createQueue(); + queue.setDocument("a.md", { + documentId: "A", + parentVersionId: 1, + hash: "hash-a" }); + + queue.enqueue({ type: SyncEventType.SyncLocal, documentId: "A" }); + queue.enqueue({ type: SyncEventType.SyncLocal, documentId: "A" }); + queue.enqueue({ + type: SyncEventType.Delete, + documentId: "A", + path: "a.md", + }); + + const event = queue.next(); + assert.strictEqual(event?.type, SyncEventType.Delete); + if (event?.type === SyncEventType.Delete) { + assert.strictEqual(event.documentId, "A"); + } assert.strictEqual(queue.next(), undefined); }); - it("updates coalesce up to a move boundary then post-move events are processed separately", () => { - const queue = new SyncEventQueue(); - queue.enqueue({ type: "local-content-update", documentId: "X" }); - queue.enqueue({ type: "remote-content-update", documentId: "X" }); - queue.enqueue({ type: "file-create", path: "new.md" }); - queue.enqueue({ type: "local-content-update", documentId: "X" }); - queue.enqueue({ type: "move", documentId: "X" }); - queue.enqueue({ type: "remote-content-update", documentId: "X" }); - queue.enqueue({ type: "local-content-update", documentId: "X" }); + it("sync-local events for the same document coalesce to one", () => { + const queue = createQueue(); + queue.setDocument("a.md", { + documentId: "A", + parentVersionId: 1, + hash: "hash-a" + }); - assert.deepStrictEqual(queue.next(), { - type: "local-content-update", - documentId: "X" - }); - assert.deepStrictEqual(queue.next(), { type: "file-create", path: "new.md" }); - assert.deepStrictEqual(queue.next(), { type: "move", documentId: "X" }); - assert.deepStrictEqual(queue.next(), { - type: "local-content-update", - documentId: "X" - }); + queue.enqueue({ type: SyncEventType.SyncLocal, documentId: "A" }); + queue.enqueue({ type: SyncEventType.SyncLocal, documentId: "A" }); + queue.enqueue({ type: SyncEventType.SyncLocal, documentId: "A" }); + + const event = queue.next(); + assert.strictEqual(event?.type, SyncEventType.SyncLocal); assert.strictEqual(queue.next(), undefined); }); + + it("sync-remote events for the same documentId coalesce to the last one", () => { + const queue = createQueue(); + + queue.enqueue({ + type: SyncEventType.SyncRemote, + remoteVersion: fakeRemoteVersion("A", { vaultUpdateId: 1 }) + }); + queue.enqueue({ + type: SyncEventType.SyncRemote, + remoteVersion: fakeRemoteVersion("A", { vaultUpdateId: 2 }) + }); + queue.enqueue({ + type: SyncEventType.SyncRemote, + remoteVersion: fakeRemoteVersion("A", { vaultUpdateId: 3 }) + }); + + const event = queue.next(); + assert.strictEqual(event?.type, SyncEventType.SyncRemote); + if (event?.type === SyncEventType.SyncRemote) { + assert.strictEqual(event.remoteVersion.vaultUpdateId, 3); + } + assert.strictEqual(queue.next(), undefined); + }); + + it("create events are returned FIFO", () => { + const queue = createQueue(); + queue.enqueue({ type: SyncEventType.Create, path: "a.md" }); + queue.enqueue({ type: SyncEventType.Create, path: "b.md" }); + + const first = queue.next(); + assert.strictEqual(first?.type, SyncEventType.Create); + if (first?.type === SyncEventType.Create) { + assert.strictEqual(first.path, "a.md"); + } + + const second = queue.next(); + assert.strictEqual(second?.type, SyncEventType.Create); + if (second?.type === SyncEventType.Create) { + assert.strictEqual(second.path, "b.md"); + } + }); + + it("duplicate creates for the same path are skipped", () => { + const queue = createQueue(); + queue.enqueue({ type: SyncEventType.Create, path: "a.md" }); + queue.enqueue({ type: SyncEventType.Create, path: "a.md" }); + assert.strictEqual(queue.size, 1); + }); + + it("create is skipped if the path already has a tracked document", () => { + const queue = createQueue(); + queue.setDocument("a.md", { + documentId: "A", + parentVersionId: 1, + hash: "hash-a" + }); + + queue.enqueue({ type: SyncEventType.Create, path: "a.md" }); + assert.strictEqual(queue.size, 0); + }); + + it("delete uses the provided documentId", () => { + const queue = createQueue(); + + queue.enqueue({ + type: SyncEventType.Delete, + documentId: "A", + path: "a.md", + }); + + const event = queue.next(); + assert.strictEqual(event?.type, SyncEventType.Delete); + if (event?.type === SyncEventType.Delete) { + assert.strictEqual(event.documentId, "A"); + } + }); + + it("updateCreatePath updates the path of a create event in the queue", () => { + const queue = createQueue(); + queue.enqueue({ type: SyncEventType.Create, path: "old.md" }); + + const updated = queue.updateCreatePath("old.md", "new.md"); + assert.strictEqual(updated, true); + assert.strictEqual(queue.hasCreateEvent("old.md"), false); + assert.strictEqual(queue.hasCreateEvent("new.md"), true); + + const event = queue.next(); + assert.strictEqual(event?.type, SyncEventType.Create); + if (event?.type === SyncEventType.Create) { + assert.strictEqual(event.path, "new.md"); + } + }); + + it("updateCreatePath returns false when no create event exists", () => { + const queue = createQueue(); + const updated = queue.updateCreatePath("old.md", "new.md"); + assert.strictEqual(updated, false); + }); + + it("hasCreateEvent detects pending creates", () => { + const queue = createQueue(); + assert.strictEqual(queue.hasCreateEvent("a.md"), false); + + queue.enqueue({ type: SyncEventType.Create, path: "a.md" }); + assert.strictEqual(queue.hasCreateEvent("a.md"), true); + + queue.next(); + assert.strictEqual(queue.hasCreateEvent("a.md"), false); + }); + + it("document store CRUD operations work correctly", () => { + const queue = createQueue(); + + assert.strictEqual(queue.getDocument("a.md"), undefined); + assert.strictEqual(queue.documentCount, 0); + + queue.setDocument("a.md", { + documentId: "A", + parentVersionId: 1, + hash: "hash-a" + }); + assert.strictEqual(queue.documentCount, 1); + assert.deepStrictEqual(queue.getDocument("a.md"), { + documentId: "A", + parentVersionId: 1, + hash: "hash-a" + }); + + const found = queue.getDocumentByDocumentId("A"); + assert.strictEqual(found?.path, "a.md"); + assert.strictEqual(found?.record.documentId, "A"); + + queue.removeDocument("a.md"); + assert.strictEqual(queue.documentCount, 0); + assert.strictEqual(queue.getDocument("a.md"), undefined); + }); + + it("moveDocument moves a document and returns displaced documentId", () => { + const queue = createQueue(); + queue.setDocument("a.md", { + documentId: "A", + parentVersionId: 1, + hash: "hash-a" + }); + queue.setDocument("b.md", { + documentId: "B", + parentVersionId: 2, + hash: "hash-b" + }); + + const displacedId = queue.moveDocument("a.md", "b.md"); + assert.strictEqual(displacedId, "B"); + assert.strictEqual(queue.getDocument("a.md"), undefined); + assert.strictEqual(queue.getDocument("b.md")?.documentId, "A"); + assert.strictEqual(queue.documentCount, 1); + }); + + it("moveDocument returns undefined when target is unoccupied", () => { + const queue = createQueue(); + queue.setDocument("a.md", { + documentId: "A", + parentVersionId: 1, + hash: "hash-a" + }); + + const displacedId = queue.moveDocument("a.md", "b.md"); + assert.strictEqual(displacedId, undefined); + assert.strictEqual(queue.getDocument("b.md")?.documentId, "A"); + }); + + it("interleaved events for different documents are not confused", () => { + const queue = createQueue(); + queue.setDocument("a.md", { + documentId: "A", + parentVersionId: 1, + hash: "hash-a" + }); + queue.setDocument("b.md", { + documentId: "B", + parentVersionId: 2, + hash: "hash-b" + }); + + queue.enqueue({ type: SyncEventType.SyncLocal, documentId: "A" }); + queue.enqueue({ type: SyncEventType.SyncLocal, documentId: "B" }); + queue.enqueue({ + type: SyncEventType.Delete, + documentId: "A", + path: "a.md", + }); + queue.enqueue({ type: SyncEventType.SyncLocal, documentId: "B" }); + + // First next() should see the delete for A (coalescing sync-local + delete) + const first = queue.next(); + assert.strictEqual(first?.type, SyncEventType.Delete); + if (first?.type === SyncEventType.Delete) { + assert.strictEqual(first.documentId, "A"); + } + + // Remaining should be the coalesced sync-local for B + const second = queue.next(); + assert.strictEqual(second?.type, SyncEventType.SyncLocal); + if (second?.type === SyncEventType.SyncLocal) { + assert.strictEqual(second.documentId, "B"); + } + + assert.strictEqual(queue.next(), undefined); + }); + + it("delete discards subsequent sync-remote events for the same document", () => { + const queue = createQueue(); + + queue.enqueue({ + type: SyncEventType.Delete, + documentId: "A", + path: "a.md", + }); + queue.enqueue({ + type: SyncEventType.SyncRemote, + remoteVersion: fakeRemoteVersion("A", { vaultUpdateId: 5 }) + }); + + const event = queue.next(); + assert.strictEqual(event?.type, SyncEventType.Delete); + assert.strictEqual(queue.next(), undefined); + }); + + it("delete discards subsequent sync-local and sync-remote for the same document", () => { + const queue = createQueue(); + queue.setDocument("a.md", { + documentId: "A", + parentVersionId: 1, + hash: "hash-a" + }); + + queue.enqueue({ + type: SyncEventType.Delete, + documentId: "A", + path: "a.md", + }); + queue.enqueue({ type: SyncEventType.SyncLocal, documentId: "A" }); + queue.enqueue({ type: SyncEventType.Create, path: "b.md" }); + queue.enqueue({ + type: SyncEventType.SyncRemote, + remoteVersion: fakeRemoteVersion("A", { vaultUpdateId: 5 }) + }); + + const first = queue.next(); + assert.strictEqual(first?.type, SyncEventType.Delete); + + // Only the unrelated create should remain + const second = queue.next(); + assert.strictEqual(second?.type, SyncEventType.Create); + assert.strictEqual(queue.next(), undefined); + }); + + it("delete with empty documentId does not discard other events", () => { + const queue = createQueue(); + queue.setDocument("a.md", { + documentId: "A", + parentVersionId: 1, + hash: "hash-a" + }); + + queue.enqueue({ + type: SyncEventType.Delete, + documentId: "", + path: "unknown.md", + }); + queue.enqueue({ type: SyncEventType.SyncLocal, documentId: "A" }); + + queue.next(); + const second = queue.next(); + assert.strictEqual(second?.type, SyncEventType.SyncLocal); + }); + + it("create can be re-enqueued after being dequeued", () => { + const queue = createQueue(); + queue.enqueue({ type: SyncEventType.Create, path: "a.md" }); + queue.next(); + + queue.enqueue({ type: SyncEventType.Create, path: "a.md" }); + assert.strictEqual(queue.size, 1); + }); + + it("silently ignores create events matching ignore patterns", () => { + const queue = createQueue(["*.tmp", ".hidden/**"]); + + queue.enqueue({ type: SyncEventType.Create, path: "scratch.tmp" }); + queue.enqueue({ + type: SyncEventType.Create, + path: ".hidden/secret.md", + }); + assert.strictEqual(queue.size, 0); + + queue.enqueue({ type: SyncEventType.Create, path: "notes-new.md" }); + assert.strictEqual(queue.size, 1); + + queue.enqueue({ + type: SyncEventType.SyncRemote, + remoteVersion: fakeRemoteVersion("N") + }); + assert.strictEqual(queue.size, 2); + }); + + it("clear removes events but keeps documents", () => { + const queue = createQueue(); + queue.setDocument("a.md", { + documentId: "A", + parentVersionId: 1, + hash: "hash-a" + }); + queue.enqueue({ type: SyncEventType.Create, path: "b.md" }); + queue.enqueue({ type: SyncEventType.SyncLocal, documentId: "A" }); + + assert.strictEqual(queue.size, 2); + + queue.clear(); + + assert.strictEqual(queue.size, 0); + assert.strictEqual(queue.documentCount, 1); + assert.strictEqual(queue.getDocument("a.md")?.documentId, "A"); + }); + + it("allDocuments returns all tracked documents", () => { + const queue = createQueue(); + queue.setDocument("a.md", { + documentId: "A", + parentVersionId: 1, + hash: "hash-a" + }); + queue.setDocument("b.md", { + documentId: "B", + parentVersionId: 2, + hash: "hash-b" + }); + + const docs = queue.allDocuments(); + assert.strictEqual(docs.length, 2); + const paths = docs.map(([p]) => p).sort(); + assert.deepStrictEqual(paths, ["a.md", "b.md"]); + }); + + it("loads initial state from persistence", () => { + const logger = new Logger(); + const settings = new Settings(logger, {}, async () => {}); + const queue = new SyncEventQueue(settings, logger, { + documents: [ + { + relativePath: "a.md", + documentId: "A", + parentVersionId: 5, + hash: "hash-a" + }, + { + relativePath: "b.md", + documentId: "B", + parentVersionId: 3, + hash: "hash-b" + } + ], + lastSeenUpdateId: 4 + }, async () => {}); + + assert.strictEqual(queue.documentCount, 2); + assert.strictEqual(queue.getDocument("a.md")?.documentId, "A"); + assert.strictEqual(queue.getDocument("b.md")?.documentId, "B"); + assert.strictEqual(queue.getLastSeenUpdateId(), 5); + }); }); diff --git a/frontend/sync-client/src/sync-operations/sync-event-queue.ts b/frontend/sync-client/src/sync-operations/sync-event-queue.ts index c3d8af82..362c35dc 100644 --- a/frontend/sync-client/src/sync-operations/sync-event-queue.ts +++ b/frontend/sync-client/src/sync-operations/sync-event-queue.ts @@ -1,85 +1,336 @@ -import type { DocumentId, RelativePath } from "../persistence/database"; - -export type SyncEvent = - | { type: "file-create"; path: RelativePath } - | { type: "local-content-update"; documentId: DocumentId } - | { type: "remote-content-update"; documentId: DocumentId } - | { type: "move"; documentId: DocumentId } - | { type: "delete"; documentId: DocumentId }; +import type { Settings } from "../persistence/settings"; +import type { Logger } from "../tracing/logger"; +import { globsToRegexes } from "../utils/globs-to-regexes"; +import { CoveredValues } from "../utils/data-structures/min-covered"; +import { removeFromArray } from "../utils/remove-from-array"; +import { + SyncEventType, + type DocumentId, + type DocumentRecord, + type RelativePath, + type StoredSyncState, + type SyncEvent, + type VaultUpdateId, +} from "./types"; export class SyncEventQueue { private readonly events: SyncEvent[] = []; + private readonly documents = new Map(); + private readonly recentlyDeletedDocumentIds = new Set(); + private lastSeenUpdateIds: CoveredValues; + private ignorePatterns: RegExp[]; + + public constructor( + private readonly settings: Settings, + private readonly logger: Logger, + initialState: Partial | undefined, + private readonly saveData: (data: StoredSyncState) => Promise + ) { + this.ignorePatterns = globsToRegexes( + this.settings.getSettings().ignorePatterns, + this.logger + ); + + this.settings.onSettingsChanged.add((newSettings) => { + this.ignorePatterns = globsToRegexes( + newSettings.ignorePatterns, + this.logger + ); + }); + + initialState ??= {}; + + if (initialState.documents !== undefined) { + for (const { relativePath, ...record } of initialState.documents) { + this.documents.set(relativePath, record); + } + } + + const { lastSeenUpdateId } = initialState; + this.lastSeenUpdateIds = new CoveredValues( + Math.max(0, lastSeenUpdateId ?? 0) + ); + + for (const [, record] of this.documents) { + this.lastSeenUpdateIds.add(record.parentVersionId); + } + + this.logger.debug(`Loaded ${this.documents.size} documents`); + } public get size(): number { return this.events.length; } + public get documentCount(): number { + return this.documents.size; + } + + public getLastSeenUpdateId(): VaultUpdateId { + return this.lastSeenUpdateIds.min; + } + + public addSeenUpdateId(value: number): void { + const previousMin = this.lastSeenUpdateIds.min; + this.lastSeenUpdateIds.add(value); + if (previousMin !== this.lastSeenUpdateIds.min) { + this.saveInTheBackground(); + } + } + + public setLastSeenUpdateId(value: number): void { + this.lastSeenUpdateIds.min = value; + this.saveInTheBackground(); + } + + public getDocument(path: RelativePath): DocumentRecord | undefined { + return this.documents.get(path); + } + + public getDocumentByDocumentId( + target: DocumentId + ): { path: RelativePath; record: DocumentRecord } | undefined { + for (const [path, record] of this.documents) { + if (record.documentId === target) { + return { path, record }; + } + } + return undefined; + } + + public setDocument(path: RelativePath, record: DocumentRecord): void { + this.documents.set(path, record); + this.saveInTheBackground(); + } + + public removeDocument(path: RelativePath): void { + const record = this.documents.get(path); + if (record !== undefined) { + this.recentlyDeletedDocumentIds.add(record.documentId); + } + this.documents.delete(path); + this.saveInTheBackground(); + } + + /** + * Move a document from oldPath to newPath. + * If the target path is occupied by a different document, it is removed + * and its documentId is returned so the caller can handle the displacement. + */ + public moveDocument( + oldPath: RelativePath, + newPath: RelativePath + ): DocumentId | undefined { + const record = this.documents.get(oldPath); + if (record === undefined) return undefined; + + let displacedDocumentId: DocumentId | undefined = undefined; + const existingAtTarget = this.documents.get(newPath); + if ( + existingAtTarget !== undefined && + existingAtTarget.documentId !== record.documentId + ) { + displacedDocumentId = existingAtTarget.documentId; + this.recentlyDeletedDocumentIds.add(displacedDocumentId); + this.documents.delete(newPath); + } + + this.documents.delete(oldPath); + this.documents.set(newPath, record); + this.saveInTheBackground(); + return displacedDocumentId; + } + + public wasRecentlyDeleted(documentId: DocumentId): boolean { + return this.recentlyDeletedDocumentIds.has(documentId); + } + + public unmarkRecentlyDeleted(documentId: DocumentId): void { + this.recentlyDeletedDocumentIds.delete(documentId); + } + + + public allDocuments(): [RelativePath, DocumentRecord][] { + return Array.from(this.documents.entries()); + } + + public hasCreateEvent(path: RelativePath): boolean { + return this.events.some( + (e) => e.type === SyncEventType.Create && e.path === path + ); + } + + public updateCreatePath( + oldPath: RelativePath, + newPath: RelativePath + ): boolean { + for (const event of this.events) { + if (event.type === SyncEventType.Create && event.path === oldPath) { + event.path = newPath; + return true; + } + } + return false; + } + + public hasPendingEventsForPath(path: RelativePath): boolean { + const record = this.documents.get(path); + const docId = record?.documentId; + return this.events.some( + (e) => + (e.type === SyncEventType.Create && e.path === path) || + (e.type === SyncEventType.SyncLocal && + docId !== undefined && + e.documentId === docId) || + (e.type === SyncEventType.Delete && + docId !== undefined && + e.documentId === docId) || + (e.type === SyncEventType.SyncRemote && + e.remoteVersion.relativePath === path) + ); + } + + public async save(): Promise { + return this.saveData({ + documents: Array.from(this.documents.entries()).map( + ([relativePath, record]) => ({ + relativePath, + ...record + }) + ), + lastSeenUpdateId: this.lastSeenUpdateIds.min + }); + } + + public resetState(): void { + this.documents.clear(); + this.recentlyDeletedDocumentIds.clear(); + this.lastSeenUpdateIds = new CoveredValues(0); + this.saveInTheBackground(); + } + public clear(): void { this.events.length = 0; + this.recentlyDeletedDocumentIds.clear(); } public enqueue(event: SyncEvent): void { + if (this.isIgnored(event)) return; + + if (event.type === SyncEventType.Create) { + if (this.documents.has(event.path)) return; + if (this.hasCreateEvent(event.path)) return; + } + this.events.push(event); } + + public next(): SyncEvent | undefined { if (this.events.length === 0) return undefined; - const first = this.events[0]; - if (first.type === "file-create") { + const [first] = this.events; + + // Creates are always returned immediately (FIFO) + if (first.type === SyncEventType.Create) { this.events.shift(); return first; } - const { documentId } = first; - - // If there's an eventual delete, discard everything for this document - const deleteEvent = this.events.find( - (e) => e.type === "delete" && e.documentId === documentId - ); - if (deleteEvent) { - this.removeAllForDocument(documentId); - return deleteEvent; - } - - // Coalesce updates: return the last update before the next move for this document. - // Moves act as barriers since they depend on each other - const moveIndex = this.events.findIndex( - (e) => e.type === "move" && e.documentId === documentId - ); - const boundary = moveIndex === -1 ? this.events.length : moveIndex; - - const updateIndices: number[] = []; - for (let i = 0; i < boundary; i++) { - const e = this.events[i]; - if ( - (e.type === "local-content-update" || - e.type === "remote-content-update") && - e.documentId === documentId - ) { - updateIndices.push(i); + // Deletes are returned immediately; also discard any subsequent + // events for the same documentId so stale broadcasts don't + // resurrect the document + if (first.type === SyncEventType.Delete) { + this.events.shift(); + const { documentId } = first; + if (documentId !== "") { + this.removeAllEventsForDocumentId(documentId); } + return first; } - if (updateIndices.length > 0) { - const result = this.events[updateIndices[updateIndices.length - 1]]; - for (let i = updateIndices.length - 1; i >= 0; i--) { - this.events.splice(updateIndices[i], 1); + if (first.type === SyncEventType.SyncLocal) { + const { documentId } = first; + + // If there's a later delete for the same documentId, discard + // all sync-locals for that document and return the delete + const deleteEvent = this.events.find( + (e) => + e.type === SyncEventType.Delete && + e.documentId === documentId + ); + if (deleteEvent !== undefined) { + this.removeAllSyncLocalsForDocumentId(documentId); + removeFromArray(this.events, deleteEvent); + return deleteEvent; + } + + // Coalesce multiple sync-locals for the same documentId to the last one + const matching = this.events.filter( + (e) => + e.type === SyncEventType.SyncLocal && + e.documentId === documentId + ); + const result = matching[matching.length - 1]; + for (const item of matching) { + removeFromArray(this.events, item); } return result; } - // First event is a move with no preceding updates - this.events.shift(); - return first; + // SyncRemote: coalesce multiple events for the same documentId to the last one + const { documentId } = first.remoteVersion; + const matching = this.events.filter( + (e) => + e.type === SyncEventType.SyncRemote && + e.remoteVersion.documentId === documentId + ); + const result = matching[matching.length - 1]; + for (const item of matching) { + removeFromArray(this.events, item); + } + return result; } - private removeAllForDocument(documentId: DocumentId): void { + private isIgnored(event: SyncEvent): boolean { + if (event.type !== SyncEventType.Create) return false; + return this.ignorePatterns.some((pattern) => pattern.test(event.path)); + } + + private removeAllEventsForDocumentId(documentId: DocumentId): void { for (let i = this.events.length - 1; i >= 0; i--) { const e = this.events[i]; - if (e.type !== "file-create" && e.documentId === documentId) { + if ( + (e.type === SyncEventType.SyncLocal && + e.documentId === documentId) || + (e.type === SyncEventType.SyncRemote && + e.remoteVersion.documentId === documentId) || + (e.type === SyncEventType.Delete && + e.documentId === documentId) + ) { + // eslint-disable-next-line no-restricted-syntax -- Bulk removal by predicate, not single-item removal this.events.splice(i, 1); } } } + + private removeAllSyncLocalsForDocumentId(documentId: DocumentId): void { + for (let i = this.events.length - 1; i >= 0; i--) { + const e = this.events[i]; + if ( + e.type === SyncEventType.SyncLocal && + e.documentId === documentId + ) { + // eslint-disable-next-line no-restricted-syntax -- Bulk removal by predicate, not single-item removal + this.events.splice(i, 1); + } + } + } + + private saveInTheBackground(): void { + void this.save().catch((error: unknown) => { + this.logger.error(`Error saving sync state: ${error}`); + }); + } } diff --git a/frontend/sync-client/src/sync-operations/syncer.ts b/frontend/sync-client/src/sync-operations/syncer.ts index 4cf92097..9e4121e6 100644 --- a/frontend/sync-client/src/sync-operations/syncer.ts +++ b/frontend/sync-client/src/sync-operations/syncer.ts @@ -1,76 +1,73 @@ -import type { - Database, - DocumentId, - DocumentRecord, - RelativePath -} from "../persistence/database"; +import { + SyncEventType, + type DocumentId, + type DocumentRecord, + type SyncEvent, + type RelativePath, + type VaultUpdateId, +} from "./types"; import type { Logger } from "../tracing/logger"; -import PQueue from "p-queue"; -import { hash } from "../utils/hash"; +import { EMPTY_HASH, hash } from "../utils/hash"; import type { Settings } from "../persistence/settings"; import type { FileOperations } from "../file-operations/file-operations"; import { findMatchingFile } from "../utils/find-matching-file"; -import type { UnrestrictedSyncer } from "./unrestricted-syncer"; import { SyncResetError } from "../errors/sync-reset-error"; -import { Locks } from "../utils/data-structures/locks"; import type { DocumentVersionWithoutContent } from "../services/types/DocumentVersionWithoutContent"; import type { WebSocketVaultUpdate } from "../services/types/WebSocketVaultUpdate"; import type { WebSocketManager } from "../services/websocket-manager"; import type { WebSocketClientMessage } from "../services/types/WebSocketClientMessage"; -import { awaitAll } from "../utils/await-all"; import { EventListeners } from "../utils/data-structures/event-listeners"; +import type { SyncEventQueue } from "./sync-event-queue"; +import type { SyncService } from "../services/sync-service"; +import { FileNotFoundError } from "../errors/file-not-found-error"; +import { HttpClientError } from "../errors/http-client-error"; +import type { + SyncHistory +} from "../tracing/sync-history"; +import { + SyncStatus, + SyncType, + type CommonHistoryEntry +} from "../tracing/sync-history"; +import { isBinary } from "../utils/is-binary"; +import { isFileTypeMergable } from "../utils/is-file-type-mergable"; +import { diff } from "reconcile-text"; +import type { ServerConfig } from "../services/server-config"; +import type { FixedSizeDocumentCache } from "../utils/data-structures/fix-sized-cache"; +import { base64ToBytes } from "byte-base64"; +import type { DocumentUpdateResponse } from "../services/types/DocumentUpdateResponse"; export class Syncer { public readonly onRemainingOperationsCountChanged = new EventListeners< (remainingOperations: number) => unknown >(); - public readonly updatedDocumentsByPathAndKeysLocks: Locks; // can be DocumentId or RelativePath - - // FIFO to limit the number of concurrent sync operations - private readonly syncQueue: PQueue; + private readonly queue: SyncEventQueue; private _isFirstSyncComplete = false; private runningScheduleSyncForOfflineChanges: Promise | undefined; + private draining: Promise | undefined; private previousRemainingOperationsCount = 0; public constructor( private readonly deviceId: string, private readonly logger: Logger, - private readonly database: Database, private readonly settings: Settings, private readonly webSocketManager: WebSocketManager, private readonly operations: FileOperations, - private readonly unrestrictedSyncer: UnrestrictedSyncer + private readonly syncService: SyncService, + private readonly history: SyncHistory, + private readonly contentCache: FixedSizeDocumentCache, + private readonly serverConfig: ServerConfig, + queue: SyncEventQueue ) { - this.syncQueue = new PQueue({ - concurrency: settings.getSettings().syncConcurrency - }); - - this.updatedDocumentsByPathAndKeysLocks = new Locks( - Syncer.name, - this.logger - ); - - settings.onSettingsChanged.add((newSettings, oldSettings) => { - if (newSettings.syncConcurrency !== oldSettings.syncConcurrency) { - this.syncQueue.concurrency = newSettings.syncConcurrency; - } - }); - - this.syncQueue.on("active", () => { - if (this.previousRemainingOperationsCount !== this.syncQueue.size) { - this.previousRemainingOperationsCount = this.syncQueue.size; - this.onRemainingOperationsCountChanged.trigger( - this.syncQueue.size - ); - } - }); + this.queue = queue; this.webSocketManager.onWebSocketStatusChanged.add((isConnected) => { if (isConnected) { - // The JS WebSocket API doesn't support setting headers, so we have to send the token as a message this.sendHandshakeMessage(); + } else { + this.runningScheduleSyncForOfflineChanges = undefined; } }); this.webSocketManager.onRemoteVaultUpdateReceived.add( @@ -83,132 +80,82 @@ export class Syncer { } public hasPendingOperationsForDocument(relativePath: string): boolean { - return this.updatedDocumentsByPathAndKeysLocks.isLocked(relativePath); + return this.queue.hasPendingEventsForPath(relativePath); } - public async syncLocallyCreatedFile( - relativePath: RelativePath - ): Promise { - // check whether someone else has already created the document in the database - if ( - this.database.getLatestDocumentByRelativePath(relativePath) - ?.isDeleted === false - ) { - // This is likely a consequence of us creating a file because of a remote update - // which triggered a local create, so we don't need to do anything here. - this.logger.debug( - `Document ${relativePath} already exists in the database, skipping` - ); - return; - } - - const document = this.database.createNewPendingDocument(relativePath); - - await this.enqueueSyncOperation( - async () => - this.unrestrictedSyncer.unrestrictedSyncLocallyCreatedOrUpdatedFile( - { - document - } - ), - [relativePath] - ); + public syncLocallyCreatedFile(relativePath: RelativePath): void { + this.queue.enqueue({ type: SyncEventType.Create, path: relativePath }); + this.ensureDraining(); } - public async syncLocallyDeletedFile( - relativePath: RelativePath - ): Promise { - const document = - this.database.getLatestDocumentByRelativePath(relativePath); - - if (document == null || document.isDeleted) { - // This is must be a consequence of us deleting a file because of a remote update - // which triggered a local delete, so we don't need to do anything here. - this.logger.debug( - `Document ${relativePath} has already been marked as deleted, skipping` - ); - return; - } - - // We have to have a record of the delete in case there's an in-flight update for the same - // document which finishes after the delete has succeeded and would introduce a phantom metadata record. - this.database.delete(relativePath); - - await this.enqueueSyncOperation(async () => { - await this.unrestrictedSyncer.unrestrictedSyncLocallyDeletedFile( - document - ); - - this.database.removeDocument(document); - }, [document?.metadata?.documentId, relativePath]); + public syncLocallyDeletedFile(relativePath: RelativePath): void { + const record = this.queue.getDocument(relativePath); + const documentId = record?.documentId ?? ""; + this.queue.enqueue({ + type: SyncEventType.Delete, + documentId, + path: relativePath, + }); + this.ensureDraining(); } - public async syncLocallyUpdatedFile({ + public syncLocallyUpdatedFile({ oldPath, relativePath }: { oldPath?: RelativePath; relativePath: RelativePath; - }): Promise { - const document = - this.database.getLatestDocumentByRelativePath(oldPath ?? relativePath); - - // must have been removed after a successful delete - if (document === undefined) { - this.logger.debug( - `Cannot find document ${relativePath} in the database, skipping` - ); - return; - } - - if (document.isDeleted) { - this.logger.debug( - `Document ${relativePath} has been deleted locally, skipping` - ); - return; - } - - const documentAtNewPath = - this.database.getLatestDocumentByRelativePath(relativePath); - - if (oldPath !== undefined) { - // We might have moved the document in the database before calling this method, - // in that case, we mustn't move it again. - if ( - documentAtNewPath === undefined || - documentAtNewPath.isDeleted - ) { - if (oldPath === relativePath) { - throw new Error( - `Old path and new path are the same: ${oldPath}` - ); - } - - this.database.move(oldPath, relativePath); + }): void { + if (oldPath === undefined) { + const record = this.queue.getDocument(relativePath); + if (record === undefined) { + this.syncLocallyCreatedFile(relativePath); + return; } - } - - - if ( - oldPath !== undefined && - document?.metadata?.remoteRelativePath === relativePath - ) { - this.logger.debug( - `Document ${relativePath} has been moved as a result of a remote update, skipping sync` - ); + this.queue.enqueue({ + type: SyncEventType.SyncLocal, + documentId: record.documentId, + }); + this.ensureDraining(); return; } - await this.enqueueSyncOperation( - async () => - this.unrestrictedSyncer.unrestrictedSyncLocallyCreatedOrUpdatedFile( - { - oldPath, - document - } - ), - [document.metadata?.documentId, relativePath, oldPath] - ); + // Handle rename + const sourceRecord = this.queue.getDocument(oldPath); + if (sourceRecord !== undefined) { + // Capture the displaced document's version before + // moveDocument removes it from the store + const displacedRecord = this.queue.getDocument(relativePath); + const displacedDocumentId = this.queue.moveDocument( + oldPath, + relativePath + ); + if (displacedDocumentId !== undefined) { + this.queue.enqueue({ + type: SyncEventType.Delete, + documentId: displacedDocumentId, + path: relativePath, + displacedAtVersion: displacedRecord?.parentVersionId, + }); + } + this.queue.enqueue({ + type: SyncEventType.SyncLocal, + documentId: sourceRecord.documentId, + }); + } else if (this.queue.hasCreateEvent(oldPath)) { + const updated = this.queue.updateCreatePath(oldPath, relativePath); + if (!updated) { + this.syncLocallyCreatedFile(relativePath); + } + } else { + // The create event may have already been dequeued and + // processed (e.g. skipped due to a concurrent rename + // deleting the file at the old path). Treat the file at + // the new path as a fresh create + this.syncLocallyCreatedFile(relativePath); + } + + this.ensureDraining(); } public async scheduleSyncForOfflineChanges(): Promise { @@ -238,7 +185,14 @@ export class Syncer { public async waitUntilFinished(): Promise { await this.runningScheduleSyncForOfflineChanges; - await this.syncQueue.onIdle(); // Wait for queue to be empty and running tasks to finish + // Loop until the draining promise stabilises — new drains can be + // chained by events enqueued during processing + let current = this.draining; + while (current !== undefined) { + await current; + if (this.draining === current) break; + current = this.draining; + } } public async syncRemotelyUpdatedFile( @@ -247,66 +201,63 @@ export class Syncer { try { await this.scheduleSyncForOfflineChanges(); - const handlerPromise = awaitAll( - message.documents.map(async (document) => - this.internalSyncRemotelyUpdatedFile(document) - ) - ); - - await handlerPromise; - - if (message.isInitialSync && message.documents.length > 0) { - this.database.setLastSeenUpdateId( - message.documents - .map((document) => document.vaultUpdateId) - .reduce((a, b) => Math.max(a, b)) - ); + for (const remoteVersion of message.documents) { + this.queue.enqueue({ + type: SyncEventType.SyncRemote, + remoteVersion + }); } - this._isFirstSyncComplete = true; + // The initial sync is a complete snapshot so we can jump the + // minimum straight to the max vaultUpdateId. Subsequent + // broadcasts use addSeenUpdateId (called per-event inside each + // processor) which tracks contiguous coverage and won't advance + // past gaps — correct for incremental updates but wrong for a + // snapshot whose IDs are intentionally sparse + if (message.isInitialSync) { + this.queue.setLastSeenUpdateId( + Math.max( + ...message.documents.map((d) => d.vaultUpdateId), + this.queue.getLastSeenUpdateId() + ) + ); + this._isFirstSyncComplete = true; + } + + await this.scheduleDrain(); } catch (e) { + if (e instanceof SyncResetError) { + this.logger.info( + "Failed to sync remotely updated file due to a reset" + ); + return; + } this.logger.error(`Failed to sync remotely updated file: ${e}`); } } public reset(): void { this._isFirstSyncComplete = false; - this.syncQueue.clear(); - this.updatedDocumentsByPathAndKeysLocks.reset(); + this.queue.clear(); this.runningScheduleSyncForOfflineChanges = undefined; + // Do not set this.draining = undefined — the in-flight drain will + // exit naturally (SyncResetError or empty queue) and the promise + // chain stays intact, preventing concurrent drain invocations } + + private sendHandshakeMessage(): void { const message: WebSocketClientMessage = { type: "handshake", deviceId: this.deviceId, token: this.settings.getSettings().token, - lastSeenVaultUpdateId: this.database.getLastSeenUpdateId() + lastSeenVaultUpdateId: this.queue.getLastSeenUpdateId() }; this.webSocketManager.sendHandshakeMessage(message); } - private async internalSyncRemotelyUpdatedFile( - remoteVersion: DocumentVersionWithoutContent - ): Promise { - const document = this.database.getDocumentByDocumentId( - remoteVersion.documentId - ); - await this.enqueueSyncOperation( - async () => - this.unrestrictedSyncer.unrestrictedSyncRemotelyUpdatedFile( - remoteVersion, - document - ), - [ - document?.relativePath, - remoteVersion.relativePath, - remoteVersion.documentId - ] - ); - this.database.addSeenUpdateId(remoteVersion.vaultUpdateId); - } private async internalScheduleSyncForOfflineChanges(): Promise { const allLocalFiles = await this.operations.listFilesRecursively(); @@ -314,14 +265,40 @@ export class Syncer { `Scheduling sync for ${allLocalFiles.length} local files` ); - let locallyPossiblyDeletedFiles: DocumentRecord[] = []; + // Clear stale event tracking from any previous drain + this.queue.clear(); - for (const document of this.database.resolvedDocuments) { - if ( - !document.isDeleted && - !(await this.operations.exists(document.relativePath)) - ) { - locallyPossiblyDeletedFiles.push(document); + // Detect documents whose local path diverges from the server path. + // This happens when a rename was recorded while sync was disabled. + const allDocuments = this.queue.allDocuments(); + const locallyRenamedPaths = new Set(); + + for (const [path, record] of allDocuments) { + const remoteRelPath = record.remoteRelativePath; + const hasLocalRename = + remoteRelPath !== undefined && remoteRelPath !== path; + + if (hasLocalRename) { + // Enqueue a sync-local at the current (renamed) path; + // the processSyncLocal handler will detect the path + // divergence and send an update with the new path + this.queue.enqueue({ + type: SyncEventType.SyncLocal, + documentId: record.documentId, + }); + locallyRenamedPaths.add(path); + } + } + + // Find files that have been deleted locally + interface DocumentWithPath { + path: RelativePath; + record: DocumentRecord; + } + let locallyPossiblyDeletedFiles: DocumentWithPath[] = []; + for (const [path, record] of allDocuments) { + if (!(await this.operations.exists(path))) { + locallyPossiblyDeletedFiles.push({ path, record }); } } @@ -330,132 +307,934 @@ export class Syncer { relativePath: string; oldPath?: string; } - const instructions: (Instruction | undefined)[] = await awaitAll( - allLocalFiles.map(async (relativePath) => { - if ( - this.database.getLatestDocumentByRelativePath(relativePath) - ?.metadata !== undefined - ) { - this.logger.debug( - `Document ${relativePath} might have been updated locally, scheduling sync to validate and update it` - ); + const instructions: Instruction[] = []; - return { type: "update", relativePath } as Instruction; - } + for (const relativePath of allLocalFiles) { + if (locallyRenamedPaths.has(relativePath)) { + continue; + } - // Perhaps the file has been moved; let's check by looking at the deleted files - const contentHash = await this.syncQueue.add(async () => { + const existingRecord = this.queue.getDocument(relativePath); + + if (existingRecord !== undefined) { + // Verify the content actually belongs to this document. + // A file might exist at a known path but actually be a + // different document that was renamed here while offline + if (locallyPossiblyDeletedFiles.length > 0) { + let contentHash: string | undefined; try { - const contentBytes = - await this.operations.read(relativePath); // this can throw FileNotFoundError - return await hash(contentBytes); + const bytes = + await this.operations.read(relativePath); + contentHash = await hash(bytes); } catch (e) { - if ( - e instanceof Error && - e.name === "FileNotFoundError" - ) { - return undefined; - } + if (e instanceof FileNotFoundError) continue; throw e; } - }); - if (contentHash == undefined) { - // The file was deleted before we had a chance to read it, no need to sync it here - return; + if (contentHash !== existingRecord.hash) { + const originalFile = await findMatchingFile( + contentHash, + locallyPossiblyDeletedFiles + ); + if (originalFile !== undefined) { + // This file was moved here from a different path + locallyPossiblyDeletedFiles.push({ + path: relativePath, + record: existingRecord + }); + locallyPossiblyDeletedFiles = + locallyPossiblyDeletedFiles.filter( + (item) => + item.path !== originalFile.path + ); + + this.logger.debug( + `Document '${originalFile.path}' was moved to ${relativePath} (displacing existing document), scheduling sync to move it` + ); + instructions.push({ + type: "update", + oldPath: originalFile.path, + relativePath + }); + continue; + } + } } - const originalFile = findMatchingFile( - contentHash, - locallyPossiblyDeletedFiles + this.logger.debug( + `Document ${relativePath} might have been updated locally, scheduling sync to validate and update it` ); - if (originalFile !== undefined) { - // `originalFile` hasn't been deleted but it got moved instead - /* eslint-disable no-restricted-syntax -- Comparing by property, not direct equality */ - locallyPossiblyDeletedFiles = - locallyPossiblyDeletedFiles.filter( - (item) => - item.relativePath !== originalFile.relativePath - ); - /* eslint-enable no-restricted-syntax */ + instructions.push({ type: "update", relativePath }); + continue; + } - this.logger.debug( - `Document '${originalFile.relativePath}' was not found under its current path in the database but was found under a different path (${relativePath}), scheduling sync to move it` + // Perhaps the file has been moved; check by looking at the deleted files + let contentHash: string | undefined = undefined; + try { + const contentBytes = await this.operations.read(relativePath); + contentHash = await hash(contentBytes); + } catch (e) { + if (e instanceof FileNotFoundError) { + continue; + } + throw e; + } + + const originalFile = await findMatchingFile( + contentHash, + locallyPossiblyDeletedFiles + ); + if (originalFile !== undefined) { + locallyPossiblyDeletedFiles = + locallyPossiblyDeletedFiles.filter( + (item) => item.path !== originalFile.path ); - return { - type: "update", - oldPath: originalFile.relativePath, - relativePath - } as Instruction; - } - this.logger.debug( - `Document ${relativePath} not found in database, scheduling sync to create it` + `Document '${originalFile.path}' was not found under its current path in the database but was found under a different path (${relativePath}), scheduling sync to move it` ); - return { - type: "create", + instructions.push({ + type: "update", + oldPath: originalFile.path, relativePath - } as Instruction; - }) - ); + }); + continue; + } - // this has to happen strictly after the previous awaitAll, as that one - // might have removed some of the documents from the list - await awaitAll( - locallyPossiblyDeletedFiles.map(async ({ relativePath }) => { - this.logger.debug( - `Document ${relativePath} has been deleted locally, scheduling sync to delete it` - ); + this.logger.debug( + `Document ${relativePath} not found in database, scheduling sync to create it` + ); + instructions.push({ type: SyncEventType.Create, relativePath }); + } - // We're outside of the pqueue, so we need to call the public wrapper - return this.syncLocallyDeletedFile(relativePath); - }) - ); + // Enqueue deletes first + for (const { path } of locallyPossiblyDeletedFiles) { + this.logger.debug( + `Document ${path} has been deleted locally, scheduling sync to delete it` + ); + this.syncLocallyDeletedFile(path); + } - await awaitAll( - instructions.map(async (instruction) => { - if (instruction === undefined) { - return; - } + // Then updates/moves + for (const instruction of instructions) { + if (instruction.type === "update") { + this.syncLocallyUpdatedFile({ + oldPath: instruction.oldPath, + relativePath: instruction.relativePath + }); + } + } - if (instruction.type === "update") { - // We're outside of the pqueue, so we need to call the public wrapper - await this.syncLocallyUpdatedFile({ - oldPath: instruction.oldPath, - relativePath: instruction.relativePath - }); - return; - } - }) - ); + // Creates last so the server can merge with existing documents + for (const instruction of instructions) { + if (instruction.type === "create") { + this.syncLocallyCreatedFile(instruction.relativePath); + } + } - // we have to ensure the deletes & updates have finished before starting creates, - // otherwise the server might return an existing document (that we're about to delete) - // instead of actually creating a new one - await awaitAll( - instructions.map(async (instruction) => { - if (instruction === undefined) { - return; - } + await this.scheduleDrain(); + } - if (instruction.type === "create") { - // We're outside of the pqueue, so we need to call the public wrapper - await this.syncLocallyCreatedFile(instruction.relativePath); - return; - } - }) + + + private ensureDraining(): void { + this.draining = (this.draining ?? Promise.resolve()).then( + async () => this.drain() ); } - private async enqueueSyncOperation( - operation: () => Promise, - keys: (string | undefined | null)[] - ): Promise { - return this.updatedDocumentsByPathAndKeysLocks.withLock( - keys.filter((k) => k !== undefined && k !== null), - async () => this.syncQueue.add(operation) + private async scheduleDrain(): Promise { + this.ensureDraining(); + await this.draining; + } + + private async drain(): Promise { + let event = this.queue.next(); + while (event !== undefined) { + try { + await this.processEvent(event); + } catch (e) { + if (e instanceof SyncResetError) { + this.logger.info("Drain interrupted by sync reset"); + return; + } + this.logger.error( + `Failed to process sync event ${event.type}: ${e}` + ); + } + this.notifyRemainingOperationsChanged(); + event = this.queue.next(); + } + } + + private async processEvent(event: SyncEvent): Promise { + if (!this.settings.getSettings().isSyncEnabled) { + this.logger.info( + `Skipping sync operation because sync is disabled` + ); + return; + } + + try { + switch (event.type) { + case SyncEventType.Create: + await this.processCreate(event); + break; + case SyncEventType.Delete: + await this.processDelete(event); + break; + case SyncEventType.SyncLocal: + await this.processSyncLocal(event); + break; + case SyncEventType.SyncRemote: + await this.processSyncRemote(event); + break; + } + } catch (e) { + if (e instanceof FileNotFoundError) { + this.logger.info( + `Skipping sync event '${event.type}' because the file no longer exists` + ); + return; + } + if ( + e instanceof HttpClientError && + event.type === SyncEventType.SyncLocal + ) { + // The server rejected the update (e.g. document was + // deleted). Re-create only if local content differs + // from the last synced version — otherwise the remote + // delete should win + const doc = this.queue.getDocumentByDocumentId( + event.documentId + ); + if (doc === undefined) return; + const { path: eventPath, record } = doc; + if (await this.operations.exists(eventPath)) { + const localBytes = + await this.operations.read(eventPath); + const localHash = await hash(localBytes); + if (localHash !== record.hash) { + this.logger.info( + `Server rejected update for ${eventPath} but local content changed, re-creating` + ); + this.queue.removeDocument(eventPath); + this.syncLocallyCreatedFile(eventPath); + return; + } + } + this.logger.info( + `Server rejected update for ${eventPath} (${e.message}), removing local copy` + ); + this.queue.removeDocument(eventPath); + await this.operations.delete(eventPath); + return; + } + if (e instanceof HttpClientError) { + // Server rejected a request (e.g. updating a deleted + // document during sync-remote processing). Not an + // error — the next offline scan will reconcile + this.logger.info( + `Server rejected ${event.type} request: ${e.message}` + ); + return; + } + throw e; + } + } + + + + private async processCreate( + event: Extract + ): Promise { + const effectivePath = event.path; + const contentBytes = await this.operations.read(effectivePath); + const contentHash = await hash(contentBytes); + + const oversizedEntry = this.getHistoryEntryForSkippedOversizedFile( + contentBytes.byteLength, + effectivePath ); + if (oversizedEntry !== undefined) { + this.history.addHistoryEntry(oversizedEntry); + return; + } + + const response = await this.syncService.create({ + relativePath: effectivePath, + contentBytes + }); + + // Handle concurrent move & creation: the server merged our create + // with an existing document that we also have locally at a different path + const existingDoc = this.queue.getDocumentByDocumentId( + response.documentId + ); + if (existingDoc !== undefined && existingDoc.path !== effectivePath) { + this.logger.info( + `Merging existing document ${existingDoc.path} into ${effectivePath} after concurrent move & creation` + ); + await this.operations.delete(existingDoc.path); + this.queue.removeDocument(existingDoc.path); + } + + // When the server deconflicts the create to a different path, another + // document may now occupy the original path (downloaded while the + // create was in flight). handleMaybeMergingResponse would move the + // file AND the foreign document's record to the deconflicted path, + // then overwrite it — orphaning the foreign document. Handle this + // by writing directly to the deconflicted path instead of moving + const foreignRecord = this.queue.getDocument(effectivePath); + const pathOccupiedByForeignDocument = + response.relativePath !== effectivePath && + foreignRecord !== undefined && + foreignRecord.documentId !== response.documentId; + + if (pathOccupiedByForeignDocument) { + const actualPath = response.relativePath; + + if ("type" in response && response.type === "MergingUpdate") { + const responseBytes = base64ToBytes(response.contentBase64); + await this.operations.create(actualPath, responseBytes); + const afterWriteBytes = + await this.operations.read(actualPath); + const afterWriteHash = await hash(afterWriteBytes); + this.queue.setDocument(actualPath, { + documentId: response.documentId, + parentVersionId: response.vaultUpdateId, + hash: afterWriteHash, + remoteRelativePath: response.relativePath + }); + await this.updateCache( + response.vaultUpdateId, + responseBytes, + actualPath + ); + } else { + await this.operations.create(actualPath, contentBytes); + this.queue.setDocument(actualPath, { + documentId: response.documentId, + parentVersionId: response.vaultUpdateId, + hash: contentHash, + remoteRelativePath: response.relativePath + }); + await this.updateCache( + response.vaultUpdateId, + contentBytes, + actualPath + ); + } + } else { + await this.handleMaybeMergingResponse({ + path: effectivePath, + response, + contentHash, + originalContentBytes: contentBytes + }); + } + + this.queue.addSeenUpdateId(response.vaultUpdateId); + + this.history.addHistoryEntry({ + status: SyncStatus.SUCCESS, + details: { type: SyncType.CREATE, relativePath: effectivePath }, + message: response.type === "MergingUpdate" + ? "Created file and merged with existing remote version" + : "Successfully created file on the server", + author: response.userId, + timestamp: new Date(response.updatedDate) + }); + } + + private async processDelete( + event: Extract + ): Promise { + let { documentId } = event; + const { path } = event; + + // Empty string means the documentId wasn't known when the + // delete was enqueued (e.g. a create was still in flight). + // Try to resolve it from the store now that the create may + // have completed + if (documentId === "") { + const record = this.queue.getDocument(path); + if (record === undefined) { + this.logger.debug( + "Skipping delete for a document whose create was cancelled" + ); + return; + } + documentId = record.documentId; + } + + // For displacement deletes (side effect of a rename), check + // if another client updated the document since our last known + // version. If so, skip the delete to preserve their edits + if (event.displacedAtVersion !== undefined) { + const latest = await this.syncService.get({ documentId }); + if ( + !latest.isDeleted && + latest.vaultUpdateId > event.displacedAtVersion + ) { + this.logger.info( + `Skipping displacement delete for ${documentId} — document was updated by another client` + ); + // Allow broadcasts for this document to be processed + // normally so the updated content is downloaded + this.queue.unmarkRecentlyDeleted(documentId); + return; + } + } + + // Use the document's current path from the store if available, + // otherwise fall back to the path from the event (e.g. when the + // document was displaced by a move and already removed from the store) + const doc = this.queue.getDocumentByDocumentId(documentId); + const relativePath = doc?.path ?? path; + + const response = await this.syncService.delete({ + documentId, + relativePath + }); + + // Only remove the document record if it still belongs to this + // documentId; the path may have been reused by a different document + // (e.g. after a move-to-occupied-path) + if (doc !== undefined) { + this.queue.removeDocument(doc.path); + } + this.queue.addSeenUpdateId(response.vaultUpdateId); + + this.history.addHistoryEntry({ + status: SyncStatus.SUCCESS, + details: { + type: SyncType.DELETE, + relativePath + }, + message: "Successfully deleted file on the server", + author: response.userId + }); + } + + private async processSyncLocal( + event: Extract + ): Promise { + const doc = this.queue.getDocumentByDocumentId(event.documentId); + + if (doc === undefined) { + this.logger.debug( + `Skipping sync-local for unknown document ${event.documentId}` + ); + return; + } + + const { path: eventPath, record } = doc; + + // Read file and compare hash + const contentBytes = await this.operations.read(eventPath); + const contentHash = await hash(contentBytes); + + const pathChanged = + record.remoteRelativePath !== undefined && + record.remoteRelativePath !== eventPath; + + if (contentHash === record.hash && !pathChanged) { + this.logger.debug( + `File hash of ${eventPath} matches last synced version; no need to sync` + ); + return; + } + + const response = await this.sendUpdate( + record, + eventPath, + contentBytes + ); + + await this.handleMaybeMergingResponse({ + path: eventPath, + response, + contentHash, + originalContentBytes: contentBytes + }); + + this.queue.addSeenUpdateId(response.vaultUpdateId); + + const isMerge = + "type" in response && response.type === "MergingUpdate"; + this.history.addHistoryEntry({ + status: SyncStatus.SUCCESS, + details: { + type: SyncType.UPDATE, + relativePath: eventPath + }, + message: isMerge + ? "Updated file and merged with remote changes" + : "Successfully updated file on the server", + author: response.userId, + timestamp: new Date(response.updatedDate) + }); + } + + private async processSyncRemote( + event: Extract + ): Promise { + const { remoteVersion } = event; + const existingDoc = this.queue.getDocumentByDocumentId( + remoteVersion.documentId + ); + + if (existingDoc !== undefined) { + if ( + existingDoc.record.parentVersionId >= + remoteVersion.vaultUpdateId + ) { + this.logger.debug( + `Document ${existingDoc.path} is already up-to-date` + ); + this.queue.addSeenUpdateId(remoteVersion.vaultUpdateId); + return; + } + + await this.processRemoteUpdateForExistingDocument( + existingDoc.path, + existingDoc.record, + remoteVersion + ); + return; + } + + if (this.queue.wasRecentlyDeleted(remoteVersion.documentId)) { + this.logger.debug( + `Ignoring stale broadcast for recently-deleted document ${remoteVersion.documentId}` + ); + this.queue.addSeenUpdateId(remoteVersion.vaultUpdateId); + return; + } + + if (remoteVersion.isDeleted) { + this.logger.debug( + `Document ${remoteVersion.relativePath} has been deleted remotely, no need to sync` + ); + this.queue.addSeenUpdateId(remoteVersion.vaultUpdateId); + return; + } + + await this.processRemoteUpdateForNewDocument(remoteVersion); + } + + private async processRemoteUpdateForExistingDocument( + currentPath: RelativePath, + record: DocumentRecord, + remoteVersion: DocumentVersionWithoutContent + ): Promise { + if (remoteVersion.isDeleted) { + // Check for local changes before deleting + let hasLocalChanges = false; + try { + const contentBytes = await this.operations.read(currentPath); + const contentHash = await hash(contentBytes); + hasLocalChanges = record.hash !== contentHash; + } catch (e) { + if (!(e instanceof FileNotFoundError)) throw e; + } + + if (hasLocalChanges) { + // Local changes survive; re-upload as a new document + this.queue.removeDocument(currentPath); + this.syncLocallyCreatedFile(currentPath); + this.queue.addSeenUpdateId(remoteVersion.vaultUpdateId); + return; + } + + await this.operations.delete(currentPath); + this.queue.removeDocument(currentPath); + this.queue.addSeenUpdateId(remoteVersion.vaultUpdateId); + + this.history.addHistoryEntry({ + status: SyncStatus.SUCCESS, + details: { + type: SyncType.DELETE, + relativePath: currentPath + }, + message: + "Successfully deleted file which had been deleted remotely", + author: remoteVersion.userId, + timestamp: new Date(remoteVersion.updatedDate) + }); + return; + } + + // Fetch the latest full version from the server + const fullVersion = await this.syncService.get({ + documentId: remoteVersion.documentId + }); + + // The document may have been deleted between the broadcast + // and the fetch — handle it the same as a remote delete + if (fullVersion.isDeleted) { + const contentBytes = await this.operations.read(currentPath); + const localHash = await hash(contentBytes); + if (localHash !== record.hash) { + this.queue.removeDocument(currentPath); + this.syncLocallyCreatedFile(currentPath); + } else { + await this.operations.delete(currentPath); + this.queue.removeDocument(currentPath); + } + this.queue.addSeenUpdateId(fullVersion.vaultUpdateId); + return; + } + + const contentBytes = await this.operations.read(currentPath); + const contentHash = await hash(contentBytes); + + const hasLocalChanges = record.hash !== contentHash; + + if (hasLocalChanges) { + const response = await this.sendUpdate( + record, + currentPath, + contentBytes + ); + + await this.handleMaybeMergingResponse({ + path: currentPath, + response, + contentHash, + originalContentBytes: contentBytes + }); + + this.queue.addSeenUpdateId(response.vaultUpdateId); + + this.history.addHistoryEntry({ + status: SyncStatus.SUCCESS, + details: { + type: SyncType.UPDATE, + relativePath: currentPath + }, + message: "Merged local changes with remote update", + author: response.userId, + timestamp: new Date(response.updatedDate) + }); + } else { + const responseBytes = base64ToBytes(fullVersion.contentBase64); + + // Handle remote path change + let actualPath = currentPath; + if ( + fullVersion.relativePath !== currentPath && + record.remoteRelativePath === currentPath + ) { + actualPath = fullVersion.relativePath; + await this.operations.delete(fullVersion.relativePath); + await this.operations.move( + currentPath, + fullVersion.relativePath + ); + } + + await this.operations.write( + actualPath, + contentBytes, + responseBytes + ); + + // Re-read and re-hash after write (the 3-way merge may produce different content) + const afterWriteBytes = await this.operations.read(actualPath); + const afterWriteHash = await hash(afterWriteBytes); + + this.queue.setDocument(actualPath, { + documentId: fullVersion.documentId, + parentVersionId: fullVersion.vaultUpdateId, + hash: afterWriteHash, + remoteRelativePath: fullVersion.relativePath + }); + + // If the path changed, remove the old entry + if (actualPath !== currentPath) { + this.queue.removeDocument(currentPath); + } + + await this.updateCache( + fullVersion.vaultUpdateId, + responseBytes, + actualPath + ); + this.queue.addSeenUpdateId(fullVersion.vaultUpdateId); + + this.history.addHistoryEntry({ + status: SyncStatus.SUCCESS, + details: + actualPath !== currentPath + ? { + type: SyncType.MOVE, + relativePath: actualPath, + movedFrom: currentPath + } + : { + type: SyncType.UPDATE, + relativePath: actualPath + }, + message: + "Successfully downloaded remotely updated file from the server", + author: fullVersion.userId, + timestamp: new Date(fullVersion.updatedDate) + }); + } + } + + private async processRemoteUpdateForNewDocument( + remoteVersion: DocumentVersionWithoutContent + ): Promise { + const oversizedEntry = this.getHistoryEntryForSkippedOversizedFile( + remoteVersion.contentSize, + remoteVersion.relativePath + ); + if (oversizedEntry !== undefined) { + this.history.addHistoryEntry(oversizedEntry); + return; + } + + const contentBytes = + await this.syncService.getDocumentVersionContent({ + documentId: remoteVersion.documentId, + vaultUpdateId: remoteVersion.vaultUpdateId + }); + + // A concurrent operation may have created the document already + const existingDoc = this.queue.getDocumentByDocumentId( + remoteVersion.documentId + ); + if (existingDoc !== undefined) { + this.logger.debug( + `Document ${remoteVersion.relativePath} has already been created locally` + ); + return; + } + + const deconflictedPath = await this.operations.ensureClearPath( + remoteVersion.relativePath + ); + if (deconflictedPath !== undefined) { + // The displaced file was moved to a deconflicted path. + // Remove its document record so the offline scan treats + // it as a new file rather than an existing document that + // needs its path synced (which would create duplicates) + this.queue.removeDocument(deconflictedPath); + } + + const contentHash = await hash(contentBytes); + this.queue.setDocument(remoteVersion.relativePath, { + documentId: remoteVersion.documentId, + parentVersionId: remoteVersion.vaultUpdateId, + hash: contentHash, + remoteRelativePath: remoteVersion.relativePath + }); + + await this.operations.create( + remoteVersion.relativePath, + contentBytes + ); + + await this.updateCache( + remoteVersion.vaultUpdateId, + contentBytes, + remoteVersion.relativePath + ); + + this.queue.addSeenUpdateId(remoteVersion.vaultUpdateId); + + this.history.addHistoryEntry({ + status: SyncStatus.SUCCESS, + details: { + type: SyncType.CREATE, + relativePath: remoteVersion.relativePath + }, + message: + "Successfully downloaded remote file which hadn't existed locally", + author: remoteVersion.userId, + timestamp: new Date(remoteVersion.updatedDate) + }); + } + + + + private async sendUpdate( + record: DocumentRecord, + relativePath: RelativePath, + contentBytes: Uint8Array + ): Promise { + const isText = + !isBinary(contentBytes) && + isFileTypeMergable( + relativePath, + (await this.serverConfig.getConfig()).mergeableFileExtensions + ); + + const cachedVersion = this.contentCache.get(record.parentVersionId); + + if (isText && cachedVersion !== undefined) { + return this.syncService.putText({ + documentId: record.documentId, + parentVersionId: record.parentVersionId, + relativePath, + content: diff( + new TextDecoder().decode(cachedVersion), + new TextDecoder().decode(contentBytes) + ) + }); + } + + return this.syncService.putBinary({ + documentId: record.documentId, + parentVersionId: record.parentVersionId, + relativePath, + contentBytes + }); + } + + private async handleMaybeMergingResponse({ + path, + response, + contentHash, + originalContentBytes + }: { + path: RelativePath; + response: DocumentUpdateResponse; + contentHash: string; + originalContentBytes: Uint8Array; + }): Promise { + if (response.isDeleted) { + // If the local file has been edited, re-create it as a new + // document so local edits survive the remote delete + if (await this.operations.exists(path)) { + const localBytes = await this.operations.read(path); + const localHash = await hash(localBytes); + const record = this.queue.getDocument(path); + if (record !== undefined && localHash !== record.hash) { + this.queue.removeDocument(path); + this.queue.addSeenUpdateId(response.vaultUpdateId); + this.syncLocallyCreatedFile(path); + return; + } + } + await this.operations.delete(path); + this.queue.removeDocument(path); + return; + } + + let actualPath = path; + + // Server may have changed the path (e.g. first-rename-wins conflict) + if (response.relativePath !== path) { + actualPath = response.relativePath; + const displacedPath = await this.operations.move( + path, + response.relativePath + ); + if (displacedPath !== undefined) { + const displacedRecord = + this.queue.getDocument(displacedPath); + if (displacedRecord !== undefined) { + const displacedBytes = + await this.operations.read(displacedPath); + const displacedHash = await hash(displacedBytes); + if (displacedHash !== displacedRecord.hash) { + this.queue.enqueue({ + type: SyncEventType.SyncLocal, + documentId: displacedRecord.documentId, + }); + } + } + } + // Remove old path entry; the new path will be set below + this.queue.removeDocument(path); + } + + if ("type" in response && response.type === "MergingUpdate") { + const responseBytes = base64ToBytes(response.contentBase64); + await this.operations.write( + actualPath, + originalContentBytes, + responseBytes + ); + + // Re-read and re-hash after write (invariant #3) + const afterWriteBytes = await this.operations.read(actualPath); + const afterWriteHash = await hash(afterWriteBytes); + + this.queue.setDocument(actualPath, { + documentId: response.documentId, + parentVersionId: response.vaultUpdateId, + hash: afterWriteHash, + remoteRelativePath: response.relativePath + }); + + // Cache the SERVER's content, not local (invariant #2) + await this.updateCache( + response.vaultUpdateId, + responseBytes, + actualPath + ); + } else { + // Fast-forward update: no merge needed + this.queue.setDocument(actualPath, { + documentId: response.documentId, + parentVersionId: response.vaultUpdateId, + hash: contentHash, + remoteRelativePath: response.relativePath + }); + + await this.updateCache( + response.vaultUpdateId, + originalContentBytes, + actualPath + ); + } + } + + private async updateCache( + updateId: VaultUpdateId, + contentBytes: Uint8Array, + filePath: RelativePath + ): Promise { + if ( + isFileTypeMergable( + filePath, + (await this.serverConfig.getConfig()).mergeableFileExtensions + ) && + !isBinary(contentBytes) + ) { + this.contentCache.put(updateId, contentBytes); + } + } + + private getHistoryEntryForSkippedOversizedFile( + sizeInBytes: number, + relativePath: RelativePath + ): CommonHistoryEntry | undefined { + const sizeInMB = Math.round(sizeInBytes / 1024 / 1024); + const { maxFileSizeMB } = this.settings.getSettings(); + if (sizeInMB > maxFileSizeMB) { + return { + status: SyncStatus.SKIPPED, + details: { + type: SyncType.SKIPPED as const, + relativePath + }, + message: `File size of ${sizeInMB} MB exceeds the maximum file size limit of ${maxFileSizeMB} MB` + }; + } + } + + private notifyRemainingOperationsChanged(): void { + const currentCount = this.queue.size; + if (this.previousRemainingOperationsCount !== currentCount) { + this.previousRemainingOperationsCount = currentCount; + this.onRemainingOperationsCountChanged.trigger(currentCount); + } } } diff --git a/frontend/sync-client/src/sync-operations/types.ts b/frontend/sync-client/src/sync-operations/types.ts new file mode 100644 index 00000000..f722aa8a --- /dev/null +++ b/frontend/sync-client/src/sync-operations/types.ts @@ -0,0 +1,42 @@ +import type { DocumentVersionWithoutContent } from "../services/types/DocumentVersionWithoutContent"; + +export type VaultUpdateId = number; +export type DocumentId = string; +export type RelativePath = string; + +export interface DocumentRecord { + documentId: DocumentId; + parentVersionId: VaultUpdateId; + hash: string; + remoteRelativePath?: RelativePath; +} + +export interface StoredDocument extends DocumentRecord { + relativePath: RelativePath; +} + +export interface StoredSyncState { + documents: StoredDocument[]; + lastSeenUpdateId: VaultUpdateId | undefined; +} + +export enum SyncEventType { + Create = "create", + SyncLocal = "sync-local", + SyncRemote = "sync-remote", + Delete = "delete", +} + +export type SyncEvent = + | { type: SyncEventType.Create; path: RelativePath } + | { type: SyncEventType.SyncLocal; documentId: DocumentId } + | { + type: SyncEventType.Delete; + documentId: DocumentId; + path: RelativePath; + displacedAtVersion?: VaultUpdateId; + } + | { + type: SyncEventType.SyncRemote; + remoteVersion: DocumentVersionWithoutContent; + }; diff --git a/frontend/sync-client/src/sync-operations/unrestricted-syncer.ts b/frontend/sync-client/src/sync-operations/unrestricted-syncer.ts deleted file mode 100644 index 98a64f5d..00000000 --- a/frontend/sync-client/src/sync-operations/unrestricted-syncer.ts +++ /dev/null @@ -1,612 +0,0 @@ -import type { - Database, - DocumentRecord, - RelativePath -} from "../persistence/database"; -import { diff } from "reconcile-text"; -import type { SyncService } from "../services/sync-service"; -import type { Logger } from "../tracing/logger"; -import type { - CommonHistoryEntry, - SyncCreateDetails, - SyncDeleteDetails, - SyncDetails, - SyncHistory, - SyncMovedDetails, - SyncUpdateDetails -} from "../tracing/sync-history"; -import { SyncStatus, SyncType } from "../tracing/sync-history"; -import { EMPTY_HASH, hash } from "../utils/hash"; -import { base64ToBytes } from "byte-base64"; -import type { Settings } from "../persistence/settings"; -import type { FileOperations } from "../file-operations/file-operations"; -import { FileNotFoundError } from "../errors/file-not-found-error"; -import { SyncResetError } from "../errors/sync-reset-error"; -import { globsToRegexes } from "../utils/globs-to-regexes"; -import type { DocumentVersion } from "../services/types/DocumentVersion"; -import type { DocumentUpdateResponse } from "../services/types/DocumentUpdateResponse"; -import type { DocumentVersionWithoutContent } from "../services/types/DocumentVersionWithoutContent"; -import type { FixedSizeDocumentCache } from "../utils/data-structures/fix-sized-cache"; -import { isFileTypeMergable } from "../utils/is-file-type-mergable"; -import { isBinary } from "../utils/is-binary"; -import type { ServerConfig } from "../services/server-config"; - -export class UnrestrictedSyncer { - private ignorePatterns: RegExp[]; - - public constructor( - private readonly logger: Logger, - private readonly database: Database, - private readonly settings: Settings, - private readonly syncService: SyncService, - private readonly operations: FileOperations, - private readonly history: SyncHistory, - private readonly contentCache: FixedSizeDocumentCache, - private readonly serverConfig: ServerConfig - ) { - this.ignorePatterns = globsToRegexes( - this.settings.getSettings().ignorePatterns, - this.logger - ); - - this.settings.onSettingsChanged.add((newSettings) => { - this.ignorePatterns = globsToRegexes( - newSettings.ignorePatterns, - this.logger - ); - }); - } - - public async unrestrictedSyncLocallyCreatedOrUpdatedFile({ - oldPath, - // We use the same code path for both local and remote updates. We need to force the update - // if there are no local changes but we know that the remote version is newer. - force = false, - document - }: { - oldPath?: RelativePath; - force?: boolean; - document: DocumentRecord; - }): Promise { - const updateDetails: - | SyncCreateDetails - | SyncUpdateDetails - | SyncMovedDetails = - document.metadata === undefined - ? { - type: SyncType.CREATE, - relativePath: document.relativePath - } - : oldPath !== undefined - ? { - type: SyncType.MOVE, - relativePath: document.relativePath, - movedFrom: oldPath - } - : { - type: SyncType.UPDATE, - relativePath: document.relativePath - }; - - await this.executeSync(updateDetails, async () => { - const originalRelativePath = document.relativePath; - - if (document.isDeleted) { - this.logger.debug( - `Document ${document.relativePath} has been already deleted, no need to update it` - ); - return; - } - - const contentBytes = await this.operations.read( - document.relativePath - ); // this can throw FileNotFoundError - const contentHash = await hash(contentBytes); - - let response: DocumentVersion | DocumentUpdateResponse | undefined = - undefined; - if (document.metadata === undefined) { - response = await this.syncService.create({ - relativePath: originalRelativePath, - contentBytes - }); - - await this.handleMaybeMergingResponse({ - document, - response, - contentHash, - originalRelativePath, - originalContentBytes: contentBytes, - isCreate: true - }); - } else { - const areThereLocalChanges = - document.metadata.hash !== contentHash || - oldPath !== undefined; - - if (areThereLocalChanges) { - const isText = - !isBinary(contentBytes) && - isFileTypeMergable( - document.relativePath, - (await this.serverConfig.getConfig()) - .mergeableFileExtensions - ); - const cachedVersion = this.contentCache.get( - document.metadata.parentVersionId - ); - - response = - isText && cachedVersion !== undefined - ? await this.syncService.putText({ - documentId: document.metadata.documentId, - parentVersionId: - document.metadata.parentVersionId, - relativePath: document.relativePath, - content: diff( - new TextDecoder().decode(cachedVersion), - new TextDecoder().decode(contentBytes) - ) - }) - : await this.syncService.putBinary({ - documentId: document.metadata.documentId, - parentVersionId: - document.metadata.parentVersionId, - relativePath: document.relativePath, - contentBytes - }); - } else { - if (!force) { - this.logger.debug( - `File hash of ${document.relativePath} matches with last synced version and the path hasn't changed; no need to sync` - ); - return; - } - - // we use this code path (force == true) to sync remotely updated files which have no local changes - response = await this.syncService.get({ - documentId: document.metadata.documentId - }); - } - - await this.handleMaybeMergingResponse({ - document, - response, - contentHash, - originalRelativePath, - originalContentBytes: contentBytes - }); - } - - if (!("type" in response) || response.type === "MergingUpdate") { - if (!force) { - this.history.addHistoryEntry({ - status: SyncStatus.SUCCESS, - details: updateDetails, - message: `The file we updated had been updated remotely, so we downloaded the merged version` - }); - return; - } - } - - const actualUpdateDetails: SyncUpdateDetails | SyncMovedDetails = - oldPath !== undefined || - response.relativePath != originalRelativePath - ? { - type: SyncType.MOVE, - relativePath: response.relativePath, - movedFrom: originalRelativePath - } - : { - type: SyncType.UPDATE, - relativePath: response.relativePath - }; - - if (!response.isDeleted) { - this.history.addHistoryEntry({ - status: SyncStatus.SUCCESS, - details: actualUpdateDetails, - message: `Successfully downloaded remotely updated file from the server`, - author: response.userId, - timestamp: new Date(response.updatedDate) - }); - } else { - this.history.addHistoryEntry({ - status: SyncStatus.SUCCESS, - details: { - type: SyncType.DELETE, - relativePath: document.relativePath - }, - message: - "Successfully deleted file which had been deleted remotely", - author: response.userId, - timestamp: new Date(response.updatedDate) - }); - } - }); - } - - public async unrestrictedSyncLocallyDeletedFile( - document: DocumentRecord - ): Promise { - const updateDetails: SyncDeleteDetails = { - type: SyncType.DELETE, - relativePath: document.relativePath - }; - - await this.executeSync(updateDetails, async () => { - if (document.metadata === undefined) { - this.logger.debug( - `Document ${document.relativePath} has never been synced, no need to delete it remotely` - ); - return; - } - - const response = await this.syncService.delete({ - documentId: document.metadata.documentId, - relativePath: document.relativePath - }); - - this.database.updateDocumentMetadata( - { - documentId: response.documentId, - parentVersionId: response.vaultUpdateId, - hash: EMPTY_HASH, - remoteRelativePath: document.relativePath - }, - document - ); - - this.database.addSeenUpdateId(response.vaultUpdateId); - - this.history.addHistoryEntry({ - status: SyncStatus.SUCCESS, - details: updateDetails, - message: `Successfully deleted locally deleted file on the server`, - author: response.userId - }); - }); - } - - public async unrestrictedSyncRemotelyUpdatedFile( - remoteVersion: DocumentVersionWithoutContent, - document?: DocumentRecord - ): Promise { - const updateDetails: SyncCreateDetails = { - type: SyncType.CREATE, - relativePath: remoteVersion.relativePath - }; - - await this.executeSync(updateDetails, async () => { - if (document?.metadata !== undefined) { - // If the file exists locally, let's pretend the user has updated it - // and deal with remote update/deletion within `unrestrictedSyncLocallyUpdatedFile` - if ( - document.metadata.parentVersionId >= - remoteVersion.vaultUpdateId - ) { - this.logger.debug( - `Document ${document.relativePath} is already at least as up-to-date as the fetched version` - ); - - return; - } - - return this.unrestrictedSyncLocallyCreatedOrUpdatedFile({ - document, - force: true - }); - } else if (remoteVersion.isDeleted) { - // Either the document hasn't made it to us before and therefore we don't need to delete it, - // or we already have it, in which case the preceeding if would've dealt with it - this.logger.debug( - `Document ${remoteVersion.relativePath} has been deleted remotely, no need to sync` - ); - return; - } - - // Don't download oversized files - const historyEntryForSkippedOversizedFile = - this.getHistoryEntryForSkippedOversizedFile( - remoteVersion.contentSize, - remoteVersion.relativePath - ); - if (historyEntryForSkippedOversizedFile !== undefined) { - this.history.addHistoryEntry( - historyEntryForSkippedOversizedFile - ); - return; - } - - const contentBytes = - await this.syncService.getDocumentVersionContent({ - documentId: remoteVersion.documentId, - vaultUpdateId: remoteVersion.vaultUpdateId - }); - - // We're trying to create an entirely new document that didn't exist locally - document = this.database.getDocumentByDocumentId( - remoteVersion.documentId - ); - // It can happen that a concurrent sync operation has already created the document, so we can bail here - if (document !== undefined) { - this.logger.debug( - `Document ${remoteVersion.relativePath} has already been created locally, no need to create it again` - ); - return; - } - - await this.operations.ensureClearPath(remoteVersion.relativePath); - - this.database.updateDocumentMetadata( - { - documentId: remoteVersion.documentId, - parentVersionId: remoteVersion.vaultUpdateId, - hash: await hash(contentBytes), - remoteRelativePath: remoteVersion.relativePath - }, - this.database.createNewPendingDocument( - remoteVersion.relativePath - ) - ); - - await this.operations.create( - remoteVersion.relativePath, - contentBytes - ); - await this.updateCache( - remoteVersion.vaultUpdateId, - contentBytes, - remoteVersion.relativePath - ); - - this.history.addHistoryEntry({ - status: SyncStatus.SUCCESS, - details: updateDetails, - message: `Successfully downloaded remote file which hadn't existed locally`, - author: remoteVersion.userId, - timestamp: new Date(remoteVersion.updatedDate) - }); - }); - } - - private async executeSync( - details: SyncDetails, - fn: () => Promise - ): Promise { - if (!this.settings.getSettings().isSyncEnabled) { - this.logger.info( - `Skipping sync operation for file '${details.relativePath}' because sync is disabled` - ); - return; - } - - for (const pattern of this.ignorePatterns) { - if (pattern.test(details.relativePath)) { - this.logger.debug( - `File '${details.relativePath}' is ignored by the ignore pattern: ${pattern}` - ); - return; // bail without SKIPPED status because we were told to ignore this file and we shouldn't clutter up the history - } - } - - try { - // Only check the size of files which already exist locally. - if (await this.operations.exists(details.relativePath)) { - const sizeInBytes = await this.operations.getFileSize( - details.relativePath - ); - const historyEntryForSkippedOversizedFile = - this.getHistoryEntryForSkippedOversizedFile( - sizeInBytes, - details.relativePath - ); - if (historyEntryForSkippedOversizedFile !== undefined) { - this.history.addHistoryEntry( - historyEntryForSkippedOversizedFile - ); - return; - } - } - - return await fn(); - } catch (e) { - if (e instanceof FileNotFoundError) { - // A subsequent sync operation must have been creating to deal with this - this.logger.info( - `Skiping file '${details.relativePath}' because it no longer exists when trying to ${details.type.toLocaleLowerCase()} it` - ); - return; - } - if (e instanceof SyncResetError) { - this.logger.info( - `Interrupting sync operation because of a reset` - ); - return; - } else { - this.history.addHistoryEntry({ - status: SyncStatus.ERROR, - details, - message: `Failed to sync file '${details.relativePath}' because of ${e} when trying to ${details.type.toLocaleLowerCase()} it` - }); - throw e; - } - } - } - - private async handleMaybeMergingResponse({ - document, - response, - contentHash, - originalRelativePath, - originalContentBytes, - isCreate - }: { - document: DocumentRecord; - response: DocumentVersion | DocumentUpdateResponse; - contentHash: string; - originalRelativePath: string; - originalContentBytes: Uint8Array; - isCreate?: boolean; - }): Promise { - // `document` is mutable and reflects the latest state in the local database - if (document.isDeleted) { - this.logger.info( - `Document ${document.relativePath} has been deleted before we could finish updating it` - ); - this.database.addSeenUpdateId(response.vaultUpdateId); - return; - } - - if ( - (document.metadata?.parentVersionId ?? 0) > response.vaultUpdateId - ) { - this.logger.debug( - `Document ${document.relativePath} is already more up to date than the fetched version` - ); - this.database.addSeenUpdateId(response.vaultUpdateId); // in case the previous `vaultUpdateId` update hasn't made it through - return; - } - - if (response.isDeleted) { - return this.applyRemoteDeleteLocally(document, response); - } - - let actualPath = document.relativePath; - - if (isCreate) { - // We have a file locally that got moved by another client to the same path as the one we're trying to create. - // The server returns a merging update for the document ID that already exists locally (but at another path). - // We have to merge these two documents by extending the provenance of the existing document and deleting - // the old document that the new document already contains the content for. - const existingDocument = this.database.getDocumentByDocumentId( - response.documentId - ); - if (existingDocument !== undefined) { - this.logger.info( - `Merging existing document ${existingDocument.relativePath} into ${document.relativePath - } after concurrent move & creation` - ); - if (!existingDocument.isDeleted) { - this.database.delete(existingDocument.relativePath); // make sure syncLocallyDeletedFile doesn't actually schedule deleting the new file - this.database.removeDocument(existingDocument); - await this.operations.move(existingDocument.relativePath, document.relativePath); - } else { - this.database.removeDocument(existingDocument); - } - } - } - - // this can't happen on the creation path as we can only get a merging response if a document already exists remotely on the same path - if (response.relativePath != originalRelativePath) { - actualPath = response.relativePath; - // Make sure to update the remote relative path to avoid uploading - // the file as a result of this filesystem event. - if (document.metadata !== undefined) { - document.metadata.remoteRelativePath = response.relativePath; - } - await this.operations.move( - document.relativePath, - response.relativePath - ); // this can throw FileNotFoundError - } - - if (!("type" in response) || response.type === "MergingUpdate") { - const responseBytes = base64ToBytes(response.contentBase64); - contentHash = await hash(responseBytes); - - this.database.updateDocumentMetadata( - { - documentId: response.documentId, - parentVersionId: response.vaultUpdateId, - hash: contentHash, - remoteRelativePath: response.relativePath - }, - document - ); - - await this.operations.write( - actualPath, - originalContentBytes, - responseBytes - ); - await this.updateCache( - response.vaultUpdateId, - responseBytes, - actualPath - ); - } else { - this.database.updateDocumentMetadata( - { - documentId: response.documentId, - parentVersionId: response.vaultUpdateId, - hash: contentHash, - remoteRelativePath: response.relativePath - }, - document - ); - await this.updateCache( - response.vaultUpdateId, - originalContentBytes, - actualPath - ); - } - - this.database.addSeenUpdateId(response.vaultUpdateId); - } - - private getHistoryEntryForSkippedOversizedFile( - sizeInBytes: number, - relativePath: RelativePath - ): CommonHistoryEntry | undefined { - const sizeInMB = Math.round(sizeInBytes / 1024 / 1024); - const { maxFileSizeMB } = this.settings.getSettings(); - if (sizeInMB > maxFileSizeMB) { - return { - status: SyncStatus.SKIPPED, - details: { - type: SyncType.SKIPPED, - relativePath - }, - message: `File size of ${sizeInMB} MB exceeds the maximum file size limit of ${maxFileSizeMB - } MB` - }; - } - } - - private async updateCache( - updateId: number, - contentBytes: Uint8Array, - filePath: RelativePath - ): Promise { - if ( - isFileTypeMergable( - filePath, - (await this.serverConfig.getConfig()).mergeableFileExtensions - ) && - !isBinary(contentBytes) - ) { - this.contentCache.put(updateId, contentBytes); - } - } - - private async applyRemoteDeleteLocally( - document: DocumentRecord, - response: DocumentVersion | DocumentUpdateResponse - ): Promise { - this.database.delete(document.relativePath); - this.database.updateDocumentMetadata( - { - documentId: response.documentId, - parentVersionId: response.vaultUpdateId, - hash: EMPTY_HASH, - remoteRelativePath: response.relativePath - }, - document - ); - - await this.operations.delete(document.relativePath); - - this.database.addSeenUpdateId(response.vaultUpdateId); - } -} diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 00000000..9e0474fd --- /dev/null +++ b/package-lock.json @@ -0,0 +1,6 @@ +{ + "name": "vault-link", + "lockfileVersion": 3, + "requires": true, + "packages": {} +} diff --git a/scripts/e2e.sh b/scripts/e2e.sh index f9e84a69..d3ebefb2 100755 --- a/scripts/e2e.sh +++ b/scripts/e2e.sh @@ -29,9 +29,9 @@ echo "Stopping existing server..." pkill -f "sync_server" 2>/dev/null || true sleep 1 -# Clean databases +# Clean databases (uses tmpfs via /dev/shm for zero disk I/O) echo "Cleaning databases..." -rm -rf databases +rm -rf /host/tmp/vaultlink-e2e-databases # Start the server in the background echo "Starting server..." diff --git a/sync-server/build.rs b/sync-server/build.rs index d5068697..53bd111b 100644 --- a/sync-server/build.rs +++ b/sync-server/build.rs @@ -1,5 +1,16 @@ -// generated by `sqlx migrate build-script` fn main() { // trigger recompilation when a new migration is added println!("cargo:rerun-if-changed=migrations"); + + // Ensure the history-ui dist directory exists so rust-embed can compile + // even when the frontend hasn't been built yet. + let dist_path = std::path::Path::new("../frontend/history-ui/dist"); + if !dist_path.exists() { + std::fs::create_dir_all(dist_path).expect("Failed to create history-ui dist directory"); + std::fs::write( + dist_path.join("index.html"), + "

Run npm run build -w history-ui first.

", + ) + .expect("Failed to write placeholder index.html"); + } } diff --git a/sync-server/src/app_state/database/migrations/20260314000000_add_idempotency_key.sql b/sync-server/src/app_state/database/migrations/20260314000000_add_idempotency_key.sql new file mode 100644 index 00000000..f3ee8dd3 --- /dev/null +++ b/sync-server/src/app_state/database/migrations/20260314000000_add_idempotency_key.sql @@ -0,0 +1,2 @@ +CREATE INDEX IF NOT EXISTS idx_documents_document_id +ON documents (document_id, vault_update_id); diff --git a/sync-server/src/app_state/websocket/utils.rs b/sync-server/src/app_state/websocket/utils.rs index ce8205fa..24bc287a 100644 --- a/sync-server/src/app_state/websocket/utils.rs +++ b/sync-server/src/app_state/websocket/utils.rs @@ -33,7 +33,7 @@ pub fn get_authenticated_handshake( let user = auth(state, handshake.token.trim(), vault_id)?; Ok(AuthenticatedWebSocketHandshake { handshake, user }) } - WebSocketClientMessage::CursorPositions(_) | WebSocketClientMessage::Ping {} => Err( + WebSocketClientMessage::CursorPositions(_) => Err( unauthenticated_error(anyhow::anyhow!("Expected a handshake message")), ), } diff --git a/sync-server/src/server.rs b/sync-server/src/server.rs index 48191893..0835e9b6 100644 --- a/sync-server/src/server.rs +++ b/sync-server/src/server.rs @@ -4,24 +4,27 @@ mod delete_document; mod device_id_header; mod fetch_document_version; mod fetch_document_version_content; +mod fetch_document_versions; mod fetch_latest_document_version; mod fetch_latest_documents; +mod fetch_vault_history; mod index; +mod list_vaults; mod ping; mod rate_limit; mod requests; mod responses; +mod restore_document_version; mod update_document; mod websocket; -use anyhow::{Context as _, Result, anyhow}; +use anyhow::{Context as _, Result}; use auth::auth_middleware; use axum::{ Router, extract::{DefaultBodyLimit, Request}, http::{self, HeaderValue, Method}, middleware, - response::IntoResponse, routing::{IntoMakeService, delete, get, post, put}, }; use device_id_header::DEVICE_ID_HEADER_NAME; @@ -52,7 +55,7 @@ pub async fn create_server(config: Config) -> Result<()> { let server_config = app_state.config.server.clone(); - let app = Router::new() + let mut app = Router::new() .nest("/", get_authed_routes(app_state.clone())) .route("/", get(index::index)) .route("/assets/*path", get(index::spa_assets)) @@ -155,6 +158,10 @@ fn get_authed_routes(app_state: AppState) -> Router { "/vaults/:vault_id/documents/:document_id/text", put(update_document::update_text), ) + .route( + "/vaults/:vault_id/documents/:document_id/versions", + get(fetch_document_versions::fetch_document_versions), + ) .route( "/vaults/:vault_id/documents/:document_id/versions/:vault_update_id", get(fetch_document_version::fetch_document_version), @@ -167,6 +174,14 @@ fn get_authed_routes(app_state: AppState) -> Router { "/vaults/:vault_id/documents/:document_id", delete(delete_document::delete_document), ) + .route( + "/vaults/:vault_id/documents/:document_id/restore", + post(restore_document_version::restore_document_version), + ) + .route( + "/vaults/:vault_id/history", + get(fetch_vault_history::fetch_vault_history), + ) .layer(middleware::from_fn_with_state(app_state, auth_middleware)) } diff --git a/sync-server/src/server/fetch_document_versions.rs b/sync-server/src/server/fetch_document_versions.rs new file mode 100644 index 00000000..46d0e073 --- /dev/null +++ b/sync-server/src/server/fetch_document_versions.rs @@ -0,0 +1,42 @@ +use axum::{ + Json, + extract::{Path, State}, +}; +use log::debug; +use serde::Deserialize; + +use crate::{ + app_state::{ + AppState, + database::models::{DocumentId, DocumentVersionWithoutContent, VaultId}, + }, + errors::{SyncServerError, server_error}, + utils::normalize::normalize, +}; + +#[derive(Deserialize)] +pub struct FetchDocumentVersionsPathParams { + #[serde(deserialize_with = "normalize")] + vault_id: VaultId, + + document_id: DocumentId, +} + +#[axum::debug_handler] +pub async fn fetch_document_versions( + Path(FetchDocumentVersionsPathParams { + vault_id, + document_id, + }): Path, + State(state): State, +) -> Result>, SyncServerError> { + debug!("Fetching all versions for document `{document_id}` in vault `{vault_id}`"); + + let versions = state + .database + .get_document_versions(&vault_id, &document_id, None) + .await + .map_err(server_error)?; + + Ok(Json(versions)) +} diff --git a/sync-server/src/server/fetch_vault_history.rs b/sync-server/src/server/fetch_vault_history.rs new file mode 100644 index 00000000..42cceaa6 --- /dev/null +++ b/sync-server/src/server/fetch_vault_history.rs @@ -0,0 +1,70 @@ +use axum::{ + Json, + extract::{Path, Query, State}, +}; +use log::debug; +use serde::Deserialize; + +use super::responses::VaultHistoryResponse; +use crate::{ + app_state::{ + AppState, + database::models::{VaultId, VaultUpdateId}, + }, + errors::{SyncServerError, client_error, server_error}, + utils::normalize::normalize, +}; + +const DEFAULT_LIMIT: i64 = 50; +const MAX_LIMIT: i64 = 500; + +#[derive(Deserialize)] +pub struct FetchVaultHistoryPathParams { + #[serde(deserialize_with = "normalize")] + vault_id: VaultId, +} + +#[derive(Deserialize)] +pub struct QueryParams { + limit: Option, + before_update_id: Option, +} + +#[axum::debug_handler] +pub async fn fetch_vault_history( + Path(FetchVaultHistoryPathParams { vault_id }): Path, + Query(QueryParams { + limit, + before_update_id, + }): Query, + State(state): State, +) -> Result, SyncServerError> { + if let Some(id) = before_update_id + && id <= 0 + { + return Err(client_error(anyhow::anyhow!( + "before_update_id must be a positive integer" + ))); + } + + let limit = limit.unwrap_or(DEFAULT_LIMIT).clamp(1, MAX_LIMIT); + + debug!( + "Fetching vault history for vault `{vault_id}` (limit={limit}, before={before_update_id:?})" + ); + + // Fetch one extra row to determine if there are more results + let mut versions = state + .database + .get_vault_history(&vault_id, limit + 1, before_update_id, None) + .await + .map_err(server_error)?; + + #[allow(clippy::cast_sign_loss)] // limit is clamped to [1, 500] above + let has_more = versions.len() > limit as usize; + if has_more { + versions.pop(); + } + + Ok(Json(VaultHistoryResponse { versions, has_more })) +} diff --git a/sync-server/src/server/index.rs b/sync-server/src/server/index.rs index 64b053f7..ca8f38ff 100644 --- a/sync-server/src/server/index.rs +++ b/sync-server/src/server/index.rs @@ -1,7 +1,77 @@ -use axum::response::{Html, IntoResponse}; +use axum::{ + body::Body, + extract::{Path, State}, + http::{StatusCode, header}, + response::{Html, IntoResponse, Response}, +}; +use log::warn; +use rust_embed::Embed; -pub async fn index() -> impl IntoResponse { - const HTML_CONTENT: &str = include_str!("./assets/index.html"); - let html_content = HTML_CONTENT; - Html(html_content) +use crate::app_state::AppState; + +#[derive(Embed)] +#[folder = "../frontend/history-ui/dist/"] +struct HistoryUiAssets; + +pub async fn index(State(_state): State) -> impl IntoResponse { + if let Some(content) = HistoryUiAssets::get("index.html") { + Html( + std::str::from_utf8(content.data.as_ref()) + .inspect_err(|e| warn!("Embedded index.html is not valid UTF-8: {e}")) + .unwrap_or("

VaultLink

") + .to_owned(), + ) + .into_response() + } else { + warn!("No embedded index.html found — history UI may not have been built"); + Html("

VaultLink server

".to_owned()).into_response() + } +} + +pub async fn spa_assets(Path(path): Path) -> impl IntoResponse { + // The route is /assets/*path so path is relative to assets/. + // The embedded files include the assets/ prefix from the dist directory. + let full_path = format!("assets/{path}"); + if let Some(content) = HistoryUiAssets::get(&full_path) { + let mime = mime_guess::from_path(&full_path).first_or_octet_stream(); + return Response::builder() + .status(StatusCode::OK) + .header(header::CONTENT_TYPE, mime.as_ref()) + .body(Body::from(content.data.to_vec())) + .unwrap_or_else(|_| { + Response::builder() + .status(StatusCode::INTERNAL_SERVER_ERROR) + .body(Body::empty()) + .unwrap_or_else(|_| Response::new(Body::empty())) + }); + } + + // Asset paths must match an embedded file — no SPA fallback. + // Serving index.html here would return 200 with text/html for missing + // .css/.js files, causing the browser to silently ignore the content. + Response::builder() + .status(StatusCode::NOT_FOUND) + .body(Body::from("Not found")) + .unwrap_or_else(|_| Response::new(Body::from("Not found"))) +} + +/// SPA fallback for production: serves index.html for client-side routes +/// (e.g. `/documents/123`). +pub async fn spa_fallback() -> impl IntoResponse { + match HistoryUiAssets::get("index.html") { + Some(content) => Response::builder() + .status(StatusCode::OK) + .header(header::CONTENT_TYPE, "text/html") + .body(Body::from(content.data.to_vec())) + .unwrap_or_else(|_| { + Response::builder() + .status(StatusCode::INTERNAL_SERVER_ERROR) + .body(Body::empty()) + .unwrap_or_else(|_| Response::new(Body::empty())) + }), + None => Response::builder() + .status(StatusCode::NOT_FOUND) + .body(Body::from("Not found")) + .unwrap_or_else(|_| Response::new(Body::from("Not found"))), + } } diff --git a/sync-server/src/server/restore_document_version.rs b/sync-server/src/server/restore_document_version.rs new file mode 100644 index 00000000..f759fa59 --- /dev/null +++ b/sync-server/src/server/restore_document_version.rs @@ -0,0 +1,147 @@ +use anyhow::anyhow; +use axum::{ + Extension, Json, + extract::{Path, State}, +}; +use axum_extra::TypedHeader; +use log::{debug, info}; +use serde::Deserialize; + +use super::device_id_header::DeviceIdHeader; +use crate::{ + app_state::{ + AppState, + database::models::{ + DocumentId, DocumentVersionWithoutContent, StoredDocumentVersion, VaultId, + VaultUpdateId, + }, + }, + config::user_config::User, + errors::{SyncServerError, client_error, not_found_error, server_error, write_transaction_error}, + utils::{find_first_available_path::find_first_available_path, normalize::normalize}, +}; + +#[derive(Deserialize)] +pub struct RestorePathParams { + #[serde(deserialize_with = "normalize")] + vault_id: VaultId, + + document_id: DocumentId, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct RestoreDocumentVersionRequest { + pub vault_update_id: VaultUpdateId, +} + +#[axum::debug_handler] +pub async fn restore_document_version( + Path(RestorePathParams { + vault_id, + document_id, + }): Path, + Extension(user): Extension, + TypedHeader(device_id): TypedHeader, + State(state): State, + Json(request): Json, +) -> Result, SyncServerError> { + debug!( + "Restoring document `{document_id}` in vault `{vault_id}` to version `{}`", + request.vault_update_id + ); + + if request.vault_update_id <= 0 { + return Err(client_error(anyhow!( + "Invalid vault_update_id: `{}`", + request.vault_update_id + ))); + } + + let mut transaction = state + .database + .create_write_transaction(&vault_id) + .await + .map_err(write_transaction_error)?; + + let target_version = state + .database + .get_document_version(&vault_id, request.vault_update_id, Some(&mut *transaction)) + .await + .map_err(server_error)? + .ok_or_else(|| { + not_found_error(anyhow!("Version `{}` not found", request.vault_update_id)) + })?; + + if target_version.document_id != document_id { + transaction.rollback().await.map_err(server_error)?; + return Err(not_found_error(anyhow!( + "Version `{}` does not belong to document `{document_id}`", + request.vault_update_id, + ))); + } + + if target_version.is_deleted { + transaction.rollback().await.map_err(server_error)?; + return Err(client_error(anyhow!( + "Cannot restore to a deleted version `{}`", + request.vault_update_id, + ))); + } + + let existing = state + .database + .get_latest_non_deleted_document_by_path( + &vault_id, + &target_version.relative_path, + Some(&mut *transaction), + ) + .await + .map_err(server_error)?; + + let restore_path = if let Some(existing_doc) = &existing + && existing_doc.document_id != document_id + { + find_first_available_path( + &vault_id, + &target_version.relative_path, + &state.database, + &mut transaction, + ) + .await + .map_err(server_error)? + } else { + target_version.relative_path.clone() + }; + + let last_update_id = state + .database + .get_max_update_id_in_vault(&vault_id, Some(&mut *transaction)) + .await + .map_err(server_error)?; + + let new_version = StoredDocumentVersion { + vault_update_id: last_update_id + 1, + document_id, + relative_path: restore_path, + content: target_version.content, + updated_date: chrono::Utc::now(), + is_deleted: false, + user_id: user.name.clone(), + device_id: device_id.0.clone(), + has_been_merged: false, + }; + + state + .database + .insert_document_version(&vault_id, &new_version, Some(transaction)) + .await + .map_err(server_error)?; + + info!( + "Restored document `{document_id}` to version `{}` as new version `{}`", + request.vault_update_id, new_version.vault_update_id + ); + + Ok(Json(new_version.into())) +}