diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..7d56669b --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,27 @@ +# To get started with Dependabot version updates, you'll need to specify which +# package ecosystems to update and where the package manifests are located. +# Please see the documentation for all configuration options: +# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file + +version: 2 +updates: + - package-ecosystem: "npm" + directories: ["/frontend", "/docs"] + schedule: + interval: "daily" + + - package-ecosystem: "docker" + directories: ["**"] + schedule: + interval: "daily" + + - package-ecosystem: "cargo" + directories: ["**"] + schedule: + interval: "daily" + + # Disable this for security reasons + # - package-ecosystem: "github-actions" + # directories: ["**"] + # schedule: + # interval: "daily" diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml new file mode 100644 index 00000000..fc1b1c99 --- /dev/null +++ b/.github/workflows/check.yml @@ -0,0 +1,36 @@ +name: Check + +on: + push: + branches: ["main"] + pull_request: + branches: ["main"] + workflow_dispatch: + +env: + CARGO_TERM_COLOR: always + RUSTFLAGS: "-Dwarnings" + +jobs: + build: + runs-on: self-hosted + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Setup Node.js environment + uses: actions/setup-node@v4.2.0 + with: + node-version: "25.x" + check-latest: true + + - name: Setup Rust toolchain + uses: dtolnay/rust-toolchain@stable + with: + toolchain: "1.92.0" + components: clippy, rustfmt + + - name: Lint & test + run: scripts/check.sh diff --git a/.github/workflows/deploy-docs.yml b/.github/workflows/deploy-docs.yml new file mode 100644 index 00000000..bb25e463 --- /dev/null +++ b/.github/workflows/deploy-docs.yml @@ -0,0 +1,58 @@ +name: Deploy Documentation + +on: + push: + branches: + - main + paths: + - "docs/**" + - ".github/workflows/deploy-docs.yml" + workflow_dispatch: + +permissions: + contents: read + pages: write + id-token: write + +concurrency: + group: pages + cancel-in-progress: false + +jobs: + build: + runs-on: self-hosted + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Setup Node.js environment + uses: actions/setup-node@v4.2.0 + with: + node-version: "25.x" + check-latest: true + + - name: Setup Pages + uses: actions/configure-pages@v4 + + - name: Build docs + run: scripts/build-docs.sh + + - name: Upload artifact + uses: actions/upload-pages-artifact@v3 + with: + path: docs/.vitepress/dist + + deploy: + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + needs: build + runs-on: self-hosted + name: Deploy + steps: + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v4 diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml new file mode 100644 index 00000000..98dbfc1f --- /dev/null +++ b/.github/workflows/e2e.yml @@ -0,0 +1,72 @@ +name: E2E tests + +on: + push: + branches: ["main"] + pull_request: + branches: ["main"] + schedule: + - cron: "0 * * * *" + workflow_dispatch: + +concurrency: + group: e2e-tests + cancel-in-progress: false + +env: + RUSTFLAGS: "-Dwarnings" + +jobs: + build: + runs-on: self-hosted + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Setup Node.js environment + uses: actions/setup-node@v4.2.0 + with: + node-version: "25.x" + check-latest: true + + - name: Setup Rust toolchain + uses: dtolnay/rust-toolchain@stable + with: + toolchain: "1.92.0" + components: clippy, rustfmt + + - name: Setup rust + run: | + which sqlx || cargo install sqlx-cli + cd sync-server + sqlx database create --database-url sqlite://db.sqlite3 + sqlx migrate run --source src/app_state/database/migrations --database-url sqlite://db.sqlite3 + + - name: E2E tests + run: | + cd sync-server + cargo run config-e2e.yml --color never & + SERVER_PID=$! + cd .. + + scripts/e2e.sh 8 + EXIT_CODE=$? + + kill $SERVER_PID 2>/dev/null || true + wait $SERVER_PID 2>/dev/null || true + + exit $EXIT_CODE + + - name: Upload e2e logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: e2e-logs + path: logs/ + retention-days: 30 + + - name: Cleanup + if: always() + run: scripts/clean-up.sh diff --git a/.github/workflows/publish-cli-docker.yml b/.github/workflows/publish-cli-docker.yml new file mode 100644 index 00000000..10a7e8ba --- /dev/null +++ b/.github/workflows/publish-cli-docker.yml @@ -0,0 +1,67 @@ +name: Publish CLI + +on: + push: + branches: ["main"] + tags: ["*"] + pull_request: + branches: ["main"] + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }}-cli + +jobs: + publish-docker: + runs-on: self-hosted + + permissions: + contents: read + packages: write + id-token: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Install cosign + uses: sigstore/cosign-installer@59acb6260d9c0ba8f4a2f9d9b48431a222b68e20 #v3.5.0 + with: + cosign-release: "v2.2.4" + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0 + + - name: Log into registry ${{ env.REGISTRY }} + uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934 # v5.0.0 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + + - name: Build and push Docker image + id: build-and-push + uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v5.0.0 + with: + context: frontend + file: frontend/local-client-cli/Dockerfile + platforms: linux/amd64 + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Sign the published Docker image + env: + TAGS: ${{ steps.meta.outputs.tags }} + DIGEST: ${{ steps.build-and-push.outputs.digest }} + run: echo "${TAGS}" | xargs -I {} cosign sign --yes {}@${DIGEST} diff --git a/.github/workflows/publish-plugin.yml b/.github/workflows/publish-plugin.yml new file mode 100644 index 00000000..452bc601 --- /dev/null +++ b/.github/workflows/publish-plugin.yml @@ -0,0 +1,59 @@ +name: Publish Obsidian plugin + +on: + push: + tags: ["*"] + +env: + CARGO_TERM_COLOR: always + +jobs: + publish-plugin: + runs-on: self-hosted + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Setup Node.js environment + uses: actions/setup-node@v4.2.0 + with: + node-version: "25.x" + check-latest: true + + - name: Build plugin + run: | + cd frontend + npm ci + npm run build + + - name: Setup Rust toolchain + uses: dtolnay/rust-toolchain@stable + with: + toolchain: "1.92.0" + components: clippy, rustfmt + + - name: Install cross-compilation tools + run: | + apt update + apt install -y gcc-aarch64-linux-gnu musl-tools gcc-mingw-w64-x86-64 + + - name: Build Linux and Windows binaries + run: ./scripts/build-sync-server-binaries.sh + + - name: Create release + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + tag="${GITHUB_REF#refs/tags/}" + + mkdir -p release + cp frontend/obsidian-plugin/dist/* release/ + cp sync-server/artifacts/sync-server-* release/ + cd release + + gh release create "$tag" \ + --title="$tag" \ + --draft \ + * diff --git a/.github/workflows/publish-server-docker.yml b/.github/workflows/publish-server-docker.yml new file mode 100644 index 00000000..4a97a9e6 --- /dev/null +++ b/.github/workflows/publish-server-docker.yml @@ -0,0 +1,92 @@ +name: Publish server Docker image + +on: + push: + branches: ["main"] + tags: ["*"] + pull_request: + branches: ["main"] + +env: + # Use docker.io for Docker Hub if empty + REGISTRY: ghcr.io + # github.repository as / + IMAGE_NAME: ${{ github.repository }} + +jobs: + publish-docker: + runs-on: self-hosted + + permissions: + contents: read + packages: write + # This is used to complete the identity challenge + # with sigstore/fulcio. + id-token: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + # Install the cosign tool + # https://github.com/sigstore/cosign-installer + - name: Install cosign + if: github.ref_type == 'tag' + uses: sigstore/cosign-installer@59acb6260d9c0ba8f4a2f9d9b48431a222b68e20 #v3.5.0 + with: + cosign-release: "v2.2.4" + + # Set up BuildKit Docker container builder to be able to build + # multi-platform images and export cache + # https://github.com/docker/setup-buildx-action + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0 + + # Login against a Docker registry + # https://github.com/docker/login-action + - name: Log into registry ${{ env.REGISTRY }} + if: github.ref_type == 'tag' + uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + # Extract metadata (tags, labels) for Docker + # https://github.com/docker/metadata-action + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934 # v5.0.0 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + + # Build and push Docker image with Buildx + # https://github.com/docker/build-push-action + - name: Build and push Docker image + id: build-and-push + uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v5.0.0 + with: + context: sync-server + platforms: linux/amd64,linux/arm64 + push: ${{ github.ref_type == 'tag' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + + # Sign the resulting Docker image digest. + # This will only write to the public Rekor transparency log when the Docker + # repository is public to avoid leaking data. If you would like to publish + # transparency data even for private images, pass --force to cosign below. + # https://github.com/sigstore/cosign + - name: Sign the published Docker image + if: ${{ github.ref_type == 'tag' }} + env: + # https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions#using-an-intermediate-environment-variable + TAGS: ${{ steps.meta.outputs.tags }} + DIGEST: ${{ steps.build-and-push.outputs.digest }} + # This step uses the identity token to provision an ephemeral certificate + # against the sigstore community Fulcio instance. + run: echo "${TAGS}" | xargs -I {} cosign sign --yes {}@${DIGEST} diff --git a/frontend/deterministic-tests/README.md b/frontend/deterministic-tests/README.md deleted file mode 100644 index 6fa2848c..00000000 --- a/frontend/deterministic-tests/README.md +++ /dev/null @@ -1,118 +0,0 @@ -# Deterministic Tests - -Scripted multi-client (with an in-memory filesystem) sync tests that run against a real server. Each test defines a sequence of file operations, sync/server controls, and assertions to exercise a specific conflict or edge case. - -Complements the fuzz-based E2E tests (`test-client`): fuzz tests discover bugs through random operations; deterministic tests pin down exact reproduction sequences for known scenarios. - -## How it works - -Each test is a `TestDefinition`: a client count and an ordered list of steps. The test name is derived from the registry key (which matches the file name). The `TestRunner` spins up N `DeterministicAgent` instances (each wrapping a real `SyncClient` with an `InMemoryFileSystem`) pointed at a shared vault on the server, then executes steps one by one. - -Tests that don't pause the server share a single server process (vault-name isolation). Tests that use `pause-server`/`resume-server` (SIGSTOP/SIGCONT) each get a dedicated server, since SIGSTOP freezes the entire process. - -The runner executes two sequential phases: regular tests on the shared server, then pause-server tests on dedicated servers. Within each phase tests run in parallel up to a concurrency limit. - -## Step types - -Clients always start with syncing disabled. - -**File operations** (per-client, fire-and-forget — sync is enqueued but not awaited): - -- `create`, `update`, `rename`, `delete` -- `rename-next-write` — arm a deferred rename that fires the next time the given path is written. Lets a test race a user-rename against an in-flight remote create that's about to land at the same path. - -**Sync control:** - -- `sync` — wait for a specific client or all clients to finish pending operations -- `barrier` — retry until all clients converge to identical file state (60s timeout) -- `enable-sync` / `disable-sync` — simulate going online/offline -- `reset` — reset a client's tracked sync state (keeps disk files); equivalent to a forced re-handshake on next enable -- `sleep` — wall-clock pause; use sparingly, prefer `barrier` / `sync` - -**WebSocket control** (per-client): - -- `pause-websocket` / `resume-websocket` — buffer/release WebSocket messages for a specific client - -**Server control:** - -- `pause-server` / `resume-server` — SIGSTOP/SIGCONT the server process -- `resume-server-until-history-then-pause` — resume the server, wait until a specific client observes a matching history entry (`CREATE`/`UPDATE`/`DELETE` for a path), then re-pause. Used to land exactly one operation across the wire. - -**Fault injection** (per-client): - -- `drop-next-create-response` — arm a one-shot interceptor that lets the next `POST /documents` reach the server (commit happens) but throws `SyncResetError` before the client sees the response, simulating connection loss after server commit. -- `wait-for-dropped-create-response` — wait until the armed drop has fired. - -**Assertions:** - -- `assert-consistent` — all clients have identical files; optionally takes a custom `verify(state: AssertableState)` callback - -## Running - -```sh -# Build server first -cd sync-server && cargo build --release && cd - - -# Run all tests -cd frontend && npm run build -w sync-client && npm run test -w deterministic-tests - -# Filter by name -npm run test -w deterministic-tests -- --filter=rename - -# Control parallelism (default: number of CPU cores) -npm run test -w deterministic-tests -- -j 4 -``` - -## Adding a test - -1. Create `src/tests/my-scenario.test.ts`: - -```typescript -import type { TestDefinition } from "../test-definition"; - -export const myScenarioTest: TestDefinition = { - description: - "Client 0 creates A.md offline. After syncing, both clients should have the file.", - clients: 2, - steps: [ - { type: "create", client: 0, path: "A.md", content: "hello" }, - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - { - type: "assert-consistent", - verify: (s) => { - s.assertFileCount(1).assertContent("A.md", "hello"); - } - } - ] -}; -``` - -The `verify` callback receives an `AssertableState` object with chainable assertion methods: - -```typescript -s.assertFileCount(n); // exact file count -s.assertFileExists("path"); // file must exist -s.assertFileNotExists("path"); // file must not exist -s.assertContent("path", "expected"); // exact content match -s.assertContains("path", "a", "b"); // all substrings present in file -s.assertContainsAny("path", "a", "b"); // at least one substring present -s.assertAnyFileContains("text"); // substring present in some file -s.assertNoFileContains("text"); // substring absent from every file -s.assertSubstringCount("path", "x", 3); // substring appears exactly N times -s.assertContentInAtMostOneFile("text"); // no duplicate content -s.ifFileExists("path", (s) => { /* … */ }); // conditional block -s.getContent("path"); // raw content (or "" if missing) -``` - -2. Register it in `src/test-registry.ts`: - -```typescript -import { myScenarioTest } from "./tests/my-scenario.test"; - -const TESTS = { - // ... - "my-scenario": myScenarioTest -}; -``` diff --git a/frontend/deterministic-tests/package.json b/frontend/deterministic-tests/package.json deleted file mode 100644 index 4bd82c74..00000000 --- a/frontend/deterministic-tests/package.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "name": "deterministic-tests", - "version": "0.14.0", - "private": true, - "bin": { - "deterministic-tests": "./dist/cli.js" - }, - "scripts": { - "dev": "webpack watch --mode development", - "build": "webpack --mode production", - "test": "npm run build && node dist/cli.js" - }, - "devDependencies": { - "commander": "^14.0.2", - "@types/node": "^25.0.2", - "sync-client": "file:../sync-client", - "ts-loader": "^9.5.4", - "tslib": "2.8.1", - "typescript": "5.9.3", - "webpack": "^5.103.0", - "webpack-cli": "^6.0.1" - } -} diff --git a/frontend/deterministic-tests/src/cli.ts b/frontend/deterministic-tests/src/cli.ts deleted file mode 100644 index 6e15cac0..00000000 --- a/frontend/deterministic-tests/src/cli.ts +++ /dev/null @@ -1,243 +0,0 @@ -import { TestRunner } from "./test-runner"; -import { ServerControl } from "./server-control"; -import { ServerManager } from "./server-manager"; -import { PrefixedLogger } from "./prefixed-logger"; -import { TESTS } from "./test-registry"; -import type { TestDefinition, TestResult } from "./test-definition"; -import { parseArgs } from "./parse-args"; -import { runWithConcurrency } from "./run-with-concurrency"; -import { TOKEN, SERVER_BINARY_PATH, CONFIG_PATH } from "./consts"; -import * as path from "node:path"; -import * as fs from "node:fs"; -import { debugging, Logger } from "sync-client"; - -const logger = new Logger(); -debugging.logToConsole(logger, { useColors: true }); - -process.on("unhandledRejection", (reason) => { - logger.error(`Unhandled Rejection: ${reason}`); - process.exit(1); -}); - -process.on("uncaughtException", (error) => { - logger.error(`Uncaught Exception: ${error}`); - process.exit(1); -}); - -const serverManager = new ServerManager(logger); -serverManager.installSignalHandlers(); - -function testUsesPauseServer(test: TestDefinition): boolean { - return test.steps.some( - (step) => - step.type === "pause-server" || - step.type === "resume-server" || - step.type === "resume-server-until-history-then-pause" - ); -} - -/** - * Walk up from the CLI binary's location until we find a directory - * containing `sync-server/` and `frontend/`. - */ -function findProjectRoot(): string { - let dir = path.dirname(__filename); - const root = path.parse(dir).root; - while (dir !== root) { - if ( - fs.existsSync(path.join(dir, "sync-server")) && - fs.existsSync(path.join(dir, "frontend")) - ) { - return dir; - } - dir = path.dirname(dir); - } - throw new Error( - `Could not locate project root (no ancestor of ${__filename} contains both 'sync-server' and 'frontend')` - ); -} - -interface NamedTestResult { - name: string; - result: TestResult; -} - -async function runSharedServerTest( - name: string, - test: TestDefinition, - sharedServer: ServerControl -): Promise { - const testLogger = new PrefixedLogger(logger, name); - const runner = new TestRunner( - sharedServer, - testLogger, - TOKEN, - sharedServer.remoteUri - ); - const result = await runner.runTest(name, test); - if (result.success) { - logger.info(`PASSED: ${name} (${result.duration}ms)`); - } else { - logger.error(`FAILED: ${name} - ${result.error}`); - } - return { name, result }; -} - -/** - * Run a test with its own dedicated server (for tests that use pause-server). - * SIGSTOP/SIGCONT affects the entire server process, so these tests need - * isolated servers to avoid interfering with other tests. - */ -async function runDedicatedServerTest( - name: string, - test: TestDefinition, - serverPath: string, - configPath: string -): Promise { - const testLogger = new PrefixedLogger(logger, name); - const server = new ServerControl(serverPath, configPath, testLogger); - serverManager.track(server); - - try { - await server.start(); - const runner = new TestRunner( - server, - testLogger, - TOKEN, - server.remoteUri - ); - const result = await runner.runTest(name, test); - if (result.success) { - logger.info(`PASSED: ${name} (${result.duration}ms)`); - } else { - logger.error(`FAILED: ${name} - ${result.error}`); - } - return { name, result }; - } finally { - try { - await server.stop(); - } catch { - // best-effort cleanup - } - serverManager.untrack(server); - } -} - -async function main(): Promise { - const projectRoot = findProjectRoot(); - const serverPath = path.join(projectRoot, SERVER_BINARY_PATH); - if (!fs.existsSync(serverPath)) { - logger.error(`Server binary not found at: ${serverPath}`); - process.exit(1); - } - - const configPath = path.join(projectRoot, CONFIG_PATH); - if (!fs.existsSync(configPath)) { - logger.error(`Config file not found at: ${configPath}`); - process.exit(1); - } - - const { filter, concurrency } = parseArgs(process.argv); - - const testsToRun: [string, TestDefinition][] = []; - for (const [key, test] of Object.entries(TESTS)) { - if (test) { - if ( - filter !== undefined && - filter.length > 0 && - !key.includes(filter) - ) { - continue; - } - testsToRun.push([key, test]); - } - } - - if (testsToRun.length === 0) { - logger.error( - filter !== undefined && filter.length > 0 - ? `No tests matched filter "${filter}"` - : "No tests found" - ); - process.exit(1); - } - - const regularTests = testsToRun.filter(([, t]) => !testUsesPauseServer(t)); - const pauseTests = testsToRun.filter(([, t]) => testUsesPauseServer(t)); - - logger.info(`Server: ${serverPath}`); - logger.info(`Config: ${configPath}`); - logger.info( - `Tests: ${testsToRun.length} total (${regularTests.length} regular, ${pauseTests.length} server-pause)` - ); - logger.info(`Concurrency: ${concurrency}`); - - const allResults: NamedTestResult[] = []; - - if (regularTests.length > 0) { - logger.info( - `\n--- Running ${regularTests.length} regular tests (shared server, concurrency ${concurrency}) ---` - ); - const sharedServer = new ServerControl(serverPath, configPath, logger); - serverManager.track(sharedServer); - - try { - await sharedServer.start(); - - const results = await runWithConcurrency( - regularTests, - concurrency, - async ([name, test]) => - runSharedServerTest(name, test, sharedServer) - ); - - allResults.push(...results); - } finally { - try { - await sharedServer.stop(); - } catch (error) { - logger.warn( - `Error stopping shared server: ${error instanceof Error ? error.message : String(error)}` - ); - } - serverManager.untrack(sharedServer); - } - } - - if (pauseTests.length > 0) { - logger.info( - `\n--- Running ${pauseTests.length} server-pause tests (dedicated servers, concurrency ${concurrency}) ---` - ); - - const results = await runWithConcurrency( - pauseTests, - concurrency, - async ([name, test]) => - runDedicatedServerTest(name, test, serverPath, configPath) - ); - - allResults.push(...results); - } - - const passed = allResults.filter((r) => r.result.success); - const failed = allResults.filter((r) => !r.result.success); - - logger.info( - `\n--- Results: ${passed.length}/${allResults.length} passed ---` - ); - - if (failed.length > 0) { - for (const { name, result } of failed) { - logger.error(` FAILED: ${name}: ${result.error}`); - } - process.exit(1); - } else { - logger.info("All tests passed!"); - process.exit(0); - } -} - -main().catch((err: unknown) => { - logger.error(`Unexpected error: ${err}`); - process.exit(1); -}); diff --git a/frontend/deterministic-tests/src/consts.ts b/frontend/deterministic-tests/src/consts.ts deleted file mode 100644 index d9a2498f..00000000 --- a/frontend/deterministic-tests/src/consts.ts +++ /dev/null @@ -1,17 +0,0 @@ -export const TOKEN = "test-token-change-me"; -export const SERVER_BINARY_PATH = "sync-server/target/release/sync_server"; -export const CONFIG_PATH = "sync-server/config-e2e.yml"; - -export const STOP_TIMEOUT_MS = 5_000; -export const CONVERGENCE_TIMEOUT_MS = 60_000; -export const CONVERGENCE_RETRY_DELAY_MS = 500; -export const AGENT_INIT_TIMEOUT_MS = 30_000; -export const IS_SYNC_ENABLED_BY_DEFAULT = false; - -export const WAIT_TIMEOUT_MS = 60_000; -export const WEBSOCKET_CONNECT_TIMEOUT_MS = 10_000; -export const WEBSOCKET_POLL_INTERVAL_MS = 50; - -export const SERVER_READY_POLL_INTERVAL_MS = 100; -export const SERVER_READY_MAX_ATTEMPTS = 50; -export const SERVER_START_MAX_ATTEMPTS = 5; diff --git a/frontend/deterministic-tests/src/deterministic-agent.ts b/frontend/deterministic-tests/src/deterministic-agent.ts deleted file mode 100644 index b32b01c2..00000000 --- a/frontend/deterministic-tests/src/deterministic-agent.ts +++ /dev/null @@ -1,483 +0,0 @@ -import type { - HistoryEntry, - StoredDatabase, - SyncSettings, - RelativePath, - TextWithCursors -} from "sync-client"; -import { - SyncClient, - SyncResetError, - debugging, - LogLevel, - utils -} from "sync-client"; -import { assert } from "./utils/assert"; -import { sleep } from "./utils/sleep"; -import { withTimeout } from "./utils/with-timeout"; -import { - IS_SYNC_ENABLED_BY_DEFAULT, - WAIT_TIMEOUT_MS, - WEBSOCKET_CONNECT_TIMEOUT_MS, - WEBSOCKET_POLL_INTERVAL_MS -} from "./consts"; -import { ManagedWebSocketFactory } from "./managed-websocket"; - -export class DeterministicAgent extends debugging.InMemoryFileSystem { - public readonly clientId: number; - private readonly logger: (msg: string) => void; - private client!: SyncClient; - private data: Partial<{ - settings: Partial; - database: Partial; - }> = {}; - private isSyncEnabled = IS_SYNC_ENABLED_BY_DEFAULT; - private readonly syncErrors: Error[] = []; - private readonly pendingSyncOperations = new Set>(); - private readonly wsFactory = new ManagedWebSocketFactory(); - private nextWriteRename: - | { - oldPath: RelativePath; - newPath: RelativePath; - } - | undefined; - private nextCreateResponseDrop: - | { - dropped: Promise; - resolveDropped: () => void; - } - | undefined; - - public constructor( - clientId: number, - initialSettings: Partial, - logger: (msg: string) => void - ) { - super(); - this.clientId = clientId; - this.logger = logger; - this.data.settings = { ...initialSettings }; - } - - public async init( - fetchImplementation: typeof globalThis.fetch - ): Promise { - this.client = await SyncClient.create({ - fs: this, - persistence: { - load: async () => this.data, - save: async (data) => void (this.data = data) - }, - fetch: this.wrapFetch(fetchImplementation), - webSocket: this.wsFactory.constructorFn - }); - - this.client.logger.onLogEmitted.add((line) => { - const prefix = `[Client ${this.clientId}]`; - switch (line.level) { - case LogLevel.ERROR: - this.logger(`${prefix} ERROR: ${line.message}`); - break; - case LogLevel.WARNING: - this.logger(`${prefix} WARN: ${line.message}`); - break; - case LogLevel.INFO: - this.logger(`${prefix} INFO: ${line.message}`); - break; - case LogLevel.DEBUG: - this.logger(`${prefix} DEBUG: ${line.message}`); - break; - } - }); - - await this.client.start(); - - const connectionCheck = await this.client.checkConnection(); - assert( - connectionCheck.isSuccessful, - `Client ${this.clientId} connection check failed` - ); - - if (this.isSyncEnabled) { - await this.waitForWebSocket(); - } - } - - public pauseWebSocket(): void { - this.log("Pausing WebSocket message delivery"); - this.wsFactory.pause(); - } - - public resumeWebSocket(): void { - this.log("Resuming WebSocket message delivery"); - this.wsFactory.resume(); - } - - public dropNextCreateResponse(): void { - assert( - this.nextCreateResponseDrop === undefined, - `Client ${this.clientId} already has a create response drop armed` - ); - let resolveDropped!: () => void; - const dropped = new Promise((resolve) => { - resolveDropped = resolve; - }); - this.nextCreateResponseDrop = { - dropped, - resolveDropped - }; - this.log("Armed next create response drop"); - } - - public async waitForDroppedCreateResponse(): Promise { - assert( - this.nextCreateResponseDrop !== undefined, - `Client ${this.clientId} has no create response drop armed` - ); - await withTimeout( - this.nextCreateResponseDrop.dropped, - WAIT_TIMEOUT_MS, - `Client ${this.clientId} timed out waiting for create response drop` - ); - this.log("Create response was dropped after server commit"); - } - - public async waitForHistoryEntry( - matches: (entry: HistoryEntry) => boolean, - onMatch?: (entry: HistoryEntry) => void - ): Promise { - const existing = this.client.getHistoryEntries().find(matches); - if (existing !== undefined) { - onMatch?.(existing); - return; - } - - await withTimeout( - new Promise((resolve) => { - const unsubscribe = this.client.onSyncHistoryUpdated.add(() => { - const entry = this.client - .getHistoryEntries() - .find(matches); - if (entry === undefined) { - return; - } - - unsubscribe(); - onMatch?.(entry); - resolve(); - }); - }), - WAIT_TIMEOUT_MS, - `Client ${this.clientId} timed out waiting for history entry` - ); - } - - public async waitForSync(): Promise { - this.log("Waiting for sync to complete..."); - // Drain agent-level sync operations first. These are the fire-and-forget - // promises from enqueueSync() that call into the SyncClient's methods. - // Without this, waitUntilFinished() might return before the SyncClient - // has even been told about the operation. - await this.drainPendingSyncOperations(); - await withTimeout( - this.client.waitUntilFinished(), - WAIT_TIMEOUT_MS, - `Client ${this.clientId} waitForSync timed out after ${WAIT_TIMEOUT_MS}ms` - ); - if (this.syncErrors.length > 0) { - const errors = this.syncErrors.splice(0); - throw new Error( - `Client ${this.clientId} had ${errors.length} sync error(s):\n${errors.map((e) => e.message).join("\n")}` - ); - } - this.log("Sync complete"); - } - - public async reset(): Promise { - this.log("Resetting client (clears tracked state, keeps disk files)"); - await this.drainPendingSyncOperations(); - await this.client.reset(); - if (this.isSyncEnabled) { - await this.waitForWebSocket(); - } - } - - public async disableSync(): Promise { - this.log("Disabling sync"); - // Drain pending enqueued operations before disabling so the SyncClient - // knows about all operations that were enqueued while sync was enabled. - await this.drainPendingSyncOperations(); - await this.client.setSetting("isSyncEnabled", false); - this.isSyncEnabled = false; - // Wait for in-flight operations to drain. Disabling sync triggers - // a reset, which aborts in-flight fetches with SyncResetError. - try { - await withTimeout( - this.client.waitUntilFinished(), - WAIT_TIMEOUT_MS, - `Client ${this.clientId} disableSync drain timed out` - ); - } catch (error) { - if (error instanceof Error && error.name === "SyncResetError") { - this.log("Disable sync drain interrupted by reset (expected)"); - } else { - throw error; - } - } - } - - public async enableSync(): Promise { - this.log("Enabling sync"); - await this.client.setSetting("isSyncEnabled", true); - this.isSyncEnabled = true; - await this.waitForWebSocket(); - } - - public async getFileContent(path: string): Promise { - const bytes = await this.read(path); - return new TextDecoder().decode(bytes); - } - - public renameNextWrite(oldPath: RelativePath, newPath: RelativePath): void { - assert( - this.nextWriteRename === undefined, - `Client ${this.clientId} already has a next-write rename armed` - ); - this.nextWriteRename = { oldPath, newPath }; - this.log(`Armed next write rename: ${oldPath} -> ${newPath}`); - } - - public async cleanup(): Promise { - this.log("Cleaning up..."); - // Guard against uninitialized client (init() failed partway). - // The class field uses `!:` so TS thinks this is always defined, - // but at runtime it can be undefined when init() throws partway. - const maybeClient = this.client as SyncClient | undefined; - if (maybeClient === undefined) { - this.log("Client not initialized, nothing to clean up"); - return; - } - try { - await this.drainPendingSyncOperations(); - await withTimeout( - this.client.waitUntilFinished(), - WAIT_TIMEOUT_MS, - `Client ${this.clientId} cleanup waitUntilFinished timed out` - ); - } catch (error) { - if (error instanceof Error && error.name === "SyncResetError") { - this.log(`Cleanup interrupted by reset (expected): ${error}`); - } else { - this.log(`Cleanup waitUntilFinished failed: ${error}`); - } - } - // Surface any background sync errors that arrived after the last - // waitForSync (e.g. between the final assert-consistent and here). - // Without this, regressions that fault the engine during the very - // last step of a test would be silently swallowed. - const pendingErrors = this.syncErrors.splice(0); - await this.client.destroy(); - this.log("Cleanup complete"); - if (pendingErrors.length > 0) { - throw new Error( - `Client ${this.clientId} had ${pendingErrors.length} background sync error(s) during cleanup:\n${pendingErrors.map((e) => e.message).join("\n")}` - ); - } - } - - public override async read(path: RelativePath): Promise { - await Promise.resolve(); - return super.read(path); - } - - public override async write( - path: RelativePath, - content: Uint8Array - ): Promise { - await Promise.resolve(); - const isNew = !this.files.has(path); - await super.write(path, content); - - if (this.isSyncEnabled && isNew) { - this.enqueueSync(async () => { - this.client.syncLocallyCreatedFile(path); - }); - } - - const nextWriteRename = this.nextWriteRename; - if ( - nextWriteRename !== undefined && - nextWriteRename.oldPath === path - ) { - this.nextWriteRename = undefined; - await super.rename( - nextWriteRename.oldPath, - nextWriteRename.newPath - ); - if (this.isSyncEnabled) { - this.enqueueSync(async () => { - this.client.syncLocallyUpdatedFile({ - oldPath: nextWriteRename.oldPath, - relativePath: nextWriteRename.newPath - }); - }); - } - // The rename consumed `path`. Skip the post-update enqueue below - // — it would send a syncLocallyUpdatedFile for a path that no - // longer exists. - return; - } - - if (!this.isSyncEnabled) { - return; - } - - if (!isNew) { - this.enqueueSync(async () => { - this.client.syncLocallyUpdatedFile({ relativePath: path }); - }); - } - } - - public override async atomicUpdateText( - path: RelativePath, - updater: (current: TextWithCursors) => TextWithCursors - ): Promise { - const result = await super.atomicUpdateText(path, updater); - if (this.isSyncEnabled) { - this.enqueueSync(async () => { - this.client.syncLocallyUpdatedFile({ relativePath: path }); - }); - } - return result; - } - - public override async delete(path: RelativePath): Promise { - await super.delete(path); - if (this.isSyncEnabled) { - this.enqueueSync(async () => { - this.client.syncLocallyDeletedFile(path); - }); - } - } - - public override async rename( - oldPath: RelativePath, - newPath: RelativePath - ): Promise { - await super.rename(oldPath, newPath); - if (this.isSyncEnabled) { - this.enqueueSync(async () => { - this.client.syncLocallyUpdatedFile({ - oldPath, - relativePath: newPath - }); - }); - } - } - - private async waitForWebSocket(): Promise { - const deadline = Date.now() + WEBSOCKET_CONNECT_TIMEOUT_MS; - while (!this.client.isWebSocketConnected && Date.now() < deadline) { - await sleep(WEBSOCKET_POLL_INTERVAL_MS); - } - assert( - this.client.isWebSocketConnected, - `Client ${this.clientId} WebSocket failed to connect within ${WEBSOCKET_CONNECT_TIMEOUT_MS}ms` - ); - } - - /** - * Wait until all agent-level enqueued sync operations have completed. - * Uses a loop because completing one operation can trigger new enqueues. - */ - private async drainPendingSyncOperations(): Promise { - while (this.pendingSyncOperations.size > 0) { - await utils.awaitAll([...this.pendingSyncOperations]); - } - } - - private enqueueSync(operation: () => Promise): void { - const promise = this.executeSyncOperation(operation).catch( - (error: unknown) => { - const err = - error instanceof Error ? error : new Error(String(error)); - this.log(`Background sync failed: ${err.message}`); - this.syncErrors.push(err); - } - ); - this.pendingSyncOperations.add(promise); - void promise.finally(() => { - this.pendingSyncOperations.delete(promise); - }); - } - - private async executeSyncOperation( - operation: () => Promise - ): Promise { - try { - await operation(); - } catch (error) { - if (error instanceof Error && error.name === "SyncResetError") { - this.log(`Sync operation interrupted by reset: ${error}`); - return; - } - if ( - error instanceof Error && - error.message.includes("has been destroyed") - ) { - this.log(`Sync operation interrupted by destroy: ${error}`); - return; - } - - throw error; - } - } - - private log(message: string): void { - this.logger(`[Client ${this.clientId}] ${message}`); - } - - private wrapFetch( - fetchImplementation: typeof globalThis.fetch - ): typeof globalThis.fetch { - return async (input, init) => { - const response = await fetchImplementation(input, init); - const drop = this.nextCreateResponseDrop; - if ( - drop !== undefined && - DeterministicAgent.isCreateDocumentRequest(input, init) - ) { - this.nextCreateResponseDrop = undefined; - try { - await response.body?.cancel(); - } catch { - // Best-effort — body may already be consumed/closed. - } - drop.resolveDropped(); - throw new SyncResetError(); - } - return response; - }; - } - - private static isCreateDocumentRequest( - input: RequestInfo | URL, - init: RequestInit | undefined - ): boolean { - const method = - init?.method ?? - (typeof Request !== "undefined" && input instanceof Request - ? input.method - : "GET"); - if (method.toUpperCase() !== "POST") { - return false; - } - - const url = - input instanceof URL - ? input - : new URL(typeof input === "string" ? input : input.url); - return /\/documents\/?$/.test(url.pathname); - } -} diff --git a/frontend/deterministic-tests/src/managed-websocket.ts b/frontend/deterministic-tests/src/managed-websocket.ts deleted file mode 100644 index c759891b..00000000 --- a/frontend/deterministic-tests/src/managed-websocket.ts +++ /dev/null @@ -1,245 +0,0 @@ -/** - * A WebSocket wrapper that can pause and resume message delivery. - * When paused, incoming messages are buffered. When resumed, buffered - * messages are delivered in order via the onmessage handler. - * - * Member layout follows typescript-eslint default member-ordering: all - * accessor properties are declared with `declare` and wired through the - * constructor using Object.defineProperty so we don't need conflicting - * get/set accessor pairs. - */ -class ManagedWebSocket implements WebSocket { - public static readonly CONNECTING = WebSocket.CONNECTING; - public static readonly OPEN = WebSocket.OPEN; - public static readonly CLOSING = WebSocket.CLOSING; - public static readonly CLOSED = WebSocket.CLOSED; - - public readonly CONNECTING = WebSocket.CONNECTING; - public readonly OPEN = WebSocket.OPEN; - public readonly CLOSING = WebSocket.CLOSING; - public readonly CLOSED = WebSocket.CLOSED; - - declare public readonly readyState: number; - declare public readonly url: string; - declare public readonly protocol: string; - declare public readonly extensions: string; - declare public readonly bufferedAmount: number; - declare public binaryType: BinaryType; - declare public onopen: ((this: WebSocket, ev: Event) => unknown) | null; - declare public onclose: - | ((this: WebSocket, ev: CloseEvent) => unknown) - | null; - declare public onerror: ((this: WebSocket, ev: Event) => unknown) | null; - declare public onmessage: - | ((this: WebSocket, ev: MessageEvent) => unknown) - | null; - - private readonly ws: WebSocket; - private readonly bufferedMessages: MessageEvent[] = []; - private paused = false; - private externalOnMessage: ((event: MessageEvent) => unknown) | null = null; - - public constructor(url: string | URL, protocols?: string | string[]) { - this.ws = new WebSocket(url, protocols); - - const { ws } = this; - Object.defineProperties(this, { - readyState: { - get: (): number => ws.readyState, - enumerable: true, - configurable: true - }, - url: { - get: (): string => ws.url, - enumerable: true, - configurable: true - }, - protocol: { - get: (): string => ws.protocol, - enumerable: true, - configurable: true - }, - extensions: { - get: (): string => ws.extensions, - enumerable: true, - configurable: true - }, - bufferedAmount: { - get: (): number => ws.bufferedAmount, - enumerable: true, - configurable: true - }, - binaryType: { - get: (): BinaryType => ws.binaryType, - set: (v: BinaryType): void => { - ws.binaryType = v; - }, - enumerable: true, - configurable: true - }, - onopen: { - get: (): ((this: WebSocket, ev: Event) => unknown) | null => - ws.onopen, - set: ( - h: ((this: WebSocket, ev: Event) => unknown) | null - ): void => { - ws.onopen = h; - }, - enumerable: true, - configurable: true - }, - onclose: { - get: (): - | ((this: WebSocket, ev: CloseEvent) => unknown) - | null => ws.onclose, - set: ( - h: ((this: WebSocket, ev: CloseEvent) => unknown) | null - ): void => { - ws.onclose = h; - }, - enumerable: true, - configurable: true - }, - onerror: { - get: (): ((this: WebSocket, ev: Event) => unknown) | null => - ws.onerror, - set: ( - h: ((this: WebSocket, ev: Event) => unknown) | null - ): void => { - ws.onerror = h; - }, - enumerable: true, - configurable: true - }, - onmessage: { - get: (): - | ((this: WebSocket, ev: MessageEvent) => unknown) - | null => this.externalOnMessage, - set: ( - h: ((this: WebSocket, ev: MessageEvent) => unknown) | null - ): void => { - this.externalOnMessage = h; - }, - enumerable: true, - configurable: true - } - }); - - this.ws.onmessage = (event: MessageEvent): void => { - if (this.paused) { - this.bufferedMessages.push(event); - } else { - this.externalOnMessage?.(event); - } - }; - } - - public pause(): void { - this.paused = true; - } - - public resume(): void { - // Drain buffered messages BEFORE flipping `paused` to false. - // If `externalOnMessage` is async (its return type is `unknown`), - // dispatch yields control between buffered messages, and a fresh - // live `ws.onmessage` event firing during that yield would jump - // ahead of unprocessed buffered messages — silently reordering - // events relative to the wire. Keeping `paused = true` during the - // drain forces the live handler to keep buffering, so we splice - // those late arrivals onto the tail and dispatch them in order. - while (this.bufferedMessages.length > 0) { - const messages = this.bufferedMessages.splice(0); - for (const msg of messages) { - this.externalOnMessage?.(msg); - } - } - this.paused = false; - } - - public send(data: string | ArrayBufferLike | Blob | ArrayBufferView): void { - this.ws.send(data); - } - - public close(code?: number, reason?: string): void { - this.ws.close(code, reason); - } - - public addEventListener( - ...args: Parameters - ): void { - // Only the `.onmessage` setter routes through the pause buffer. - // If sync-client ever attaches "message" listeners via - // addEventListener instead, those messages would bypass pause/resume - // and deterministic tests would silently lose their fault injection. - if (args[0] === "message") { - throw new Error( - "ManagedWebSocket: addEventListener('message') bypasses the " + - "pause buffer. Use the .onmessage setter instead, or " + - "extend ManagedWebSocket to route message listeners." - ); - } - this.ws.addEventListener(...args); - } - - public removeEventListener( - ...args: Parameters - ): void { - this.ws.removeEventListener(...args); - } - - public dispatchEvent(event: Event): boolean { - return this.ws.dispatchEvent(event); - } -} - -/** - * Factory that creates ManagedWebSocket instances and tracks them - * for pause/resume control from the test harness - */ -export class ManagedWebSocketFactory { - // Append-only: closed sockets stay tracked. Bounded per test (one - // factory per agent, each test discards its agents on cleanup), so - // not a real leak — but iterating over closed instances on - // pause/resume is a deliberate no-op since their `.onmessage` is - // already detached. - private readonly instances: ManagedWebSocket[] = []; - // Sticky pause state: applied to current instances on `pause()` AND - // to any new instance created later (e.g. WS reconnect after a - // `disable-sync` / `reset` cycle). Without this, a test pausing the - // WS before the agent reconnects would silently see the new socket - // start un-paused and miss the messages it meant to buffer. - private currentlyPaused = false; - - public get constructorFn(): typeof globalThis.WebSocket { - const trackInstance = (instance: ManagedWebSocket): void => { - this.instances.push(instance); - if (this.currentlyPaused) { - instance.pause(); - } - }; - class TrackedManagedWebSocket extends ManagedWebSocket { - public constructor( - url: string | URL, - protocols?: string | string[] - ) { - super(url, protocols); - trackInstance(this); - } - } - return TrackedManagedWebSocket; - } - - public pause(): void { - this.currentlyPaused = true; - for (const ws of this.instances) { - ws.pause(); - } - } - - public resume(): void { - this.currentlyPaused = false; - for (const ws of this.instances) { - ws.resume(); - } - } -} diff --git a/frontend/deterministic-tests/src/parse-args.ts b/frontend/deterministic-tests/src/parse-args.ts deleted file mode 100644 index 11c56f19..00000000 --- a/frontend/deterministic-tests/src/parse-args.ts +++ /dev/null @@ -1,43 +0,0 @@ -import * as os from "node:os"; -import { Command, InvalidArgumentError } from "commander"; - -export interface CliArgs { - filter: string | undefined; - concurrency: number; -} - -function parsePositiveInt(value: string): number { - const n = parseInt(value, 10); - if (isNaN(n) || n <= 0) { - throw new InvalidArgumentError("must be a positive integer"); - } - return n; -} - -export function parseArgs(argv: string[]): CliArgs { - const program = new Command(); - - program - .name("deterministic-tests") - .description("Scripted multi-client sync tests against a real server") - .option( - "-f, --filter ", - "Run only tests whose name contains this substring" - ) - .option( - "-j, --concurrency ", - "Number of tests to run in parallel", - parsePositiveInt, - os.cpus().length - ); - - program.parse(argv); - - /* eslint-disable @typescript-eslint/no-unsafe-type-assertion */ - const opts = program.opts(); - const filter = opts.filter as string | undefined; - const concurrency = opts.concurrency as number; - /* eslint-enable @typescript-eslint/no-unsafe-type-assertion */ - - return { filter, concurrency }; -} diff --git a/frontend/deterministic-tests/src/prefixed-logger.ts b/frontend/deterministic-tests/src/prefixed-logger.ts deleted file mode 100644 index 769d7545..00000000 --- a/frontend/deterministic-tests/src/prefixed-logger.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { Logger } from "sync-client"; - -export class PrefixedLogger extends Logger { - private readonly base: Logger; - private readonly prefix: string; - - public constructor(base: Logger, prefix: string) { - super(); - this.base = base; - this.prefix = prefix; - } - - public override debug(message: string): void { - this.base.debug(`[${this.prefix}] ${message}`); - } - - public override info(message: string): void { - this.base.info(`[${this.prefix}] ${message}`); - } - - public override warn(message: string): void { - this.base.warn(`[${this.prefix}] ${message}`); - } - - public override error(message: string): void { - this.base.error(`[${this.prefix}] ${message}`); - } -} diff --git a/frontend/deterministic-tests/src/run-with-concurrency.ts b/frontend/deterministic-tests/src/run-with-concurrency.ts deleted file mode 100644 index f5bcf745..00000000 --- a/frontend/deterministic-tests/src/run-with-concurrency.ts +++ /dev/null @@ -1,33 +0,0 @@ -export async function runWithConcurrency( - items: T[], - concurrency: number, - fn: (item: T) => Promise -): Promise { - const results: R[] = []; - const errors: unknown[] = []; - const executing = new Set>(); - - for (let i = 0; i < items.length; i++) { - const index = i; - const p = fn(items[index]) - .then((result) => { - results[index] = result; - }) - .catch((error: unknown) => { - errors.push(error); - }) - .finally(() => executing.delete(p)); - executing.add(p); - if (executing.size >= concurrency) { - await Promise.race(executing); - } - } - - // eslint-disable-next-line no-restricted-properties - await Promise.all(executing); - - if (errors.length > 0) { - throw errors[0]; - } - return results; -} diff --git a/frontend/deterministic-tests/src/server-control.ts b/frontend/deterministic-tests/src/server-control.ts deleted file mode 100644 index 9cb4cde0..00000000 --- a/frontend/deterministic-tests/src/server-control.ts +++ /dev/null @@ -1,296 +0,0 @@ -import { spawn, type ChildProcess } from "node:child_process"; -import * as fs from "node:fs"; -import * as os from "node:os"; -import * as path from "node:path"; -import { sleep } from "./utils/sleep"; -import { findFreePort } from "./utils/find-free-port"; -import type { Logger } from "sync-client"; -import { - STOP_TIMEOUT_MS, - SERVER_READY_POLL_INTERVAL_MS, - SERVER_READY_MAX_ATTEMPTS, - SERVER_START_MAX_ATTEMPTS -} from "./consts"; - -export class ServerControl { - private process: ChildProcess | null = null; - private readonly serverPath: string; - private readonly baseConfigPath: string; - private readonly logger: Logger; - private _port: number | undefined; - private tempDir: string | undefined; - private _isPaused = false; - - public constructor(serverPath: string, configPath: string, logger: Logger) { - this.serverPath = serverPath; - this.baseConfigPath = configPath; - this.logger = logger; - } - - public get port(): number { - if (this._port === undefined) { - throw new Error("Server has not been started yet"); - } - return this._port; - } - - public get remoteUri(): string { - return `http://localhost:${this.port}`; - } - - public async start(): Promise { - if (this.process !== null) { - throw new Error("Server is already running"); - } - - // Retry on bind failure: findFreePort closes its probe before we - // spawn, so under heavy parallelism another process can grab the - // same port. Each attempt picks a fresh port. - let lastError: unknown; - for (let attempt = 1; attempt <= SERVER_START_MAX_ATTEMPTS; attempt++) { - try { - await this.startOnce(); - return; - } catch (error) { - lastError = error; - this.logger.warn( - `Server start attempt ${attempt}/${SERVER_START_MAX_ATTEMPTS} failed: ${error instanceof Error ? error.message : String(error)}` - ); - // startOnce already cleaned up its child + tempdir on failure. - } - } - throw new Error( - `Server failed to start after ${SERVER_START_MAX_ATTEMPTS} attempts: ${lastError instanceof Error ? lastError.message : String(lastError)}`, - { cause: lastError instanceof Error ? lastError : undefined } - ); - } - - private async startOnce(): Promise { - const reservation = await findFreePort(); - this._port = reservation.port; - const tmpBase = os.tmpdir(); - this.tempDir = fs.mkdtempSync(path.join(tmpBase, "vault-link-test-")); - const tempConfigPath = path.join(this.tempDir, "config.yml"); - const dbDir = path.join(this.tempDir, "databases"); - - this.writeConfigFile(tempConfigPath, dbDir); - - this.logger.info( - `Starting server: ${this.serverPath} (port ${this._port})` - ); - - // Release the port reservation right before spawning to minimize - // the TOCTOU window between port discovery and server binding. - reservation.release(); - - this.process = spawn(this.serverPath, [tempConfigPath], { - stdio: ["ignore", "pipe", "pipe"], - detached: false - }); - - this.process.stdout?.on("data", (data: Buffer) => { - this.logger.info(`[SERVER] ${data.toString().trim()}`); - }); - - this.process.stderr?.on("data", (data: Buffer) => { - this.logger.info(`[SERVER] ${data.toString().trim()}`); - }); - - this.process.on("error", (err) => { - this.logger.error(`[SERVER] Process error: ${err.message}`); - }); - - const currentProcess = this.process; - currentProcess.on("exit", (code, signal) => { - this.logger.info( - `Server exited with code ${code}, signal ${signal}` - ); - // Only clear state if this handler is for the current process. - // A fast stop→start cycle could create a new process before this - // handler fires — clearing state here would corrupt the new one. - if (this.process === currentProcess) { - this.process = null; - this._isPaused = false; - } - }); - - try { - await this.waitForReady(); - } catch (error) { - // Kill the spawned process if it failed to become ready, - // preventing a zombie process from lingering. - try { - await this.stop(); - } catch { - // Best-effort cleanup - } - throw error; - } - } - - public async waitForReady( - maxAttempts: number = SERVER_READY_MAX_ATTEMPTS - ): Promise { - const pingUrl = `${this.remoteUri}/vaults/test/ping`; - for (let i = 0; i < maxAttempts; i++) { - if (this.process?.exitCode !== null) { - throw new Error( - "Server process died while waiting for it to become ready" - ); - } - try { - const response = await fetch(pingUrl); - if (response.ok) { - this.logger.info("[SERVER] Ready"); - return; - } - } catch { - // Server not ready yet, continue polling - } - await sleep(SERVER_READY_POLL_INTERVAL_MS); - } - throw new Error("Server failed to start within timeout"); - } - - public pause(): void { - if (this.process?.pid === undefined) { - throw new Error("Server is not running"); - } - if (this._isPaused) { - this.logger.warn("Server is already paused, skipping double-pause"); - return; - } - this.logger.info("Server pausing..."); - try { - process.kill(this.process.pid, "SIGSTOP"); - this._isPaused = true; - this.logger.info("Server paused (SIGSTOP sent)"); - } catch (error) { - throw new Error( - `Failed to pause server (pid ${this.process.pid}): ${error instanceof Error ? error.message : String(error)}` - ); - } - } - - public resume(): void { - if (this.process?.pid === undefined) { - throw new Error("Server is not running"); - } - if (!this._isPaused) { - return; - } - this.logger.info("Server resuming..."); - try { - process.kill(this.process.pid, "SIGCONT"); - this._isPaused = false; - this.logger.info("Server resumed (SIGCONT sent)"); - } catch (error) { - throw new Error( - `Failed to resume server (pid ${this.process.pid}): ${error instanceof Error ? error.message : String(error)}` - ); - } - } - - public async stop(): Promise { - const proc = this.process; - if (proc?.pid === undefined) { - this.cleanupTempDir(); - return; - } - - // Resume if paused — a SIGSTOP'd process ignores SIGKILL - if (this._isPaused) { - try { - process.kill(proc.pid, "SIGCONT"); - } catch { - // Process may already be gone - } - this._isPaused = false; - } - - this.logger.info("Server stopping..."); - - // Set up a promise that resolves when the process actually exits. - const exitPromise = new Promise((resolve) => { - if (proc.exitCode !== null) { - resolve(); - return; - } - proc.on("exit", () => { - resolve(); - }); - }); - - try { - process.kill(proc.pid, "SIGKILL"); - } catch { - // Process already gone - } - - // Wait for the process to actually exit before cleaning up, - // with a 5s safety timeout to avoid hanging forever. - await Promise.race([exitPromise, sleep(STOP_TIMEOUT_MS)]); - - this.process = null; - this._isPaused = false; - this.cleanupTempDir(); - } - - public isRunning(): boolean { - const proc = this.process; - return ( - proc !== null && - proc.pid !== undefined && - proc.exitCode === null && - proc.signalCode === null - ); - } - - /** - * Synchronously SIGCONT-then-SIGKILL the child process. Safe to call - * from a `process.on("exit", ...)` handler, where async work cannot - * run. Used as a last-resort cleanup so a SIGSTOP'd server doesn't - * outlive the test runner and wedge the next CI invocation. - */ - public forceKillSync(): void { - const proc = this.process; - if (proc?.pid === undefined) { - return; - } - try { - process.kill(proc.pid, "SIGCONT"); - } catch { - // Process may already be gone or never paused. - } - try { - process.kill(proc.pid, "SIGKILL"); - } catch { - // Process already gone. - } - } - - private writeConfigFile(destPath: string, dbDir: string): void { - // Assumes config-e2e.yml has exactly one 2-space-indented `port:` and - // one `databases_directory_path:` (under `server:` and `database:` - // respectively) - const baseConfig = fs.readFileSync(this.baseConfigPath, "utf-8"); - const config = baseConfig - .replace(/^\s*port:\s*\d+/m, ` port: ${this._port}`) - .replace( - /^\s*databases_directory_path:\s*.+/m, - ` databases_directory_path: ${dbDir}` - ); - fs.writeFileSync(destPath, config); - } - - private cleanupTempDir(): void { - if (this.tempDir !== undefined) { - try { - fs.rmSync(this.tempDir, { recursive: true, force: true }); - } catch { - // Best-effort cleanup - } - this.tempDir = undefined; - } - } -} diff --git a/frontend/deterministic-tests/src/server-manager.ts b/frontend/deterministic-tests/src/server-manager.ts deleted file mode 100644 index 76c624f7..00000000 --- a/frontend/deterministic-tests/src/server-manager.ts +++ /dev/null @@ -1,71 +0,0 @@ -import type { ServerControl } from "./server-control"; -import type { Logger } from "sync-client"; - -export class ServerManager { - private readonly activeServers = new Set(); - private readonly logger: Logger; - private isShuttingDown = false; - - public constructor(logger: Logger) { - this.logger = logger; - } - - public track(server: ServerControl): void { - this.activeServers.add(server); - } - - public untrack(server: ServerControl): void { - this.activeServers.delete(server); - } - - public async stopAll(): Promise { - if (this.isShuttingDown) { - return; - } - this.isShuttingDown = true; - - const servers = Array.from(this.activeServers); - // eslint-disable-next-line no-restricted-properties - await Promise.all( - servers.map(async (server) => { - try { - await server.stop(); - } catch { - // Best-effort cleanup during shutdown - } - }) - ); - } - - public installSignalHandlers(): void { - process.on("SIGINT", () => { - this.logger.info("Received SIGINT, shutting down..."); - void this.stopAll() - .catch(() => { - /* no-op */ - }) - .then(() => process.exit(130)); - }); - - process.on("SIGTERM", () => { - this.logger.info("Received SIGTERM, shutting down..."); - void this.stopAll() - .catch(() => { - /* no-op */ - }) - .then(() => process.exit(143)); - }); - - // Last-resort synchronous cleanup. Runs even when the process is - // exiting via process.exit() from unhandledRejection / - // uncaughtException — paths where async stopAll() cannot complete. - // SIGSTOP'd servers MUST receive SIGCONT before SIGKILL or the - // kernel keeps them as zombies holding the test's tmpdir, and the - // next CI run can't reuse the port. - process.on("exit", () => { - for (const server of this.activeServers) { - server.forceKillSync(); - } - }); - } -} diff --git a/frontend/deterministic-tests/src/test-definition.ts b/frontend/deterministic-tests/src/test-definition.ts deleted file mode 100644 index bd832a50..00000000 --- a/frontend/deterministic-tests/src/test-definition.ts +++ /dev/null @@ -1,49 +0,0 @@ -import type { AssertableState } from "./utils/assertable-state"; - -export interface ClientState { - files: Map; - clientFiles: Map[]; -} - -export type TestStep = - | { type: "create"; client: number; path: string; content: string } - | { type: "update"; client: number; path: string; content: string } - | { type: "rename"; client: number; oldPath: string; newPath: string } - | { - type: "rename-next-write"; - client: number; - oldPath: string; - newPath: string; - } - | { type: "delete"; client: number; path: string } - | { type: "sync"; client?: number } - | { type: "disable-sync"; client: number } - | { type: "enable-sync"; client: number } - | { type: "pause-server" } - | { type: "resume-server" } - | { - type: "resume-server-until-history-then-pause"; - client: number; - syncType: "CREATE" | "UPDATE" | "DELETE"; - path: string; - } - | { type: "barrier" } - | { type: "assert-consistent"; verify?: (state: AssertableState) => void } - | { type: "pause-websocket"; client: number } - | { type: "resume-websocket"; client: number } - | { type: "drop-next-create-response"; client: number } - | { type: "wait-for-dropped-create-response"; client: number } - | { type: "sleep"; ms: number } - | { type: "reset"; client: number }; - -export interface TestDefinition { - description?: string; - clients: number; - steps: TestStep[]; -} - -export interface TestResult { - success: boolean; - error?: string; - duration?: number; -} diff --git a/frontend/deterministic-tests/src/test-registry.ts b/frontend/deterministic-tests/src/test-registry.ts deleted file mode 100644 index 1a07b411..00000000 --- a/frontend/deterministic-tests/src/test-registry.ts +++ /dev/null @@ -1,245 +0,0 @@ -import type { TestDefinition } from "./test-definition"; -import { renameCreateConflictTest } from "./tests/rename-create-conflict.test"; -import { renameChainTest } from "./tests/rename-chain.test"; -import { renameUpdateConflictTest } from "./tests/rename-update-conflict.test"; -import { deleteRenameConflictTest } from "./tests/delete-rename-conflict.test"; -import { multiFileOperationsTest } from "./tests/multi-file-operations.test"; -import { deleteRecreateSamePathTest } from "./tests/delete-recreate-same-path.test"; -import { offlineRenameAndEditTest } from "./tests/offline-rename-and-edit.test"; -import { simultaneousCreateDeleteSamePathTest } from "./tests/simultaneous-create-delete-same-path.test"; -import { idempotencyAfterServerPauseTest } from "./tests/idempotency-after-server-pause.test"; -import { sequentialCreateDuplicateContentTest } from "./tests/sequential-create-duplicate-content.test"; -import { mcThreeClientRenameOfflineUpdateTest } from "./tests/mc-three-client-rename-offline-update.test"; -import { mcMultiDeleteOfflineRenameTest } from "./tests/mc-multi-delete-offline-rename.test"; -import { mcCrossCreateRenameSameTargetTest } from "./tests/mc-cross-create-rename-same-target.test"; -import { mcDeleteThenOfflineRenameTest } from "./tests/mc-delete-then-offline-rename.test"; -import { offlineMixedOperationsTest } from "./tests/offline-mixed-operations.test"; -import { offlineConcurrentRenamesTest } from "./tests/offline-concurrent-renames.test"; -import { offlineMultipleEditsTest } from "./tests/offline-multiple-edits.test"; -import { serverPauseBothClientsCreateTest } from "./tests/server-pause-both-clients-create.test"; -import { serverPauseUpdateAndCreateTest } from "./tests/server-pause-update-and-create.test"; -import { renameSwapTest } from "./tests/rename-swap.test"; -import { renameCircularTest } from "./tests/rename-circular.test"; -import { renameRoundtripTest } from "./tests/rename-roundtrip.test"; -import { offlineRenameRemoteCreateOldPathTest } from "./tests/offline-rename-remote-create-old-path.test"; -import { offlineEditRemoteRenameTest } from "./tests/offline-edit-remote-rename.test"; -import { renameChainThenDeleteTest } from "./tests/rename-chain-then-delete.test"; -import { offlineDeleteRemoteRenameTest } from "./tests/offline-delete-remote-rename.test"; -import { overlappingEditsSameSectionTest } from "./tests/overlapping-edits-same-section.test"; -import { rapidUpdatesAfterMergeTest } from "./tests/rapid-updates-after-merge.test"; -import { deleteRecreateConcurrentUpdateTest } from "./tests/delete-recreate-concurrent-update.test"; -import { moveAndConcurrentRemoteUpdateTest } from "./tests/move-and-concurrent-remote-update.test"; -import { offlineDeleteVsRemoteUpdateTest } from "./tests/offline-delete-vs-remote-update.test"; -import { doubleOfflineCycleTest } from "./tests/double-offline-cycle.test"; -import { serverPauseRenameEditResumeTest } from "./tests/server-pause-rename-edit-resume.test"; -import { offlineUpdateBothThenDeleteOneTest } from "./tests/offline-update-both-then-delete-one.test"; -import { offlineCreateSamePathMergeableTest } from "./tests/offline-create-same-path-mergeable.test"; -import { deleteDuringPendingCreateTest } from "./tests/delete-during-pending-create.test"; -import { threeClientRenameCreateDeleteTest } from "./tests/three-client-rename-create-delete.test"; -import { renameToPathOfUnconfirmedDeleteTest } from "./tests/rename-to-path-of-unconfirmed-delete.test"; -import { offlineEditThenMoveSameContentTest } from "./tests/offline-edit-then-move-same-content.test"; -import { rapidCreateUpdateDeleteCycleTest } from "./tests/rapid-create-update-delete-cycle.test"; -import { serverPauseBothEditSameFileTest } from "./tests/server-pause-both-edit-same-file.test"; -import { deleteRecreateDifferentContentTest } from "./tests/delete-recreate-different-content.test"; -import { updateDuringCreateProcessingTest } from "./tests/update-during-create-processing.test"; -import { offlineMoveThenRemoteDeleteTest } from "./tests/offline-move-then-remote-delete.test"; -import { resetClearsRecentlyDeletedResurrectionTest } from "./tests/reset-clears-recently-deleted-resurrection.test"; -import { moveThenDeleteStalePathTest } from "./tests/move-then-delete-stale-path.test"; -import { interruptedDeleteRetryTest } from "./tests/interrupted-delete-retry.test"; -import { updateDoesNotSurviveRemoteDeleteTest } from "./tests/update-does-not-survive-remote-delete.test"; -import { movePreservesRemoteUpdateTest } from "./tests/move-preserves-remote-update.test"; -import { recentlyDeletedClearedOnReconnectTest } from "./tests/recently-deleted-cleared-on-reconnect.test"; -import { watermarkAdvancesOnSkipTest } from "./tests/watermark-advances-on-skip.test"; -import { watermarkGapRemoteUpdateNotRecordedTest } from "./tests/watermark-gap-remote-update-not-recorded.test"; -import { queueResetLosesCoalescedLocalEditTest } from "./tests/queue-reset-loses-coalesced-local-edit.test"; -import { renameToPendingPathFallbackTest } from "./tests/rename-to-pending-path-fallback.test"; -import { moveRemoteUpdateRevertsRenameTest } from "./tests/move-remote-update-reverts-rename.test"; -import { localEditLostDuringCreateMergeTest } from "./tests/local-edit-lost-during-create-merge.test"; -import { renamePendingCreateBeforeResponseTest } from "./tests/rename-pending-create-before-response.test"; -import { createRenameResponseSkipsFileTest } from "./tests/create-rename-response-skips-file.test"; -import { onlineCreateRenameConcurrentCreateOrphanTest } from "./tests/online-create-rename-concurrent-create-orphan.test"; -import { concurrentRenameFirstWinsTest } from "./tests/concurrent-rename-first-wins.test"; -import { binaryToTextTransitionTest } from "./tests/binary-to-text-transition.test"; -import { textPendingCreateNotDisplacedTest } from "./tests/text-pending-create-not-displaced.test"; -import { binaryPendingCreateNotDisplacedTest } from "./tests/binary-pending-create-not-displaced.test"; -import { coalesceUpdateRemoteUpdateDataLossTest } from "./tests/coalesce-update-remote-update-data-loss.test"; -import { coalescedRemoteUpdateWatermarkLossTest } from "./tests/coalesced-remote-update-watermark-loss.test"; -import { concurrentDeleteDuringRemoteUpdateTest } from "./tests/concurrent-delete-during-remote-update.test"; -import { concurrentEditExactSamePositionTest } from "./tests/concurrent-edit-exact-same-position.test"; -import { concurrentRenameAndCreateAtTargetRenameFirstTest } from "./tests/concurrent-rename-and-create-at-target-rename-first.test"; -import { concurrentRenameAndCreateAtTargetCreateFirstTest } from "./tests/concurrent-rename-and-create-at-target-create-first.test"; -import { concurrentRenameSameTargetTest } from "./tests/concurrent-rename-same-target.test"; -import { concurrentUpdateDiffConsistencyTest } from "./tests/concurrent-update-diff-consistency.test"; -import { userParenthesizedFileNotDeletedTest } from "./tests/user-parenthesized-file-not-deleted.test"; -import { createDeleteNoopTest } from "./tests/create-delete-noop.test"; -import { createMergeDeleteTest } from "./tests/create-merge-delete.test"; -import { moveIdenticalContentAmbiguityTest } from "./tests/move-identical-content-ambiguity.test"; -import { createUpdateCoalesceServerPauseTest } from "./tests/create-update-coalesce-server-pause.test"; -import { createDuringReconciliationTest } from "./tests/create-during-reconciliation.test"; -import { createMergePreservesRenamedUpdateTest } from "./tests/create-merge-preserves-renamed-update.test"; -import { createRenameCreateSamePathTest } from "./tests/create-rename-create-same-path.test"; -import { moveChainThreeFilesTest } from "./tests/move-chain-three-files.test"; -import { deleteByOtherClientThenRecreateTest } from "./tests/delete-by-other-client-then-recreate.test"; -import { onlineDeleteRecreateRapidCycleTest } from "./tests/online-delete-recreate-rapid-cycle.test"; -import { onlineEditVsDeleteConvergenceTest } from "./tests/online-edit-vs-delete-convergence.test"; -import { rapidEditDeleteOnlineConvergenceTest } from "./tests/rapid-edit-delete-online-convergence.test"; -import { serverPauseDeleteRecreateTest } from "./tests/server-pause-delete-recreate.test"; -import { onlineBothCreateSamePathDeconflictTest } from "./tests/online-both-create-same-path-deconflict.test"; -import { onlineCreateUpdateWhileOtherCreatesSamePathTest } from "./tests/online-create-update-while-other-creates-same-path.test"; -import { displacedFileNotMarkedDeletedTest } from "./tests/displaced-file-not-marked-deleted.test"; -import { remoteUpdateResurrectsDeletedDocTest } from "./tests/remote-update-resurrects-deleted-doc.test"; -import { localUpdateSurvivesRemoteRenameTest } from "./tests/local-update-survives-remote-rename.test"; -import { mergingUpdateResponseSurvivesUserRenameTest } from "./tests/merging-update-response-survives-user-rename.test"; -import { catchupCreateAndUpdateNotSkippedTest } from "./tests/catchup-create-and-update-not-skipped.test"; -import { localRenameSurvivesRemoteRenameTest } from "./tests/local-rename-survives-remote-rename.test"; -import { renameChainDuringPendingCreateTest } from "./tests/rename-chain-during-pending-create.test"; -import { remoteRenameCollidesWithPendingLocalCreateTest } from "./tests/remote-rename-collides-with-pending-local-create.test"; -import { remoteUpdateSurvivesUserRenameTest } from "./tests/remote-update-survives-user-rename.test"; -import { sameDocIdCollapseOnLocalCreateAfterRemoteCreateTest } from "./tests/same-doc-id-collapse-on-local-create-after-remote-create.test"; -import { sameDocIdCollapseAfterRemoteQuickWriteAndPendingRenameTest } from "./tests/same-doc-id-collapse-after-remote-quick-write-and-pending-rename.test"; -import { renameOverwritesPendingCreateThenDeleteTest } from "./tests/rename-overwrites-pending-create-then-delete.test"; -import { deleteRecreatedPendingCreateWithStaleDeletingRecordTest } from "./tests/delete-recreated-pending-create-with-stale-deleting-record.test"; -import { queuedCreateDeleteDoesNotHijackReusedPathTest } from "./tests/queued-create-delete-does-not-hijack-reused-path.test"; -import { renamedPendingCreateReusedPathThenDeleteTest } from "./tests/renamed-pending-create-reused-path-then-delete.test"; -import { renamePendingCreateOntoPendingDeletePathTest } from "./tests/rename-pending-create-onto-pending-delete-path.test"; -import { remoteQuickWriteRenameBeforeRecordTest } from "./tests/remote-quick-write-rename-before-record.test"; -import { selfMergePendingRenameAliasesSecondCreateTest } from "./tests/self-merge-pending-rename-aliases-second-create.test"; - -export const TESTS: Partial> = { - "rename-create-conflict": renameCreateConflictTest, - "rename-chain": renameChainTest, - "rename-update-conflict": renameUpdateConflictTest, - "delete-rename-conflict": deleteRenameConflictTest, - "multi-file-operations": multiFileOperationsTest, - "delete-recreate-same-path": deleteRecreateSamePathTest, - "offline-rename-and-edit": offlineRenameAndEditTest, - "simultaneous-create-delete-same-path": - simultaneousCreateDeleteSamePathTest, - "idempotency-after-server-pause": idempotencyAfterServerPauseTest, - "sequential-create-duplicate-content": sequentialCreateDuplicateContentTest, - "mc-three-client-rename-offline-update": - mcThreeClientRenameOfflineUpdateTest, - "mc-multi-delete-offline-rename": mcMultiDeleteOfflineRenameTest, - "mc-cross-create-rename-same-target": mcCrossCreateRenameSameTargetTest, - "mc-delete-then-offline-rename": mcDeleteThenOfflineRenameTest, - "offline-mixed-operations": offlineMixedOperationsTest, - "offline-concurrent-renames": offlineConcurrentRenamesTest, - "offline-multiple-edits": offlineMultipleEditsTest, - "server-pause-both-clients-create": serverPauseBothClientsCreateTest, - "server-pause-update-and-create": serverPauseUpdateAndCreateTest, - "rename-swap": renameSwapTest, - "rename-circular": renameCircularTest, - "rename-roundtrip": renameRoundtripTest, - "offline-rename-remote-create-old-path": - offlineRenameRemoteCreateOldPathTest, - "offline-edit-remote-rename": offlineEditRemoteRenameTest, - "rename-chain-then-delete": renameChainThenDeleteTest, - "offline-delete-remote-rename": offlineDeleteRemoteRenameTest, - "overlapping-edits-same-section": overlappingEditsSameSectionTest, - "rapid-updates-after-merge": rapidUpdatesAfterMergeTest, - "delete-recreate-concurrent-update": deleteRecreateConcurrentUpdateTest, - "move-and-concurrent-remote-update": moveAndConcurrentRemoteUpdateTest, - "double-offline-cycle": doubleOfflineCycleTest, - "server-pause-rename-edit-resume": serverPauseRenameEditResumeTest, - "offline-update-both-then-delete-one": offlineUpdateBothThenDeleteOneTest, - "offline-create-same-path-mergeable": offlineCreateSamePathMergeableTest, - "delete-during-pending-create": deleteDuringPendingCreateTest, - "three-client-rename-create-delete": threeClientRenameCreateDeleteTest, - "rename-to-path-of-unconfirmed-delete": renameToPathOfUnconfirmedDeleteTest, - "offline-edit-then-move-same-content": offlineEditThenMoveSameContentTest, - "rapid-create-update-delete-cycle": rapidCreateUpdateDeleteCycleTest, - "server-pause-both-edit-same-file": serverPauseBothEditSameFileTest, - "delete-recreate-different-content": deleteRecreateDifferentContentTest, - "update-during-create-processing": updateDuringCreateProcessingTest, - "offline-move-then-remote-delete": offlineMoveThenRemoteDeleteTest, - "reset-clears-recently-deleted-resurrection": - resetClearsRecentlyDeletedResurrectionTest, - "move-then-delete-stale-path": moveThenDeleteStalePathTest, - "offline-delete-vs-remote-update": offlineDeleteVsRemoteUpdateTest, - "interrupted-delete-retry": interruptedDeleteRetryTest, - "update-does-not-survive-remote-delete": updateDoesNotSurviveRemoteDeleteTest, - "move-preserves-remote-update": movePreservesRemoteUpdateTest, - "recently-deleted-cleared-on-reconnect": - recentlyDeletedClearedOnReconnectTest, - "watermark-advances-on-skip": watermarkAdvancesOnSkipTest, - "watermark-gap-remote-update-not-recorded": - watermarkGapRemoteUpdateNotRecordedTest, - "queue-reset-loses-coalesced-local-edit": - queueResetLosesCoalescedLocalEditTest, - "rename-to-pending-path-fallback": renameToPendingPathFallbackTest, - "move-remote-update-reverts-rename": moveRemoteUpdateRevertsRenameTest, - "local-edit-lost-during-create-merge": localEditLostDuringCreateMergeTest, - "rename-pending-create-before-response": - renamePendingCreateBeforeResponseTest, - "create-rename-response-skips-file": createRenameResponseSkipsFileTest, - "online-create-rename-concurrent-create-orphan": - onlineCreateRenameConcurrentCreateOrphanTest, - "concurrent-rename-first-wins": concurrentRenameFirstWinsTest, - "binary-to-text-transition": binaryToTextTransitionTest, - "text-pending-create-not-displaced": textPendingCreateNotDisplacedTest, - "binary-pending-create-not-displaced": binaryPendingCreateNotDisplacedTest, - "coalesce-update-remote-update-data-loss": - coalesceUpdateRemoteUpdateDataLossTest, - "coalesced-remote-update-watermark-loss": - coalescedRemoteUpdateWatermarkLossTest, - "concurrent-delete-during-remote-update": - concurrentDeleteDuringRemoteUpdateTest, - "concurrent-edit-exact-same-position": concurrentEditExactSamePositionTest, - "concurrent-rename-and-create-at-target-rename-first": - concurrentRenameAndCreateAtTargetRenameFirstTest, - "concurrent-rename-and-create-at-target-create-first": - concurrentRenameAndCreateAtTargetCreateFirstTest, - "concurrent-rename-same-target": concurrentRenameSameTargetTest, - "concurrent-update-diff-consistency": concurrentUpdateDiffConsistencyTest, - "user-parenthesized-file-not-deleted": userParenthesizedFileNotDeletedTest, - "create-delete-noop": createDeleteNoopTest, - "create-merge-delete": createMergeDeleteTest, - "move-identical-content-ambiguity": moveIdenticalContentAmbiguityTest, - "create-update-coalesce-server-pause": createUpdateCoalesceServerPauseTest, - "create-during-reconciliation": createDuringReconciliationTest, - "create-merge-preserves-renamed-update": - createMergePreservesRenamedUpdateTest, - "create-rename-create-same-path": createRenameCreateSamePathTest, - "move-chain-three-files": moveChainThreeFilesTest, - "delete-by-other-client-then-recreate": deleteByOtherClientThenRecreateTest, - "online-delete-recreate-rapid-cycle": onlineDeleteRecreateRapidCycleTest, - "online-edit-vs-delete-convergence": onlineEditVsDeleteConvergenceTest, - "rapid-edit-delete-online-convergence": - rapidEditDeleteOnlineConvergenceTest, - "server-pause-delete-recreate": serverPauseDeleteRecreateTest, - "online-both-create-same-path-deconflict": - onlineBothCreateSamePathDeconflictTest, - "online-create-update-while-other-creates-same-path": - onlineCreateUpdateWhileOtherCreatesSamePathTest, - "displaced-file-not-marked-deleted": displacedFileNotMarkedDeletedTest, - "remote-update-resurrects-deleted-doc": - remoteUpdateResurrectsDeletedDocTest, - "local-update-survives-remote-rename": localUpdateSurvivesRemoteRenameTest, - "merging-update-response-survives-user-rename": - mergingUpdateResponseSurvivesUserRenameTest, - "catchup-create-and-update-not-skipped": - catchupCreateAndUpdateNotSkippedTest, - "local-rename-survives-remote-rename": localRenameSurvivesRemoteRenameTest, - "rename-chain-during-pending-create": renameChainDuringPendingCreateTest, - "remote-rename-collides-with-pending-local-create": - remoteRenameCollidesWithPendingLocalCreateTest, - "remote-update-survives-user-rename": remoteUpdateSurvivesUserRenameTest, - "same-doc-id-collapse-on-local-create-after-remote-create": - sameDocIdCollapseOnLocalCreateAfterRemoteCreateTest, - "renamed-pending-create-reused-path-then-delete": - renamedPendingCreateReusedPathThenDeleteTest, - "rename-pending-create-onto-pending-delete-path": - renamePendingCreateOntoPendingDeletePathTest, - "rename-overwrites-pending-create-then-delete": - renameOverwritesPendingCreateThenDeleteTest, - "same-doc-id-collapse-after-remote-quick-write-and-pending-rename": - sameDocIdCollapseAfterRemoteQuickWriteAndPendingRenameTest, - "delete-recreated-pending-create-with-stale-deleting-record": - deleteRecreatedPendingCreateWithStaleDeletingRecordTest, - "queued-create-delete-does-not-hijack-reused-path": - queuedCreateDeleteDoesNotHijackReusedPathTest, - "remote-quick-write-rename-before-record": - remoteQuickWriteRenameBeforeRecordTest, - "self-merge-pending-rename-aliases-second-create": - selfMergePendingRenameAliasesSecondCreateTest -}; diff --git a/frontend/deterministic-tests/src/test-runner.ts b/frontend/deterministic-tests/src/test-runner.ts deleted file mode 100644 index 411e9b08..00000000 --- a/frontend/deterministic-tests/src/test-runner.ts +++ /dev/null @@ -1,399 +0,0 @@ -import type { TestDefinition, TestResult, TestStep } from "./test-definition"; -import { DeterministicAgent } from "./deterministic-agent"; -import type { ServerControl } from "./server-control"; -import type { SyncSettings, Logger } from "sync-client"; -import { assert } from "./utils/assert"; -import { AssertableState } from "./utils/assertable-state"; -import { sleep } from "./utils/sleep"; -import { withTimeout } from "./utils/with-timeout"; -import { - CONVERGENCE_TIMEOUT_MS, - CONVERGENCE_RETRY_DELAY_MS, - AGENT_INIT_TIMEOUT_MS, - IS_SYNC_ENABLED_BY_DEFAULT -} from "./consts"; -import { randomUUID } from "node:crypto"; - -export class TestRunner { - private agents: DeterministicAgent[] = []; - private readonly serverControl: ServerControl; - private readonly token: string; - private readonly remoteUri: string; - private readonly logger: Logger; - - public constructor( - serverControl: ServerControl, - logger: Logger, - token: string, - remoteUri: string - ) { - this.serverControl = serverControl; - this.logger = logger; - this.token = token; - this.remoteUri = remoteUri; - } - - public async runTest( - name: string, - test: TestDefinition - ): Promise { - const startTime = Date.now(); - this.logger.info(`Running test: ${name}`); - if (test.description !== undefined && test.description !== "") { - this.logger.info(`Description: ${test.description}`); - } - this.logger.info(`Clients: ${test.clients}`); - this.logger.info(`Steps: ${test.steps.length}`); - - try { - assert( - this.serverControl.isRunning(), - "Server is not running before test start" - ); - - await this.initializeAgents(test.clients); - - for (let i = 0; i < test.steps.length; i++) { - const step = test.steps[i]; - this.logger.info( - `Step ${i + 1}/${test.steps.length}: ${JSON.stringify(step)}` - ); - await this.executeStep(step); - } - - await this.cleanup(); - - const duration = Date.now() - startTime; - this.logger.info(`\n✓ Test passed: ${name} (${duration}ms)`); - - return { - success: true, - duration - }; - } catch (error) { - const duration = Date.now() - startTime; - const errorMessage = - error instanceof Error ? error.message : String(error); - this.logger.info(`\n✗ Test failed: ${name}`); - this.logger.info(`Error: ${errorMessage}`); - - await this.cleanup(); - - return { - success: false, - error: errorMessage, - duration - }; - } - } - - private async initializeAgents(count: number): Promise { - assert(count > 0, `Client count must be positive, got ${count}`); - const vaultName = `test-${randomUUID()}`; - this.logger.info( - `Initializing ${count} agents with vault: ${vaultName}` - ); - - for (let i = 0; i < count; i++) { - const settings: Partial = { - isSyncEnabled: IS_SYNC_ENABLED_BY_DEFAULT, - token: this.token, - vaultName, - remoteUri: this.remoteUri - }; - - const agent = new DeterministicAgent(i, settings, (msg) => { - this.logger.info(msg); - }); - - // Push before init so cleanup() handles this agent if init fails - this.agents.push(agent); - await withTimeout( - agent.init(fetch), - AGENT_INIT_TIMEOUT_MS, - `Client ${i} init timed out after ${AGENT_INIT_TIMEOUT_MS}ms` - ); - this.logger.info(`Initialized client ${i}`); - } - - this.logger.info("All agents initialized"); - } - - private getAgent(index: number): DeterministicAgent { - assert( - index >= 0 && index < this.agents.length, - `Client index ${index} out of bounds (have ${this.agents.length} agents)` - ); - return this.agents[index]; - } - - private async executeStep(step: TestStep): Promise { - switch (step.type) { - case "create": - case "update": - await this.getAgent(step.client).write( - step.path, - new TextEncoder().encode(step.content) - ); - break; - - case "rename": - await this.getAgent(step.client).rename( - step.oldPath, - step.newPath - ); - break; - - case "rename-next-write": - this.getAgent(step.client).renameNextWrite( - step.oldPath, - step.newPath - ); - break; - - case "delete": - await this.getAgent(step.client).delete(step.path); - break; - - case "sync": - if (step.client !== undefined) { - await this.getAgent(step.client).waitForSync(); - } else { - for (const agent of this.agents) { - await agent.waitForSync(); - } - } - break; - - case "disable-sync": - await this.getAgent(step.client).disableSync(); - break; - - case "enable-sync": - await this.getAgent(step.client).enableSync(); - break; - - case "pause-server": - this.serverControl.pause(); - break; - - case "resume-server": - this.serverControl.resume(); - // Verify the server is actually responsive before proceeding. - // This replaces relying solely on hardcoded waits. - await this.serverControl.waitForReady(); - break; - - case "resume-server-until-history-then-pause": { - const agent = this.getAgent(step.client); - const historySeen = agent.waitForHistoryEntry( - (entry) => - entry.details.type === step.syncType && - entry.details.relativePath === step.path, - () => this.serverControl.pause() - ); - this.serverControl.resume(); - await historySeen; - break; - } - - case "barrier": - await this.waitForConvergence(); - break; - - case "assert-consistent": - await this.assertConsistent(step.verify); - break; - - case "pause-websocket": - this.getAgent(step.client).pauseWebSocket(); - break; - - case "resume-websocket": - this.getAgent(step.client).resumeWebSocket(); - break; - - case "drop-next-create-response": - this.getAgent(step.client).dropNextCreateResponse(); - break; - - case "wait-for-dropped-create-response": - await this.getAgent(step.client).waitForDroppedCreateResponse(); - break; - - case "sleep": - await sleep(step.ms); - break; - - case "reset": - await this.getAgent(step.client).reset(); - break; - - default: { - const unknownStep = step as { type: string }; - throw new Error(`Unknown step type: ${unknownStep.type}`); - } - } - } - - /** - * Wait for all agents to reach a consistent state. - * - * Waiting for agents is done in two full rounds: the first round - * drains in-flight operations, but completing those operations can - * trigger new work on OTHER agents via server broadcasts. The second - * round waits for that cascading work to settle. Deeper cascades - * are handled by the outer retry loop. - */ - private async waitForConvergence(): Promise { - this.logger.info("Barrier: waiting for convergence..."); - - const deadline = Date.now() + CONVERGENCE_TIMEOUT_MS; - let lastError: Error | undefined = undefined; - - while (Date.now() < deadline) { - await this.waitAllAgentsSettled(); - - try { - await this.assertConsistent(); - this.logger.info("Barrier complete: all clients converged"); - return; - } catch (error) { - lastError = - error instanceof Error ? error : new Error(String(error)); - this.logger.info("Barrier: not yet converged, retrying..."); - await sleep(CONVERGENCE_RETRY_DELAY_MS); - } - } - - throw new Error( - `Convergence timed out after ${CONVERGENCE_TIMEOUT_MS}ms: ${lastError?.message ?? "no consistency check ran"}`, - { cause: lastError } - ); - } - - /** - * Wait for all agents to be simultaneously idle. - * - * Completing work on agent A can trigger a server broadcast that - * enqueues new work on agent B, which can cascade further. With N - * agents the worst-case cascade depth is N (a chain A→B→C→…→A), - * so we run N+1 sequential passes to drain it. Extra passes are - * essentially free when there is no outstanding work. - * - * The outer {@link waitForConvergence} loop with consistency checks - * remains the ultimate guarantee — this method just minimizes how - * many slow retry iterations are needed. - */ - private async waitAllAgentsSettled(): Promise { - const rounds = this.agents.length + 1; - for (let round = 0; round < rounds; round++) { - for (const agent of this.agents) { - await agent.waitForSync(); - } - } - } - - private async assertConsistent( - verify?: (state: AssertableState) => void - ): Promise { - this.logger.info("Asserting all clients are consistent..."); - assert( - this.agents.length >= 2, - "Need at least 2 agents for consistency check" - ); - - // Snapshot all agents' file states upfront to minimize the window - // where background sync could mutate state between reads. - const clientFiles: Map[] = []; - for (const agent of this.agents) { - const sortedFiles = (await agent.listFilesRecursively()).sort(); - const fileMap = new Map(); - for (const file of sortedFiles) { - const content = await agent.getFileContent(file); - fileMap.set(file, content); - } - clientFiles.push(fileMap); - } - - const referenceFiles = Array.from(clientFiles[0].keys()); - - this.logger.info( - `Reference client has ${referenceFiles.length} files: ${referenceFiles.join(", ")}` - ); - - for (let i = 1; i < clientFiles.length; i++) { - const agentFileKeys = Array.from(clientFiles[i].keys()); - - this.logger.info( - `Client ${i} has ${agentFileKeys.length} files: ${agentFileKeys.join(", ")}` - ); - - assert( - agentFileKeys.length === referenceFiles.length, - `File count mismatch: client 0 has ${referenceFiles.length} files, client ${i} has ${agentFileKeys.length} files` - ); - - for (let j = 0; j < agentFileKeys.length; j++) { - assert( - agentFileKeys[j] === referenceFiles[j], - `File list mismatch at index ${j}: client 0 has "${referenceFiles[j]}", client ${i} has "${agentFileKeys[j]}"` - ); - } - - for (const file of referenceFiles) { - const referenceContent = clientFiles[0].get(file); - const agentContent = clientFiles[i].get(file); - - assert( - referenceContent === agentContent, - `Content mismatch for ${file}:\nClient 0: "${referenceContent}"\nClient ${i}: "${agentContent}"` - ); - } - } - - this.logger.info("✓ All clients are consistent"); - - if (verify) { - this.logger.info("Running custom verification..."); - try { - verify( - new AssertableState({ - files: clientFiles[0], - clientFiles - }) - ); - } catch (error) { - const msg = - error instanceof Error ? error.message : String(error); - throw new Error(`Custom verification failed: ${msg}`); - } - this.logger.info("✓ Custom verification passed"); - } - } - - private async cleanup(): Promise { - // Always resume the server in case a test paused it and then - // failed before reaching the resume step. Without this, all - // subsequent tests would hang because the server process is - // frozen (SIGSTOP) and can't respond to HTTP or WebSocket. - try { - this.serverControl.resume(); - } catch { - // Server wasn't paused or isn't running — safe to ignore - } - - this.logger.info("\nCleaning up agents..."); - for (const agent of this.agents) { - try { - await agent.cleanup(); - } catch (error) { - this.logger.warn( - `Agent cleanup error: ${error instanceof Error ? error.message : String(error)}` - ); - } - } - this.agents = []; - this.logger.info("Cleanup complete"); - } -} diff --git a/frontend/deterministic-tests/src/tests/binary-pending-create-not-displaced.test.ts b/frontend/deterministic-tests/src/tests/binary-pending-create-not-displaced.test.ts deleted file mode 100644 index 467c19f0..00000000 --- a/frontend/deterministic-tests/src/tests/binary-pending-create-not-displaced.test.ts +++ /dev/null @@ -1,40 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const binaryPendingCreateNotDisplacedTest: TestDefinition = { - description: - "Two clients each create a binary file at the same path while offline. " + - "After syncing, both files should exist on both clients at separate paths.", - clients: 2, - steps: [ - { - type: "create", - client: 0, - path: "data.bin", - content: "binary data from client 0" - }, - { - type: "create", - client: 1, - path: "data.bin", - content: "binary data from client 1" - }, - - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertFileCount(2) - .assertFileExists("data.bin") - .assertFileExists("data (1).bin") - .assertAnyFileContains( - "binary data from client 0", - "binary data from client 1" - ); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/binary-to-text-transition.test.ts b/frontend/deterministic-tests/src/tests/binary-to-text-transition.test.ts deleted file mode 100644 index 8b934c1b..00000000 --- a/frontend/deterministic-tests/src/tests/binary-to-text-transition.test.ts +++ /dev/null @@ -1,97 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const binaryToTextTransitionTest: TestDefinition = { - description: - "A .bin file is created and synced. Both clients edit it offline " + - "(binary last-write-wins), then client 0 renames it to .md and " + - "writes a clean text baseline. Both clients edit different sections " + - "offline. The text merge should preserve both edits.", - clients: 2, - steps: [ - { - type: "create", - client: 0, - path: "data.bin", - content: "original content" - }, - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertContent("data.bin", "original content"); - } - }, - - { type: "disable-sync", client: 0 }, - { type: "disable-sync", client: 1 }, - - { type: "update", client: 0, path: "data.bin", content: "version A" }, - { type: "update", client: 1, path: "data.bin", content: "version B" }, - - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertFileCount(1).assertContainsAny( - "data.bin", - "version A", - "version B" - ); - } - }, - - { type: "disable-sync", client: 1 }, - { type: "rename", client: 0, oldPath: "data.bin", newPath: "data.md" }, - { - type: "update", - client: 0, - path: "data.md", - content: "top line\nmiddle line\nbottom line" - }, - { type: "sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertContent( - "data.md", - "top line\nmiddle line\nbottom line" - ); - } - }, - - { type: "disable-sync", client: 0 }, - { type: "disable-sync", client: 1 }, - - { - type: "update", - client: 0, - path: "data.md", - content: "alpha\nmiddle line\nbottom line" - }, - { - type: "update", - client: 1, - path: "data.md", - content: "top line\nmiddle line\nbeta" - }, - - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertFileCount(1).assertContains("data.md", "alpha", "beta"); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/catchup-create-and-update-not-skipped.test.ts b/frontend/deterministic-tests/src/tests/catchup-create-and-update-not-skipped.test.ts deleted file mode 100644 index 2d40228f..00000000 --- a/frontend/deterministic-tests/src/tests/catchup-create-and-update-not-skipped.test.ts +++ /dev/null @@ -1,66 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const catchupCreateAndUpdateNotSkippedTest: TestDefinition = { - description: - "Client 1 disconnects (sync disabled). Client 0 creates a doc and " + - "then updates it. When Client 1 reconnects, the server's catch-up " + - "stream sends only the doc's *latest* version (the update), not the " + - "full history. Pre-fix the wire's `is_new_file` was set to " + - "`creation == latest_version`, so the catch-up flagged the doc as " + - "non-new even though Client 1 had never seen its creation. Client " + - "1's `processRemoteChange` then dropped it as a 'stale RemoteChange " + - "for untracked, non-new document' and the doc was silently lost. " + - "Post-fix `is_new_file` in the catch-up stream means 'new relative " + - "to the recipient's watermark' (`creation > last_seen_vault_update_id`).", - clients: 2, - steps: [ - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - // Establish a baseline so Client 1's last_seen is non-zero before - // we take it offline. This makes the bug genuinely about catch-up - // missing the create rather than just an empty-vault first sync. - { type: "create", client: 0, path: "warmup.md", content: "w\n" }, - { type: "barrier" }, - - // Client 1 goes offline. - { type: "disable-sync", client: 1 }, - - // Client 0 creates the doc (vault_update_id v_C, after Client 1's - // watermark). Client 1 doesn't see this because it's offline. - { type: "create", client: 0, path: "doc.md", content: "v1\n" }, - // Wait for the create's HTTP to land before the update; otherwise - // both writes are coalesced into a single POST and the server - // never sees the doc as "create followed by update". - { type: "sync", client: 0 }, - - // Client 0 updates the doc (vault_update_id v_X > v_C). The - // server's `latest_document_versions` view now returns the - // *update* row — its `creation_vault_update_id != vault_update_id`. - { - type: "update", - client: 0, - path: "doc.md", - content: "v1\nupdate\n" - }, - { type: "sync", client: 0 }, - - // Client 1 reconnects. Server's catch-up replays docs with - // `vault_update_id > last_seen`. For doc.md it sends v_X with - // `is_new_file` derived from `creation_vault_update_id > - // last_seen_vault_update_id` (post-fix) — so Client 1 treats it - // as a fresh create and downloads the latest content. - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (state: AssertableState): void => { - state.assertFileCount(2); - state.assertFileExists("doc.md"); - state.assertContent("doc.md", "v1\nupdate\n"); - state.assertContent("warmup.md", "w\n"); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/coalesce-update-remote-update-data-loss.test.ts b/frontend/deterministic-tests/src/tests/coalesce-update-remote-update-data-loss.test.ts deleted file mode 100644 index 1972526a..00000000 --- a/frontend/deterministic-tests/src/tests/coalesce-update-remote-update-data-loss.test.ts +++ /dev/null @@ -1,59 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const coalesceUpdateRemoteUpdateDataLossTest: TestDefinition = { - description: - "Divergent offline edits with text-merge expectation. Client 0's " + - "remote update fully lands before Client 1 reconnects (`sync`-after " + - "the c0 update enforces this), so Client 1's offline edit merges " + - "against a server-known version, not a coalesced batch. Both " + - "additions must survive in the final merged content. (Filename's " + - "'coalesce' framing is aspirational — a true update-coalesce test " + - "would skip the c0 sync and queue overlapping local + remote " + - "updates against the same parent version.)", - clients: 2, - steps: [ - { - type: "create", - client: 0, - path: "doc.md", - content: "line 1\nline 2\nline 3" - }, - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - - { type: "disable-sync", client: 1 }, - - { - type: "update", - client: 0, - path: "doc.md", - content: "line 1\nline 2\nline 3\nclient 0 addition" - }, - { type: "sync", client: 0 }, - - { - type: "update", - client: 1, - path: "doc.md", - content: "client 1 addition\nline 1\nline 2\nline 3" - }, - - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (state: AssertableState): void => { - state - .assertFileCount(1) - .assertContains( - "doc.md", - "client 0 addition", - "client 1 addition" - ); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/coalesced-remote-update-watermark-loss.test.ts b/frontend/deterministic-tests/src/tests/coalesced-remote-update-watermark-loss.test.ts deleted file mode 100644 index aceb8baa..00000000 --- a/frontend/deterministic-tests/src/tests/coalesced-remote-update-watermark-loss.test.ts +++ /dev/null @@ -1,53 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const coalescedRemoteUpdateWatermarkLossTest: TestDefinition = { - description: - "Client 0 sends three rapid updates. After syncing, both clients " + - "disconnect and reconnect twice. Content should remain correct " + - "after each reconnect.", - clients: 2, - steps: [ - { type: "create", client: 0, path: "doc.md", content: "original" }, - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - - { type: "update", client: 0, path: "doc.md", content: "update 1" }, - { type: "update", client: 0, path: "doc.md", content: "update 2" }, - { type: "update", client: 0, path: "doc.md", content: "final update" }, - - { type: "barrier" }, - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertFileCount(1).assertContent("doc.md", "final update"); - } - }, - - { type: "disable-sync", client: 0 }, - { type: "disable-sync", client: 1 }, - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertFileCount(1).assertContent("doc.md", "final update"); - } - }, - - { type: "disable-sync", client: 0 }, - { type: "disable-sync", client: 1 }, - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertFileCount(1).assertContent("doc.md", "final update"); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/concurrent-delete-during-remote-update.test.ts b/frontend/deterministic-tests/src/tests/concurrent-delete-during-remote-update.test.ts deleted file mode 100644 index 88376f22..00000000 --- a/frontend/deterministic-tests/src/tests/concurrent-delete-during-remote-update.test.ts +++ /dev/null @@ -1,32 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const concurrentDeleteDuringRemoteUpdateTest: TestDefinition = { - description: - "One client updates a file while the other deletes it at the same " + - "time. Both clients should converge without errors.", - clients: 2, - steps: [ - { type: "create", client: 0, path: "doc.md", content: "original" }, - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - - { type: "disable-sync", client: 0 }, - { type: "disable-sync", client: 1 }, - - { type: "update", client: 0, path: "doc.md", content: "updated by 0" }, - { type: "delete", client: 1, path: "doc.md" }, - - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (state: AssertableState): void => { - state.assertFileCount(0); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/concurrent-edit-exact-same-position.test.ts b/frontend/deterministic-tests/src/tests/concurrent-edit-exact-same-position.test.ts deleted file mode 100644 index 5c141a0e..00000000 --- a/frontend/deterministic-tests/src/tests/concurrent-edit-exact-same-position.test.ts +++ /dev/null @@ -1,49 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const concurrentEditExactSamePositionTest: TestDefinition = { - description: - "Both clients replace the same word in a file with different text " + - "while offline. After syncing, the merged result should contain " + - "both replacements.", - clients: 2, - steps: [ - { - type: "create", - client: 0, - path: "doc.md", - content: "the quick brown fox" - }, - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - - { type: "disable-sync", client: 0 }, - { type: "disable-sync", client: 1 }, - { - type: "update", - client: 0, - path: "doc.md", - content: "the slow brown fox" - }, - { - type: "update", - client: 1, - path: "doc.md", - content: "the fast brown fox" - }, - - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (state: AssertableState): void => { - state - .assertFileCount(1) - .assertContains("doc.md", "slow", "fast", "brown fox"); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/concurrent-rename-and-create-at-target-create-first.test.ts b/frontend/deterministic-tests/src/tests/concurrent-rename-and-create-at-target-create-first.test.ts deleted file mode 100644 index cd8046ce..00000000 --- a/frontend/deterministic-tests/src/tests/concurrent-rename-and-create-at-target-create-first.test.ts +++ /dev/null @@ -1,49 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const concurrentRenameAndCreateAtTargetCreateFirstTest: TestDefinition = { - description: - "One client renames X to Y while another creates a new file at Y, " + - "both offline. After syncing, Y should contain merged content from " + - "both the renamed file and the newly created file.", - clients: 2, - steps: [ - { - type: "create", - client: 0, - path: "X.md", - content: "original file X" - }, - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - - { type: "disable-sync", client: 0 }, - { type: "disable-sync", client: 1 }, - - { type: "rename", client: 0, oldPath: "X.md", newPath: "Y.md" }, - - { - type: "create", - client: 1, - path: "Y.md", - content: "brand new Y content" - }, - - { type: "enable-sync", client: 1 }, - { type: "sync", client: 1 }, - - { type: "enable-sync", client: 0 }, - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (state: AssertableState): void => { - state - .assertFileCount(2) - .assertContains("Y (1).md", "original file X") - .assertContains("Y.md", "brand new Y content"); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/concurrent-rename-and-create-at-target-rename-first.test.ts b/frontend/deterministic-tests/src/tests/concurrent-rename-and-create-at-target-rename-first.test.ts deleted file mode 100644 index 0ac0b721..00000000 --- a/frontend/deterministic-tests/src/tests/concurrent-rename-and-create-at-target-rename-first.test.ts +++ /dev/null @@ -1,52 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const concurrentRenameAndCreateAtTargetRenameFirstTest: TestDefinition = { - description: - "One client renames X to Y while another creates a new file at Y, " + - "both offline. We can't merge the create because it would result in a cycle", - clients: 2, - steps: [ - { - type: "create", - client: 0, - path: "X.md", - content: "original file X" - }, - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - - { type: "disable-sync", client: 0 }, - { type: "disable-sync", client: 1 }, - - { type: "rename", client: 0, oldPath: "X.md", newPath: "Y.md" }, - - { - type: "create", - client: 1, - path: "Y.md", - content: "brand new Y content" - }, - - { type: "enable-sync", client: 0 }, - { type: "sync", client: 0 }, - - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (state: AssertableState): void => { - state - .assertFileNotExists("X.md") - .assertFileExists("Y.md") - .assertFileExists("Y (1).md") - .assertAnyFileContains( - "original file X", - "brand new Y content" - ); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/concurrent-rename-first-wins.test.ts b/frontend/deterministic-tests/src/tests/concurrent-rename-first-wins.test.ts deleted file mode 100644 index 5337649d..00000000 --- a/frontend/deterministic-tests/src/tests/concurrent-rename-first-wins.test.ts +++ /dev/null @@ -1,61 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const concurrentRenameFirstWinsTest: TestDefinition = { - description: - "Both clients start online with the same file. Both go offline, " + - "rename the file to different paths, and edit it. When they reconnect, " + - "the first rename to reach the server wins the path and both content " + - "edits are merged.", - clients: 2, - steps: [ - { - type: "create", - client: 0, - path: "A.md", - content: "line 1\nline 2\nline 3" - }, - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertContent("A.md", "line 1\nline 2\nline 3"); - } - }, - - { type: "disable-sync", client: 0 }, - { type: "disable-sync", client: 1 }, - - { type: "rename", client: 0, oldPath: "A.md", newPath: "B.md" }, - { - type: "update", - client: 0, - path: "B.md", - content: "edit from 0\nline 2\nline 3" - }, - - { type: "rename", client: 1, oldPath: "A.md", newPath: "C.md" }, - { - type: "update", - client: 1, - path: "C.md", - content: "line 1\nline 2\nedit from 1" - }, - - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertFileNotExists("A.md") - .assertFileCount(2) - .assertContent("B.md", "edit from 0\nline 2\nline 3") - .assertContent("C.md", "line 1\nline 2\nedit from 1"); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/concurrent-rename-same-target.test.ts b/frontend/deterministic-tests/src/tests/concurrent-rename-same-target.test.ts deleted file mode 100644 index 0b72c0f3..00000000 --- a/frontend/deterministic-tests/src/tests/concurrent-rename-same-target.test.ts +++ /dev/null @@ -1,39 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const concurrentRenameSameTargetTest: TestDefinition = { - description: - "One client renames A to C while the other renames B to C, both offline. " + - "After syncing, both file contents should be preserved via path deconfliction.", - clients: 2, - steps: [ - { type: "create", client: 0, path: "A.md", content: "content-a" }, - { type: "create", client: 0, path: "B.md", content: "content-b" }, - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - - { type: "disable-sync", client: 1 }, - - { type: "rename", client: 0, oldPath: "A.md", newPath: "C.md" }, - { type: "sync", client: 0 }, - - { type: "rename", client: 1, oldPath: "B.md", newPath: "C.md" }, - - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (state: AssertableState): void => { - state - .assertFileCount(2) - .assertFileNotExists("A.md") - .assertFileNotExists("B.md") - .assertFileExists("C.md") - .assertFileExists("C (1).md") - .assertAnyFileContains("content-a", "content-b"); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/concurrent-update-diff-consistency.test.ts b/frontend/deterministic-tests/src/tests/concurrent-update-diff-consistency.test.ts deleted file mode 100644 index d21ce16b..00000000 --- a/frontend/deterministic-tests/src/tests/concurrent-update-diff-consistency.test.ts +++ /dev/null @@ -1,51 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const concurrentUpdateDiffConsistencyTest: TestDefinition = { - description: - "Both clients edit different sections of the same file while offline. " + - "After syncing, the merged file should contain both edits.", - clients: 2, - steps: [ - { - type: "create", - client: 0, - path: "doc.md", - content: "header\nmiddle\nfooter" - }, - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - - { type: "disable-sync", client: 0 }, - { type: "disable-sync", client: 1 }, - { - type: "update", - client: 0, - path: "doc.md", - content: "header by 0\nmiddle\nfooter" - }, - { - type: "update", - client: 1, - path: "doc.md", - content: "header\nmiddle\nfooter by 1" - }, - - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (state: AssertableState): void => { - state - .assertFileCount(1) - .assertContent( - "doc.md", - "header by 0\nmiddle\nfooter by 1" - ); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/create-delete-noop.test.ts b/frontend/deterministic-tests/src/tests/create-delete-noop.test.ts deleted file mode 100644 index 6c766001..00000000 --- a/frontend/deterministic-tests/src/tests/create-delete-noop.test.ts +++ /dev/null @@ -1,27 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const createDeleteNoopTest: TestDefinition = { - description: - "A client creates a file, updates it multiple times, then deletes it, all while " + - "offline. After syncing, neither client should have the file.", - clients: 2, - steps: [ - { type: "enable-sync", client: 1 }, - - { type: "create", client: 0, path: "temp.md", content: "version 1" }, - { type: "update", client: 0, path: "temp.md", content: "version 2" }, - { type: "update", client: 0, path: "temp.md", content: "version 3" }, - { type: "delete", client: 0, path: "temp.md" }, - - { type: "enable-sync", client: 0 }, - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertFileNotExists("temp.md"); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/create-during-reconciliation.test.ts b/frontend/deterministic-tests/src/tests/create-during-reconciliation.test.ts deleted file mode 100644 index 0fe51106..00000000 --- a/frontend/deterministic-tests/src/tests/create-during-reconciliation.test.ts +++ /dev/null @@ -1,50 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const createDuringReconciliationTest: TestDefinition = { - description: - "Client creates two files while offline, reconnects, then immediately " + - "creates a third file. All three files should sync to the other client.", - clients: 2, - steps: [ - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - - { type: "disable-sync", client: 0 }, - { - type: "create", - client: 0, - path: "A.md", - content: "offline A" - }, - { - type: "create", - client: 0, - path: "B.md", - content: "offline B" - }, - - { type: "enable-sync", client: 0 }, - - { - type: "create", - client: 0, - path: "C.md", - content: "post-reconnect C" - }, - - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (state: AssertableState): void => { - state - .assertFileCount(3) - .assertContent("A.md", "offline A") - .assertContent("B.md", "offline B") - .assertContent("C.md", "post-reconnect C"); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/create-merge-delete.test.ts b/frontend/deterministic-tests/src/tests/create-merge-delete.test.ts deleted file mode 100644 index ef7ea5c3..00000000 --- a/frontend/deterministic-tests/src/tests/create-merge-delete.test.ts +++ /dev/null @@ -1,37 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const createMergeDeleteTest: TestDefinition = { - description: - "Two clients create A.md offline with different content. Both come online and " + - "the content is merged. Then one client deletes A.md. Both clients should " + - "converge on an empty state.", - clients: 2, - steps: [ - { type: "create", client: 0, path: "A.md", content: "from-zero" }, - { type: "create", client: 1, path: "A.md", content: "from-one" }, - - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (state: AssertableState): void => { - state - .assertFileCount(1) - .assertContains("A.md", "from-zero", "from-one"); - } - }, - - { type: "delete", client: 0, path: "A.md" }, - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertFileCount(0).assertFileNotExists("A.md"); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/create-merge-preserves-renamed-update.test.ts b/frontend/deterministic-tests/src/tests/create-merge-preserves-renamed-update.test.ts deleted file mode 100644 index a9bc37d4..00000000 --- a/frontend/deterministic-tests/src/tests/create-merge-preserves-renamed-update.test.ts +++ /dev/null @@ -1,59 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const createMergePreservesRenamedUpdateTest: TestDefinition = { - description: - "Both clients create the same file, which gets merged. One client goes " + - "offline, renames the file, updates it, and creates a new file at the " + - "original path. After reconnecting, the updated content must be preserved.", - clients: 2, - steps: [ - { type: "create", client: 0, path: "doc.md", content: "alpha" }, - { type: "create", client: 1, path: "doc.md", content: "beta" }, - - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (state: AssertableState): void => { - state.assertContains("doc.md", "alpha", "beta"); - } - }, - - { type: "disable-sync", client: 1 }, - - { - type: "rename", - client: 1, - oldPath: "doc.md", - newPath: "moved.md" - }, - { - type: "update", - client: 1, - path: "moved.md", - content: "alpha beta extra-update" - }, - - { - type: "create", - client: 1, - path: "doc.md", - content: "new-content" - }, - - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (state: AssertableState): void => { - state - .assertContent("moved.md", "alpha beta extra-update") - .assertContent("doc.md", "new-content"); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/create-rename-create-same-path.test.ts b/frontend/deterministic-tests/src/tests/create-rename-create-same-path.test.ts deleted file mode 100644 index b9e16c90..00000000 --- a/frontend/deterministic-tests/src/tests/create-rename-create-same-path.test.ts +++ /dev/null @@ -1,34 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const createRenameCreateSamePathTest: TestDefinition = { - description: - "Client creates A.md, renames to B.md, creates new A.md, renames " + - "to C.md, creates yet another A.md. All three files should exist " + - "as separate documents on both clients.", - clients: 2, - steps: [ - { type: "create", client: 0, path: "A.md", content: "first file" }, - { type: "rename", client: 0, oldPath: "A.md", newPath: "B.md" }, - - { type: "create", client: 0, path: "A.md", content: "second file" }, - { type: "rename", client: 0, oldPath: "A.md", newPath: "C.md" }, - - { type: "create", client: 0, path: "A.md", content: "third file" }, - - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (state: AssertableState): void => { - state - .assertFileCount(3) - .assertContent("B.md", "first file") - .assertContent("C.md", "second file") - .assertContent("A.md", "third file"); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/create-rename-response-skips-file.test.ts b/frontend/deterministic-tests/src/tests/create-rename-response-skips-file.test.ts deleted file mode 100644 index aa24b110..00000000 --- a/frontend/deterministic-tests/src/tests/create-rename-response-skips-file.test.ts +++ /dev/null @@ -1,36 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const createRenameResponseSkipsFileTest: TestDefinition = { - description: - "Client 0 creates a file online then immediately renames it. " + - "Client 1 must receive the file content at the renamed path.", - clients: 2, - steps: [ - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - - { - type: "create", - client: 0, - path: "doc.md", - content: "the-content" - }, - - { - type: "rename", - client: 0, - oldPath: "doc.md", - newPath: "renamed.md" - }, - - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertFileCount(1).assertAnyFileContains("the-content"); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/create-update-coalesce-server-pause.test.ts b/frontend/deterministic-tests/src/tests/create-update-coalesce-server-pause.test.ts deleted file mode 100644 index 9b752d05..00000000 --- a/frontend/deterministic-tests/src/tests/create-update-coalesce-server-pause.test.ts +++ /dev/null @@ -1,32 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const createUpdateCoalesceServerPauseTest: TestDefinition = { - description: - "Client creates a file and immediately updates it while the server is " + - "paused. When the server resumes, both clients should have the final " + - "updated content.", - clients: 2, - steps: [ - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - - { type: "pause-server" }, - - { type: "create", client: 0, path: "doc.md", content: "initial" }, - { type: "update", client: 0, path: "doc.md", content: "final version" }, - - { type: "resume-server" }, - - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (state: AssertableState): void => { - state - .assertFileCount(1) - .assertContent("doc.md", "final version"); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/delete-by-other-client-then-recreate.test.ts b/frontend/deterministic-tests/src/tests/delete-by-other-client-then-recreate.test.ts deleted file mode 100644 index dfef9961..00000000 --- a/frontend/deterministic-tests/src/tests/delete-by-other-client-then-recreate.test.ts +++ /dev/null @@ -1,40 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const deleteByOtherClientThenRecreateTest: TestDefinition = { - description: - "Client 1 deletes a file and the delete propagates. Then client 0 " + - "creates a new file at the same path. Both clients must have the file.", - clients: 2, - steps: [ - { type: "create", client: 0, path: "A.md", content: "original" }, - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - - { type: "delete", client: 1, path: "A.md" }, - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertFileNotExists("A.md"); - } - }, - - { - type: "create", - client: 0, - path: "A.md", - content: "recreated by client 0" - }, - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertContent("A.md", "recreated by client 0"); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/delete-during-pending-create.test.ts b/frontend/deterministic-tests/src/tests/delete-during-pending-create.test.ts deleted file mode 100644 index 3ba393b8..00000000 --- a/frontend/deterministic-tests/src/tests/delete-during-pending-create.test.ts +++ /dev/null @@ -1,35 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const deleteDuringPendingCreateTest: TestDefinition = { - description: - "Client 0 creates a file while the server is paused, then deletes it before the server resumes. " + - "After resume, the file should end up deleted on both clients.", - clients: 2, - steps: [ - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - - { type: "pause-server" }, - - { - type: "create", - client: 0, - path: "ephemeral.md", - content: "this will be deleted" - }, - - { type: "delete", client: 0, path: "ephemeral.md" }, - - { type: "resume-server" }, - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertFileCount(0).assertFileNotExists("ephemeral.md"); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/delete-recreate-concurrent-update.test.ts b/frontend/deterministic-tests/src/tests/delete-recreate-concurrent-update.test.ts deleted file mode 100644 index 6cb4cb98..00000000 --- a/frontend/deterministic-tests/src/tests/delete-recreate-concurrent-update.test.ts +++ /dev/null @@ -1,42 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const deleteRecreateConcurrentUpdateTest: TestDefinition = { - description: - "Client 0 deletes and recreates A.md with new content while offline. Client 1 updates A.md concurrently. " + - "After client 0 reconnects, both clients must converge with client 0's recreated content preserved.", - clients: 2, - steps: [ - { type: "create", client: 0, path: "A.md", content: "original" }, - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - - { type: "disable-sync", client: 0 }, - { type: "delete", client: 0, path: "A.md" }, - { - type: "create", - client: 0, - path: "A.md", - content: "recreated by client 0" - }, - - { - type: "update", - client: 1, - path: "A.md", - content: "updated by client 1" - }, - { type: "sync", client: 1 }, - - { type: "enable-sync", client: 0 }, - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertFileExists("A.md").assertContains("A.md", "recreated"); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/delete-recreate-different-content.test.ts b/frontend/deterministic-tests/src/tests/delete-recreate-different-content.test.ts deleted file mode 100644 index 782c3cd5..00000000 --- a/frontend/deterministic-tests/src/tests/delete-recreate-different-content.test.ts +++ /dev/null @@ -1,54 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const deleteRecreateDifferentContentTest: TestDefinition = { - description: - "Client 0 deletes and recreates A.md with new content offline while client 1 edits A.md offline. " + - "Both clients should converge with content from both sides merged.", - clients: 2, - steps: [ - { - type: "create", - client: 0, - path: "A.md", - content: "original content here" - }, - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - - { type: "disable-sync", client: 0 }, - { type: "disable-sync", client: 1 }, - - { type: "delete", client: 0, path: "A.md" }, - { - type: "create", - client: 0, - path: "A.md", - content: "brand new content" - }, - - { - type: "update", - client: 1, - path: "A.md", - content: "edit from client 1" - }, - - { type: "enable-sync", client: 0 }, - { type: "sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertFileCount(1).assertContains( - "A.md", - "brand new", - "client 1" - ); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/delete-recreate-same-path.test.ts b/frontend/deterministic-tests/src/tests/delete-recreate-same-path.test.ts deleted file mode 100644 index dde8d341..00000000 --- a/frontend/deterministic-tests/src/tests/delete-recreate-same-path.test.ts +++ /dev/null @@ -1,34 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const deleteRecreateSamePathTest: TestDefinition = { - description: - "Client 0 creates A.md, syncs. Then deletes A.md and creates a new A.md " + - "with different content. Both clients should converge on the new content.", - clients: 2, - steps: [ - { type: "create", client: 0, path: "A.md", content: "version 1" }, - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertContent("A.md", "version 1"); - } - }, - - { type: "disable-sync", client: 0 }, - { type: "delete", client: 0, path: "A.md" }, - { type: "create", client: 0, path: "A.md", content: "version 2" }, - { type: "enable-sync", client: 0 }, - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertContent("A.md", "version 2"); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/delete-recreated-pending-create-with-stale-deleting-record.test.ts b/frontend/deterministic-tests/src/tests/delete-recreated-pending-create-with-stale-deleting-record.test.ts deleted file mode 100644 index 80e95f48..00000000 --- a/frontend/deterministic-tests/src/tests/delete-recreated-pending-create-with-stale-deleting-record.test.ts +++ /dev/null @@ -1,52 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const deleteRecreatedPendingCreateWithStaleDeletingRecordTest: TestDefinition = - { - description: - "A local delete for a recreated pending create must target the " + - "new pending create, not an older same-path record whose server " + - "delete has been acked but whose WebSocket delete receipt is " + - "still paused.", - clients: 2, - steps: [ - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - - { type: "pause-websocket", client: 0 }, - { type: "pause-server" }, - { - type: "create", - client: 0, - path: "binary-14.bin", - content: "BINARY:first" - }, - { type: "sleep", ms: 100 }, - { type: "delete", client: 0, path: "binary-14.bin" }, - { type: "resume-server" }, - { type: "sync", client: 0 }, - - { type: "pause-server" }, - { - type: "create", - client: 0, - path: "binary-14.bin", - content: "BINARY:second" - }, - { type: "sleep", ms: 100 }, - { type: "delete", client: 0, path: "binary-14.bin" }, - { type: "resume-server" }, - { type: "sync", client: 0 }, - - { type: "resume-websocket", client: 0 }, - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (state: AssertableState): void => { - state.assertFileCount(0); - } - } - ] - }; diff --git a/frontend/deterministic-tests/src/tests/delete-rename-conflict.test.ts b/frontend/deterministic-tests/src/tests/delete-rename-conflict.test.ts deleted file mode 100644 index 91e6289b..00000000 --- a/frontend/deterministic-tests/src/tests/delete-rename-conflict.test.ts +++ /dev/null @@ -1,43 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const deleteRenameConflictTest: TestDefinition = { - description: - "Client 0 deletes A.md while client 1 renames A.md to C.md offline. " + - "After client 1 reconnects, both clients should converge to the same state.", - clients: 2, - steps: [ - { type: "create", client: 0, path: "A.md", content: "content-a" }, - { type: "create", client: 0, path: "B.md", content: "content-b" }, - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertFileExists("A.md").assertFileExists("B.md"); - } - }, - - { type: "disable-sync", client: 1 }, - - { type: "delete", client: 0, path: "A.md" }, - { type: "sync", client: 0 }, - - { type: "rename", client: 1, oldPath: "A.md", newPath: "C.md" }, - - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertContent("B.md", "content-b"); - s.assertFileNotExists("A.md"); - s.ifFileExists("C.md", (inner) => - inner.assertContent("C.md", "content-a") - ); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/displaced-file-not-marked-deleted.test.ts b/frontend/deterministic-tests/src/tests/displaced-file-not-marked-deleted.test.ts deleted file mode 100644 index cb995243..00000000 --- a/frontend/deterministic-tests/src/tests/displaced-file-not-marked-deleted.test.ts +++ /dev/null @@ -1,38 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const displacedFileNotMarkedDeletedTest: TestDefinition = { - description: - "Client 0 creates a new file at path B.md while client 1 renames " + - "A.md to B.md. The remote download of B.md displaces client 1's " + - "renamed file. The displaced document must not be permanently " + - "marked as recently deleted, so it can still be synced.", - clients: 2, - steps: [ - { type: "create", client: 0, path: "A.md", content: "content of A" }, - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - - { type: "disable-sync", client: 1 }, - - { type: "create", client: 0, path: "B.md", content: "content of B" }, - { type: "rename", client: 0, oldPath: "A.md", newPath: "C.md" }, - { type: "sync", client: 0 }, - - { type: "rename", client: 1, oldPath: "A.md", newPath: "B.md" }, - { type: "enable-sync", client: 1 }, - - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (state: AssertableState): void => { - state - .assertFileCount(2) - .assertContent("B.md", "content of B") - .assertContent("C.md", "content of A"); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/double-offline-cycle.test.ts b/frontend/deterministic-tests/src/tests/double-offline-cycle.test.ts deleted file mode 100644 index 744d862e..00000000 --- a/frontend/deterministic-tests/src/tests/double-offline-cycle.test.ts +++ /dev/null @@ -1,77 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const doubleOfflineCycleTest: TestDefinition = { - description: - "Client 0 goes through three offline-edit-reconnect cycles. " + - "Each offline edit must propagate to client 1 after reconnection.", - clients: 2, - steps: [ - { - type: "create", - client: 0, - path: "doc.md", - content: "initial" - }, - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertContent("doc.md", "initial"); - } - }, - - { type: "disable-sync", client: 0 }, - { - type: "update", - client: 0, - path: "doc.md", - content: "first edit" - }, - - { type: "enable-sync", client: 0 }, - { type: "barrier" }, - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertContent("doc.md", "first edit"); - } - }, - - { type: "disable-sync", client: 0 }, - { - type: "update", - client: 0, - path: "doc.md", - content: "second edit" - }, - - { type: "enable-sync", client: 0 }, - { type: "barrier" }, - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertContent("doc.md", "second edit"); - } - }, - - { type: "disable-sync", client: 0 }, - { - type: "update", - client: 0, - path: "doc.md", - content: "third edit" - }, - - { type: "enable-sync", client: 0 }, - { type: "barrier" }, - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertFileCount(1).assertContent("doc.md", "third edit"); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/idempotency-after-server-pause.test.ts b/frontend/deterministic-tests/src/tests/idempotency-after-server-pause.test.ts deleted file mode 100644 index 551c702d..00000000 --- a/frontend/deterministic-tests/src/tests/idempotency-after-server-pause.test.ts +++ /dev/null @@ -1,33 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const idempotencyAfterServerPauseTest: TestDefinition = { - description: - "Client 0 creates a file, then the server is paused mid-response. " + - "After the server resumes, both clients must converge to a single copy of the file with no duplicates.", - clients: 2, - steps: [ - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - - { - type: "create", - client: 0, - path: "doc.md", - content: "important data" - }, - { type: "pause-server" }, - - { type: "resume-server" }, - - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertFileCount(1).assertContent("doc.md", "important data"); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/interrupted-delete-retry.test.ts b/frontend/deterministic-tests/src/tests/interrupted-delete-retry.test.ts deleted file mode 100644 index 3ae7eda5..00000000 --- a/frontend/deterministic-tests/src/tests/interrupted-delete-retry.test.ts +++ /dev/null @@ -1,29 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const interruptedDeleteRetryTest: TestDefinition = { - description: - "Client 0 deletes a file, then the server is paused. " + - "After the server resumes, both clients should have zero files.", - clients: 2, - steps: [ - { type: "create", client: 0, path: "doc.md", content: "to be deleted" }, - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - - { type: "delete", client: 0, path: "doc.md" }, - - { type: "pause-server" }, - - { type: "resume-server" }, - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertFileCount(0); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/local-edit-lost-during-create-merge.test.ts b/frontend/deterministic-tests/src/tests/local-edit-lost-during-create-merge.test.ts deleted file mode 100644 index 20925889..00000000 --- a/frontend/deterministic-tests/src/tests/local-edit-lost-during-create-merge.test.ts +++ /dev/null @@ -1,41 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const localEditLostDuringCreateMergeTest: TestDefinition = { - description: - "Both clients create doc.md with different content while offline. " + - "Client 0 also edits the file before syncing. After both connect, " + - "the merged result should contain content from both clients.", - clients: 2, - steps: [ - { type: "create", client: 1, path: "doc.md", content: "from-client-1" }, - { - type: "create", - client: 0, - path: "doc.md", - content: "from-client-0" - }, - { - type: "update", - client: 0, - path: "doc.md", - content: "local-edit-during-create" - }, - - { type: "enable-sync", client: 1 }, - { type: "sync", client: 1 }, - { type: "enable-sync", client: 0 }, - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertFileCount(1).assertContains( - "doc.md", - "from-client-1", - "local-edit-during-create" - ); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/local-rename-survives-remote-rename.test.ts b/frontend/deterministic-tests/src/tests/local-rename-survives-remote-rename.test.ts deleted file mode 100644 index c2b80af3..00000000 --- a/frontend/deterministic-tests/src/tests/local-rename-survives-remote-rename.test.ts +++ /dev/null @@ -1,80 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const localRenameSurvivesRemoteRenameTest: TestDefinition = { - description: - "Drain processes a RemoteChange (remote rename for doc D) while a " + - "LocalUpdate (user rename of D) is also queued behind it. " + - "`processRemoteUpdate` moves the disk file and, because there is a " + - "pending LocalUpdate, takes the else branch — but its setDocument " + - "uses the stale `record.path` (= the user-rename target) instead of " + - "the actualPath the file just moved to. The queued LocalUpdate then " + - "reads from `record.path`, throws FileNotFoundError, and is " + - "silently dropped. Setup pins the queue order: a sentinel " + - "LocalUpdate keeps drain busy on a SIGSTOPped HTTP roundtrip while " + - "we resume client 0's WebSocket (enqueues RemoteChange) and then " + - "user-rename D (enqueues LocalUpdate after the RemoteChange). On " + - "server resume the drain pops the sentinel, then RemoteChange, then " + - "LocalUpdate — exactly the order that triggers the bug.", - clients: 2, - steps: [ - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "create", client: 0, path: "doc.md", content: "v1\n" }, - { type: "create", client: 0, path: "sentinel.md", content: "s\n" }, - { type: "barrier" }, - - // Pause client 0's WebSocket so the upcoming remote rename buffers. - { type: "pause-websocket", client: 0 }, - - // Server applies remote rename of doc.md -> remote.md. Broadcast - // is buffered on client 0's WebSocket. - { type: "rename", client: 1, oldPath: "doc.md", newPath: "remote.md" }, - { type: "sync", client: 1 }, - - // Pause the server BEFORE arming the sentinel, so the sentinel's - // HTTP request will buffer at the kernel and keep drain occupied. - { type: "pause-server" }, - - // Sentinel: a LocalUpdate on a *different* doc that drain pops - // first. Its HTTP roundtrip stalls on SIGSTOP, freezing drain - // until we resume the server. While drain is frozen we can grow - // the queue with additional events whose order we control. - { - type: "update", - client: 0, - path: "sentinel.md", - content: "s\nedit\n" - }, - - // Resume the WebSocket — buffered remote rename enqueues as a - // RemoteChange. Drain is still stuck on the sentinel HTTP. - { type: "resume-websocket", client: 0 }, - - // User renames doc.md -> local.md on client 0. queue.enqueue - // mutates the doc's record.path to "local.md" and pushes a - // LocalUpdate(rename) onto the tail of the queue. Queue is now - // [sentinel-update (in-flight), RemoteChange, LocalUpdate-rename]. - { type: "rename", client: 0, oldPath: "doc.md", newPath: "local.md" }, - - // Resume the server. Drain pops sentinel-update (succeeds), then - // RemoteChange. Pre-fix: processRemoteUpdate moves disk - // local.md -> remote.md, takes the else branch, and - // setDocument(record.path = "local.md", …) leaves record.path - // stale. Drain pops the LocalUpdate-rename and reads from the - // stale record.path, hits FileNotFoundError, silent skip. - // Post-fix: when a local event is pending, we re-queue the - // remote update without touching disk or record, so the local - // rename drains first and both ends converge. - { type: "resume-server" }, - - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (state: AssertableState): void => { - state.assertFileCount(2); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/local-update-survives-remote-rename.test.ts b/frontend/deterministic-tests/src/tests/local-update-survives-remote-rename.test.ts deleted file mode 100644 index 0d8348c0..00000000 --- a/frontend/deterministic-tests/src/tests/local-update-survives-remote-rename.test.ts +++ /dev/null @@ -1,69 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const localUpdateSurvivesRemoteRenameTest: TestDefinition = { - description: - "Client 0 has a local content edit pending while a remote rename for " + - "the same doc arrives over the WebSocket. The remote rename's internal " + - "move relocates the disk file from the old path (where the user wrote) " + - "to the new server path. Previously, the queued LocalUpdate's " + - "`event.path` was left pointing at the now-vacated old path, so " + - "`skipIfOversized`'s `getFileSize(event.path)` threw " + - "`FileNotFoundError`, which `processEvent`'s catch silently swallowed " + - "as 'Skipping sync event 'local-update' because the file no longer " + - "exists' — and the user's edit was lost. The fix routes the size " + - "check through `tracked.path` (the doc's current disk path), " + - "matching the path `processLocalUpdate` itself reads from.", - clients: 2, - steps: [ - { type: "create", client: 0, path: "doc.md", content: "v1\n" }, - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - - // Pause client 0's WebSocket so the upcoming remote rename buffers - // there until we've already enqueued client 0's local content - // edit. This guarantees the LocalUpdate sits in client 0's queue - // when the rename's RemoteChange drains. - { type: "pause-websocket", client: 0 }, - - { - type: "rename", - client: 1, - oldPath: "doc.md", - newPath: "renamed.md" - }, - { type: "sync", client: 1 }, - - // Client 0 still believes the file is at `doc.md` (its WebSocket is - // paused, so the rename hasn't reached it). The user edits content - // at `doc.md`. This pushes a LocalUpdate(D, path=doc.md, - // originalPath=doc.md, isUserRename=false) into client 0's queue. - { - type: "update", - client: 0, - path: "doc.md", - content: "v1\nclient 0 edit\n" - }, - - // Resume the WebSocket. The buffered remote rename (server-broadcast) - // drains. `processRemoteUpdate` does an internal `move(doc.md, - // renamed.md)` and, because there's a pending LocalUpdate for D, - // takes the else branch (re-enqueue v_K, setDocument(renamed.md, …)). - // Then drain reaches the LocalUpdate. Pre-fix: skipped silently. - // Post-fix: PUTs the user's content to the doc (at its new path, - // since this is a content-only edit, not a user rename). - { type: "resume-websocket", client: 0 }, - - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (state: AssertableState): void => { - state.assertFileCount(1); - state.assertFileExists("renamed.md"); - state.assertContent("renamed.md", "v1\nclient 0 edit\n"); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/mc-cross-create-rename-same-target.test.ts b/frontend/deterministic-tests/src/tests/mc-cross-create-rename-same-target.test.ts deleted file mode 100644 index d986a733..00000000 --- a/frontend/deterministic-tests/src/tests/mc-cross-create-rename-same-target.test.ts +++ /dev/null @@ -1,46 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const mcCrossCreateRenameSameTargetTest: TestDefinition = { - description: - "Client 0 creates X.md, Client 1 creates Y.md. Both sync. Client 0 renames " + - "X.md -> Z.md. Client 1 (offline) renames Y.md -> Z.md. Both must converge " + - "with both contents preserved via path deconfliction.", - clients: 2, - steps: [ - { type: "create", client: 0, path: "X.md", content: "content-x" }, - { type: "create", client: 1, path: "Y.md", content: "content-y" }, - - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertFileExists("X.md").assertFileExists("Y.md"); - } - }, - - { type: "disable-sync", client: 1 }, - - { type: "rename", client: 0, oldPath: "X.md", newPath: "Z.md" }, - { type: "sync", client: 0 }, - - { type: "rename", client: 1, oldPath: "Y.md", newPath: "Z.md" }, - - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertFileCount(2) - .assertFileNotExists("X.md") - .assertFileNotExists("Y.md") - .assertFileExists("Z.md") - .assertAnyFileContains("content-x", "content-y"); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/mc-delete-then-offline-rename.test.ts b/frontend/deterministic-tests/src/tests/mc-delete-then-offline-rename.test.ts deleted file mode 100644 index 6727e99d..00000000 --- a/frontend/deterministic-tests/src/tests/mc-delete-then-offline-rename.test.ts +++ /dev/null @@ -1,39 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const mcDeleteThenOfflineRenameTest: TestDefinition = { - description: - "Client 0 creates A.md, both sync. Client 1 goes offline. Client 0 deletes " + - "A.md and syncs. Client 1 (offline) renames A.md to B.md. Client 1 reconnects. " + - "Both must converge. C.md (unrelated) must be unaffected.", - clients: 2, - steps: [ - { type: "create", client: 0, path: "A.md", content: "original" }, - { type: "create", client: 0, path: "C.md", content: "unrelated" }, - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - - { type: "disable-sync", client: 1 }, - - { type: "delete", client: 0, path: "A.md" }, - { type: "sync", client: 0 }, - - { type: "rename", client: 1, oldPath: "A.md", newPath: "B.md" }, - - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertContent("C.md", "unrelated").assertFileNotExists( - "A.md" - ); - s.ifFileExists("B.md", (inner) => - inner.assertContent("B.md", "original") - ); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/mc-multi-delete-offline-rename.test.ts b/frontend/deterministic-tests/src/tests/mc-multi-delete-offline-rename.test.ts deleted file mode 100644 index 8db90aab..00000000 --- a/frontend/deterministic-tests/src/tests/mc-multi-delete-offline-rename.test.ts +++ /dev/null @@ -1,49 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const mcMultiDeleteOfflineRenameTest: TestDefinition = { - description: - "Client 0 creates 5 files. Client 1 deletes 2 while Client 0 (offline) " + - "renames one of the deleted files. Both must converge.", - clients: 2, - steps: [ - { type: "create", client: 0, path: "file-1.md", content: "content-1" }, - { type: "create", client: 0, path: "file-2.md", content: "content-2" }, - { type: "create", client: 0, path: "file-3.md", content: "content-3" }, - { type: "create", client: 0, path: "file-4.md", content: "content-4" }, - { type: "create", client: 0, path: "file-5.md", content: "content-5" }, - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - - { type: "disable-sync", client: 0 }, - - { type: "delete", client: 1, path: "file-2.md" }, - { type: "delete", client: 1, path: "file-4.md" }, - { type: "sync", client: 1 }, - - { - type: "rename", - client: 0, - oldPath: "file-2.md", - newPath: "renamed.md" - }, - - { type: "enable-sync", client: 0 }, - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertFileExists("file-1.md") - .assertFileExists("file-3.md") - .assertFileExists("file-5.md") - .assertFileNotExists("file-2.md") - .assertFileNotExists("file-4.md"); - s.ifFileExists("renamed.md", (inner) => - inner.assertContent("renamed.md", "content-2") - ); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/mc-three-client-rename-offline-update.test.ts b/frontend/deterministic-tests/src/tests/mc-three-client-rename-offline-update.test.ts deleted file mode 100644 index 4167b925..00000000 --- a/frontend/deterministic-tests/src/tests/mc-three-client-rename-offline-update.test.ts +++ /dev/null @@ -1,41 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const mcThreeClientRenameOfflineUpdateTest: TestDefinition = { - description: - "Client 0 creates A.md. Client 1 renames to B.md. Client 2 (offline) " + - "updates A.md. All three converge with updated content at B.md.", - clients: 3, - steps: [ - { type: "create", client: 0, path: "A.md", content: "original" }, - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "enable-sync", client: 2 }, - { type: "barrier" }, - - { type: "disable-sync", client: 2 }, - - { type: "rename", client: 1, oldPath: "A.md", newPath: "B.md" }, - { type: "sync", client: 1 }, - { type: "sync", client: 0 }, - - { - type: "update", - client: 2, - path: "A.md", - content: "updated-by-client-2" - }, - - { type: "enable-sync", client: 2 }, - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertFileCount(1) - .assertFileNotExists("A.md") - .assertContains("B.md", "updated-by-client-2"); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/merging-update-response-survives-user-rename.test.ts b/frontend/deterministic-tests/src/tests/merging-update-response-survives-user-rename.test.ts deleted file mode 100644 index e93240f9..00000000 --- a/frontend/deterministic-tests/src/tests/merging-update-response-survives-user-rename.test.ts +++ /dev/null @@ -1,77 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const mergingUpdateResponseSurvivesUserRenameTest: TestDefinition = { - description: - "Client 1 sends a content update with a stale `parent_version_id` " + - "(its WebSocket is paused, so it hasn't seen Client 0's intervening " + - "edit). The server merges and replies with `MergingUpdate` carrying " + - "the merged text. Before the response lands, the user renames the " + - "doc on Client 1, vacating the disk path the in-flight " + - "`processLocalUpdate` captured. Pre-fix: " + - "`handleMaybeMergingResponse`'s `operations.write(diskPath, …)` " + - "hits the `we wont recreate it` early-return inside `write`, " + - "silently dropping the server-merged content — Client 0's edit is " + - "lost on Client 1's disk, and Client 1's next local-update PUT " + - "(rebased on the now-untracked merged version) deletes Client 0's " + - "edit on the server too. Post-fix: the response is written to the " + - "doc's current tracked disk path, preserving both edits.", - clients: 2, - steps: [ - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "create", client: 0, path: "doc.md", content: "0\n" }, - { type: "barrier" }, - - // Stop Client 1 from seeing Client 0's next edit, so its next - // outbound PUT carries a stale `parent_version_id` and the server - // is forced to merge. - { type: "pause-websocket", client: 1 }, - - // Server now holds v_b = "0\nA\n". Client 1's tracked parent - // version stays at v_a = "0\n". - { type: "update", client: 0, path: "doc.md", content: "0\nA\n" }, - { type: "sync", client: 0 }, - - // Pause the server. Subsequent HTTP PUTs from Client 1 buffer at - // the OS layer until resume. This guarantees the merge response - // for Client 1's update is still in flight when the rename below - // mutates `queue.documents`. - { type: "pause-server" }, - - // Client 1 edits doc.md with "B". The drain pops the LocalUpdate, - // captures `diskPath = "doc.md"`, reads the file, and sends the - // HTTP PUT — which buffers because the server is SIGSTOPped. - { type: "update", client: 1, path: "doc.md", content: "0\nB\n" }, - - // User renames the file while the previous PUT is still in flight. - // `queue.enqueue`'s rename branch updates `documents` to point at - // `renamed.md` synchronously, but `processLocalUpdate`'s captured - // `diskPath` ("doc.md") is a local — it can't be retargeted. - { type: "rename", client: 1, oldPath: "doc.md", newPath: "renamed.md" }, - - // Resume the server. It reconciles parent=v_a, latest=v_b, - // new="0\nB\n" → v_c with both edits, replies `MergingUpdate`. - // Pre-fix: write("doc.md", …) sees no file at that path - // (renamed.md now holds the data) and bails out without ever - // writing the merged bytes. Post-fix: the merged bytes land at - // the tracked path (renamed.md). - { type: "resume-server" }, - { type: "resume-websocket", client: 1 }, - - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (state: AssertableState): void => { - state.assertFileCount(1); - state.assertFileExists("renamed.md"); - state.assertFileNotExists("doc.md"); - // Both edits survive: Client 0's "A" and Client 1's "B". - // The reconcile may interleave them either way; assert - // both tokens are present in the converged content. - state.assertContains("renamed.md", "A", "B"); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/move-and-concurrent-remote-update.test.ts b/frontend/deterministic-tests/src/tests/move-and-concurrent-remote-update.test.ts deleted file mode 100644 index 86657f0f..00000000 --- a/frontend/deterministic-tests/src/tests/move-and-concurrent-remote-update.test.ts +++ /dev/null @@ -1,43 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const moveAndConcurrentRemoteUpdateTest: TestDefinition = { - description: - "Client 0 renames A.md to B.md offline while client 1 updates A.md. " + - "After client 0 reconnects, both should have B.md with client 1's updated content.", - clients: 2, - steps: [ - { - type: "create", - client: 0, - path: "A.md", - content: "original content" - }, - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - - { type: "disable-sync", client: 0 }, - { type: "rename", client: 0, oldPath: "A.md", newPath: "B.md" }, - - { - type: "update", - client: 1, - path: "A.md", - content: "updated by client 1" - }, - { type: "sync", client: 1 }, - - { type: "enable-sync", client: 0 }, - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertFileCount(1) - .assertFileNotExists("A.md") - .assertContains("B.md", "updated by client 1"); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/move-chain-three-files.test.ts b/frontend/deterministic-tests/src/tests/move-chain-three-files.test.ts deleted file mode 100644 index fe9267d4..00000000 --- a/frontend/deterministic-tests/src/tests/move-chain-three-files.test.ts +++ /dev/null @@ -1,42 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const moveChainThreeFilesTest: TestDefinition = { - description: - "Three files have their contents rotated (A gets C's content, B gets A's, C gets B's) " + - "while offline. After reconnecting, both clients should converge with the rotated contents.", - clients: 2, - steps: [ - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - - { type: "create", client: 0, path: "A.md", content: "was A" }, - { type: "create", client: 0, path: "B.md", content: "was B" }, - { type: "create", client: 0, path: "C.md", content: "was C" }, - { type: "barrier" }, - - { type: "disable-sync", client: 0 }, - - { type: "delete", client: 0, path: "A.md" }, - { type: "delete", client: 0, path: "B.md" }, - { type: "delete", client: 0, path: "C.md" }, - - { type: "create", client: 0, path: "A.md", content: "was C" }, - { type: "create", client: 0, path: "B.md", content: "was A" }, - { type: "create", client: 0, path: "C.md", content: "was B" }, - - { type: "enable-sync", client: 0 }, - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (state: AssertableState): void => { - state - .assertFileCount(3) - .assertContent("A.md", "was C") - .assertContent("B.md", "was A") - .assertContent("C.md", "was B"); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/move-identical-content-ambiguity.test.ts b/frontend/deterministic-tests/src/tests/move-identical-content-ambiguity.test.ts deleted file mode 100644 index 2a9ce0b4..00000000 --- a/frontend/deterministic-tests/src/tests/move-identical-content-ambiguity.test.ts +++ /dev/null @@ -1,44 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const moveIdenticalContentAmbiguityTest: TestDefinition = { - description: - "Two files with identical content exist. One is deleted and the other renamed " + - "while offline. The system should still converge correctly despite the ambiguity.", - clients: 2, - steps: [ - { - type: "create", - client: 0, - path: "A.md", - content: "identical content" - }, - { - type: "create", - client: 0, - path: "B.md", - content: "identical content" - }, - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - - { type: "disable-sync", client: 1 }, - { type: "delete", client: 1, path: "A.md" }, - { type: "rename", client: 1, oldPath: "B.md", newPath: "C.md" }, - - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (state: AssertableState): void => { - state - .assertFileCount(1) - .assertFileNotExists("A.md") - .assertFileNotExists("B.md") - .assertContent("C.md", "identical content"); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/move-preserves-remote-update.test.ts b/frontend/deterministic-tests/src/tests/move-preserves-remote-update.test.ts deleted file mode 100644 index 13e27349..00000000 --- a/frontend/deterministic-tests/src/tests/move-preserves-remote-update.test.ts +++ /dev/null @@ -1,48 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const movePreservesRemoteUpdateTest: TestDefinition = { - description: - "Client 0 renames a file offline while client 1 edits it offline. " + - "After both reconnect, the renamed file should contain client 1's edit.", - clients: 2, - steps: [ - { - type: "create", - client: 0, - path: "doc.md", - content: "line 1\nline 2" - }, - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - - { type: "disable-sync", client: 0 }, - { type: "disable-sync", client: 1 }, - - { type: "rename", client: 0, oldPath: "doc.md", newPath: "renamed.md" }, - { - type: "update", - client: 1, - path: "doc.md", - content: "line 1\nclient 1 edit\nline 2" - }, - - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertFileCount(1); - const [content] = Array.from(s.files.values()); - if (!content.includes("client 1 edit")) { - throw new Error( - `Expected merged content to include "client 1 edit", got: "${content}"` - ); - } - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/move-remote-update-reverts-rename.test.ts b/frontend/deterministic-tests/src/tests/move-remote-update-reverts-rename.test.ts deleted file mode 100644 index 433bf01b..00000000 --- a/frontend/deterministic-tests/src/tests/move-remote-update-reverts-rename.test.ts +++ /dev/null @@ -1,38 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const moveRemoteUpdateRevertsRenameTest: TestDefinition = { - description: - "Client 1 updates a file while client 0 is offline. Client 0 reconnects and renames the file. " + - "Both clients should converge with client 1's updated content.", - clients: 2, - steps: [ - { type: "create", client: 0, path: "doc.md", content: "original" }, - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - - { type: "disable-sync", client: 0 }, - { - type: "update", - client: 1, - path: "doc.md", - content: "updated by client 1" - }, - { type: "sync", client: 1 }, - - { type: "enable-sync", client: 0 }, - { type: "rename", client: 0, oldPath: "doc.md", newPath: "renamed.md" }, - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertFileCount(1).assertContent( - "renamed.md", - "updated by client 1" - ); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/move-then-delete-stale-path.test.ts b/frontend/deterministic-tests/src/tests/move-then-delete-stale-path.test.ts deleted file mode 100644 index 4f5feab5..00000000 --- a/frontend/deterministic-tests/src/tests/move-then-delete-stale-path.test.ts +++ /dev/null @@ -1,34 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const moveThenDeleteStalePathTest: TestDefinition = { - description: - "Client 0 renames A.md to B.md and immediately deletes B.md. " + - "Both clients should end up with zero files.", - clients: 2, - steps: [ - { - type: "create", - client: 0, - path: "A.md", - content: "content to delete" - }, - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - - { type: "rename", client: 0, oldPath: "A.md", newPath: "B.md" }, - { type: "delete", client: 0, path: "B.md" }, - - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertFileCount(0) - .assertFileNotExists("A.md") - .assertFileNotExists("B.md"); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/multi-file-operations.test.ts b/frontend/deterministic-tests/src/tests/multi-file-operations.test.ts deleted file mode 100644 index a47f5a2a..00000000 --- a/frontend/deterministic-tests/src/tests/multi-file-operations.test.ts +++ /dev/null @@ -1,45 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const multiFileOperationsTest: TestDefinition = { - description: - "Client 0 deletes A.md while client 1 is offline. Client 1 updates B.md and renames A.md to D.md offline. " + - "After client 1 reconnects, both clients must converge with B.md updated and C.md intact.", - clients: 2, - steps: [ - { type: "create", client: 0, path: "A.md", content: "content-a" }, - { type: "create", client: 0, path: "B.md", content: "content-b" }, - { type: "create", client: 0, path: "C.md", content: "content-c" }, - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - - { type: "disable-sync", client: 1 }, - - { type: "delete", client: 0, path: "A.md" }, - { type: "sync", client: 0 }, - - { - type: "update", - client: 1, - path: "B.md", - content: "updated by client 1" - }, - { type: "rename", client: 1, oldPath: "A.md", newPath: "D.md" }, - - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertContains("B.md", "updated") - .assertFileExists("C.md") - .assertFileNotExists("A.md"); - s.ifFileExists("D.md", (inner) => - inner.assertContent("D.md", "content-a") - ); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/offline-concurrent-renames.test.ts b/frontend/deterministic-tests/src/tests/offline-concurrent-renames.test.ts deleted file mode 100644 index 6c946b9c..00000000 --- a/frontend/deterministic-tests/src/tests/offline-concurrent-renames.test.ts +++ /dev/null @@ -1,59 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const offlineConcurrentRenamesTest: TestDefinition = { - description: - "Client 0 creates A.md and syncs to both clients. Both clients go offline. " + - "Client 0 renames A.md to B.md. Client 1 renames A.md to C.md. " + - "Both reconnect. The system must converge -- both clients should " + - "agree on the final state and the content must not be lost.", - clients: 2, - steps: [ - { type: "create", client: 0, path: "A.md", content: "shared-content" }, - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertContent("A.md", "shared-content"); - } - }, - - { type: "disable-sync", client: 0 }, - { type: "disable-sync", client: 1 }, - - { - type: "rename", - client: 0, - oldPath: "A.md", - newPath: "B.md" - }, - - { - type: "rename", - client: 1, - oldPath: "A.md", - newPath: "C.md" - }, - - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertFileNotExists("A.md") - .assertFileCount(1) - .assertAnyFileContains("shared-content"); - s.ifFileExists("B.md", (inner) => - inner.assertContent("B.md", "shared-content") - ); - s.ifFileExists("C.md", (inner) => - inner.assertContent("C.md", "shared-content") - ); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/offline-create-same-path-mergeable.test.ts b/frontend/deterministic-tests/src/tests/offline-create-same-path-mergeable.test.ts deleted file mode 100644 index cbd59a4a..00000000 --- a/frontend/deterministic-tests/src/tests/offline-create-same-path-mergeable.test.ts +++ /dev/null @@ -1,41 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const offlineCreateSamePathMergeableTest: TestDefinition = { - description: - "Both clients create a file at the same path while offline with different text content. " + - "After both sync, both clients must converge to a merged result containing both contributions.", - clients: 2, - steps: [ - { - type: "create", - client: 0, - path: "notes.md", - content: "alpha wrote this line" - }, - { - type: "create", - client: 1, - path: "notes.md", - content: "beta wrote this different line" - }, - - { type: "enable-sync", client: 0 }, - { type: "sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertFileCount(1) - .assertFileExists("notes.md") - .assertContains( - "notes.md", - "alpha wrote this line", - "beta wrote this different line" - ); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/offline-delete-remote-rename.test.ts b/frontend/deterministic-tests/src/tests/offline-delete-remote-rename.test.ts deleted file mode 100644 index 1e9ea8f7..00000000 --- a/frontend/deterministic-tests/src/tests/offline-delete-remote-rename.test.ts +++ /dev/null @@ -1,38 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const offlineDeleteRemoteRenameTest: TestDefinition = { - description: - "Client 0 deletes A.md offline while client 1 renames it to A_renamed.md. " + - "After client 0 reconnects, both clients must converge.", - clients: 2, - steps: [ - { type: "create", client: 0, path: "A.md", content: "content-a" }, - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - - { type: "disable-sync", client: 0 }, - { type: "delete", client: 0, path: "A.md" }, - - { - type: "rename", - client: 1, - oldPath: "A.md", - newPath: "A_renamed.md" - }, - { type: "sync", client: 1 }, - - { type: "enable-sync", client: 0 }, - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertFileNotExists("A.md").assertFileNotExists( - "A_renamed.md" - ); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/offline-delete-vs-remote-update.test.ts b/frontend/deterministic-tests/src/tests/offline-delete-vs-remote-update.test.ts deleted file mode 100644 index 21e81aa6..00000000 --- a/frontend/deterministic-tests/src/tests/offline-delete-vs-remote-update.test.ts +++ /dev/null @@ -1,46 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const offlineDeleteVsRemoteUpdateTest: TestDefinition = { - description: - "Client 0 deletes A.md offline while client 1 updates it. Both clients must converge.", - clients: 2, - steps: [ - { - type: "create", - client: 0, - path: "A.md", - content: "original content" - }, - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertContent("A.md", "original content"); - } - }, - - { type: "disable-sync", client: 0 }, - { type: "delete", client: 0, path: "A.md" }, - - { - type: "update", - client: 1, - path: "A.md", - content: "important update by client 1" - }, - { type: "sync", client: 1 }, - - { type: "enable-sync", client: 0 }, - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertFileCount(0); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/offline-edit-remote-rename.test.ts b/frontend/deterministic-tests/src/tests/offline-edit-remote-rename.test.ts deleted file mode 100644 index ffc41b89..00000000 --- a/frontend/deterministic-tests/src/tests/offline-edit-remote-rename.test.ts +++ /dev/null @@ -1,49 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const offlineEditRemoteRenameTest: TestDefinition = { - description: - "Client 0 edits A.md offline while client 1 renames A.md to B.md. " + - "After client 0 reconnects, the edit must appear in B.md and A.md must not exist.", - clients: 2, - steps: [ - { type: "create", client: 0, path: "A.md", content: "original" }, - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertContent("A.md", "original"); - } - }, - - { type: "disable-sync", client: 0 }, - { - type: "update", - client: 0, - path: "A.md", - content: "edited by client 0" - }, - - { - type: "rename", - client: 1, - oldPath: "A.md", - newPath: "B.md" - }, - { type: "sync", client: 1 }, - - { type: "enable-sync", client: 0 }, - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertFileNotExists("A.md") - .assertFileCount(1) - .assertContains("B.md", "edited by client 0"); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/offline-edit-then-move-same-content.test.ts b/frontend/deterministic-tests/src/tests/offline-edit-then-move-same-content.test.ts deleted file mode 100644 index 970eabd3..00000000 --- a/frontend/deterministic-tests/src/tests/offline-edit-then-move-same-content.test.ts +++ /dev/null @@ -1,51 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const offlineEditThenMoveSameContentTest: TestDefinition = { - description: - "A file is renamed and edited to match a deleted file's content. Both clients must converge despite the ambiguity.", - clients: 2, - steps: [ - { - type: "create", - client: 0, - path: "A.md", - content: "content A" - }, - { - type: "create", - client: 0, - path: "B.md", - content: "content B" - }, - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - - { type: "disable-sync", client: 0 }, - - { type: "delete", client: 0, path: "A.md" }, - - { type: "rename", client: 0, oldPath: "B.md", newPath: "C.md" }, - - { - type: "update", - client: 0, - path: "C.md", - content: "content A" - }, - - { type: "enable-sync", client: 0 }, - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertFileNotExists("A.md") - .assertFileNotExists("B.md") - .assertContent("C.md", "content A") - .assertFileCount(1); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/offline-mixed-operations.test.ts b/frontend/deterministic-tests/src/tests/offline-mixed-operations.test.ts deleted file mode 100644 index da875b6e..00000000 --- a/frontend/deterministic-tests/src/tests/offline-mixed-operations.test.ts +++ /dev/null @@ -1,57 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const offlineMixedOperationsTest: TestDefinition = { - description: - "Client 0 creates 3 files, syncs to both clients. Client 0 goes offline, " + - "deletes file 1, renames file 2 to a new name, and edits file 3. " + - "When Client 0 reconnects, all three operations should propagate to Client 1.", - clients: 2, - steps: [ - { type: "create", client: 0, path: "file1.md", content: "content-1" }, - { type: "create", client: 0, path: "file2.md", content: "content-2" }, - { type: "create", client: 0, path: "file3.md", content: "content-3" }, - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertContent("file1.md", "content-1") - .assertContent("file2.md", "content-2") - .assertContent("file3.md", "content-3"); - } - }, - - { type: "disable-sync", client: 0 }, - - { type: "delete", client: 0, path: "file1.md" }, - { - type: "rename", - client: 0, - oldPath: "file2.md", - newPath: "moved.md" - }, - { - type: "update", - client: 0, - path: "file3.md", - content: "updated-content-3" - }, - - { type: "enable-sync", client: 0 }, - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertFileNotExists("file1.md") - .assertFileNotExists("file2.md") - .assertContent("moved.md", "content-2") - .assertContent("file3.md", "updated-content-3") - .assertFileCount(2); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/offline-move-then-remote-delete.test.ts b/frontend/deterministic-tests/src/tests/offline-move-then-remote-delete.test.ts deleted file mode 100644 index f8e92bd9..00000000 --- a/frontend/deterministic-tests/src/tests/offline-move-then-remote-delete.test.ts +++ /dev/null @@ -1,36 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const offlineMoveThenRemoteDeleteTest: TestDefinition = { - description: - "Client 0 renames A.md to B.md offline while client 1 deletes A.md. " + - "Both clients must converge to having no files.", - clients: 2, - steps: [ - { - type: "create", - client: 0, - path: "A.md", - content: "content to delete" - }, - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - - { type: "disable-sync", client: 0 }, - { type: "rename", client: 0, oldPath: "A.md", newPath: "B.md" }, - - { type: "delete", client: 1, path: "A.md" }, - { type: "sync", client: 1 }, - - { type: "enable-sync", client: 0 }, - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertFileCount(0); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/offline-multiple-edits.test.ts b/frontend/deterministic-tests/src/tests/offline-multiple-edits.test.ts deleted file mode 100644 index 6341fe8f..00000000 --- a/frontend/deterministic-tests/src/tests/offline-multiple-edits.test.ts +++ /dev/null @@ -1,40 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const offlineMultipleEditsTest: TestDefinition = { - description: - "Client 0 creates a file and syncs. Client 0 goes offline, edits the file " + - "5 times with different content. When Client 0 reconnects, both clients " + - "must converge to the final version.", - clients: 2, - steps: [ - { type: "create", client: 0, path: "doc.md", content: "original" }, - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertContent("doc.md", "original"); - } - }, - - { type: "disable-sync", client: 0 }, - - { type: "update", client: 0, path: "doc.md", content: "edit-1" }, - { type: "update", client: 0, path: "doc.md", content: "edit-2" }, - { type: "update", client: 0, path: "doc.md", content: "edit-3" }, - { type: "update", client: 0, path: "doc.md", content: "edit-4" }, - { type: "update", client: 0, path: "doc.md", content: "edit-5-final" }, - - { type: "enable-sync", client: 0 }, - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertFileCount(1).assertContent("doc.md", "edit-5-final"); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/offline-rename-and-edit.test.ts b/frontend/deterministic-tests/src/tests/offline-rename-and-edit.test.ts deleted file mode 100644 index 836c7fb2..00000000 --- a/frontend/deterministic-tests/src/tests/offline-rename-and-edit.test.ts +++ /dev/null @@ -1,43 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const offlineRenameAndEditTest: TestDefinition = { - description: - "Client 0 creates A.md and syncs. Client 0 goes offline, renames A.md " + - "to B.md, then edits B.md. When Client 0 reconnects, the rename and edit " + - "should both propagate to Client 1.", - clients: 2, - steps: [ - { type: "create", client: 0, path: "A.md", content: "original" }, - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertContent("A.md", "original"); - } - }, - - { type: "disable-sync", client: 0 }, - { type: "rename", client: 0, oldPath: "A.md", newPath: "B.md" }, - { - type: "update", - client: 0, - path: "B.md", - content: "edited after rename" - }, - - { type: "enable-sync", client: 0 }, - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertFileNotExists("A.md") - .assertFileCount(1) - .assertContent("B.md", "edited after rename"); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/offline-rename-remote-create-old-path.test.ts b/frontend/deterministic-tests/src/tests/offline-rename-remote-create-old-path.test.ts deleted file mode 100644 index c1b2913a..00000000 --- a/frontend/deterministic-tests/src/tests/offline-rename-remote-create-old-path.test.ts +++ /dev/null @@ -1,51 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const offlineRenameRemoteCreateOldPathTest: TestDefinition = { - description: - "Client 0 renames X.md to Y.md while offline. Client 1 updates X.md " + - "(same document). When Client 0 reconnects, the rename and update " + - "should merge. Y.md should exist with Client 1's content.", - clients: 2, - steps: [ - { type: "create", client: 0, path: "X.md", content: "original" }, - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertContent("X.md", "original"); - } - }, - - { type: "disable-sync", client: 0 }, - { - type: "rename", - client: 0, - oldPath: "X.md", - newPath: "Y.md" - }, - - { - type: "update", - client: 1, - path: "X.md", - content: "updated-by-client-1" - }, - { type: "sync", client: 1 }, - - { type: "enable-sync", client: 0 }, - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertFileCount(1).assertContains( - "Y.md", - "updated-by-client-1" - ); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/offline-update-both-then-delete-one.test.ts b/frontend/deterministic-tests/src/tests/offline-update-both-then-delete-one.test.ts deleted file mode 100644 index 3442cda7..00000000 --- a/frontend/deterministic-tests/src/tests/offline-update-both-then-delete-one.test.ts +++ /dev/null @@ -1,75 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const offlineUpdateBothThenDeleteOneTest: TestDefinition = { - description: - "Client 0 goes offline, updates A.md and B.md, then deletes B.md. " + - "Client 1 updates B.md while Client 0 is offline. When Client 0 " + - "reconnects, A.md should have the update and B.md should be " + - "consistently resolved (delete wins).", - clients: 2, - steps: [ - { - type: "create", - client: 0, - path: "A.md", - content: "A original" - }, - { - type: "create", - client: 0, - path: "B.md", - content: "B original" - }, - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertContent("A.md", "A original").assertContent( - "B.md", - "B original" - ); - } - }, - - { type: "disable-sync", client: 0 }, - - { - type: "update", - client: 0, - path: "A.md", - content: "A updated by client 0" - }, - { - type: "update", - client: 0, - path: "B.md", - content: "B updated by client 0" - }, - - { type: "delete", client: 0, path: "B.md" }, - - { - type: "update", - client: 1, - path: "B.md", - content: "B updated by client 1" - }, - { type: "sync", client: 1 }, - - { type: "enable-sync", client: 0 }, - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertContent( - "A.md", - "A updated by client 0" - ).assertFileNotExists("B.md"); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/online-both-create-same-path-deconflict.test.ts b/frontend/deterministic-tests/src/tests/online-both-create-same-path-deconflict.test.ts deleted file mode 100644 index b951b0be..00000000 --- a/frontend/deterministic-tests/src/tests/online-both-create-same-path-deconflict.test.ts +++ /dev/null @@ -1,34 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const onlineBothCreateSamePathDeconflictTest: TestDefinition = { - description: - "Both clients create a file at the same path while online. " + - "One client's create gets deconflicted by the server. " + - "Both files must exist on both clients after convergence.", - clients: 2, - steps: [ - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - - { type: "pause-websocket", client: 1 }, - { type: "create", client: 0, path: "A.md", content: " from-client-0 " }, - { type: "update", client: 0, path: "A.md", content: " updated-by-0 " }, - { type: "sync" }, - - { type: "create", client: 1, path: "A.md", content: " from-client-1 " }, - { type: "resume-websocket", client: 1 }, - - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (state: AssertableState): void => { - state - .assertFileCount(1) - .assertContains("A.md", "updated-by-0", "from-client-1 "); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/online-create-rename-concurrent-create-orphan.test.ts b/frontend/deterministic-tests/src/tests/online-create-rename-concurrent-create-orphan.test.ts deleted file mode 100644 index f86b3347..00000000 --- a/frontend/deterministic-tests/src/tests/online-create-rename-concurrent-create-orphan.test.ts +++ /dev/null @@ -1,41 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const onlineCreateRenameConcurrentCreateOrphanTest: TestDefinition = { - description: - "Client 0 creates a binary file and renames it while offline, then reconnects and immediately deletes it. " + - "Both clients must converge to zero files.", - clients: 2, - steps: [ - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - - { type: "disable-sync", client: 0 }, - - { - type: "create", - client: 0, - path: "data.bin", - content: "BINARY:offline-content" - }, - { - type: "rename", - client: 0, - oldPath: "data.bin", - newPath: "moved.bin" - }, - - { type: "enable-sync", client: 0 }, - { type: "delete", client: 0, path: "moved.bin" }, - - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (state: AssertableState): void => { - state.assertFileCount(0); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/online-create-update-while-other-creates-same-path.test.ts b/frontend/deterministic-tests/src/tests/online-create-update-while-other-creates-same-path.test.ts deleted file mode 100644 index e0ddc21a..00000000 --- a/frontend/deterministic-tests/src/tests/online-create-update-while-other-creates-same-path.test.ts +++ /dev/null @@ -1,48 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const onlineCreateUpdateWhileOtherCreatesSamePathTest: TestDefinition = { - description: - "Client 0 creates a binary file and updates it while client 1 also " + - "creates a binary file at the same path. Both clients are online. " + - "Both clients must end up with the same file set.", - clients: 2, - steps: [ - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - - { type: "pause-websocket", client: 1 }, - { - type: "create", - client: 0, - path: "data.bin", - content: "BINARY:content-v1" - }, - { - type: "update", - client: 0, - path: "data.bin", - content: "BINARY:content-v2" - }, - { - type: "create", - client: 1, - path: "data.bin", - content: "BINARY:other-content" - }, - { type: "resume-websocket", client: 1 }, - - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (state: AssertableState): void => { - state - .assertFileCount(2) - .assertNoFileContains("content-v1") - .assertAnyFileContains("content-v2") - .assertAnyFileContains("other-content"); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/online-delete-recreate-rapid-cycle.test.ts b/frontend/deterministic-tests/src/tests/online-delete-recreate-rapid-cycle.test.ts deleted file mode 100644 index de5d6c89..00000000 --- a/frontend/deterministic-tests/src/tests/online-delete-recreate-rapid-cycle.test.ts +++ /dev/null @@ -1,37 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const onlineDeleteRecreateRapidCycleTest: TestDefinition = { - description: - "A file is deleted and recreated multiple times by alternating clients while both are online. " + - "Both clients must converge after each cycle.", - clients: 2, - steps: [ - { type: "create", client: 0, path: "A.md", content: "round 0" }, - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - - { type: "delete", client: 1, path: "A.md" }, - { type: "barrier" }, - { type: "create", client: 0, path: "A.md", content: "round 1" }, - { type: "barrier" }, - - { type: "delete", client: 0, path: "A.md" }, - { type: "barrier" }, - { type: "create", client: 1, path: "A.md", content: "round 2" }, - { type: "barrier" }, - - { type: "delete", client: 1, path: "A.md" }, - { type: "barrier" }, - { type: "create", client: 0, path: "A.md", content: "round 3" }, - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertContent("A.md", "round 3"); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/online-edit-vs-delete-convergence.test.ts b/frontend/deterministic-tests/src/tests/online-edit-vs-delete-convergence.test.ts deleted file mode 100644 index d3a9d84e..00000000 --- a/frontend/deterministic-tests/src/tests/online-edit-vs-delete-convergence.test.ts +++ /dev/null @@ -1,31 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const onlineEditVsDeleteConvergenceTest: TestDefinition = { - description: - "Both clients are online. Client 0 edits a file while client 1 " + - "deletes it. The clients must converge to the same state.", - clients: 2, - steps: [ - { type: "create", client: 0, path: "A.md", content: "original" }, - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - - { - type: "update", - client: 0, - path: "A.md", - content: "edited by client 0" - }, - { type: "delete", client: 1, path: "A.md" }, - - { type: "barrier" }, - { - type: "assert-consistent", - verify: (state: AssertableState): void => { - state.assertFileCount(0); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/overlapping-edits-same-section.test.ts b/frontend/deterministic-tests/src/tests/overlapping-edits-same-section.test.ts deleted file mode 100644 index a93a6f69..00000000 --- a/frontend/deterministic-tests/src/tests/overlapping-edits-same-section.test.ts +++ /dev/null @@ -1,54 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const overlappingEditsSameSectionTest: TestDefinition = { - description: - "Both clients go offline and edit different parts of the same document. " + - "After both reconnect, both edits must be preserved without data loss.", - clients: 2, - steps: [ - { - type: "create", - client: 0, - path: "doc.md", - content: "# Title\n\nfooter" - }, - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - - { type: "disable-sync", client: 0 }, - { type: "disable-sync", client: 1 }, - - { - type: "update", - client: 0, - path: "doc.md", - content: "# Title\nalpha addition\n\nfooter" - }, - - { - type: "update", - client: 1, - path: "doc.md", - content: "# Title\n\nbeta addition\nfooter" - }, - - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertFileCount(1).assertContains( - "doc.md", - "# Title", - "alpha addition", - "beta addition", - "footer" - ); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/queue-reset-loses-coalesced-local-edit.test.ts b/frontend/deterministic-tests/src/tests/queue-reset-loses-coalesced-local-edit.test.ts deleted file mode 100644 index 6d89acf4..00000000 --- a/frontend/deterministic-tests/src/tests/queue-reset-loses-coalesced-local-edit.test.ts +++ /dev/null @@ -1,36 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const queueResetLosesCoalescedLocalEditTest: TestDefinition = { - description: - "Client 0 goes offline, both clients edit doc.md concurrently, " + - "then client 0 reconnects. Both edits must be preserved.", - clients: 2, - steps: [ - { type: "create", client: 0, path: "doc.md", content: "original" }, - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - - { type: "disable-sync", client: 0 }, - - { type: "update", client: 1, path: "doc.md", content: "alpha bravo" }, - { type: "sync", client: 1 }, - - { type: "update", client: 0, path: "doc.md", content: "charlie delta" }, - - { type: "enable-sync", client: 0 }, - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertFileCount(1).assertContains( - "doc.md", - "alpha", - "charlie" - ); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/queued-create-delete-does-not-hijack-reused-path.test.ts b/frontend/deterministic-tests/src/tests/queued-create-delete-does-not-hijack-reused-path.test.ts deleted file mode 100644 index a29f8314..00000000 --- a/frontend/deterministic-tests/src/tests/queued-create-delete-does-not-hijack-reused-path.test.ts +++ /dev/null @@ -1,56 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const queuedCreateDeleteDoesNotHijackReusedPathTest: TestDefinition = { - description: - "A create/delete pair that is still queued behind another request " + - "must collapse locally. It must not later read a different file " + - "that reused the same path before the queued create drained.", - clients: 2, - steps: [ - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - - { type: "pause-server" }, - { - type: "create", - client: 1, - path: "blocker.bin", - content: "BINARY:blocker" - }, - { type: "sleep", ms: 100 }, - { - type: "create", - client: 1, - path: "target.bin", - content: "BINARY:old" - }, - { type: "delete", client: 1, path: "target.bin" }, - { - type: "create", - client: 1, - path: "source.bin", - content: "BINARY:new" - }, - { - type: "rename", - client: 1, - oldPath: "source.bin", - newPath: "target.bin" - }, - { type: "resume-server" }, - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (state: AssertableState): void => { - state - .assertFileCount(2) - .assertContent("blocker.bin", "BINARY:blocker") - .assertContent("target.bin", "BINARY:new") - .assertFileNotExists("source.bin"); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/rapid-create-update-delete-cycle.test.ts b/frontend/deterministic-tests/src/tests/rapid-create-update-delete-cycle.test.ts deleted file mode 100644 index f9c58753..00000000 --- a/frontend/deterministic-tests/src/tests/rapid-create-update-delete-cycle.test.ts +++ /dev/null @@ -1,52 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const rapidCreateUpdateDeleteCycleTest: TestDefinition = { - description: - "Client 0 rapidly creates, updates, deletes, then re-creates a file while the server is paused. " + - "After the server resumes, client 1 must see only the final file.", - clients: 2, - steps: [ - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - - { type: "pause-server" }, - - { - type: "create", - client: 0, - path: "cycle.md", - content: "version 1" - }, - { - type: "update", - client: 0, - path: "cycle.md", - content: "version 2" - }, - { type: "delete", client: 0, path: "cycle.md" }, - - { type: "resume-server" }, - { type: "sync" }, - - { - type: "create", - client: 0, - path: "cycle.md", - content: "final creation" - }, - - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertFileCount(1).assertContent( - "cycle.md", - "final creation" - ); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/rapid-edit-delete-online-convergence.test.ts b/frontend/deterministic-tests/src/tests/rapid-edit-delete-online-convergence.test.ts deleted file mode 100644 index 48c062e0..00000000 --- a/frontend/deterministic-tests/src/tests/rapid-edit-delete-online-convergence.test.ts +++ /dev/null @@ -1,48 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const rapidEditDeleteOnlineConvergenceTest: TestDefinition = { - description: - "Client 0 rapidly edits multiple files while client 1 deletes some of them, all while both are online. " + - "Both clients must converge to a consistent state.", - clients: 2, - steps: [ - { type: "create", client: 0, path: "A.md", content: "content A" }, - { type: "create", client: 0, path: "B.md", content: "content B" }, - { type: "create", client: 0, path: "C.md", content: "content C" }, - { type: "create", client: 0, path: "D.md", content: "content D" }, - { type: "create", client: 0, path: "E.md", content: "content E" }, - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - - { type: "update", client: 0, path: "A.md", content: "A edit 1" }, - { type: "update", client: 0, path: "B.md", content: "B edit 1" }, - { type: "update", client: 0, path: "C.md", content: "C edit 1" }, - { type: "delete", client: 1, path: "A.md" }, - { type: "delete", client: 1, path: "C.md" }, - { type: "delete", client: 1, path: "E.md" }, - { type: "update", client: 0, path: "A.md", content: "A edit 2" }, - { type: "update", client: 0, path: "B.md", content: "B edit 2" }, - { type: "update", client: 0, path: "C.md", content: "C edit 2" }, - - { type: "barrier" }, - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - for (const [path, content] of s.files) { - for (const clientFiles of s.clientFiles) { - if ( - clientFiles.has(path) && - clientFiles.get(path) !== content - ) { - throw new Error( - `Content mismatch for ${path}: "${clientFiles.get(path)}" vs "${content}"` - ); - } - } - } - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/rapid-updates-after-merge.test.ts b/frontend/deterministic-tests/src/tests/rapid-updates-after-merge.test.ts deleted file mode 100644 index 6f97ff05..00000000 --- a/frontend/deterministic-tests/src/tests/rapid-updates-after-merge.test.ts +++ /dev/null @@ -1,49 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const rapidUpdatesAfterMergeTest: TestDefinition = { - description: - "Both clients create the same file offline, triggering a merge on sync. " + - "Client 0 then rapidly sends three updates. Both clients must converge to the final update.", - clients: 2, - steps: [ - { type: "create", client: 0, path: "doc.md", content: "from client 0" }, - { type: "create", client: 1, path: "doc.md", content: "from client 1" }, - - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - - { - type: "update", - client: 0, - path: "doc.md", - content: "update 1" - }, - { type: "sync", client: 0 }, - - { - type: "update", - client: 0, - path: "doc.md", - content: "update 2" - }, - { type: "sync", client: 0 }, - - { - type: "update", - client: 0, - path: "doc.md", - content: "update 3" - }, - - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertFileCount(1).assertContains("doc.md", "update 3"); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/recently-deleted-cleared-on-reconnect.test.ts b/frontend/deterministic-tests/src/tests/recently-deleted-cleared-on-reconnect.test.ts deleted file mode 100644 index c8e70243..00000000 --- a/frontend/deterministic-tests/src/tests/recently-deleted-cleared-on-reconnect.test.ts +++ /dev/null @@ -1,45 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const recentlyDeletedClearedOnReconnectTest: TestDefinition = { - description: - "After a client deletes a document and reconnects, it should " + - "accept new documents from other clients even if they happen to " + - "arrive at the same path as the deleted document.", - clients: 2, - steps: [ - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - - { type: "create", client: 0, path: "doc.md", content: "original" }, - { type: "sync" }, - - { type: "delete", client: 0, path: "doc.md" }, - { type: "barrier" }, - - { type: "disable-sync", client: 0 }, - { type: "disable-sync", client: 1 }, - - { - type: "create", - client: 1, - path: "doc.md", - content: "new content from client 1" - }, - - { type: "enable-sync", client: 1 }, - { type: "sync", client: 1 }, - { type: "enable-sync", client: 0 }, - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertFileCount(1).assertContent( - "doc.md", - "new content from client 1" - ); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/remote-quick-write-rename-before-record.test.ts b/frontend/deterministic-tests/src/tests/remote-quick-write-rename-before-record.test.ts deleted file mode 100644 index ca184b27..00000000 --- a/frontend/deterministic-tests/src/tests/remote-quick-write-rename-before-record.test.ts +++ /dev/null @@ -1,36 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const remoteQuickWriteRenameBeforeRecordTest: TestDefinition = { - description: - "Client 0 receives a remote create and the user renames the new " + - "file immediately after the syncer writes it. The watcher event " + - "must bind to the new document instead of being dropped before " + - "the remote-create handler persists the record.", - clients: 2, - steps: [ - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - - { - type: "rename-next-write", - client: 0, - oldPath: "doc.md", - newPath: "renamed.md" - }, - - { type: "create", client: 1, path: "doc.md", content: "v1\n" }, - { type: "sync", client: 1 }, - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertFileCount(1); - s.assertFileExists("renamed.md"); - s.assertFileNotExists("doc.md"); - s.assertContent("renamed.md", "v1\n"); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/remote-rename-collides-with-pending-local-create.test.ts b/frontend/deterministic-tests/src/tests/remote-rename-collides-with-pending-local-create.test.ts deleted file mode 100644 index d30fdc67..00000000 --- a/frontend/deterministic-tests/src/tests/remote-rename-collides-with-pending-local-create.test.ts +++ /dev/null @@ -1,76 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const remoteRenameCollidesWithPendingLocalCreateTest: TestDefinition = { - // TODO(refactor): the failure mode described below is the - // pre-refactor "deflect-to-conflict-uuid" path that no longer - // exists. Under the new model the wire loop never moves files for - // path placement, so the remote rename can't deflect anywhere; the - // reconciler waits for the slot to free. Convergence assertion is - // still valid (no conflict-uuid stashes, both files present, the - // local create lands at a server-deconflicted sibling). - description: - "Client 0 has doc D tracked at `original.md`. Client 1 owns doc E " + - "and renames it to `target.md` server-side. Before client 0's " + - "drain processes the WS broadcast for E, the user creates a new " + - "local file `target.md` (a different doc, untracked). When the " + - "buffered RemoteChange for E drains, the engine has to reconcile " + - "doc E onto `target.md` even though the slot is held by client " + - "0's pending LocalCreate. Convergence requires both clients end " + - "up with [target.md = E] and the local create lands at a " + - "server-deconflicted sibling (e.g. `target (1).md`).", - clients: 2, - steps: [ - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - - { type: "create", client: 1, path: "original.md", content: "v1\n" }, - { type: "barrier" }, - - // Pause client 0's WS so the upcoming remote rename buffers and - // we can stage a colliding local create before the rename - // drains on client 0. - { type: "pause-websocket", client: 0 }, - - // Client 1 renames the doc. Server commits, broadcasts to - // client 0 (buffered). - { - type: "rename", - client: 1, - oldPath: "original.md", - newPath: "target.md" - }, - { type: "sync", client: 1 }, - - // Client 0 still believes the doc is at `original.md`. The user - // creates a NEW file at `target.md` (an unrelated untracked - // doc). Disk on client 0 now has both `original.md` (the - // tracked doc) and `target.md` (the new untracked file). - { type: "create", client: 0, path: "target.md", content: "extra\n" }, - - // Resume client 0's WS. The buffered RemoteChange drains. - // The reconciler must converge without ever leaving a - // conflict-uuid stash on disk. - { type: "resume-websocket", client: 0 }, - - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (state: AssertableState): void => { - state.assertFileCount(2); - for (const path of state.files.keys()) { - if (path.startsWith("conflict-")) { - throw new Error( - `Unexpected conflict-uuid stash on a converged client: ${path}` - ); - } - } - state.assertFileExists("target.md"); - state.assertContent("target.md", "v1\n"); - // The local create gets server-deconflicted to a - // sibling path (e.g. `target (1).md`). - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/remote-update-resurrects-deleted-doc.test.ts b/frontend/deterministic-tests/src/tests/remote-update-resurrects-deleted-doc.test.ts deleted file mode 100644 index eb2ed86d..00000000 --- a/frontend/deterministic-tests/src/tests/remote-update-resurrects-deleted-doc.test.ts +++ /dev/null @@ -1,59 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const remoteUpdateResurrectsDeletedDocTest: TestDefinition = { - description: - "Client 1 updates, deletes, and recreates P (with a new docId D2). " + - "While the buffered remote events are being processed by client 0, " + - "client 0 also makes a local edit to P. The local edit lands in the " + - "queue while v17 is mid-process, sending v17 down processRemoteUpdate's " + - "re-enqueue branch. The deferred v17 must NOT later resurrect D1 as a " + - "conflict-… file at P after the delete and the D2 create have drained.", - clients: 2, - steps: [ - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - - { type: "create", client: 1, path: "P.md", content: "v8 content\n" }, - { type: "barrier" }, - - { type: "pause-websocket", client: 0 }, - - { - type: "update", - client: 1, - path: "P.md", - content: "v17 content from client 1\n" - }, - { type: "sync", client: 1 }, - { type: "delete", client: 1, path: "P.md" }, - { type: "sync", client: 1 }, - { - type: "create", - client: 1, - path: "P.md", - content: "v21 content (D2)\n" - }, - { type: "sync", client: 1 }, - - { type: "resume-websocket", client: 0 }, - - { - type: "update", - client: 0, - path: "P.md", - content: "local edit by client 0\n" - }, - - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (state: AssertableState): void => { - state - .assertFileCount(1) - .assertContent("P.md", "v21 content (D2)\n"); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/remote-update-survives-user-rename.test.ts b/frontend/deterministic-tests/src/tests/remote-update-survives-user-rename.test.ts deleted file mode 100644 index b78ad143..00000000 --- a/frontend/deterministic-tests/src/tests/remote-update-survives-user-rename.test.ts +++ /dev/null @@ -1,84 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const remoteUpdateSurvivesUserRenameTest: TestDefinition = { - description: - "Client 0 updates a tracked doc; while Client 1 is processing the " + - "broadcast and parked on the GET for the new version's content, the " + - "user renames the doc on Client 1. Pre-fix: `processRemoteUpdate` " + - "captures `actualPath` before the await and, after the GET returns, " + - "calls `write(actualPath, …)` (no-op — file was renamed away), " + - "`updateCache(actualPath, …)`, and `setDocument(actualPath, …)`. " + - "`setDocument` mutates the same record in place so its `path` is " + - "yanked from the user's renamed slot back to the pre-rename path, " + - "wiping the rename out of the queue's documents map. The queued " + - "`LocalUpdate` then reads from the now-stale `record.path`, hits " + - "`FileNotFoundError`, and is silently dropped — the user's rename " + - "never reaches the server. Post-fix: the handler defers when a " + - "local event landed mid-await, so the rename drains first and " + - "the deferred remote update is folded into the broadcast that " + - "follows the rename round-trip.", - clients: 2, - steps: [ - { type: "create", client: 0, path: "doc.md", content: "v1\n" }, - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - - // Buffer Client 1's incoming broadcasts so it doesn't see - // Client 0's update until we've paused the server. - { type: "pause-websocket", client: 1 }, - - // Server now holds v=2 of doc.md. - { type: "update", client: 0, path: "doc.md", content: "v2\n" }, - { type: "sync", client: 0 }, - - // Pause the server. Client 1's upcoming GET for the new version - // content blocks at the OS layer until resume. - { type: "pause-server" }, - - // Release the buffered broadcast. Client 1's drain enters - // `processRemoteUpdate`, captures `actualPath`, fires the GET, - // and parks awaiting the response. - { type: "resume-websocket", client: 1 }, - - // Yield long enough for the drain to traverse all microtask - // hops between the WS handler and the GET, so the HTTP request - // is queued at the (paused) server before the rename runs. - // Without this yield the rename would be enqueued before - // `processRemoteUpdate`'s entry-time `hasPendingLocalEvents` - // check and the early-defer branch would mask the bug. - { type: "sleep", ms: 50 }, - - // While the GET is in flight the user renames the doc. The queue - // mutates `record.path` to "renamed.md" in place and pushes a - // LocalUpdate carrying the rename target. - { - type: "rename", - client: 1, - oldPath: "doc.md", - newPath: "renamed.md" - }, - - // Resume the server. The GET response unblocks - // `processRemoteUpdate`. With the fix in place it sees the - // queued LocalUpdate and defers; without the fix it walks past - // the rename and clobbers the documents map, dropping the - // pending LocalUpdate's read on the way back through. - { type: "resume-server" }, - - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertFileCount(1); - s.assertFileExists("renamed.md"); - s.assertFileNotExists("doc.md"); - // Both edits survive: the user's rename and Client 0's - // content update at v=2. - s.assertContent("renamed.md", "v2\n"); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/rename-chain-during-pending-create.test.ts b/frontend/deterministic-tests/src/tests/rename-chain-during-pending-create.test.ts deleted file mode 100644 index 822e83df..00000000 --- a/frontend/deterministic-tests/src/tests/rename-chain-during-pending-create.test.ts +++ /dev/null @@ -1,64 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const renameChainDuringPendingCreateTest: TestDefinition = { - description: - "User creates a doc, then renames it twice while the LocalCreate's " + - "HTTP roundtrip is still in flight (server paused). Each rename " + - "pushes a LocalUpdate whose `documentId` is the create's Promise " + - "(see `pendingDocumentId` in `SyncEventQueue.enqueue`). After the " + - "create resolves, the first rename drains successfully and " + - "`setDocument` walks `events[]` to retarget queued LocalUpdates' " + - "`event.path` to the new disk location — but the comparison " + - "`e.documentId === record.documentId` mismatches the still-Promise " + - "references, so the second rename's `event.path` stays at the " + - "vacated previous slot. On the next drain step `skipIfOversized`'s " + - "`getFileSize(event.path)` throws FileNotFoundError, which " + - "`processEvent` swallows as 'Skipping sync event ... because the " + - "file no longer exists' — losing the user's final rename. " + - "Post-fix: `resolveCreate` (and the displacement-merge branch in " + - "`processCreate`) swap the Promise references for the resolved id " + - "before `setDocument` runs, so retarget works.", - clients: 2, - steps: [ - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - - // Pause the server so client 0's create stalls on the HTTP PUT - // while we queue rename events behind it. - { type: "pause-server" }, - - { type: "create", client: 0, path: "first.md", content: "v1\n" }, - { - type: "rename", - client: 0, - oldPath: "first.md", - newPath: "second.md" - }, - { - type: "rename", - client: 0, - oldPath: "second.md", - newPath: "third.md" - }, - - // Resume — drain pops LocalCreate (now resolves), then the two - // queued LocalUpdates. Pre-fix: only the first rename's - // file-system effect lands; the second is silently dropped. - // The server ends up with the doc at second.md, leaving - // client 0's local third.md untracked / out-of-sync. - { type: "resume-server" }, - - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (state: AssertableState): void => { - state.assertFileCount(1); - state.assertFileExists("third.md"); - state.assertContent("third.md", "v1\n"); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/rename-chain-then-delete.test.ts b/frontend/deterministic-tests/src/tests/rename-chain-then-delete.test.ts deleted file mode 100644 index 03196919..00000000 --- a/frontend/deterministic-tests/src/tests/rename-chain-then-delete.test.ts +++ /dev/null @@ -1,50 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const renameChainThenDeleteTest: TestDefinition = { - description: - "Client 0 renames X.md to Y.md to Z.md, then deletes Z.md while client 1 is offline. " + - "After client 1 reconnects, both clients must have no files.", - clients: 2, - steps: [ - { type: "create", client: 0, path: "X.md", content: "chain-content" }, - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertContent("X.md", "chain-content"); - } - }, - - { type: "disable-sync", client: 1 }, - - { - type: "rename", - client: 0, - oldPath: "X.md", - newPath: "Y.md" - }, - { type: "sync", client: 0 }, - { - type: "rename", - client: 0, - oldPath: "Y.md", - newPath: "Z.md" - }, - { type: "sync", client: 0 }, - { type: "delete", client: 0, path: "Z.md" }, - { type: "sync", client: 0 }, - - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertFileCount(0); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/rename-chain.test.ts b/frontend/deterministic-tests/src/tests/rename-chain.test.ts deleted file mode 100644 index 8f9d7a7f..00000000 --- a/frontend/deterministic-tests/src/tests/rename-chain.test.ts +++ /dev/null @@ -1,34 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const renameChainTest: TestDefinition = { - description: - "Client 0 (offline) creates A.md, renames to B.md, then renames to C.md. " + - "When sync is enabled, only C.md should exist. Client 1 should receive C.md " + - "with the original content. Intermediate paths should never appear.", - clients: 2, - steps: [ - { type: "enable-sync", client: 1 }, - - { - type: "create", - client: 0, - path: "A.md", - content: "important content" - }, - { type: "rename", client: 0, oldPath: "A.md", newPath: "B.md" }, - { type: "rename", client: 0, oldPath: "B.md", newPath: "C.md" }, - - { type: "enable-sync", client: 0 }, - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertFileNotExists("A.md") - .assertFileNotExists("B.md") - .assertContent("C.md", "important content"); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/rename-circular.test.ts b/frontend/deterministic-tests/src/tests/rename-circular.test.ts deleted file mode 100644 index 44a65149..00000000 --- a/frontend/deterministic-tests/src/tests/rename-circular.test.ts +++ /dev/null @@ -1,44 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const renameCircularTest: TestDefinition = { - description: - "Client 0 creates three files, syncs, then goes offline and performs a circular rename via a temp file (A->temp, C->A, B->C, temp->B). After reconnecting, all three contents should exist across three files but paths may be deconflicted.", - clients: 2, - steps: [ - { type: "create", client: 0, path: "A.md", content: "content-a" }, - { type: "create", client: 0, path: "B.md", content: "content-b" }, - { type: "create", client: 0, path: "C.md", content: "content-c" }, - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertContent("A.md", "content-a") - .assertContent("B.md", "content-b") - .assertContent("C.md", "content-c"); - } - }, - - { type: "disable-sync", client: 0 }, - { type: "rename", client: 0, oldPath: "A.md", newPath: "temp-a.md" }, - { type: "rename", client: 0, oldPath: "C.md", newPath: "A.md" }, - { type: "rename", client: 0, oldPath: "B.md", newPath: "C.md" }, - { type: "rename", client: 0, oldPath: "temp-a.md", newPath: "B.md" }, - - { type: "enable-sync", client: 0 }, - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertFileNotExists("temp-a.md") - .assertFileCount(3) - .assertAnyFileContains("content-c") - .assertAnyFileContains("content-a") - .assertAnyFileContains("content-b"); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/rename-create-conflict.test.ts b/frontend/deterministic-tests/src/tests/rename-create-conflict.test.ts deleted file mode 100644 index fc6a00a7..00000000 --- a/frontend/deterministic-tests/src/tests/rename-create-conflict.test.ts +++ /dev/null @@ -1,34 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const renameCreateConflictTest: TestDefinition = { - description: - "Client 0 creates A.md and syncs. Client 1 renames A.md to B.md and syncs. Client 0 (offline) creates B.md with the same content. After reconnecting, both clients should converge with only B.md.", - clients: 2, - steps: [ - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "create", client: 0, path: "A.md", content: "hi" }, - { type: "barrier" }, - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertContent("A.md", "hi"); - } - }, - { type: "disable-sync", client: 0 }, - { type: "rename", client: 1, oldPath: "A.md", newPath: "B.md" }, - { type: "sync", client: 1 }, - { type: "create", client: 0, path: "B.md", content: "hi" }, - { type: "enable-sync", client: 0 }, - { type: "barrier" }, - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertFileCount(2) - .assertContent("B.md", "hi") - .assertContent("B (1).md", "hi"); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/rename-overwrites-pending-create-then-delete.test.ts b/frontend/deterministic-tests/src/tests/rename-overwrites-pending-create-then-delete.test.ts deleted file mode 100644 index 0b47c781..00000000 --- a/frontend/deterministic-tests/src/tests/rename-overwrites-pending-create-then-delete.test.ts +++ /dev/null @@ -1,51 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const renameOverwritesPendingCreateThenDeleteTest: TestDefinition = { - description: - "A pending local create at a path must not mask a synced document renamed onto that path; later rename/delete events still belong to the synced document.", - clients: 2, - steps: [ - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { - type: "create", - client: 0, - path: "tracked.bin", - content: "BINARY:tracked" - }, - { type: "barrier" }, - - { type: "pause-server" }, - - { - type: "create", - client: 0, - path: "pending.bin", - content: "BINARY:pending" - }, - { - type: "rename", - client: 0, - oldPath: "tracked.bin", - newPath: "pending.bin" - }, - { - type: "rename", - client: 0, - oldPath: "pending.bin", - newPath: "final.bin" - }, - { type: "delete", client: 0, path: "final.bin" }, - - { type: "resume-server" }, - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (state: AssertableState): void => { - state.assertFileCount(0); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/rename-pending-create-before-response.test.ts b/frontend/deterministic-tests/src/tests/rename-pending-create-before-response.test.ts deleted file mode 100644 index 26623c43..00000000 --- a/frontend/deterministic-tests/src/tests/rename-pending-create-before-response.test.ts +++ /dev/null @@ -1,42 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const renamePendingCreateBeforeResponseTest: TestDefinition = { - description: - "Client 0 creates a file while the server is paused, then renames it before the create completes. After the server resumes, both clients should converge with the file at the renamed path.", - clients: 2, - steps: [ - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - - { type: "pause-server" }, - - { - type: "create", - client: 0, - path: "doc.md", - content: "original-content" - }, - - { - type: "rename", - client: 0, - oldPath: "doc.md", - newPath: "renamed.md" - }, - - { type: "resume-server" }, - - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertFileCount(1).assertContent( - "renamed.md", - "original-content" - ); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/rename-pending-create-onto-pending-delete-path.test.ts b/frontend/deterministic-tests/src/tests/rename-pending-create-onto-pending-delete-path.test.ts deleted file mode 100644 index 0906f209..00000000 --- a/frontend/deterministic-tests/src/tests/rename-pending-create-onto-pending-delete-path.test.ts +++ /dev/null @@ -1,59 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const renamePendingCreateOntoPendingDeletePathTest: TestDefinition = { - description: - "A pending create is renamed onto a path whose old server document " + - "has a queued delete. The delete must reach the server before the " + - "new create so the new generation is not merged into the soon-to-be " + - "deleted document.", - clients: 2, - steps: [ - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - - { - type: "create", - client: 1, - path: "file-17.md", - content: "old\n" - }, - { type: "barrier" }, - - { type: "pause-server" }, - { - type: "create", - client: 1, - path: "blocker.md", - content: "blocker\n" - }, - { type: "sleep", ms: 100 }, - { - type: "create", - client: 1, - path: "file-23.md", - content: "new\n" - }, - { type: "delete", client: 1, path: "file-17.md" }, - { - type: "rename", - client: 1, - oldPath: "file-23.md", - newPath: "file-17.md" - }, - { type: "resume-server" }, - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (state: AssertableState): void => { - state - .assertFileCount(2) - .assertContent("blocker.md", "blocker\n") - .assertContent("file-17.md", "new\n") - .assertFileNotExists("file-23.md"); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/rename-roundtrip.test.ts b/frontend/deterministic-tests/src/tests/rename-roundtrip.test.ts deleted file mode 100644 index 0373debf..00000000 --- a/frontend/deterministic-tests/src/tests/rename-roundtrip.test.ts +++ /dev/null @@ -1,40 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const renameRoundtripTest: TestDefinition = { - description: - "Client 0 creates A.md, renames it to B.md, then renames it back to A.md. After each step both clients sync. Both should end with only A.md at the original path.", - clients: 2, - steps: [ - { type: "create", client: 0, path: "A.md", content: "original" }, - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertContent("A.md", "original"); - } - }, - - { type: "rename", client: 0, oldPath: "A.md", newPath: "B.md" }, - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertFileNotExists("A.md").assertContent("B.md", "original"); - } - }, - - { type: "rename", client: 0, oldPath: "B.md", newPath: "A.md" }, - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertFileNotExists("B.md").assertContent("A.md", "original"); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/rename-swap.test.ts b/frontend/deterministic-tests/src/tests/rename-swap.test.ts deleted file mode 100644 index 9910e8ef..00000000 --- a/frontend/deterministic-tests/src/tests/rename-swap.test.ts +++ /dev/null @@ -1,44 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const renameSwapTest: TestDefinition = { - description: - "Client 0 has A.md and B.md synced. Goes offline and swaps them using " + - "a temp file: A.md -> temp.md, B.md -> A.md, temp.md -> B.md. " + - "When Client 0 reconnects, both contents should exist across two files.", - clients: 2, - steps: [ - { type: "create", client: 0, path: "A.md", content: "content-a" }, - { type: "create", client: 0, path: "B.md", content: "content-b" }, - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertContent("A.md", "content-a").assertContent( - "B.md", - "content-b" - ); - } - }, - - { type: "disable-sync", client: 0 }, - { type: "rename", client: 0, oldPath: "A.md", newPath: "temp.md" }, - { type: "rename", client: 0, oldPath: "B.md", newPath: "A.md" }, - { type: "rename", client: 0, oldPath: "temp.md", newPath: "B.md" }, - - { type: "enable-sync", client: 0 }, - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertFileNotExists("temp.md") - .assertFileCount(2) - .assertAnyFileContains("content-b") - .assertAnyFileContains("content-a"); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/rename-to-path-of-unconfirmed-delete.test.ts b/frontend/deterministic-tests/src/tests/rename-to-path-of-unconfirmed-delete.test.ts deleted file mode 100644 index 34a3867c..00000000 --- a/frontend/deterministic-tests/src/tests/rename-to-path-of-unconfirmed-delete.test.ts +++ /dev/null @@ -1,44 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const renameToPathOfUnconfirmedDeleteTest: TestDefinition = { - description: - "Client 0 deletes A.md then renames B.md to A.md. After syncing, " + - "B's content should exist and the old A.md content should be gone. " + - "The server may deconflict the path if the delete and move arrive " + - "in the same transaction.", - clients: 2, - steps: [ - { - type: "create", - client: 0, - path: "A.md", - content: "content A" - }, - { - type: "create", - client: 0, - path: "B.md", - content: "content B" - }, - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "sync" }, - - { type: "delete", client: 0, path: "A.md" }, - { type: "barrier" }, - - { type: "rename", client: 0, oldPath: "B.md", newPath: "A.md" }, - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertFileNotExists("B.md").assertContains( - "A.md", - "content B" - ); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/rename-to-pending-path-fallback.test.ts b/frontend/deterministic-tests/src/tests/rename-to-pending-path-fallback.test.ts deleted file mode 100644 index 8747218a..00000000 --- a/frontend/deterministic-tests/src/tests/rename-to-pending-path-fallback.test.ts +++ /dev/null @@ -1,43 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const renameToPendingPathFallbackTest: TestDefinition = { - description: - "Client 0 creates B.md and syncs. Goes offline, creates A.md, then renames B.md to A.md (overwriting the unsynced A). After reconnecting, B.md should be gone and A.md should have B's content.", - clients: 2, - steps: [ - { - type: "create", - client: 0, - path: "B.md", - content: "tracked B content" - }, - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - - { type: "disable-sync", client: 0 }, - - { - type: "create", - client: 0, - path: "A.md", - content: "pending A content" - }, - - { type: "rename", client: 0, oldPath: "B.md", newPath: "A.md" }, - - { type: "enable-sync", client: 0 }, - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertFileNotExists("B.md").assertContains( - "A.md", - "tracked B content" - ); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/rename-update-conflict.test.ts b/frontend/deterministic-tests/src/tests/rename-update-conflict.test.ts deleted file mode 100644 index 18d4c101..00000000 --- a/frontend/deterministic-tests/src/tests/rename-update-conflict.test.ts +++ /dev/null @@ -1,42 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const renameUpdateConflictTest: TestDefinition = { - description: - "Client 0 renames A.md to B.md while client 1 updates A.md offline. After client 1 reconnects, both should converge with the update at B.md.", - clients: 2, - steps: [ - { type: "create", client: 0, path: "A.md", content: "original" }, - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertContent("A.md", "original"); - } - }, - - { type: "disable-sync", client: 1 }, - - { type: "rename", client: 0, oldPath: "A.md", newPath: "B.md" }, - { type: "sync", client: 0 }, - - { - type: "update", - client: 1, - path: "A.md", - content: "updated by client 1" - }, - - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertFileNotExists("A.md").assertContains("B.md", "updated"); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/renamed-pending-create-reused-path-then-delete.test.ts b/frontend/deterministic-tests/src/tests/renamed-pending-create-reused-path-then-delete.test.ts deleted file mode 100644 index 3ffb376e..00000000 --- a/frontend/deterministic-tests/src/tests/renamed-pending-create-reused-path-then-delete.test.ts +++ /dev/null @@ -1,65 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const renamedPendingCreateReusedPathThenDeleteTest: TestDefinition = { - description: - "A queued create is renamed away from file-59.md, a newer local " + - "file reuses file-59.md before the queued create drains, and the " + - "renamed-away generation is deleted. The delete must not erase or " + - "orphan the newer file-59.md generation.", - clients: 2, - steps: [ - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - - { type: "pause-server" }, - { - type: "create", - client: 1, - path: "blocker.md", - content: "blocker\n" - }, - { type: "sleep", ms: 100 }, - - { - type: "create", - client: 1, - path: "file-59.md", - content: "old\n" - }, - { - type: "rename", - client: 1, - oldPath: "file-59.md", - newPath: "file-33.md" - }, - { - type: "create", - client: 1, - path: "file-59.md", - content: "new\n" - }, - - { - type: "resume-server-until-history-then-pause", - client: 1, - syncType: "CREATE", - path: "file-33.md" - }, - { type: "delete", client: 1, path: "file-33.md" }, - { type: "resume-server" }, - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (state: AssertableState): void => { - state - .assertFileCount(2) - .assertContent("blocker.md", "blocker\n") - .assertContent("file-59.md", "new\n") - .assertFileNotExists("file-33.md"); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/reset-clears-recently-deleted-resurrection.test.ts b/frontend/deterministic-tests/src/tests/reset-clears-recently-deleted-resurrection.test.ts deleted file mode 100644 index e0a1565c..00000000 --- a/frontend/deterministic-tests/src/tests/reset-clears-recently-deleted-resurrection.test.ts +++ /dev/null @@ -1,43 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const resetClearsRecentlyDeletedResurrectionTest: TestDefinition = { - description: - "Client 0 deletes a file. Client 1 toggles sync off and on " + - "(simulating reconnect). The deleted file should NOT reappear " + - "on Client 1 after the sync reset.", - clients: 2, - steps: [ - { - type: "create", - client: 0, - path: "ghost.md", - content: "should be deleted" - }, - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - - { type: "delete", client: 0, path: "ghost.md" }, - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertFileNotExists("ghost.md"); - } - }, - - { type: "disable-sync", client: 1 }, - { type: "enable-sync", client: 1 }, - - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertFileCount(0); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/same-doc-id-collapse-after-remote-quick-write-and-pending-rename.test.ts b/frontend/deterministic-tests/src/tests/same-doc-id-collapse-after-remote-quick-write-and-pending-rename.test.ts deleted file mode 100644 index 2a3b5de4..00000000 --- a/frontend/deterministic-tests/src/tests/same-doc-id-collapse-after-remote-quick-write-and-pending-rename.test.ts +++ /dev/null @@ -1,82 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const sameDocIdCollapseAfterRemoteQuickWriteAndPendingRenameTest: TestDefinition = - { - description: - "A remote create starts quick-writing at doc.md while a local " + - "create for the same path is queued and renamed to renamed.md. " + - "Because the local create was renamed before it reached the " + - "server, the two generations should remain separate tracked " + - "documents.", - clients: 2, - steps: [ - { type: "enable-sync", client: 0 }, - - // Create a deleted latest version before client 1 joins. - // Catch-up will advance MinCovered with a non-contiguous id, - // keeping client 1's create lastSeen low enough to exercise - // the server's same-doc merge path from the e2e failure. - { - type: "create", - client: 0, - path: "history.md", - content: "history-v1" - }, - { type: "sync", client: 0 }, - { - type: "update", - client: 0, - path: "history.md", - content: "history-v2" - }, - { type: "sync", client: 0 }, - { type: "delete", client: 0, path: "history.md" }, - { type: "sync", client: 0 }, - - { type: "enable-sync", client: 1 }, - { type: "sync", client: 1 }, - - { type: "pause-websocket", client: 1 }, - - { - type: "create", - client: 0, - path: "doc.md", - content: "remote\n" - }, - { type: "sync", client: 0 }, - - // Let client 1's buffered RemoteCreate enter the quick-write - // path, but hold the content fetch until the local create has - // appeared and moved away from doc.md. - { type: "pause-server" }, - { type: "resume-websocket", client: 1 }, - { type: "sleep", ms: 100 }, - - { - type: "create", - client: 1, - path: "doc.md", - content: "local\n" - }, - { - type: "rename", - client: 1, - oldPath: "doc.md", - newPath: "renamed.md" - }, - - { type: "resume-server" }, - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (state: AssertableState): void => { - state.assertFileCount(2); - state.assertContent("doc.md", "remote\n"); - state.assertContent("renamed.md", "local\n"); - } - } - ] - }; diff --git a/frontend/deterministic-tests/src/tests/same-doc-id-collapse-on-local-create-after-remote-create.test.ts b/frontend/deterministic-tests/src/tests/same-doc-id-collapse-on-local-create-after-remote-create.test.ts deleted file mode 100644 index dee3a9ad..00000000 --- a/frontend/deterministic-tests/src/tests/same-doc-id-collapse-on-local-create-after-remote-create.test.ts +++ /dev/null @@ -1,121 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const sameDocIdCollapseOnLocalCreateAfterRemoteCreateTest: TestDefinition = - { - description: - "Client B creates X with content C2; the server commits and " + - "broadcasts. Client A's WS is paused so the RemoteCreate buffers. " + - "Server is then paused so A's about-to-POST LocalCreate will " + - "hang. A creates X with content C1: file lands on disk, " + - "LocalCreate enqueues, drain starts the POST, the POST stalls " + - "at the paused server. A's WS is resumed: the buffered " + - "RemoteCreate for doc-X is delivered to A and enqueues behind " + - "the in-flight LocalCreate. Per the lazy-paths model, when " + - "the RemoteCreate is processed it observes that path X is " + - "occupied locally by A's pending-create bytes, so it tracks " + - "doc-X with `localPath = undefined` / `remoteRelativePath = " + - "X` and does NOT fetch content. The server is then resumed: " + - "A's LocalCreate POST returns. The server, finding X already " + - "taken by doc-X, replies with doc-X's existing documentId " + - "(typically a MergingUpdate carrying the merged bytes). A's " + - "processCreate handler detects that response.documentId " + - "matches the no-localPath record built from the RemoteCreate " + - "and collapses the two: it sets localPath = X on that " + - "record, writes the merged bytes, and resolves the pending " + - "create promise. Final state: exactly one file at X on both " + - "clients, both pointing at doc-X's documentId, content " + - "carrying both contributions, and no conflict-- " + - "stash anywhere.", - clients: 2, - steps: [ - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - - // Buffer broadcasts to client 0 (A) so client 1's create - // doesn't reach A's WS handler until we say so. - { type: "pause-websocket", client: 0 }, - - // Client 1 (B) commits doc-X at path X with content C2. - // The server commits, broadcasts (broadcast queued at A's - // paused WS). - { - type: "create", - client: 1, - path: "X.md", - content: "from-client-1 " - }, - { type: "sync", client: 1 }, - - // Pause the server so A's upcoming LocalCreate POST hangs. - // This holds A's drain on the in-flight POST while we - // release the WS so the RemoteCreate enqueues behind it. - { type: "pause-server" }, - - // Client 0 (A) creates X locally with content C1. The - // file lands on A's disk; LocalCreate enqueues; drain - // starts the POST; POST stalls at the paused server. - { - type: "create", - client: 0, - path: "X.md", - content: "from-client-0 " - }, - - // Release A's WS. The buffered RemoteCreate for doc-X is - // delivered to A and enqueues behind the in-flight - // LocalCreate. Whichever of (RemoteCreate processed first - // → no-localPath record, then LocalCreate POST returns - // with merging response that collapses) or (LocalCreate - // POST returns first with merging response that creates - // the canonical record, then RemoteCreate finds the doc - // already tracked by id and no-ops) actually plays out - // depends on the fine-grained interleaving the runtime - // produces, but both paths are required to converge to - // the same single-record same-docId state. - { type: "resume-websocket", client: 0 }, - - // Resume the server: A's LocalCreate POST completes. - // Server returns doc-X's existing documentId (MergingUpdate - // with merged content). processCreate runs the collapse - // path. - { type: "resume-server" }, - - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (state: AssertableState): void => { - state.assertFileCount(1); - state.assertFileExists("X.md"); - // Server-side merge of the two text creates must - // carry both contributions through to the - // converged file. - state.assertContains( - "X.md", - "from-client-0", - "from-client-1" - ); - // The lazy-paths collapse path must not leave a - // conflict-- stash on either client. - for (const path of state.files.keys()) { - if (path.startsWith("conflict-")) { - throw new Error( - `Unexpected conflict-uuid stash on a converged client: ${path}` - ); - } - } - for (const perClient of state.clientFiles) { - for (const path of perClient.keys()) { - if (path.startsWith("conflict-")) { - throw new Error( - `Unexpected conflict-uuid stash on a per-client view: ${path}` - ); - } - } - } - } - } - ] - }; diff --git a/frontend/deterministic-tests/src/tests/self-merge-pending-rename-aliases-second-create.test.ts b/frontend/deterministic-tests/src/tests/self-merge-pending-rename-aliases-second-create.test.ts deleted file mode 100644 index ac8ed3ed..00000000 --- a/frontend/deterministic-tests/src/tests/self-merge-pending-rename-aliases-second-create.test.ts +++ /dev/null @@ -1,152 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const selfMergePendingRenameAliasesSecondCreateTest: TestDefinition = { - description: - "Single client makes two distinct creates that briefly share a path. " + - "Client 0 POSTs the first create at primary.md while the server is " + - "paused. While that POST is in flight: a second create is queued at " + - "staging.md, primary.md is renamed to moved.md (rewriting the in- " + - "flight create's event.path to moved.md and pushing a rename " + - "LocalUpdate at the queue tail), and staging.md is renamed onto the " + - "now-vacated primary.md slot (rewriting the second create's " + - "event.path to primary.md and pushing another rename LocalUpdate). " + - "Client 0's WS is paused throughout, so its watermark stays at 0. " + - "On resume the first POST commits Doc-X at primary.md (creation_vuid " + - "= N). The drain then processes the second LocalCreate (POST " + - "relativePath=primary.md, last_seen=0); the server's path-based " + - "dedup sees N > 0 and merges the second create into Doc-X " + - "(MergingUpdate). The buggy behaviour: processCreate's resolveCreate " + - "calls upsertRecord with localPath=primary.md, but the existing " + - "record (from the first create) already holds localPath=moved.md, " + - "and upsertRecord's `existing.localPath !== undefined` guard " + - "silently drops the new claim. The file at primary.md is left " + - "orphaned: tracked by no record, never broadcast, never deleted. " + - "After the user's renames the expected user-visible state is two " + - "distinct files at moved.md and primary.md — both clients must " + - "converge to that.", - clients: 2, - steps: [ - // Both clients online so the WS connection is established before - // the test starts pausing things. - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - - // Pause client 0's WS so its MinCovered watermark stays at 0 - // through the whole bug sequence. The merge condition the - // server is going to fire is `creation_vuid > last_seen`; with - // a non-zero gap the same-device second create gets merged - // into the same-device first create. - { type: "pause-websocket", client: 0 }, - - // Client 1 commits a doc to push the server's vuid above 0. - // Without this filler, Doc-X's create vuid could be 1 and - // client 0's last_seen.add(1) would advance min to 1, killing - // the watermark gap that triggers the merge. - { - type: "create", - client: 1, - path: "filler.md", - content: "filler-content " - }, - { type: "sync", client: 1 }, - - // Pause the server so client 0's first create POST hangs in - // flight, giving us a deterministic window in which to enqueue - // the second create and the renames. - { type: "pause-server" }, - - // First create — Doc-X. The wire-loop drains it, captures - // requestPath = event.path = "primary.md", reads the bytes, - // sends the POST, and stalls on the response. - { - type: "create", - client: 0, - path: "primary.md", - content: "primary content " - }, - - // Make sure the POST is actually on the wire with - // relativePath="primary.md" before we rewrite event.path. - // Without this delay the rename can win the race, the POST - // goes out with relativePath="moved.md", and the server-side - // path-collision merge never fires. - { type: "sleep", ms: 100 }, - - // Second create at a staging path. The wire-loop is still - // blocked on Doc-X's POST, so this LocalCreate just queues at - // index 1. - { - type: "create", - client: 0, - path: "staging.md", - content: "secondary content " - }, - - // Rename Doc-X's path. enqueue's pending-create branch - // rewrites Doc-X's event.path in place (moved.md) and pushes - // a LocalUpdate(rename, originalPath=moved.md) at the END of - // the queue. Note the ordering: this LocalUpdate is enqueued - // AFTER the staging LocalCreate above. That ordering is - // load-bearing — it is what makes the second create's POST - // drain (and trigger the server-side merge) before Doc-X's - // rename PUT moves the doc away from primary.md on the - // server. - { - type: "rename", - client: 0, - oldPath: "primary.md", - newPath: "moved.md" - }, - - // Rename the staging file onto Doc-X's now-vacated primary.md - // slot. enqueue rewrites the staging LocalCreate's event.path - // to primary.md and pushes a LocalUpdate(rename, - // originalPath=primary.md) at the queue tail. After this the - // disk has: moved.md = Doc-X's bytes, primary.md = Doc-Y's - // bytes. - { - type: "rename", - client: 0, - oldPath: "staging.md", - newPath: "primary.md" - }, - - // Let everything fly: server processes the queued POSTs; - // client 0 catches up on broadcasts. - { type: "resume-server" }, - { type: "resume-websocket", client: 0 }, - - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (state: AssertableState): void => { - // The user did two distinct creates (Doc-X and Doc-Y); - // both contents must survive on both clients. - state.assertFileCount(3); - state.assertFileExists("filler.md"); - state.assertFileExists("moved.md"); - state.assertFileExists("primary.md"); - - // After the renames the user expects: - // - moved.md = the file that was originally created - // at primary.md (Doc-X's content). - // - primary.md = the file that was originally created - // at staging.md (Doc-Y's content). - state.assertContains("moved.md", "primary content"); - state.assertContains("primary.md", "secondary content"); - - // No content cross-contamination: each contribution - // should land in exactly one of the user-visible - // files. Under the bug, the orphan at primary.md - // carries Doc-X's content (because Doc-Y's PUT was - // aliased onto Doc-X's record and read Doc-X's bytes - // from moved.md), so this catches the leak too. - state.assertContentInAtMostOneFile("primary content"); - state.assertContentInAtMostOneFile("secondary content"); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/sequential-create-duplicate-content.test.ts b/frontend/deterministic-tests/src/tests/sequential-create-duplicate-content.test.ts deleted file mode 100644 index 611e1ae3..00000000 --- a/frontend/deterministic-tests/src/tests/sequential-create-duplicate-content.test.ts +++ /dev/null @@ -1,43 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const sequentialCreateDuplicateContentTest: TestDefinition = { - description: - "Client 0 creates A.md, syncs, then creates B.md with identical content. Both files must remain as separate documents on both clients.", - clients: 2, - steps: [ - { - type: "create", - client: 0, - path: "A.md", - content: "identical content here" - }, - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertContent("A.md", "identical content here"); - } - }, - - { - type: "create", - client: 0, - path: "B.md", - content: "identical content here" - }, - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertFileCount(2) - .assertContent("A.md", "identical content here") - .assertContent("B.md", "identical content here"); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/server-pause-both-clients-create.test.ts b/frontend/deterministic-tests/src/tests/server-pause-both-clients-create.test.ts deleted file mode 100644 index f99cf92d..00000000 --- a/frontend/deterministic-tests/src/tests/server-pause-both-clients-create.test.ts +++ /dev/null @@ -1,42 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const serverPauseBothClientsCreateTest: TestDefinition = { - description: - "Client 0 creates a file, then the server is paused. Client 1 creates a different file while the server is paused. After the server resumes, both files should exist on both clients.", - clients: 2, - steps: [ - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - - { - type: "create", - client: 0, - path: "alpha.md", - content: "from client 0" - }, - { type: "pause-server" }, - - { - type: "create", - client: 1, - path: "beta.md", - content: "from client 1" - }, - - { type: "resume-server" }, - - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertContains("alpha.md", "from client 0").assertContains( - "beta.md", - "from client 1" - ); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/server-pause-both-edit-same-file.test.ts b/frontend/deterministic-tests/src/tests/server-pause-both-edit-same-file.test.ts deleted file mode 100644 index ff8cf194..00000000 --- a/frontend/deterministic-tests/src/tests/server-pause-both-edit-same-file.test.ts +++ /dev/null @@ -1,68 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const serverPauseBothEditSameFileTest: TestDefinition = { - description: - "Both clients edit different sections of the same file while the server is paused. After resuming and converging, client 0 makes another edit to verify further updates still work correctly.", - clients: 2, - steps: [ - { - type: "create", - client: 0, - path: "shared.md", - content: "line 1: original\nline 2: original\nline 3: original" - }, - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - - { type: "pause-server" }, - - { - type: "update", - client: 0, - path: "shared.md", - content: - "line 1: edited by client 0\nline 2: original\nline 3: original" - }, - { - type: "update", - client: 1, - path: "shared.md", - content: - "line 1: original\nline 2: original\nline 3: edited by client 1" - }, - - { type: "resume-server" }, - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertFileCount(1).assertContains( - "shared.md", - "edited by client 0", - "edited by client 1" - ); - } - }, - - { - type: "update", - client: 0, - path: "shared.md", - content: "post-merge edit from client 0" - }, - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertFileCount(1).assertContains( - "shared.md", - "post-merge edit from client 0" - ); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/server-pause-delete-recreate.test.ts b/frontend/deterministic-tests/src/tests/server-pause-delete-recreate.test.ts deleted file mode 100644 index 5ac97f0d..00000000 --- a/frontend/deterministic-tests/src/tests/server-pause-delete-recreate.test.ts +++ /dev/null @@ -1,38 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const serverPauseDeleteRecreateTest: TestDefinition = { - description: - "Client 1 deletes a file and syncs. The server is paused, then client 0 creates at the same path. After the server resumes, both clients should have the recreated file.", - clients: 2, - steps: [ - { type: "create", client: 0, path: "A.md", content: "original" }, - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - - { type: "delete", client: 1, path: "A.md" }, - { type: "barrier" }, - - { type: "pause-server" }, - - { - type: "create", - client: 0, - path: "A.md", - content: "recreated during contention" - }, - - { type: "resume-server" }, - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (state: AssertableState): void => { - state - .assertFileCount(1) - .assertContent("A.md", "recreated during contention"); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/server-pause-rename-edit-resume.test.ts b/frontend/deterministic-tests/src/tests/server-pause-rename-edit-resume.test.ts deleted file mode 100644 index b1739135..00000000 --- a/frontend/deterministic-tests/src/tests/server-pause-rename-edit-resume.test.ts +++ /dev/null @@ -1,50 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const serverPauseRenameEditResumeTest: TestDefinition = { - description: - "Client 0 creates A.md and syncs. Server is paused. Client 0 " + - "renames A.md to B.md and edits B.md. Server resumes. Both the " + - "rename and edit should propagate to Client 1.", - clients: 2, - steps: [ - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { - type: "create", - client: 0, - path: "A.md", - content: "original content" - }, - { type: "barrier" }, - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertContent("A.md", "original content"); - } - }, - - { type: "pause-server" }, - - { type: "rename", client: 0, oldPath: "A.md", newPath: "B.md" }, - { - type: "update", - client: 0, - path: "B.md", - content: "edited after rename during pause" - }, - - { type: "resume-server" }, - - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertFileCount(1) - .assertFileNotExists("A.md") - .assertContent("B.md", "edited after rename during pause"); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/server-pause-update-and-create.test.ts b/frontend/deterministic-tests/src/tests/server-pause-update-and-create.test.ts deleted file mode 100644 index 2389ccf5..00000000 --- a/frontend/deterministic-tests/src/tests/server-pause-update-and-create.test.ts +++ /dev/null @@ -1,54 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const serverPauseUpdateAndCreateTest: TestDefinition = { - description: - "Client 0 updates a shared file while client 1 creates a new file, both during a server pause. After the server resumes, both operations should complete and propagate to both clients.", - clients: 2, - steps: [ - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { - type: "create", - client: 0, - path: "shared.md", - content: "initial content" - }, - { type: "barrier" }, - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertContent("shared.md", "initial content"); - } - }, - - { type: "pause-server" }, - - { - type: "update", - client: 0, - path: "shared.md", - content: "updated during pause" - }, - { - type: "create", - client: 1, - path: "new-file.md", - content: "created by client 1" - }, - - { type: "resume-server" }, - - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertContent( - "shared.md", - "updated during pause" - ).assertContent("new-file.md", "created by client 1"); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/simultaneous-create-delete-same-path.test.ts b/frontend/deterministic-tests/src/tests/simultaneous-create-delete-same-path.test.ts deleted file mode 100644 index 7ec116ac..00000000 --- a/frontend/deterministic-tests/src/tests/simultaneous-create-delete-same-path.test.ts +++ /dev/null @@ -1,38 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const simultaneousCreateDeleteSamePathTest: TestDefinition = { - description: - "Client 0 creates A.md and syncs to both clients. Client 0 deletes A.md while " + - "Client 1 (offline) updates A.md with different content. When Client 1 reconnects, " + - "the update and delete must be reconciled. Both clients must converge.", - clients: 2, - steps: [ - { type: "create", client: 0, path: "A.md", content: "original from 0" }, - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - - { type: "disable-sync", client: 1 }, - - { type: "delete", client: 0, path: "A.md" }, - { type: "sync", client: 0 }, - - { - type: "update", - client: 1, - path: "A.md", - content: "modified by 1 while offline" - }, - - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertFileCount(0); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/text-pending-create-not-displaced.test.ts b/frontend/deterministic-tests/src/tests/text-pending-create-not-displaced.test.ts deleted file mode 100644 index 28243525..00000000 --- a/frontend/deterministic-tests/src/tests/text-pending-create-not-displaced.test.ts +++ /dev/null @@ -1,36 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const textPendingCreateNotDisplacedTest: TestDefinition = { - description: - "Two clients each create a text file at the same path while offline. " + - "After syncing, the file should contain merged content from both clients.", - clients: 2, - steps: [ - { - type: "create", - client: 0, - path: "data.txt", - content: "text data from client-0" - }, - { - type: "create", - client: 1, - path: "data.txt", - content: "text data from client-1" - }, - - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertFileCount(1) - .assertFileExists("data.txt") - .assertAnyFileContains("client-0", "client-1"); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/three-client-rename-create-delete.test.ts b/frontend/deterministic-tests/src/tests/three-client-rename-create-delete.test.ts deleted file mode 100644 index 80478adc..00000000 --- a/frontend/deterministic-tests/src/tests/three-client-rename-create-delete.test.ts +++ /dev/null @@ -1,55 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const threeClientRenameCreateDeleteTest: TestDefinition = { - description: - "Client 0 renames X -> Y, Client 1 deletes X, Client 2 creates Y. " + - "All three operations happen while the other clients are offline. " + - "Tests that the system handles the three-way conflict and converges.", - clients: 3, - steps: [ - { - type: "create", - client: 0, - path: "X.md", - content: "original from A" - }, - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "enable-sync", client: 2 }, - { type: "barrier" }, - - { type: "disable-sync", client: 0 }, - { type: "disable-sync", client: 1 }, - { type: "disable-sync", client: 2 }, - - { type: "rename", client: 0, oldPath: "X.md", newPath: "Y.md" }, - - { type: "delete", client: 1, path: "X.md" }, - - { - type: "create", - client: 2, - path: "Y.md", - content: "new from C" - }, - - { type: "enable-sync", client: 0 }, - { type: "sync", client: 0 }, - - { type: "enable-sync", client: 1 }, - { type: "sync", client: 1 }, - - { type: "enable-sync", client: 2 }, - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertFileNotExists("X.md").assertAnyFileContains( - "new from C" - ); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/update-does-not-survive-remote-delete.test.ts b/frontend/deterministic-tests/src/tests/update-does-not-survive-remote-delete.test.ts deleted file mode 100644 index 70a2fc8c..00000000 --- a/frontend/deterministic-tests/src/tests/update-does-not-survive-remote-delete.test.ts +++ /dev/null @@ -1,36 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const updateDoesNotSurvivesRemoteDeleteTest: TestDefinition = { - description: - "Client 0 deletes a file while client 1 edits it offline. Client 0 syncs the delete first, then client 1 reconnects. Deletes always win.", - clients: 2, - steps: [ - { type: "create", client: 0, path: "doc.md", content: "original" }, - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - - { type: "disable-sync", client: 0 }, - { type: "disable-sync", client: 1 }, - - { type: "delete", client: 0, path: "doc.md" }, - { - type: "update", - client: 1, - path: "doc.md", - content: "edited by client 1" - }, - - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertFileCount(0); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/update-during-create-processing.test.ts b/frontend/deterministic-tests/src/tests/update-during-create-processing.test.ts deleted file mode 100644 index ca53244e..00000000 --- a/frontend/deterministic-tests/src/tests/update-during-create-processing.test.ts +++ /dev/null @@ -1,42 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const updateDuringCreateProcessingTest: TestDefinition = { - description: - "Client 0 creates a file while the server is paused, then immediately updates it. After the server resumes, both clients should converge with the updated content.", - clients: 2, - steps: [ - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - - { type: "pause-server" }, - - { - type: "create", - client: 0, - path: "file.md", - content: "initial" - }, - - { - type: "update", - client: 0, - path: "file.md", - content: "updated during create" - }, - - { type: "resume-server" }, - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertFileCount(1).assertContent( - "file.md", - "updated during create" - ); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/user-parenthesized-file-not-deleted.test.ts b/frontend/deterministic-tests/src/tests/user-parenthesized-file-not-deleted.test.ts deleted file mode 100644 index ef6cd771..00000000 --- a/frontend/deterministic-tests/src/tests/user-parenthesized-file-not-deleted.test.ts +++ /dev/null @@ -1,47 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const userParenthesizedFileNotDeletedTest: TestDefinition = { - description: - "A user-created file named 'Chapter (1).bin' alongside 'Chapter.bin' should not " + - "be mistakenly removed when another client creates a conflicting file.", - clients: 2, - steps: [ - { type: "enable-sync", client: 0 }, - - { - type: "create", - client: 0, - path: "Chapter.bin", - content: "chapter one" - }, - { - type: "create", - client: 0, - path: "Chapter (1).bin", - content: "chapter one notes" - }, - - { type: "sync", client: 0 }, - - { - type: "create", - client: 1, - path: "Chapter.bin", - content: "chapter one notes" - }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (state: AssertableState): void => { - state - .assertFileCount(3) - .assertFileExists("Chapter.bin") - .assertFileExists("Chapter (1).bin") - .assertFileExists("Chapter (2).bin"); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/watermark-advances-on-skip.test.ts b/frontend/deterministic-tests/src/tests/watermark-advances-on-skip.test.ts deleted file mode 100644 index 063faff4..00000000 --- a/frontend/deterministic-tests/src/tests/watermark-advances-on-skip.test.ts +++ /dev/null @@ -1,35 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const watermarkAdvancesOnSkipTest: TestDefinition = { - description: - "Both clients create the same file offline. After syncing, both disconnect and reconnect. The reconnect should not replay already-processed updates.", - clients: 2, - steps: [ - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - - { type: "disable-sync", client: 0 }, - { type: "disable-sync", client: 1 }, - { type: "create", client: 0, path: "doc.md", content: "from client 0" }, - { type: "create", client: 1, path: "doc.md", content: "from client 1" }, - - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - - { type: "disable-sync", client: 0 }, - { type: "disable-sync", client: 1 }, - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertFileCount(1).assertFileExists("doc.md"); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/watermark-gap-remote-update-not-recorded.test.ts b/frontend/deterministic-tests/src/tests/watermark-gap-remote-update-not-recorded.test.ts deleted file mode 100644 index ac9ba467..00000000 --- a/frontend/deterministic-tests/src/tests/watermark-gap-remote-update-not-recorded.test.ts +++ /dev/null @@ -1,37 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const watermarkGapRemoteUpdateNotRecordedTest: TestDefinition = { - description: - "Client 0 sends two rapid updates. Client 1 processes both, then disconnects and reconnects. Both clients should still converge to the latest content after reconnect.", - clients: 2, - steps: [ - { type: "create", client: 0, path: "doc.md", content: "original" }, - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - - { type: "update", client: 0, path: "doc.md", content: "update 1" }, - { type: "sync", client: 0 }, - { type: "update", client: 0, path: "doc.md", content: "update 2" }, - - { type: "barrier" }, - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertFileCount(1).assertContent("doc.md", "update 2"); - } - }, - - { type: "disable-sync", client: 1 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertFileCount(1).assertContent("doc.md", "update 2"); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/utils/assert.ts b/frontend/deterministic-tests/src/utils/assert.ts deleted file mode 100644 index 4e709060..00000000 --- a/frontend/deterministic-tests/src/utils/assert.ts +++ /dev/null @@ -1,5 +0,0 @@ -export function assert(value: boolean, message: string): asserts value { - if (!value) { - throw new Error(message); - } -} diff --git a/frontend/deterministic-tests/src/utils/assertable-state.ts b/frontend/deterministic-tests/src/utils/assertable-state.ts deleted file mode 100644 index 7c6f192c..00000000 --- a/frontend/deterministic-tests/src/utils/assertable-state.ts +++ /dev/null @@ -1,150 +0,0 @@ -import type { ClientState } from "../test-definition"; - -export class AssertableState { - public readonly files: Map; - public readonly clientFiles: Map[]; - - public constructor(state: ClientState) { - this.files = state.files; - this.clientFiles = state.clientFiles; - } - - public assertFileCount(expected: number): this { - if (this.files.size !== expected) { - const keys = Array.from(this.files.keys()).join(", "); - throw new Error( - `Expected ${expected} file(s), got ${this.files.size}: [${keys}]` - ); - } - return this; - } - - public assertFileExists(path: string): this { - if (!this.files.has(path)) { - const keys = Array.from(this.files.keys()).join(", "); - throw new Error(`Expected "${path}" to exist. Files: [${keys}]`); - } - return this; - } - - public assertFileNotExists(path: string): this { - if (this.files.has(path)) { - const keys = Array.from(this.files.keys()).join(", "); - throw new Error( - `Expected "${path}" not to exist. Files: [${keys}]` - ); - } - return this; - } - - public assertContent(path: string, expected: string): this { - this.assertFileExists(path); - const actual = this.files.get(path) ?? ""; - if (actual !== expected) { - throw new Error( - `Expected "${path}" to have content "${expected}", got: "${actual}"` - ); - } - return this; - } - - public assertContains(path: string, ...substrings: string[]): this { - this.assertFileExists(path); - const content = this.files.get(path) ?? ""; - const missing = substrings.filter((s) => !content.includes(s)); - if (missing.length > 0) { - throw new Error( - `Expected "${path}" to contain ${missing.map((s) => `"${s}"`).join(", ")}. Content: "${content}"` - ); - } - return this; - } - - public assertContainsAny(path: string, ...substrings: string[]): this { - this.assertFileExists(path); - const content = this.files.get(path) ?? ""; - const found = substrings.some((s) => content.includes(s)); - if (!found) { - throw new Error( - `Expected "${path}" to contain at least one of ${substrings.map((s) => `"${s}"`).join(", ")}. Content: "${content}"` - ); - } - return this; - } - - public assertAnyFileContains(...substrings: string[]): this { - const allContent = Array.from(this.files.values()).join("\n"); - const missing = substrings.filter((s) => !allContent.includes(s)); - if (missing.length > 0) { - const dump = Array.from(this.files.entries()) - .map(([k, v]) => ` ${k}: "${v}"`) - .join("\n"); - throw new Error( - `Expected some file to contain ${missing.map((s) => `"${s}"`).join(", ")}.\nFiles:\n${dump}` - ); - } - return this; - } - - public assertNoFileContains(...substrings: string[]): this { - const offenders: { path: string; substring: string }[] = []; - for (const [path, content] of this.files) { - for (const s of substrings) { - if (content.includes(s)) { - offenders.push({ path, substring: s }); - } - } - } - if (offenders.length > 0) { - const dump = Array.from(this.files.entries()) - .map(([k, v]) => ` ${k}: "${v}"`) - .join("\n"); - throw new Error( - `Expected no file to contain ${substrings.map((s) => `"${s}"`).join(", ")}, but found ${offenders.map((o) => `"${o.substring}" in "${o.path}"`).join(", ")}.\nFiles:\n${dump}` - ); - } - return this; - } - - public assertSubstringCount( - path: string, - substring: string, - expected: number - ): this { - this.assertFileExists(path); - const content = this.files.get(path) ?? ""; - const actual = content.split(substring).length - 1; - if (actual !== expected) { - throw new Error( - `Expected "${substring}" to appear ${expected} time(s) in "${path}", found ${actual}. Content: "${content}"` - ); - } - return this; - } - - public assertContentInAtMostOneFile(substring: string): this { - const matches = Array.from(this.files.entries()).filter(([, content]) => - content.includes(substring) - ); - if (matches.length > 1) { - const dump = Array.from(this.files.entries()) - .map(([k, v]) => ` ${k}: "${v}"`) - .join("\n"); - throw new Error( - `Expected "${substring}" in at most 1 file, found in ${matches.length}: [${matches.map(([p]) => p).join(", ")}].\nFiles:\n${dump}` - ); - } - return this; - } - - public ifFileExists(path: string, fn: (state: this) => void): this { - if (this.files.has(path)) { - fn(this); - } - return this; - } - - public getContent(path: string): string { - return this.files.get(path) ?? ""; - } -} diff --git a/frontend/deterministic-tests/src/utils/find-free-port.ts b/frontend/deterministic-tests/src/utils/find-free-port.ts deleted file mode 100644 index 0734c1a9..00000000 --- a/frontend/deterministic-tests/src/utils/find-free-port.ts +++ /dev/null @@ -1,29 +0,0 @@ -import * as net from "node:net"; - -interface PortReservation { - port: number; - release: () => void; -} - -/** - * Find a free port and keep it reserved until the caller explicitly releases it. - */ -export async function findFreePort(): Promise { - return new Promise((resolve, reject) => { - const server = net.createServer(); - server.listen(0, "127.0.0.1", () => { - const addr = server.address(); - if (addr === null || typeof addr === "string") { - server.close(); - reject(new Error("Failed to get port from server")); - return; - } - const { port } = addr; - resolve({ - port, - release: () => server.close() - }); - }); - server.on("error", reject); - }); -} diff --git a/frontend/deterministic-tests/src/utils/sleep.ts b/frontend/deterministic-tests/src/utils/sleep.ts deleted file mode 100644 index ff474799..00000000 --- a/frontend/deterministic-tests/src/utils/sleep.ts +++ /dev/null @@ -1,3 +0,0 @@ -export async function sleep(ms: number): Promise { - return new Promise((resolve) => setTimeout(resolve, ms)); -} diff --git a/frontend/deterministic-tests/src/utils/with-timeout.ts b/frontend/deterministic-tests/src/utils/with-timeout.ts deleted file mode 100644 index 14ee3f27..00000000 --- a/frontend/deterministic-tests/src/utils/with-timeout.ts +++ /dev/null @@ -1,15 +0,0 @@ -export async function withTimeout( - promise: Promise, - timeoutMs: number, - message: string -): Promise { - let timeoutId: ReturnType | undefined = undefined; - const timeoutPromise = new Promise((_resolve, reject) => { - timeoutId = setTimeout(() => { - reject(new Error(message)); - }, timeoutMs); - }); - return Promise.race([promise, timeoutPromise]).finally(() => { - clearTimeout(timeoutId); - }); -} diff --git a/frontend/deterministic-tests/tsconfig.json b/frontend/deterministic-tests/tsconfig.json deleted file mode 100644 index 7558871d..00000000 --- a/frontend/deterministic-tests/tsconfig.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "compilerOptions": { - "baseUrl": ".", - "strict": true, - "target": "ES2022", - "module": "CommonJS", - "esModuleInterop": true, - "lib": ["DOM", "ES2024"], - "moduleResolution": "node" - }, - "exclude": ["./dist"] -} diff --git a/frontend/deterministic-tests/webpack.config.js b/frontend/deterministic-tests/webpack.config.js deleted file mode 100644 index 6aee1547..00000000 --- a/frontend/deterministic-tests/webpack.config.js +++ /dev/null @@ -1,30 +0,0 @@ -const path = require("path"); -const webpack = require("webpack"); - -module.exports = { - entry: "./src/cli.ts", - target: "node", - mode: "production", - optimization: { - minimize: false - }, - module: { - rules: [ - { - test: /\.ts$/, - use: "ts-loader" - } - ] - }, - resolve: { - extensions: [".ts", ".js"] - }, - output: { - globalObject: "this", - filename: "cli.js", - path: path.resolve(__dirname, "dist") - }, - plugins: [ - new webpack.BannerPlugin({ banner: "#!/usr/bin/env node", raw: true }) - ] -}; diff --git a/frontend/history-ui/src/lib/types/ClientCursors.ts b/frontend/history-ui/src/lib/types/ClientCursors.ts new file mode 100644 index 00000000..14298431 --- /dev/null +++ b/frontend/history-ui/src/lib/types/ClientCursors.ts @@ -0,0 +1,8 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { DocumentWithCursors } from "./DocumentWithCursors"; + +export type ClientCursors = { + userName: string; + deviceId: string; + documentsWithCursors: Array; +}; diff --git a/frontend/sync-client/src/services/types/UpdateDocumentVersion.ts b/frontend/history-ui/src/lib/types/CreateDocumentVersion.ts similarity index 55% rename from frontend/sync-client/src/services/types/UpdateDocumentVersion.ts rename to frontend/history-ui/src/lib/types/CreateDocumentVersion.ts index 4e57a297..389d8e88 100644 --- a/frontend/sync-client/src/services/types/UpdateDocumentVersion.ts +++ b/frontend/history-ui/src/lib/types/CreateDocumentVersion.ts @@ -1,7 +1,7 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. -export interface UpdateDocumentVersion { - parent_version_id: bigint; +export type CreateDocumentVersion = { relative_path: string; - content: number[]; -} + last_seen_vault_update_id: number; + content: Array; +}; diff --git a/frontend/history-ui/src/lib/types/CursorPositionFromClient.ts b/frontend/history-ui/src/lib/types/CursorPositionFromClient.ts new file mode 100644 index 00000000..5846843e --- /dev/null +++ b/frontend/history-ui/src/lib/types/CursorPositionFromClient.ts @@ -0,0 +1,6 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { DocumentWithCursors } from "./DocumentWithCursors"; + +export type CursorPositionFromClient = { + documentsWithCursors: Array; +}; diff --git a/frontend/history-ui/src/lib/types/CursorPositionFromServer.ts b/frontend/history-ui/src/lib/types/CursorPositionFromServer.ts new file mode 100644 index 00000000..3a72c706 --- /dev/null +++ b/frontend/history-ui/src/lib/types/CursorPositionFromServer.ts @@ -0,0 +1,4 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { ClientCursors } from "./ClientCursors"; + +export type CursorPositionFromServer = { clients: Array }; diff --git a/frontend/sync-client/src/services/types/DeleteDocumentVersion.ts b/frontend/history-ui/src/lib/types/CursorSpan.ts similarity index 61% rename from frontend/sync-client/src/services/types/DeleteDocumentVersion.ts rename to frontend/history-ui/src/lib/types/CursorSpan.ts index 99ecc9e7..916019ce 100644 --- a/frontend/sync-client/src/services/types/DeleteDocumentVersion.ts +++ b/frontend/history-ui/src/lib/types/CursorSpan.ts @@ -1,5 +1,3 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. -export interface DeleteDocumentVersion { - relativePath: string; -} +export type CursorSpan = { start: number; end: number }; diff --git a/frontend/history-ui/src/lib/types/DocumentUpdateResponse.ts b/frontend/history-ui/src/lib/types/DocumentUpdateResponse.ts new file mode 100644 index 00000000..dd7eadda --- /dev/null +++ b/frontend/history-ui/src/lib/types/DocumentUpdateResponse.ts @@ -0,0 +1,10 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { DocumentVersion } from "./DocumentVersion"; +import type { DocumentVersionWithoutContent } from "./DocumentVersionWithoutContent"; + +/** + * Response to a create/update document request. + */ +export type DocumentUpdateResponse = + | ({ type: "FastForwardUpdate" } & DocumentVersionWithoutContent) + | ({ type: "MergingUpdate" } & DocumentVersion); diff --git a/frontend/history-ui/src/lib/types/DocumentVersion.ts b/frontend/history-ui/src/lib/types/DocumentVersion.ts new file mode 100644 index 00000000..50a6c591 --- /dev/null +++ b/frontend/history-ui/src/lib/types/DocumentVersion.ts @@ -0,0 +1,12 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +export type DocumentVersion = { + vaultUpdateId: number; + documentId: string; + relativePath: string; + updatedDate: string; + contentBase64: string; + isDeleted: boolean; + userId: string; + deviceId: string; +}; diff --git a/frontend/history-ui/src/lib/types/DocumentVersionWithoutContent.ts b/frontend/history-ui/src/lib/types/DocumentVersionWithoutContent.ts new file mode 100644 index 00000000..e3ed828a --- /dev/null +++ b/frontend/history-ui/src/lib/types/DocumentVersionWithoutContent.ts @@ -0,0 +1,16 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +export type DocumentVersionWithoutContent = { + vaultUpdateId: number; + documentId: string; + relativePath: string; + updatedDate: string; + isDeleted: boolean; + userId: string; + deviceId: string; + contentSize: number; + /** + * True iff this is the first version of the document + */ + isNewFile: boolean; +}; diff --git a/frontend/history-ui/src/lib/types/DocumentWithCursors.ts b/frontend/history-ui/src/lib/types/DocumentWithCursors.ts new file mode 100644 index 00000000..ca6a2155 --- /dev/null +++ b/frontend/history-ui/src/lib/types/DocumentWithCursors.ts @@ -0,0 +1,9 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { CursorSpan } from "./CursorSpan"; + +export type DocumentWithCursors = { + vaultUpdateId: number | null; + documentId: string; + relativePath: string; + cursors: Array; +}; diff --git a/frontend/history-ui/src/lib/types/FetchLatestDocumentsResponse.ts b/frontend/history-ui/src/lib/types/FetchLatestDocumentsResponse.ts new file mode 100644 index 00000000..141c2565 --- /dev/null +++ b/frontend/history-ui/src/lib/types/FetchLatestDocumentsResponse.ts @@ -0,0 +1,13 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { DocumentVersionWithoutContent } from "./DocumentVersionWithoutContent"; + +/** + * Response to a fetch latest documents request. + */ +export type FetchLatestDocumentsResponse = { + latestDocuments: Array; + /** + * The update ID of the latest document in the response. + */ + lastUpdateId: bigint; +}; diff --git a/frontend/history-ui/src/lib/types/ListVaultsResponse.ts b/frontend/history-ui/src/lib/types/ListVaultsResponse.ts new file mode 100644 index 00000000..604ad958 --- /dev/null +++ b/frontend/history-ui/src/lib/types/ListVaultsResponse.ts @@ -0,0 +1,11 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { VaultInfo } from "./VaultInfo"; + +/** + * Response to listing vaults accessible to the authenticated user. + */ +export type ListVaultsResponse = { + vaults: Array; + hasMore: boolean; + userName: string; +}; diff --git a/frontend/history-ui/src/lib/types/PingResponse.ts b/frontend/history-ui/src/lib/types/PingResponse.ts new file mode 100644 index 00000000..7e5ac4f8 --- /dev/null +++ b/frontend/history-ui/src/lib/types/PingResponse.ts @@ -0,0 +1,25 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +/** + * Response to a ping request. + */ +export type PingResponse = { + /** + * Semantic version of the server. + */ + serverVersion: string; + /** + * Whether the client is authenticated based on the sent Authorization + * header. + */ + isAuthenticated: boolean; + /** + * List of file extensions that are allowed to be merged. + */ + mergeableFileExtensions: Array; + /** + * API version ensuring backwards & forwards compatibility between the client + * and server. + */ + supportedApiVersion: number; +}; diff --git a/frontend/history-ui/src/lib/types/SerializedError.ts b/frontend/history-ui/src/lib/types/SerializedError.ts new file mode 100644 index 00000000..354305f6 --- /dev/null +++ b/frontend/history-ui/src/lib/types/SerializedError.ts @@ -0,0 +1,7 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +export type SerializedError = { + errorType: string; + message: string; + causes: Array; +}; diff --git a/frontend/history-ui/src/lib/types/UpdateTextDocumentVersion.ts b/frontend/history-ui/src/lib/types/UpdateTextDocumentVersion.ts new file mode 100644 index 00000000..5a1978eb --- /dev/null +++ b/frontend/history-ui/src/lib/types/UpdateTextDocumentVersion.ts @@ -0,0 +1,7 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +export type UpdateTextDocumentVersion = { + parentVersionId: number; + relativePath: string | null; + content: Array; +}; diff --git a/frontend/history-ui/src/lib/types/VaultHistoryResponse.ts b/frontend/history-ui/src/lib/types/VaultHistoryResponse.ts new file mode 100644 index 00000000..e69366f0 --- /dev/null +++ b/frontend/history-ui/src/lib/types/VaultHistoryResponse.ts @@ -0,0 +1,10 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { DocumentVersionWithoutContent } from "./DocumentVersionWithoutContent"; + +/** + * Response to a vault history request (paginated). + */ +export type VaultHistoryResponse = { + versions: Array; + hasMore: boolean; +}; diff --git a/frontend/history-ui/src/lib/types/VaultInfo.ts b/frontend/history-ui/src/lib/types/VaultInfo.ts new file mode 100644 index 00000000..3f630ae9 --- /dev/null +++ b/frontend/history-ui/src/lib/types/VaultInfo.ts @@ -0,0 +1,10 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +/** + * Summary of a single vault returned by the list-vaults endpoint. + */ +export type VaultInfo = { + name: string; + documentCount: number; + createdAt: string | null; +}; diff --git a/frontend/history-ui/src/lib/types/WebSocketClientMessage.ts b/frontend/history-ui/src/lib/types/WebSocketClientMessage.ts new file mode 100644 index 00000000..9608f3af --- /dev/null +++ b/frontend/history-ui/src/lib/types/WebSocketClientMessage.ts @@ -0,0 +1,7 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { CursorPositionFromClient } from "./CursorPositionFromClient"; +import type { WebSocketHandshake } from "./WebSocketHandshake"; + +export type WebSocketClientMessage = + | ({ type: "handshake" } & WebSocketHandshake) + | ({ type: "cursorPositions" } & CursorPositionFromClient); diff --git a/frontend/history-ui/src/lib/types/WebSocketHandshake.ts b/frontend/history-ui/src/lib/types/WebSocketHandshake.ts new file mode 100644 index 00000000..8e51a121 --- /dev/null +++ b/frontend/history-ui/src/lib/types/WebSocketHandshake.ts @@ -0,0 +1,7 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +export type WebSocketHandshake = { + token: string; + deviceId: string; + lastSeenVaultUpdateId: number | null; +}; diff --git a/frontend/history-ui/src/lib/types/WebSocketServerMessage.ts b/frontend/history-ui/src/lib/types/WebSocketServerMessage.ts new file mode 100644 index 00000000..fd250b7b --- /dev/null +++ b/frontend/history-ui/src/lib/types/WebSocketServerMessage.ts @@ -0,0 +1,7 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { CursorPositionFromServer } from "./CursorPositionFromServer"; +import type { WebSocketVaultUpdate } from "./WebSocketVaultUpdate"; + +export type WebSocketServerMessage = + | ({ type: "vaultUpdate" } & WebSocketVaultUpdate) + | ({ type: "cursorPositions" } & CursorPositionFromServer); diff --git a/frontend/history-ui/src/lib/types/WebSocketVaultUpdate.ts b/frontend/history-ui/src/lib/types/WebSocketVaultUpdate.ts new file mode 100644 index 00000000..94d70c0a --- /dev/null +++ b/frontend/history-ui/src/lib/types/WebSocketVaultUpdate.ts @@ -0,0 +1,4 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { DocumentVersionWithoutContent } from "./DocumentVersionWithoutContent"; + +export type WebSocketVaultUpdate = { document: DocumentVersionWithoutContent }; diff --git a/frontend/local-client-cli/Dockerfile b/frontend/local-client-cli/Dockerfile index 0dfa7055..695ab587 100644 --- a/frontend/local-client-cli/Dockerfile +++ b/frontend/local-client-cli/Dockerfile @@ -1,4 +1,4 @@ -FROM node:25-slim AS builder +FROM node:22-slim AS builder WORKDIR /build @@ -7,7 +7,7 @@ COPY . . RUN npm ci RUN npm run build -FROM node:25-alpine +FROM node:22-alpine LABEL org.opencontainers.image.title="VaultLink Local CLI" LABEL org.opencontainers.image.description="Standalone CLI for VaultLink sync client" diff --git a/frontend/local-client-cli/README.md b/frontend/local-client-cli/README.md index e91322f9..0585bacc 100644 --- a/frontend/local-client-cli/README.md +++ b/frontend/local-client-cli/README.md @@ -47,25 +47,24 @@ vaultlink \ ### Required -| Option | Description | -| ------------------------- | --------------------------------------------- | -| `-l, --local-path ` | Local directory to sync | -| `-r, --remote-uri ` | Remote server WebSocket URI (ws:// or wss://) | -| `-t, --token ` | Authentication token | -| `-v, --vault-name ` | Vault name on server | +| Option | Description | +|--------|-------------| +| `-l, --local-path ` | Local directory to sync | +| `-r, --remote-uri ` | Remote server WebSocket URI (ws:// or wss://) | +| `-t, --token ` | Authentication token | +| `-v, --vault-name ` | Vault name on server | ### Optional -| Option | Default | Description | -| ------------------------------------ | ------- | ----------------------------------------------- | -| `--max-file-size-mb ` | `10` | Maximum file size in MB | -| `--ignore-pattern ` | - | Glob pattern to ignore (repeatable) | -| `--websocket-retry-interval-ms ` | `3500` | WebSocket reconnection interval | -| `--log-level ` | `INFO` | Log level: DEBUG, INFO, WARNING, ERROR | -| `--line-endings ` | `auto` | Line ending style: auto, lf, crlf | -| `-q, --quiet` | - | Suppress startup banner for non-interactive use | -| `-h, --help` | - | Show help | -| `-V, --version` | - | Show version | +| Option | Default | Description | +|--------|---------|-------------| +| `--sync-concurrency ` | `1` | Concurrent sync operations | +| `--max-file-size-mb ` | `10` | Maximum file size in MB | +| `--ignore-pattern ` | - | Glob pattern to ignore (repeatable) | +| `--websocket-retry-interval-ms ` | `3500` | WebSocket reconnection interval | +| `--log-level ` | `INFO` | Log level: DEBUG, INFO, WARNING, ERROR | +| `-h, --help` | - | Show help | +| `-V, --version` | - | Show version | ### Auto-Ignored Patterns @@ -75,32 +74,22 @@ vaultlink \ ### Examples Basic usage: - ```bash vaultlink -l ./vault -r wss://sync.example.com -t token123 -v default ``` With ignore patterns: - ```bash vaultlink -l ./vault -r wss://sync.example.com -t token123 -v default \ - --ignore-pattern "**/*.tmp" \ + --ignore-pattern "*.tmp" \ --ignore-pattern ".DS_Store" \ --ignore-pattern "node_modules/**" ``` -With debug logging and quiet startup: - +With debug logging: ```bash vaultlink -l ./vault -r wss://sync.example.com -t token123 -v default \ - --log-level DEBUG --quiet -``` - -Force LF line endings (useful for cross-platform vaults): - -```bash -vaultlink -l ./vault -r wss://sync.example.com -t token123 -v default \ - --line-endings lf + --log-level DEBUG ``` ## Docker Deployment @@ -187,7 +176,6 @@ services: ## Development Build: - ```bash npm run build # or from the parent folder, run @@ -195,13 +183,11 @@ docker build -f local-client-cli/Dockerfile . ``` Test: - ```bash npm test ``` Docker build: - ```bash cd frontend docker build -f local-client-cli/Dockerfile -t vault-link-cli:test . diff --git a/frontend/local-client-cli/package.json b/frontend/local-client-cli/package.json index a862b297..cade4990 100644 --- a/frontend/local-client-cli/package.json +++ b/frontend/local-client-cli/package.json @@ -11,16 +11,18 @@ "build": "webpack --mode production", "test": "tsx --test 'src/**/*.test.ts'" }, - "devDependencies": { + "dependencies": { "commander": "^14.0.2", - "watcher": "^2.3.1", - "@types/node": "^25.0.2", + "watcher": "^2.3.1" + }, + "devDependencies": { + "@types/node": "^24.8.1", "sync-client": "file:../sync-client", - "ts-loader": "^9.5.4", + "ts-loader": "^9.5.2", "tslib": "2.8.1", - "tsx": "^4.21.0", - "typescript": "5.9.3", - "webpack": "^5.103.0", + "tsx": "^4.20.6", + "typescript": "5.8.3", + "webpack": "^5.99.9", "webpack-cli": "^6.0.1" } } diff --git a/frontend/local-client-cli/src/args.test.ts b/frontend/local-client-cli/src/args.test.ts index fdf0b6c8..eb195538 100644 --- a/frontend/local-client-cli/src/args.test.ts +++ b/frontend/local-client-cli/src/args.test.ts @@ -55,10 +55,13 @@ test("parseArgs - parse with optional arguments", () => { "mytoken", "-v", "default", + "--sync-concurrency", + "5", "--max-file-size-mb", "20" ]); + assert.equal(args.syncConcurrency, 5); assert.equal(args.maxFileSizeMB, 20); }); @@ -150,6 +153,25 @@ test("parseArgs - default log level is INFO", () => { assert.equal(args.logLevel, LogLevel.INFO); }); +test("parseArgs - parse DEBUG log level", () => { + const args = parseArgs([ + "node", + "cli.js", + "-l", + "/path/to/vault", + "-r", + "https://sync.example.com", + "-t", + "mytoken", + "-v", + "default", + "--log-level", + "DEBUG" + ]); + + assert.equal(args.logLevel, LogLevel.DEBUG); +}); + test("parseArgs - parse ERROR log level", () => { const args = parseArgs([ "node", @@ -169,32 +191,28 @@ test("parseArgs - parse ERROR log level", () => { assert.equal(args.logLevel, LogLevel.ERROR); }); +test("parseArgs - log level is case insensitive", () => { + const args = parseArgs([ + "node", + "cli.js", + "-l", + "/path/to/vault", + "-r", + "https://sync.example.com", + "-t", + "mytoken", + "-v", + "default", + "--log-level", + "debug" + ]); -test("parseArgs - reads required options from environment variables", () => { - process.env.VAULTLINK_LOCAL_PATH = "/env/path"; - process.env.VAULTLINK_REMOTE_URI = "https://env.example.com"; - process.env.VAULTLINK_TOKEN = "env-token"; - process.env.VAULTLINK_VAULT_NAME = "env-vault"; - - try { - const args = parseArgs(["node", "cli.js"]); - assert.equal(args.localPath, "/env/path"); - assert.equal(args.remoteUri, "https://env.example.com"); - assert.equal(args.token, "env-token"); - assert.equal(args.vaultName, "env-vault"); - } finally { - delete process.env.VAULTLINK_LOCAL_PATH; - delete process.env.VAULTLINK_REMOTE_URI; - delete process.env.VAULTLINK_TOKEN; - delete process.env.VAULTLINK_VAULT_NAME; - } + assert.equal(args.logLevel, LogLevel.DEBUG); }); -test("parseArgs - CLI arguments take precedence over environment variables", () => { - process.env.VAULTLINK_TOKEN = "env-token"; - - try { - const args = parseArgs([ +test("parseArgs - throws on invalid log level", () => { + assert.throws(() => { + parseArgs([ "node", "cli.js", "-l", @@ -202,12 +220,11 @@ test("parseArgs - CLI arguments take precedence over environment variables", () "-r", "https://sync.example.com", "-t", - "cli-token", + "mytoken", "-v", - "default" + "default", + "--log-level", + "INVALID" ]); - assert.equal(args.token, "cli-token"); - } finally { - delete process.env.VAULTLINK_TOKEN; - } + }, /Invalid log level/); }); diff --git a/frontend/local-client-cli/src/args.ts b/frontend/local-client-cli/src/args.ts index 34d839b1..615b9d71 100644 --- a/frontend/local-client-cli/src/args.ts +++ b/frontend/local-client-cli/src/args.ts @@ -1,54 +1,19 @@ -import { Command, Option } from "commander"; +import { Command } from "commander"; import packageJson from "../package.json"; import { LogLevel } from "sync-client"; -export const LINE_ENDING_MODES = ["auto", "lf", "crlf"] as const; -export type LineEndingMode = (typeof LINE_ENDING_MODES)[number]; - -interface CliArgs { +export interface CliArgs { remoteUri: string; token: string; vaultName: string; localPath: string; + syncConcurrency?: number; maxFileSizeMB?: number; ignorePatterns?: string[]; webSocketRetryIntervalMs?: number; logLevel: LogLevel; health?: string; enableTelemetry?: boolean; - quiet: boolean; - lineEndings: LineEndingMode; -} - -const VALID_PROTOCOLS = ["http://", "https://", "ws://", "wss://"]; - -const REQUIRED_OPTIONS = { - localPath: { - flags: "-l, --local-path ", - env: "VAULTLINK_LOCAL_PATH" - }, - remoteUri: { - flags: "-r, --remote-uri ", - env: "VAULTLINK_REMOTE_URI" - }, - token: { flags: "-t, --token ", env: "VAULTLINK_TOKEN" }, - vaultName: { - flags: "-v, --vault-name ", - env: "VAULTLINK_VAULT_NAME" - } -} as const; - -function requireOption( - value: T | undefined, - name: keyof typeof REQUIRED_OPTIONS -): T { - if (value === undefined) { - const { flags, env } = REQUIRED_OPTIONS[name]; - throw new Error( - `required option '${flags}' not specified (or set ${env})` - ); - } - return value; } export function parseArgs(argv: string[]): CliArgs { @@ -60,85 +25,41 @@ export function parseArgs(argv: string[]): CliArgs { "VaultLink Local CLI - Sync your vault to the local filesystem" ) .version(packageJson.version) - .addOption( - new Option( - REQUIRED_OPTIONS.localPath.flags, - "Local directory path to sync" - ).env(REQUIRED_OPTIONS.localPath.env) + .option("-l, --local-path ", "Local directory path to sync") + .option("-r, --remote-uri ", "Remote server URI") + .option("-t, --token ", "Authentication token") + .option("-v, --vault-name ", "Vault name") + .option( + "--sync-concurrency ", + "[OPTIONAL] Number of concurrent sync operations", + parseInt ) - .addOption( - new Option( - REQUIRED_OPTIONS.remoteUri.flags, - "Remote server URI" - ).env(REQUIRED_OPTIONS.remoteUri.env) + .option( + "--max-file-size-mb ", + "[OPTIONAL] Maximum file size in MB", + parseInt ) - .addOption( - new Option( - REQUIRED_OPTIONS.token.flags, - "Authentication token" - ).env(REQUIRED_OPTIONS.token.env) + .option( + "--ignore-pattern ", + "[OPTIONAL] Patterns to ignore (can be specified multiple times)" ) - .addOption( - new Option(REQUIRED_OPTIONS.vaultName.flags, "Vault name").env( - REQUIRED_OPTIONS.vaultName.env - ) + .option( + "--websocket-retry-interval-ms ", + "[OPTIONAL] WebSocket retry interval in milliseconds", + parseInt ) - .addOption( - new Option( - "--max-file-size-mb ", - "[OPTIONAL] Maximum file size in MB" - ) - .argParser(parseInt) - .env("VAULTLINK_MAX_FILE_SIZE_MB") + .option( + "--log-level ", + "[OPTIONAL] Log level (DEBUG, INFO, WARNING, ERROR)", + "INFO" ) - .addOption( - new Option( - "--ignore-pattern ", - "[OPTIONAL] Patterns to ignore (can be specified multiple times)" - ).env("VAULTLINK_IGNORE_PATTERNS") + .option( + "--health ", + "[OPTIONAL] Path to health status file for Docker healthcheck" ) - .addOption( - new Option( - "--websocket-retry-interval-ms ", - "[OPTIONAL] WebSocket retry interval in milliseconds" - ) - .argParser(parseInt) - .env("VAULTLINK_WEBSOCKET_RETRY_INTERVAL_MS") - ) - .addOption( - new Option( - "--log-level ", - "[OPTIONAL] Log level (DEBUG, INFO, WARNING, ERROR)" - ) - .default("INFO") - .env("VAULTLINK_LOG_LEVEL") - ) - .addOption( - new Option( - "--health ", - "[OPTIONAL] Path to health status file for Docker healthcheck" - ).env("VAULTLINK_HEALTH") - ) - .addOption( - new Option( - "--enable-telemetry", - "[OPTIONAL] Enable telemetry (disabled by default)" - ).env("VAULTLINK_ENABLE_TELEMETRY") - ) - .addOption( - new Option( - "-q, --quiet", - "[OPTIONAL] Suppress startup banner for non-interactive use" - ).env("VAULTLINK_QUIET") - ) - .addOption( - new Option( - "--line-endings ", - "[OPTIONAL] Line ending style: auto (platform default), lf, crlf" - ) - .default("auto") - .choices([...LINE_ENDING_MODES]) - .env("VAULTLINK_LINE_ENDINGS") + .option( + "--enable-telemetry", + "[OPTIONAL] Enable telemetry (disabled by default)" ) .addHelpText( "after", @@ -146,13 +67,9 @@ export function parseArgs(argv: string[]): CliArgs { Examples: $ vaultlink -l ./my-vault -r https://sync.example.com -t mytoken -v default $ vaultlink -l ./my-vault -r https://sync.example.com -t mytoken -v default \\ - --ignore-pattern ".git/**" --ignore-pattern "**/*.tmp" + --ignore-pattern ".git/**" --ignore-pattern "*.tmp" $ vaultlink -l ./my-vault -r https://sync.example.com -t mytoken -v default \\ - --log-level DEBUG --quiet - -Environment variables: - All options can be configured via VAULTLINK_ prefixed environment variables. - CLI arguments take precedence over environment variables. + --log-level DEBUG ` ); @@ -164,6 +81,7 @@ Environment variables: const remoteUri = opts.remoteUri as string | undefined; const token = opts.token as string | undefined; const vaultName = opts.vaultName as string | undefined; + const syncConcurrency = opts.syncConcurrency as number | undefined; const maxFileSizeMb = opts.maxFileSizeMb as number | undefined; const ignorePattern = opts.ignorePattern as string[] | undefined; const websocketRetryIntervalMs = opts.websocketRetryIntervalMs as @@ -172,23 +90,22 @@ Environment variables: const logLevelStr = (opts.logLevel as string | undefined) ?? "INFO"; const health = opts.health as string | undefined; const enableTelemetry = opts.enableTelemetry as boolean | undefined; - const quiet = (opts.quiet as boolean | undefined) ?? false; - const lineEndingsStr = (opts.lineEndings as string | undefined) ?? "auto"; /* eslint-enable @typescript-eslint/no-unsafe-type-assertion */ - const requiredLocalPath = requireOption(localPath, "localPath"); - const requiredRemoteUri = requireOption(remoteUri, "remoteUri"); - const requiredToken = requireOption(token, "token"); - const requiredVaultName = requireOption(vaultName, "vaultName"); - - // Validate remote URI protocol - if ( - !VALID_PROTOCOLS.some((prefix) => requiredRemoteUri.startsWith(prefix)) - ) { + if (localPath === undefined) { throw new Error( - `Invalid remote URI '${requiredRemoteUri}'. Must start with ${VALID_PROTOCOLS.join(", ")}` + "required option '-l, --local-path ' not specified" ); } + if (remoteUri === undefined) { + throw new Error("required option '--remote-uri ' not specified"); + } + if (token === undefined) { + throw new Error("required option '--token ' not specified"); + } + if (vaultName === undefined) { + throw new Error("required option '--vault-name ' not specified"); + } // Validate and parse log level const logLevelUpper = logLevelStr.toUpperCase(); @@ -203,27 +120,17 @@ Environment variables: } const logLevel = logLevelUpper; - const isLineEndingMode = (value: string): value is LineEndingMode => - (LINE_ENDING_MODES as readonly string[]).includes(value); - if (!isLineEndingMode(lineEndingsStr)) { - throw new Error( - `Invalid line endings mode '${lineEndingsStr}'. Valid values are: ${LINE_ENDING_MODES.join(", ")}` - ); - } - const lineEndings = lineEndingsStr; - return { - localPath: requiredLocalPath, - remoteUri: requiredRemoteUri, - token: requiredToken, - vaultName: requiredVaultName, + localPath, + remoteUri, + token, + vaultName, + syncConcurrency, maxFileSizeMB: maxFileSizeMb, ignorePatterns: ignorePattern, webSocketRetryIntervalMs: websocketRetryIntervalMs, logLevel, health, - enableTelemetry, - quiet, - lineEndings + enableTelemetry }; } diff --git a/frontend/local-client-cli/src/cli.ts b/frontend/local-client-cli/src/cli.ts index 31c81d5c..48fd8954 100644 --- a/frontend/local-client-cli/src/cli.ts +++ b/frontend/local-client-cli/src/cli.ts @@ -5,27 +5,24 @@ import type { NetworkConnectionStatus } from "sync-client"; import { SyncClient, DEFAULT_SETTINGS, - Logger, LogLevel, - LogLine, type SyncSettings, type StoredDatabase } from "sync-client"; -import { parseArgs, type LineEndingMode } from "./args"; -import { NodeFileSystemOperations, VAULTLINK_DIR } from "./node-filesystem"; +import { parseArgs } from "./args"; +import { NodeFileSystemOperations } from "./node-filesystem"; import { FileWatcher } from "./file-watcher"; -import { formatLogLine } from "./logger-formatter"; +import { formatLogLine, colorize, styleText } from "./logger-formatter"; import packageJson from "../package.json"; function writeHealthStatus( - logger: Logger, filePath: string, connectionStatus: NetworkConnectionStatus ): void { try { fsSync.writeFileSync(filePath, JSON.stringify(connectionStatus)); } catch (error) { - logger.error( + console.error( `Failed to write health status to ${filePath}: ${error instanceof Error ? error.message : String(error)}` ); } @@ -38,41 +35,12 @@ const LOG_LEVEL_ORDER = { [LogLevel.ERROR]: 3 }; -function createLogHandler(minLevel: LogLevel): (logLine: LogLine) => void { - return (logLine: LogLine): void => { - if (LOG_LEVEL_ORDER[logLine.level] >= LOG_LEVEL_ORDER[minLevel]) { - // eslint-disable-next-line no-console - console.log(formatLogLine(logLine)); - } - }; -} - const HEALTH_CHECK_INTERVAL_MS = 30 * 1000; -const PROGRESS_LOG_INTERVAL_MS = 2000; - -function resolveLineEndings(mode: LineEndingMode): string { - switch (mode) { - case "lf": - return "\n"; - case "crlf": - return "\r\n"; - case "auto": - return process.platform === "win32" ? "\r\n" : "\n"; - } -} async function main(): Promise { const args = parseArgs(process.argv); const absolutePath = path.resolve(args.localPath); - const logHandler = createLogHandler(args.logLevel); - // Boot-time messages are emitted directly through logHandler before the - // SyncClient (and its Logger) exist; afterwards every log line flows - // through client.logger. - const emitBoot = (level: LogLevel, message: string): void => { - logHandler(new LogLine(level, message)); - }; - if (!fsSync.existsSync(absolutePath)) { fsSync.mkdirSync(absolutePath, { recursive: true }); } @@ -80,31 +48,38 @@ async function main(): Promise { try { const stats = await fs.stat(absolutePath); if (!stats.isDirectory()) { - emitBoot(LogLevel.ERROR, `${absolutePath} is not a directory`); + console.error( + colorize(`Error: ${absolutePath} is not a directory`, "red") + ); process.exit(1); } } catch (error) { - emitBoot( - LogLevel.ERROR, - `Cannot access directory ${absolutePath}: ${error instanceof Error ? error.message : String(error)}` + console.error( + colorize( + `Error: Cannot access directory ${absolutePath}: ${error instanceof Error ? error.message : String(error)}`, + "red" + ) ); process.exit(1); } - if (!args.quiet) { - emitBoot(LogLevel.INFO, `VaultLink Local CLI v${packageJson.version}`); - emitBoot(LogLevel.INFO, `Local path: ${absolutePath}`); - emitBoot(LogLevel.INFO, `Remote URI: ${args.remoteUri}`); - emitBoot(LogLevel.INFO, `Vault name: ${args.vaultName}`); - if (args.lineEndings !== "auto") { - emitBoot( - LogLevel.INFO, - `Line endings: ${args.lineEndings.toUpperCase()}` - ); - } - } + console.log( + styleText("VaultLink Local CLI", "bold", "cyan") + + colorize(` v${packageJson.version}`, "dim") + ); + console.log(colorize("=".repeat(50), "dim")); + console.log( + `${colorize("Local path:", "dim")} ${colorize(absolutePath, "green")}` + ); + console.log( + `${colorize("Remote URI:", "dim")} ${colorize(args.remoteUri, "cyan")}` + ); + console.log( + `${colorize("Vault name:", "dim")} ${colorize(args.vaultName, "green")}` + ); + console.log(""); - const dataDir = path.join(absolutePath, VAULTLINK_DIR); + const dataDir = path.join(absolutePath, ".vaultlink"); const dataFile = path.join(dataDir, "sync-data.json"); await fs.mkdir(dataDir, { recursive: true }); @@ -113,7 +88,8 @@ async function main(): Promise { const ignorePatterns = [ ...(args.ignorePatterns ?? []), - `${VAULTLINK_DIR}/**` + ".vaultlink/**", + ".git/**" ]; const settings: SyncSettings = { @@ -121,6 +97,8 @@ async function main(): Promise { remoteUri: args.remoteUri, token: args.token, vaultName: args.vaultName, + syncConcurrency: + args.syncConcurrency ?? DEFAULT_SETTINGS.syncConcurrency, maxFileSizeMB: args.maxFileSizeMB ?? DEFAULT_SETTINGS.maxFileSizeMB, ignorePatterns, webSocketRetryIntervalMs: @@ -141,9 +119,11 @@ async function main(): Promise { // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion database = JSON.parse(content) as Partial; } catch { - emitBoot( - LogLevel.WARNING, - `Cannot read data file at ${dataFile}` + console.error( + colorize( + `Cannot read data file at ${dataFile}`, + "yellow" + ) ); } @@ -153,27 +133,23 @@ async function main(): Promise { }; }, save: async ({ database: persistedDatabase }) => { + // settings can't be updated when running with this CLI await fs.writeFile( dataFile, JSON.stringify(persistedDatabase, null, 2) ); } }, - nativeLineEndings: resolveLineEndings(args.lineEndings) + nativeLineEndings: process.platform === "win32" ? "\r\n" : "\n" }); if (args.health !== undefined) { const healthFile = args.health; - const writeHealth = (): void => { + const healthInterval = setInterval(() => { void client.checkConnection().then((status) => { - writeHealthStatus(client.logger, healthFile, status); + writeHealthStatus(healthFile, status); }); - }; - writeHealth(); - const healthInterval = setInterval( - writeHealth, - HEALTH_CHECK_INTERVAL_MS - ); + }, HEALTH_CHECK_INTERVAL_MS); const clearHealthInterval = (): void => { clearInterval(healthInterval); }; @@ -182,10 +158,17 @@ async function main(): Promise { process.on("exit", clearHealthInterval); } - client.logger.onLogEmitted.add(logHandler); + // Add colored log formatter with level filtering + client.logger.onLogEmitted.add((logLine) => { + // Only show messages at or above the configured log level + if (LOG_LEVEL_ORDER[logLine.level] >= LOG_LEVEL_ORDER[args.logLevel]) { + console.log(formatLogLine(logLine)); + } + }); + client.logger.info("Starting sync client"); - const fileWatcher = new FileWatcher(absolutePath, client, ignorePatterns); + const fileWatcher = new FileWatcher(absolutePath, client); client.onWebSocketStatusChanged.add(() => { const isConnected = client.isWebSocketConnected; @@ -194,54 +177,26 @@ async function main(): Promise { ); }); - let syncBatchSize = 0; - let totalSyncOps = 0; - let lastProgressLogTime = 0; - client.onRemainingOperationsCountChanged.add((remaining) => { - if (remaining > syncBatchSize) { - syncBatchSize = remaining; - } - if (remaining === 0) { - if (syncBatchSize > 0) { - totalSyncOps += syncBatchSize; - client.logger.info( - `Sync batch complete (${syncBatchSize} operations)` - ); - syncBatchSize = 0; - } + client.logger.info("All sync operations completed"); } else { - const now = Date.now(); - if (now - lastProgressLogTime >= PROGRESS_LOG_INTERVAL_MS) { - client.logger.info( - `Syncing: ${remaining} operations remaining` - ); - lastProgressLogTime = now; - } + client.logger.info(`${remaining} sync operations remaining`); } }); - let isShuttingDown = false; const gracefulShutdown = async (signal: string): Promise => { - if (isShuttingDown) { - return; - } - isShuttingDown = true; - - client.logger.info(`${signal} received, shutting down gracefully`); + console.log( + colorize( + `\n${signal} received. Shutting down gracefully...`, + "yellow" + ) + ); fileWatcher.stop(); await client.waitUntilFinished(); await client.destroy(); - - if (totalSyncOps > 0) { - client.logger.info( - `Shutdown complete (${totalSyncOps} operations synced)` - ); - } else { - client.logger.info("Shutdown complete"); - } + console.log(colorize("Shutdown complete", "green")); process.exit(0); }; @@ -255,21 +210,27 @@ async function main(): Promise { try { const connectionStatus = await client.checkConnection(); if (!connectionStatus.isSuccessful) { - client.logger.error( - `Cannot connect to server: ${connectionStatus.serverMessage}` + console.error( + colorize( + `Error: Cannot connect to server: ${connectionStatus.serverMessage}`, + "red" + ) ); process.exit(1); } - if (!args.quiet) { - client.logger.info("Server connection successful"); - } + console.log(`${colorize("✓", "green")} Server connection successful`); + console.log(colorize("Press Ctrl+C to stop", "dim")); + console.log(""); await client.start(); fileWatcher.start(); } catch (error) { - client.logger.error( - `Fatal error: ${error instanceof Error ? error.message : String(error)}` + console.error( + colorize( + `Fatal error: ${error instanceof Error ? error.message : String(error)}`, + "red" + ) ); fileWatcher.stop(); @@ -279,9 +240,11 @@ async function main(): Promise { } main().catch((error: unknown) => { - // eslint-disable-next-line no-console console.error( - `Unexpected error: ${error instanceof Error ? error.message : String(error)}` + colorize( + `Unexpected error: ${error instanceof Error ? error.message : String(error)}`, + "red" + ) ); process.exit(1); }); diff --git a/frontend/local-client-cli/src/file-watcher.ts b/frontend/local-client-cli/src/file-watcher.ts index c273a412..e781d18f 100644 --- a/frontend/local-client-cli/src/file-watcher.ts +++ b/frontend/local-client-cli/src/file-watcher.ts @@ -1,20 +1,15 @@ import Watcher from "watcher"; import * as path from "path"; import type { SyncClient, RelativePath } from "sync-client"; -import { toUnixPath, matchesGlob } from "./path-utils"; export class FileWatcher { private watcher: Watcher | undefined; private isRunning = false; - private readonly ignorePatterns: string[]; public constructor( private readonly basePath: string, - private readonly client: SyncClient, - ignorePatterns: string[] = [] - ) { - this.ignorePatterns = ignorePatterns; - } + private readonly client: SyncClient + ) {} public start(): void { if (this.isRunning) { @@ -27,8 +22,7 @@ export class FileWatcher { recursive: true, renameDetection: true, renameTimeout: 125, - ignoreInitial: true, - ignore: (filePath: string): boolean => this.shouldIgnore(filePath) + ignoreInitial: true }); this.watcher.on("add", (filePath: string) => { @@ -62,32 +56,66 @@ export class FileWatcher { this.client.logger.info("File watcher stopped"); } - private shouldIgnore(filePath: string): boolean { - const rel = toUnixPath(path.relative(this.basePath, filePath)); - return this.ignorePatterns.some((pattern) => matchesGlob(rel, pattern)); - } - private handleCreate(relativePath: RelativePath): void { - this.client.syncLocallyCreatedFile(relativePath); + this.client + .syncLocallyCreatedFile(relativePath) + .catch((err: unknown) => { + this.client.logger.error( + `Failed to sync created file ${relativePath}: ${this.formatError(err)}` + ); + }); } private handleChange(relativePath: RelativePath): void { - this.client.syncLocallyUpdatedFile({ relativePath }); + this.client + .syncLocallyUpdatedFile({ relativePath }) + .catch((err: unknown) => { + this.client.logger.error( + `Failed to sync updated file ${relativePath}: ${this.formatError(err)}` + ); + }); } private handleDelete(relativePath: RelativePath): void { - this.client.syncLocallyDeletedFile(relativePath); + this.client + .syncLocallyDeletedFile(relativePath) + .catch((err: unknown) => { + this.client.logger.error( + `Failed to sync deleted file ${relativePath}: ${this.formatError(err)}` + ); + }); } private handleRename(oldPath: RelativePath, newPath: RelativePath): void { this.client.logger.info(`File renamed: ${oldPath} -> ${newPath}`); - this.client.syncLocallyUpdatedFile({ - oldPath, - relativePath: newPath - }); + this.client + .syncLocallyUpdatedFile({ + oldPath, + relativePath: newPath + }) + .catch((err: unknown) => { + this.client.logger.error( + `Failed to sync renamed file ${oldPath} -> ${newPath}: ${this.formatError(err)}` + ); + }); } private toRelativePath(absolutePath: string): RelativePath { - return toUnixPath(path.relative(this.basePath, absolutePath)); + const relative = path.relative(this.basePath, absolutePath); + return this.toUnixPath(relative); + } + + /** + * Convert a native platform path to forward slashes + */ + private toUnixPath(nativePath: string): string { + if (path.sep === "\\") { + return nativePath.replace(/\\/g, "/"); + } + return nativePath; + } + + private formatError(err: unknown): string { + return err instanceof Error ? err.message : String(err); } } diff --git a/frontend/local-client-cli/src/healthcheck.ts b/frontend/local-client-cli/src/healthcheck.ts index d7211c88..2dd9e721 100644 --- a/frontend/local-client-cli/src/healthcheck.ts +++ b/frontend/local-client-cli/src/healthcheck.ts @@ -1,5 +1,4 @@ #!/usr/bin/env node -/* eslint-disable no-console */ /** * Healthcheck script for Docker container diff --git a/frontend/local-client-cli/src/logger-formatter.test.ts b/frontend/local-client-cli/src/logger-formatter.test.ts deleted file mode 100644 index f3078242..00000000 --- a/frontend/local-client-cli/src/logger-formatter.test.ts +++ /dev/null @@ -1,50 +0,0 @@ -import { test } from "node:test"; -import * as assert from "node:assert/strict"; -import { formatLogLine } from "./logger-formatter"; -import { LogLevel } from "sync-client"; - -test("formatLogLine - includes level and message", () => { - const logLine = { - timestamp: new Date("2024-01-15T10:30:45.123Z"), - level: LogLevel.INFO, - message: "Test message" - }; - - const result = formatLogLine(logLine); - assert.ok(result.includes("INFO")); - assert.ok(result.includes("Test message")); -}); - -test("formatLogLine - ERROR level messages contain bold escape", () => { - const logLine = { - timestamp: new Date("2024-01-15T10:30:45.123Z"), - level: LogLevel.ERROR, - message: "Error occurred" - }; - - const result = formatLogLine(logLine); - assert.ok(result.includes("\x1b[1m")); -}); - -test("formatLogLine - highlights file paths in quotes", () => { - const logLine = { - timestamp: new Date("2024-01-15T10:30:45.123Z"), - level: LogLevel.INFO, - message: 'Syncing "notes/test.md"' - }; - - const result = formatLogLine(logLine); - assert.ok(result.includes("\x1b[35m")); -}); - -test("formatLogLine - highlights standalone numbers but not numbers in versions", () => { - const logLine = { - timestamp: new Date("2024-01-15T10:30:45.123Z"), - level: LogLevel.INFO, - message: "Listed 42 files from v1.2.3" - }; - - const result = formatLogLine(logLine); - assert.ok(result.includes("\x1b[36m42\x1b[0m")); - assert.ok(!result.includes("\x1b[36m1\x1b[0m.")); -}); diff --git a/frontend/local-client-cli/src/logger-formatter.ts b/frontend/local-client-cli/src/logger-formatter.ts index b98415b6..9f237103 100644 --- a/frontend/local-client-cli/src/logger-formatter.ts +++ b/frontend/local-client-cli/src/logger-formatter.ts @@ -1,21 +1,36 @@ import { LogLevel, type LogLine } from "sync-client"; -const colors = { +// ANSI color codes +export const colors = { reset: "\x1b[0m", bold: "\x1b[1m", + dim: "\x1b[2m", + // Foreground colors red: "\x1b[31m", green: "\x1b[32m", yellow: "\x1b[33m", + blue: "\x1b[34m", magenta: "\x1b[35m", cyan: "\x1b[36m", gray: "\x1b[90m" } as const; -function colorize(text: string, color: keyof typeof colors): string { +export function colorize(text: string, color: keyof typeof colors): string { return `${colors[color]}${text}${colors.reset}`; } +/** + * Helper function to apply multiple color modifiers to text + */ +export function styleText( + text: string, + ...modifiers: (keyof typeof colors)[] +): string { + const prefix = modifiers.map((m) => colors[m]).join(""); + return `${prefix}${text}${colors.reset}`; +} + function formatTimestamp(date: Date): string { const [time] = date.toTimeString().split(" "); const ms = date.getMilliseconds().toString().padStart(3, "0"); diff --git a/frontend/local-client-cli/src/node-filesystem.ts b/frontend/local-client-cli/src/node-filesystem.ts index ba95ab6a..3da8fc3a 100644 --- a/frontend/local-client-cli/src/node-filesystem.ts +++ b/frontend/local-client-cli/src/node-filesystem.ts @@ -1,32 +1,31 @@ import * as fs from "fs/promises"; import type { Dirent } from "fs"; import * as path from "path"; -import { randomUUID } from "crypto"; import type { FileSystemOperations, RelativePath, TextWithCursors } from "sync-client"; -import { toUnixPath } from "./path-utils"; - -// VaultLink's per-vault metadata directory. Holds the persisted sync database -// and the tmp files atomicWrite renames into place; the matching `${VAULTLINK_DIR}/**` -// ignore pattern keeps everything in here invisible to the file watcher. -export const VAULTLINK_DIR = ".vaultlink"; export class NodeFileSystemOperations implements FileSystemOperations { - public constructor(private readonly basePath: string) { } + public constructor(private readonly basePath: string) {} public async listFilesRecursively( directory: RelativePath | undefined ): Promise { const files: RelativePath[] = []; - await this.walkDirectory(directory ?? "", files); + await this.walkDirectory( + directory !== undefined ? this.toNativePath(directory) : "", + files + ); return files; } public async read(relativePath: RelativePath): Promise { - const fullPath = path.join(this.basePath, relativePath); + const fullPath = path.join( + this.basePath, + this.toNativePath(relativePath) + ); try { return await fs.readFile(fullPath); } catch (error) { @@ -40,12 +39,15 @@ export class NodeFileSystemOperations implements FileSystemOperations { relativePath: RelativePath, content: Uint8Array ): Promise { - const fullPath = path.join(this.basePath, relativePath); + const fullPath = path.join( + this.basePath, + this.toNativePath(relativePath) + ); const dir = path.dirname(fullPath); try { await fs.mkdir(dir, { recursive: true }); - await this.atomicWrite(fullPath, content); + await fs.writeFile(fullPath, content); } catch (error) { throw new Error( `Failed to write file ${fullPath}: ${error instanceof Error ? error.message : String(error)}` @@ -57,12 +59,15 @@ export class NodeFileSystemOperations implements FileSystemOperations { relativePath: RelativePath, updater: (current: TextWithCursors) => TextWithCursors ): Promise { - const fullPath = path.join(this.basePath, relativePath); + const fullPath = path.join( + this.basePath, + this.toNativePath(relativePath) + ); try { const currentContent = await fs.readFile(fullPath, "utf-8"); const result = updater({ text: currentContent, cursors: [] }); - await this.atomicWrite(fullPath, result.text, "utf-8"); + await fs.writeFile(fullPath, result.text, "utf-8"); return result.text; } catch (error) { throw new Error( @@ -72,7 +77,10 @@ export class NodeFileSystemOperations implements FileSystemOperations { } public async getFileSize(relativePath: RelativePath): Promise { - const fullPath = path.join(this.basePath, relativePath); + const fullPath = path.join( + this.basePath, + this.toNativePath(relativePath) + ); try { const stats = await fs.stat(fullPath); return stats.size; @@ -84,7 +92,10 @@ export class NodeFileSystemOperations implements FileSystemOperations { } public async exists(relativePath: RelativePath): Promise { - const fullPath = path.join(this.basePath, relativePath); + const fullPath = path.join( + this.basePath, + this.toNativePath(relativePath) + ); try { await fs.access(fullPath); return true; @@ -94,7 +105,10 @@ export class NodeFileSystemOperations implements FileSystemOperations { } public async createDirectory(relativePath: RelativePath): Promise { - const fullPath = path.join(this.basePath, relativePath); + const fullPath = path.join( + this.basePath, + this.toNativePath(relativePath) + ); try { await fs.mkdir(fullPath, { recursive: false }); } catch (error) { @@ -105,7 +119,10 @@ export class NodeFileSystemOperations implements FileSystemOperations { } public async delete(relativePath: RelativePath): Promise { - const fullPath = path.join(this.basePath, relativePath); + const fullPath = path.join( + this.basePath, + this.toNativePath(relativePath) + ); try { await fs.unlink(fullPath); } catch (error) { @@ -119,8 +136,14 @@ export class NodeFileSystemOperations implements FileSystemOperations { oldPath: RelativePath, newPath: RelativePath ): Promise { - const oldFullPath = path.join(this.basePath, oldPath); - const newFullPath = path.join(this.basePath, newPath); + const oldFullPath = path.join( + this.basePath, + this.toNativePath(oldPath) + ); + const newFullPath = path.join( + this.basePath, + this.toNativePath(newPath) + ); const newDir = path.dirname(newFullPath); try { @@ -133,44 +156,6 @@ export class NodeFileSystemOperations implements FileSystemOperations { } } - private async atomicWrite( - fullPath: string, - content: Uint8Array | string, - encoding?: BufferEncoding - ): Promise { - const tmpDir = path.join(this.basePath, VAULTLINK_DIR); - await fs.mkdir(tmpDir, { recursive: true }); - const tmpPath = path.join(tmpDir, `atomic-write-${randomUUID()}.tmp`); - try { - await fs.writeFile(tmpPath, content, encoding); - const fd = await fs.open(tmpPath, "r"); - try { - await fd.datasync(); - } finally { - await fd.close(); - } - await fs.rename(tmpPath, fullPath); - await this.syncDirectory(path.dirname(fullPath)); - } catch (error) { - await fs.unlink(tmpPath).catch(() => undefined); - throw error; - } - } - - // Make the rename durable by fsync'ing the destination's parent directory. - // Skipped on Windows: fsync on a directory handle isn't supported there - private async syncDirectory(dir: string): Promise { - if (process.platform === "win32") { - return; - } - const fd = await fs.open(dir, "r"); - try { - await fd.sync(); - } finally { - await fd.close(); - } - } - private async walkDirectory( relativePath: string, files: RelativePath[] @@ -194,8 +179,28 @@ export class NodeFileSystemOperations implements FileSystemOperations { await this.walkDirectory(entryRelativePath, files); } else if (entry.isFile()) { // Always return forward slashes - files.push(toUnixPath(entryRelativePath)); + files.push(this.toUnixPath(entryRelativePath)); } } } + + /** + * Convert a forward-slash path to native platform path separators + */ + private toNativePath(relativePath: string): string { + if (path.sep === "\\") { + return relativePath.replace(/\//g, "\\"); + } + return relativePath; + } + + /** + * Convert a native platform path to forward slashes + */ + private toUnixPath(nativePath: string): string { + if (path.sep === "\\") { + return nativePath.replace(/\\/g, "/"); + } + return nativePath; + } } diff --git a/frontend/local-client-cli/src/path-utils.test.ts b/frontend/local-client-cli/src/path-utils.test.ts deleted file mode 100644 index 13d33e6e..00000000 --- a/frontend/local-client-cli/src/path-utils.test.ts +++ /dev/null @@ -1,60 +0,0 @@ -import { test } from "node:test"; -import * as assert from "node:assert/strict"; -import { matchesGlob, toUnixPath } from "./path-utils"; - -test("matchesGlob - exact match", () => { - assert.equal(matchesGlob(".DS_Store", ".DS_Store"), true); - assert.equal(matchesGlob("other", ".DS_Store"), false); -}); - -test("matchesGlob - dir/** matches directory and contents", () => { - assert.equal(matchesGlob(".git", ".git/**"), true); - assert.equal(matchesGlob(".git/config", ".git/**"), true); - assert.equal(matchesGlob(".git/refs/heads/main", ".git/**"), true); - assert.equal(matchesGlob(".gitignore", ".git/**"), false); -}); - -test("matchesGlob - * matches within a single segment", () => { - assert.equal(matchesGlob("foo.tmp", "*.tmp"), true); - assert.equal(matchesGlob("bar.tmp", "*.tmp"), true); - assert.equal(matchesGlob("foo.md", "*.tmp"), false); - assert.equal(matchesGlob("dir/foo.tmp", "*.tmp"), false); -}); - -test("matchesGlob - **/*.ext matches at any depth", () => { - assert.equal(matchesGlob("foo.tmp", "**/*.tmp"), true); - assert.equal(matchesGlob("dir/foo.tmp", "**/*.tmp"), true); - assert.equal(matchesGlob("a/b/c/foo.tmp", "**/*.tmp"), true); - assert.equal(matchesGlob("foo.md", "**/*.tmp"), false); -}); - -test("matchesGlob - ? matches single character", () => { - assert.equal(matchesGlob("a.md", "?.md"), true); - assert.equal(matchesGlob("ab.md", "?.md"), false); - assert.equal(matchesGlob(".md", "?.md"), false); -}); - -test("matchesGlob - dots are literal", () => { - assert.equal(matchesGlob(".DS_Store", ".DS_Store"), true); - assert.equal(matchesGlob("xDS_Store", ".DS_Store"), false); -}); - -test("matchesGlob - node_modules/** matches directory tree", () => { - assert.equal(matchesGlob("node_modules", "node_modules/**"), true); - assert.equal(matchesGlob("node_modules/foo", "node_modules/**"), true); - assert.equal( - matchesGlob("node_modules/foo/bar/baz.js", "node_modules/**"), - true - ); - assert.equal(matchesGlob("not_node_modules", "node_modules/**"), false); -}); - -test("matchesGlob - **/ prefix matches zero or more segments", () => { - assert.equal(matchesGlob("test.log", "**/test.log"), true); - assert.equal(matchesGlob("dir/test.log", "**/test.log"), true); - assert.equal(matchesGlob("a/b/test.log", "**/test.log"), true); -}); - -test("toUnixPath - forward slashes unchanged", () => { - assert.equal(toUnixPath("foo/bar/baz"), "foo/bar/baz"); -}); diff --git a/frontend/local-client-cli/src/path-utils.ts b/frontend/local-client-cli/src/path-utils.ts deleted file mode 100644 index 1ead144c..00000000 --- a/frontend/local-client-cli/src/path-utils.ts +++ /dev/null @@ -1,21 +0,0 @@ -import * as path from "path"; - -// Convert a native platform path to forward slashes (no-op on non-Windows) -export function toUnixPath(nativePath: string): string { - return nativePath.split(path.sep).join(path.posix.sep); -} - -// Match a file path against a glob pattern. -// -// Behaves like Node's path.matchesGlob with one extension: `dir/**` matches -// the directory `dir` itself, not only its descendants. The watcher feeds us -// a directory's relative path (e.g. ".git") at the same time it's about to -// recurse into it, and the natural way for users to write the ignore pattern -// is `.git/**` — under stdlib semantics that pattern would let the directory -// through and only block its children, defeating the prune. -export function matchesGlob(filePath: string, pattern: string): boolean { - if (pattern.endsWith("/**") && filePath === pattern.slice(0, -3)) { - return true; - } - return path.matchesGlob(filePath, pattern); -} diff --git a/frontend/local-client-cli/tsconfig.json b/frontend/local-client-cli/tsconfig.json index b07ec41a..25f249c9 100644 --- a/frontend/local-client-cli/tsconfig.json +++ b/frontend/local-client-cli/tsconfig.json @@ -18,5 +18,7 @@ "declarationMap": true, "sourceMap": true }, - "exclude": ["dist"] + "exclude": [ + "dist" + ] } diff --git a/frontend/local-client-cli/webpack.config.js b/frontend/local-client-cli/webpack.config.js index 9226b9dc..f8f48534 100644 --- a/frontend/local-client-cli/webpack.config.js +++ b/frontend/local-client-cli/webpack.config.js @@ -2,32 +2,32 @@ const path = require("path"); const webpack = require("webpack"); module.exports = { - entry: { - cli: "./src/cli.ts", - healthcheck: "./src/healthcheck.ts" - }, - target: "node", - mode: "production", - optimization: { - minimize: false - }, - module: { - rules: [ - { - test: /\.ts$/, - use: "ts-loader" - } - ] - }, - resolve: { - extensions: [".ts", ".js"] - }, - output: { - globalObject: "this", - filename: "[name].js", - path: path.resolve(__dirname, "dist") - }, - plugins: [ + entry: { + cli: "./src/cli.ts", + healthcheck: "./src/healthcheck.ts" + }, + target: "node", + mode: "production", + optimization: { + minimize: false + }, + module: { + rules: [ + { + test: /\.ts$/, + use: "ts-loader" + } + ] + }, + resolve: { + extensions: [".ts", ".js"] + }, + output: { + globalObject: "this", + filename: "[name].js", + path: path.resolve(__dirname, "dist") + }, + plugins: [ new webpack.BannerPlugin({ banner: "#!/usr/bin/env node", raw: true }) - ] + ] }; diff --git a/frontend/obsidian-plugin/README.md b/frontend/obsidian-plugin/README.md index 68e10a83..93c2cba7 100644 --- a/frontend/obsidian-plugin/README.md +++ b/frontend/obsidian-plugin/README.md @@ -8,7 +8,6 @@ The repo depends on the latest plugin API (obsidian.d.ts) in TypeScript Definiti **Note:** The Obsidian API is still in early alpha and is subject to change at any time! This sample plugin demonstrates some of the basic functionality the plugin API can do. - - Adds a ribbon icon, which shows a Notice when clicked. - Adds a command "Open Sample Modal" which opens a Modal. - Adds a plugin setting tab to the settings page. @@ -58,6 +57,31 @@ Quick starting guide for new plugin devs: - Copy over `main.js`, `styles.css`, `manifest.json` to your vault `VaultFolder/.obsidian/plugins/your-plugin-id/`. + +## Funding URL + +You can include funding URLs where people who use your plugin can financially support it. + +The simple way is to set the `fundingUrl` field to your link in your `manifest.json` file: + +```json +{ + "fundingUrl": "https://buymeacoffee.com" +} +``` + +If you have multiple URLs, you can also do: + +```json +{ + "fundingUrl": { + "Buy Me a Coffee": "https://buymeacoffee.com", + "GitHub Sponsor": "https://github.com/sponsors", + "Patreon": "https://www.patreon.com/" + } +} +``` + ## API Documentation See https://github.com/obsidianmd/obsidian-api diff --git a/frontend/obsidian-plugin/package.json b/frontend/obsidian-plugin/package.json index d24e537b..b7ae4909 100644 --- a/frontend/obsidian-plugin/package.json +++ b/frontend/obsidian-plugin/package.json @@ -13,25 +13,25 @@ "author": "", "license": "MIT", "devDependencies": { - "@types/node": "^25.0.2", + "@types/node": "^24.8.1", "css-loader": "^7.1.2", "date-fns": "^4.1.0", "file-loader": "^6.2.0", - "fs-extra": "^11.3.2", - "mini-css-extract-plugin": "^2.9.4", - "obsidian": "1.11.0", - "reconcile-text": "^0.11.0", + "fs-extra": "^11.3.0", + "mini-css-extract-plugin": "^2.9.2", + "obsidian": "1.10.2", + "reconcile-text": "^0.8.0", "resolve-url-loader": "^5.0.0", - "sass": "^1.96.0", + "sass": "^1.91.0", "sass-loader": "^16.0.6", "sync-client": "file:../sync-client", - "terser-webpack-plugin": "^5.3.16", - "ts-loader": "^9.5.4", + "terser-webpack-plugin": "^5.3.14", + "ts-loader": "^9.5.2", "tslib": "2.8.1", - "tsx": "^4.21.0", - "typescript": "5.9.3", + "tsx": "^4.20.6", + "typescript": "5.8.3", "url": "^0.11.4", - "webpack": "^5.103.0", + "webpack": "^5.99.9", "webpack-cli": "^6.0.1" } } diff --git a/frontend/obsidian-plugin/src/vault-link-plugin.ts b/frontend/obsidian-plugin/src/vault-link-plugin.ts index e222796b..7d91b9f5 100644 --- a/frontend/obsidian-plugin/src/vault-link-plugin.ts +++ b/frontend/obsidian-plugin/src/vault-link-plugin.ts @@ -135,14 +135,14 @@ export default class VaultLinkPlugin extends Plugin { nativeLineEndings: Platform.isWin ? "\r\n" : "\n", ...(IS_DEBUG_BUILD ? { - fetch: debugging.slowFetchFactory(1), - webSocket: debugging.slowWebSocketFactory(1, new Logger()) - } + fetch: debugging.slowFetchFactory(1), + webSocket: debugging.slowWebSocketFactory(1, new Logger()) + } : {}) }); if (IS_DEBUG_BUILD) { - debugging.logToConsole(client.logger); + debugging.logToConsole(client); } return client; @@ -231,9 +231,9 @@ export default class VaultLinkPlugin extends Plugin { } } ), - this.app.vault.on("create", (file: TAbstractFile) => { + this.app.vault.on("create", async (file: TAbstractFile) => { if (file instanceof TFile) { - client.syncLocallyCreatedFile(file.path); + await client.syncLocallyCreatedFile(file.path); } }), this.app.vault.on("modify", async (file: TAbstractFile) => { @@ -241,14 +241,14 @@ export default class VaultLinkPlugin extends Plugin { await this.rateLimitedUpdate(file.path, client); } }), - this.app.vault.on("delete", (file: TAbstractFile) => { - client.syncLocallyDeletedFile(file.path); + this.app.vault.on("delete", async (file: TAbstractFile) => { + await client.syncLocallyDeletedFile(file.path); }), this.app.vault.on( "rename", - (file: TAbstractFile, oldPath: string) => { + async (file: TAbstractFile, oldPath: string) => { if (file instanceof TFile) { - client.syncLocallyUpdatedFile({ + await client.syncLocallyUpdatedFile({ oldPath, relativePath: file.path }); @@ -267,11 +267,13 @@ export default class VaultLinkPlugin extends Plugin { if (!this.rateLimitedUpdatesPerFile.has(path)) { this.rateLimitedUpdatesPerFile.set( path, - rateLimit(async () => { - client.syncLocallyUpdatedFile({ - relativePath: path - }); - }, MIN_WAIT_BETWEEN_UPDATES_IN_MS) + rateLimit( + async () => + client.syncLocallyUpdatedFile({ + relativePath: path + }), + MIN_WAIT_BETWEEN_UPDATES_IN_MS + ) ); } await this.rateLimitedUpdatesPerFile.get(path)?.(); diff --git a/frontend/obsidian-plugin/src/views/cursors/file-explorer.ts b/frontend/obsidian-plugin/src/views/cursors/file-explorer.ts index 409402a4..3088c640 100644 --- a/frontend/obsidian-plugin/src/views/cursors/file-explorer.ts +++ b/frontend/obsidian-plugin/src/views/cursors/file-explorer.ts @@ -14,9 +14,7 @@ export function renderCursorsInFileExplorer( app: App ): void { const fileExplorers = app.workspace.getLeavesOfType("file-explorer"); - if (fileExplorers.length == 0) { - return; - } + if (fileExplorers.length == 0) return; const [fileExplorer] = fileExplorers; @@ -36,7 +34,7 @@ export function renderCursorsInFileExplorer( (parent) => { cursors.forEach((cursor) => { cursor.documentsWithCursors.forEach((document) => { - if (document.relativePath.startsWith(key)) { + if (document.relative_path.startsWith(key)) { parent.appendChild( createSpan({ text: cursor.userName, diff --git a/frontend/obsidian-plugin/src/views/cursors/remote-cursors-plugin.ts b/frontend/obsidian-plugin/src/views/cursors/remote-cursors-plugin.ts index 4200a72a..1191d9a2 100644 --- a/frontend/obsidian-plugin/src/views/cursors/remote-cursors-plugin.ts +++ b/frontend/obsidian-plugin/src/views/cursors/remote-cursors-plugin.ts @@ -61,7 +61,7 @@ export class RemoteCursorsPluginValue implements PluginValue { return clientCursors.flatMap((cursor) => cursor.cursors.map((span) => ({ name: client.userName, - path: cursor.relativePath, + path: cursor.relative_path, deviceId: client.deviceId, isOutdated: client.isOutdated, span: { ...span } @@ -132,8 +132,7 @@ export class RemoteCursorsPluginValue implements PluginValue { ] ) }, - edited, - "Markdown" + edited ); reconciled.cursors.forEach(({ id, position }) => { diff --git a/frontend/obsidian-plugin/src/views/settings/settings-tab.ts b/frontend/obsidian-plugin/src/views/settings/settings-tab.ts index 5a5823c2..213c0d2c 100644 --- a/frontend/obsidian-plugin/src/views/settings/settings-tab.ts +++ b/frontend/obsidian-plugin/src/views/settings/settings-tab.ts @@ -266,8 +266,9 @@ export class SyncSettingsTab extends PluginSettingTab { new Notice("Checking connection to the server..."); new Notice( - (await this.syncClient.checkConnection()) - .serverMessage + ( + await this.syncClient.checkConnection() + ).serverMessage ); await this.statusDescription.updateConnectionState(); } else { @@ -350,6 +351,22 @@ export class SyncSettingsTab extends PluginSettingTab { }) ); + new Setting(containerEl) + .setName("Sync concurrency") + .setDesc( + "How many concurrent sync operations to run. Setting this value higher may increase the overall performance, however, it will require more memory as well. If you notice frequent crashes, especially on mobile, set this to 1." + ) + .addSlider((text) => + text + .setLimits(1, 16, 1) + .setDynamicTooltip() + .setInstant(false) + .setValue(this.syncClient.getSettings().syncConcurrency) + .onChange(async (value) => + this.syncClient.setSetting("syncConcurrency", value) + ) + ); + new Setting(containerEl) .setName("Maximum file size to be uploaded (MB)") .setDesc( @@ -467,6 +484,40 @@ export class SyncSettingsTab extends PluginSettingTab { ); }) ); + + new Setting(containerEl) + .setName("Minimum save interval (ms)") + .setDesc( + "The minimum time between saving settings and database to disk, in milliseconds. Lower values save more frequently but may impact performance." + ) + .addText((input) => + input + .setValue( + this.syncClient + .getSettings() + .minimumSaveIntervalMs.toString() + ) + .onChange(async (value) => { + if (value === "") { + return; + } + let parsedValue = Number.parseInt(value, 10); + if (Number.isNaN(parsedValue) || parsedValue < 0) { + parsedValue = + this.syncClient.getSettings() + .minimumSaveIntervalMs; + } + + if (value !== parsedValue.toString()) { + input.setValue(parsedValue.toString()); + } + + return this.syncClient.setSetting( + "minimumSaveIntervalMs", + parsedValue + ); + }) + ); } private setStatusDescriptionSubscription( diff --git a/frontend/obsidian-plugin/src/views/status-description/status-description.ts b/frontend/obsidian-plugin/src/views/status-description/status-description.ts index 6d8d74fe..53fea486 100644 --- a/frontend/obsidian-plugin/src/views/status-description/status-description.ts +++ b/frontend/obsidian-plugin/src/views/status-description/status-description.ts @@ -88,7 +88,7 @@ export class StatusDescription { text: ` and has indexed approximately ` }); container.createSpan({ - text: `${this.syncClient.syncedDocumentCount}`, + text: `${this.syncClient.documentCount}`, cls: "number" }); container.createSpan({ diff --git a/frontend/obsidian-plugin/tsconfig.json b/frontend/obsidian-plugin/tsconfig.json index 7ec2a9cd..81af03a7 100644 --- a/frontend/obsidian-plugin/tsconfig.json +++ b/frontend/obsidian-plugin/tsconfig.json @@ -6,7 +6,12 @@ "strict": true, "moduleResolution": "bundler", "allowSyntheticDefaultImports": true, - "lib": ["DOM", "ES2024"] + "lib": [ + "DOM", + "ES2024" + ] }, - "exclude": ["./dist"] + "exclude": [ + "./dist" + ] } diff --git a/frontend/obsidian-plugin/webpack.config.js b/frontend/obsidian-plugin/webpack.config.js index 12844fd7..b749b20d 100644 --- a/frontend/obsidian-plugin/webpack.config.js +++ b/frontend/obsidian-plugin/webpack.config.js @@ -46,7 +46,8 @@ module.exports = (env, argv) => ({ const source = path.resolve(__dirname, "dist"); const destinations = [ "/volumes/syncthing/Desktop/test/test/.obsidian/plugins/vault-link", - "/volumes/syncthing/Desktop/test/test2/.obsidian/plugins/vault-link" + "/volumes/syncthing/Desktop/test/test2/.obsidian/plugins/vault-link", + // "/home/andras/obsidian-test/.obsidian/plugins/vault-link" ]; destinations.forEach((destination) => { fs.copy(source, destination) diff --git a/frontend/package.json b/frontend/package.json index 0dd9057d..df167a5e 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -5,7 +5,6 @@ "sync-client", "obsidian-plugin", "test-client", - "deterministic-tests", "local-client-cli" ], "prettier": { @@ -18,7 +17,7 @@ "build": "npm run build --workspaces", "dev": "concurrently --kill-others \"npm run dev -w sync-client\" \"npm run dev -w obsidian-plugin\"", "test": "npm run test --workspaces", - "lint": "eslint --fix sync-client obsidian-plugin test-client deterministic-tests local-client-cli && prettier --write \"**/*.ts\"", + "lint": "eslint --fix sync-client obsidian-plugin test-client local-client-cli && prettier --write \"**/*.ts\"", "update": "ncu -u -ws" }, "devDependencies": { diff --git a/frontend/sync-client/src/consts.ts b/frontend/sync-client/src/consts.ts index da70ba47..86319fd7 100644 --- a/frontend/sync-client/src/consts.ts +++ b/frontend/sync-client/src/consts.ts @@ -1,6 +1,6 @@ export const TIMEOUT_FOR_MERGING_HISTORY_ENTRIES_IN_SECONDS = 60; -export const DIFF_CACHE_SIZE_MB = 2; export const MAX_LOG_MESSAGE_COUNT = 100000; export const MAX_HISTORY_ENTRY_COUNT = 5000; -export const SUPPORTED_API_VERSION = 2; -export const WEBSOCKET_DISCONNECT_TIMEOUT_IN_S = 10; +export const SUPPORTED_API_VERSION = 3; +export const WEBSOCKET_DISCONNECT_TIMEOUT_IN_SECONDS = 10; +export const WEBSOCKET_CONNECTION_TIMEOUT_IN_SECONDS = 10; diff --git a/frontend/sync-client/src/services/authentication-error.ts b/frontend/sync-client/src/errors/authentication-error.ts similarity index 100% rename from frontend/sync-client/src/services/authentication-error.ts rename to frontend/sync-client/src/errors/authentication-error.ts diff --git a/frontend/sync-client/src/errors/file-already-exists-error.ts b/frontend/sync-client/src/errors/file-already-exists-error.ts new file mode 100644 index 00000000..35f51a66 --- /dev/null +++ b/frontend/sync-client/src/errors/file-already-exists-error.ts @@ -0,0 +1,9 @@ +export class FileAlreadyExistsError extends Error { + public constructor( + message: string, + public readonly filePath: string + ) { + super(message); + this.name = "FileAlreadyExistsError"; + } +} diff --git a/frontend/sync-client/src/file-operations/file-not-found-error.ts b/frontend/sync-client/src/errors/file-not-found-error.ts similarity index 100% rename from frontend/sync-client/src/file-operations/file-not-found-error.ts rename to frontend/sync-client/src/errors/file-not-found-error.ts diff --git a/frontend/sync-client/src/errors/http-client-error.ts b/frontend/sync-client/src/errors/http-client-error.ts new file mode 100644 index 00000000..2475cf35 --- /dev/null +++ b/frontend/sync-client/src/errors/http-client-error.ts @@ -0,0 +1,9 @@ +export class HttpClientError extends Error { + public constructor( + public readonly statusCode: number, + message: string + ) { + super(message); + this.name = "HttpClientError"; + } +} diff --git a/frontend/sync-client/src/services/server-version-mismatch-error.ts b/frontend/sync-client/src/errors/server-version-mismatch-error.ts similarity index 100% rename from frontend/sync-client/src/services/server-version-mismatch-error.ts rename to frontend/sync-client/src/errors/server-version-mismatch-error.ts diff --git a/frontend/sync-client/src/services/sync-reset-error.ts b/frontend/sync-client/src/errors/sync-reset-error.ts similarity index 100% rename from frontend/sync-client/src/services/sync-reset-error.ts rename to frontend/sync-client/src/errors/sync-reset-error.ts diff --git a/frontend/sync-client/src/services/types/CreateDocumentVersion.ts b/frontend/sync-client/src/services/types/CreateDocumentVersion.ts index ed921f18..2d83cd99 100644 --- a/frontend/sync-client/src/services/types/CreateDocumentVersion.ts +++ b/frontend/sync-client/src/services/types/CreateDocumentVersion.ts @@ -1,13 +1,7 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. export interface CreateDocumentVersion { - /** - * The client can decide the document id (if it wishes to) in order - * to help with syncing. If the client does not provide a document id, - * the server will generate one. If the client provides a document id - * it must not already exist in the database. - */ - document_id: string | null; relative_path: string; + last_seen_vault_update_id: number; content: number[]; } diff --git a/frontend/sync-client/src/services/types/DocumentUpdateResponse.ts b/frontend/sync-client/src/services/types/DocumentUpdateResponse.ts index 7fd06c7a..dd7eadda 100644 --- a/frontend/sync-client/src/services/types/DocumentUpdateResponse.ts +++ b/frontend/sync-client/src/services/types/DocumentUpdateResponse.ts @@ -3,7 +3,7 @@ import type { DocumentVersion } from "./DocumentVersion"; import type { DocumentVersionWithoutContent } from "./DocumentVersionWithoutContent"; /** - * Response to an update document request. + * Response to a create/update document request. */ export type DocumentUpdateResponse = | ({ type: "FastForwardUpdate" } & DocumentVersionWithoutContent) diff --git a/frontend/sync-client/src/services/types/DocumentVersionWithoutContent.ts b/frontend/sync-client/src/services/types/DocumentVersionWithoutContent.ts index 4b24e7c5..662b41e5 100644 --- a/frontend/sync-client/src/services/types/DocumentVersionWithoutContent.ts +++ b/frontend/sync-client/src/services/types/DocumentVersionWithoutContent.ts @@ -9,4 +9,8 @@ export interface DocumentVersionWithoutContent { userId: string; deviceId: string; contentSize: number; + /** + * True iff this is the first version of the document + */ + isNewFile: boolean; } diff --git a/frontend/sync-client/src/services/types/DocumentWithCursors.ts b/frontend/sync-client/src/services/types/DocumentWithCursors.ts index dcfe6e2d..8ed59067 100644 --- a/frontend/sync-client/src/services/types/DocumentWithCursors.ts +++ b/frontend/sync-client/src/services/types/DocumentWithCursors.ts @@ -2,8 +2,8 @@ import type { CursorSpan } from "./CursorSpan"; export interface DocumentWithCursors { - vault_update_id: number | null; - document_id: string; - relative_path: string; + vaultUpdateId: number | null; + documentId: string; + relativePath: string; cursors: CursorSpan[]; } diff --git a/frontend/sync-client/src/services/types/FetchLatestDocumentsResponse.ts b/frontend/sync-client/src/services/types/FetchLatestDocumentsResponse.ts index 160c9279..315d701a 100644 --- a/frontend/sync-client/src/services/types/FetchLatestDocumentsResponse.ts +++ b/frontend/sync-client/src/services/types/FetchLatestDocumentsResponse.ts @@ -7,7 +7,7 @@ import type { DocumentVersionWithoutContent } from "./DocumentVersionWithoutCont export interface FetchLatestDocumentsResponse { latestDocuments: DocumentVersionWithoutContent[]; /** - * The update ID of the latest document in the response. - */ + * The update ID of the latest document in the response. + */ lastUpdateId: bigint; } diff --git a/frontend/sync-client/src/services/types/ListVaultsResponse.ts b/frontend/sync-client/src/services/types/ListVaultsResponse.ts new file mode 100644 index 00000000..babad2d5 --- /dev/null +++ b/frontend/sync-client/src/services/types/ListVaultsResponse.ts @@ -0,0 +1,11 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { VaultInfo } from "./VaultInfo"; + +/** + * Response to listing vaults accessible to the authenticated user. + */ +export interface ListVaultsResponse { + vaults: VaultInfo[]; + hasMore: boolean; + userName: string; +} diff --git a/frontend/sync-client/src/services/types/PingResponse.ts b/frontend/sync-client/src/services/types/PingResponse.ts index 6db66354..f96520e9 100644 --- a/frontend/sync-client/src/services/types/PingResponse.ts +++ b/frontend/sync-client/src/services/types/PingResponse.ts @@ -5,21 +5,21 @@ */ export interface PingResponse { /** - * Semantic version of the server. - */ + * Semantic version of the server. + */ serverVersion: string; /** - * Whether the client is authenticated based on the sent Authorization - * header. - */ + * Whether the client is authenticated based on the sent Authorization + * header. + */ isAuthenticated: boolean; /** - * List of file extensions that are allowed to be merged. - */ + * List of file extensions that are allowed to be merged. + */ mergeableFileExtensions: string[]; /** - * API version ensuring backwards & forwards compatibility between the client - * and server. - */ + * API version ensuring backwards & forwards compatibility between the client + * and server. + */ supportedApiVersion: number; } diff --git a/frontend/sync-client/src/services/types/UpdateTextDocumentVersion.ts b/frontend/sync-client/src/services/types/UpdateTextDocumentVersion.ts index 46f36bd0..988e3b2f 100644 --- a/frontend/sync-client/src/services/types/UpdateTextDocumentVersion.ts +++ b/frontend/sync-client/src/services/types/UpdateTextDocumentVersion.ts @@ -2,6 +2,6 @@ export interface UpdateTextDocumentVersion { parentVersionId: number; - relativePath: string; + relativePath: string | null; content: (number | string)[]; } diff --git a/frontend/sync-client/src/services/types/VaultHistoryResponse.ts b/frontend/sync-client/src/services/types/VaultHistoryResponse.ts new file mode 100644 index 00000000..35531010 --- /dev/null +++ b/frontend/sync-client/src/services/types/VaultHistoryResponse.ts @@ -0,0 +1,10 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { DocumentVersionWithoutContent } from "./DocumentVersionWithoutContent"; + +/** + * Response to a vault history request (paginated). + */ +export interface VaultHistoryResponse { + versions: DocumentVersionWithoutContent[]; + hasMore: boolean; +} diff --git a/frontend/sync-client/src/services/types/VaultInfo.ts b/frontend/sync-client/src/services/types/VaultInfo.ts new file mode 100644 index 00000000..20d6811c --- /dev/null +++ b/frontend/sync-client/src/services/types/VaultInfo.ts @@ -0,0 +1,10 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +/** + * Summary of a single vault returned by the list-vaults endpoint. + */ +export interface VaultInfo { + name: string; + documentCount: number; + createdAt: string | null; +} diff --git a/frontend/sync-client/src/services/types/WebSocketVaultUpdate.ts b/frontend/sync-client/src/services/types/WebSocketVaultUpdate.ts index f1ea0f80..b4a942c8 100644 --- a/frontend/sync-client/src/services/types/WebSocketVaultUpdate.ts +++ b/frontend/sync-client/src/services/types/WebSocketVaultUpdate.ts @@ -2,6 +2,5 @@ import type { DocumentVersionWithoutContent } from "./DocumentVersionWithoutContent"; export interface WebSocketVaultUpdate { - documents: DocumentVersionWithoutContent[]; - isInitialSync: boolean; + document: DocumentVersionWithoutContent; } diff --git a/frontend/sync-client/src/utils/await-all.ts b/frontend/sync-client/src/utils/await-all.ts index 9406a6b8..43e06ce6 100644 --- a/frontend/sync-client/src/utils/await-all.ts +++ b/frontend/sync-client/src/utils/await-all.ts @@ -9,7 +9,7 @@ type ResolvedTuple = { export const awaitAll = async ( promises: PromiseTuple ): Promise> => { - // eslint-disable-next-line no-restricted-properties + // eslint-disable-next-line no-restricted-properties, @typescript-eslint/await-thenable const result = await Promise.allSettled(promises); for (const res of result) { if (res.status === "rejected") { diff --git a/frontend/sync-client/src/utils/create-client-id.ts b/frontend/sync-client/src/utils/create-client-id.ts index cfa132da..03dc2ae9 100644 --- a/frontend/sync-client/src/utils/create-client-id.ts +++ b/frontend/sync-client/src/utils/create-client-id.ts @@ -1,5 +1,3 @@ -import { v4 as uuidv4 } from "uuid"; - export function createClientId(): string { // @ts-expect-error, injected by webpack const packageVersion = __CURRENT_VERSION__; // eslint-disable-line @@ -8,8 +6,8 @@ export function createClientId(): string { typeof navigator !== "undefined" ? navigator.platform // eslint-disable-line @typescript-eslint/no-deprecated : typeof process !== "undefined" - ? process.platform - : "unknown"; + ? process.platform + : "unknown"; - return `vault-link/${packageVersion} (${uuidv4()}; ${platform})`; + return `vault-link/${packageVersion} (${Math.round(Math.random() * 1e10)}; ${platform})`; } diff --git a/frontend/sync-client/src/utils/create-promise.ts b/frontend/sync-client/src/utils/create-promise.ts deleted file mode 100644 index a49196ee..00000000 --- a/frontend/sync-client/src/utils/create-promise.ts +++ /dev/null @@ -1,25 +0,0 @@ -type ResolveFunction = undefined extends T - ? (value?: T) => unknown - : (value: T) => unknown; - -/** - * A type-safe utility function to create a Promise with resolve and reject functions. - * @returns A tuple containing a Promise, a resolve function, and a reject function. - */ -export function createPromise(): [ - Promise, - ResolveFunction, - (error: unknown) => unknown -] { - let resolve: undefined | ResolveFunction = undefined; - let reject: undefined | ((error: unknown) => unknown) = undefined; - - const creationPromise = new Promise( - (resolve_, reject_) => - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion - ((resolve = resolve_ as ResolveFunction), (reject = reject_)) - ); - - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - return [creationPromise, resolve!, reject!]; -} diff --git a/frontend/sync-client/src/utils/data-structures/event-listeners.ts b/frontend/sync-client/src/utils/data-structures/event-listeners.ts index e08ca65e..420e4e63 100644 --- a/frontend/sync-client/src/utils/data-structures/event-listeners.ts +++ b/frontend/sync-client/src/utils/data-structures/event-listeners.ts @@ -13,56 +13,64 @@ export class EventListeners any> { } /** - * Adds a new listener to the collection. - * - * @param listener The listener callback to add - * @returns An unsubscribe function that removes this listener when called - */ + * Adds a new listener to the collection. + * + * @param listener The listener callback to add + * @returns An unsubscribe function that removes this listener when called + */ public add(listener: TListener): () => void { this.listeners.push(listener); return () => this.remove(listener); } /** - * Removes a listener from the collection. - * - * @param listener The listener callback to remove - * @returns true if the listener was found and removed, false otherwise - */ + * Removes a listener from the collection. + * + * @param listener The listener callback to remove + * @returns true if the listener was found and removed, false otherwise + */ public remove(listener: TListener): boolean { return removeFromArray(this.listeners, listener); } /** - * Triggers all listeners synchronously with the provided arguments. - * Any returned promises are ignored. Use triggerAsync() to await them. - * - * @param args The arguments to pass to each listener - */ + * Triggers all listeners synchronously with the provided arguments. + * Any returned promises are ignored. Use triggerAsync() to await them. + * + * @param args The arguments to pass to each listener + */ public trigger(...args: Parameters): void { - this.listeners.forEach((listener) => { + const snapshot = this.listeners.slice(); + for (const listener of snapshot) { + // allow removing listeners during the trigger loop + if (!this.listeners.includes(listener)) { + continue; + } listener(...args); - }); + } } /** - * Triggers all listeners and awaits any promises they return. - * Synchronous listeners are called immediately, and any async listeners - * are awaited in parallel. - * - * @param args The arguments to pass to each listener - */ + * Triggers all listeners and awaits any promises they return. + * Synchronous listeners are called immediately, and any async listeners + * are awaited in parallel. + * + * @param args The arguments to pass to each listener + */ public async triggerAsync(...args: Parameters): Promise { - await awaitAll( - this.listeners - .map((listener) => { - // eslint-disable-next-line @typescript-eslint/no-unsafe-return - return listener(...args); - }) - .filter((result): result is Promise => { - return result instanceof Promise; - }) - ); + const snapshot = this.listeners.slice(); + const promises: Promise[] = []; + for (const listener of snapshot) { + if (!this.listeners.includes(listener)) { + continue; + } + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + const result = listener(...args); + if (result instanceof Promise) { + promises.push(result); + } + } + await awaitAll(promises); } public clear(): void { diff --git a/frontend/sync-client/src/utils/data-structures/fix-sized-cache.ts b/frontend/sync-client/src/utils/data-structures/fix-sized-cache.ts index 51ad41c1..44a71dc8 100644 --- a/frontend/sync-client/src/utils/data-structures/fix-sized-cache.ts +++ b/frontend/sync-client/src/utils/data-structures/fix-sized-cache.ts @@ -1,6 +1,6 @@ // Implements an in-memory fixed-size cache for document contents, -import type { VaultUpdateId } from "../../persistence/database"; +import type { VaultUpdateId } from "../../sync-operations/types"; // Doubly-linked list node for O(1) LRU operations class LRUNode { diff --git a/frontend/sync-client/src/utils/data-structures/locks.test.ts b/frontend/sync-client/src/utils/data-structures/locks.test.ts index 9beb867a..c98bda0b 100644 --- a/frontend/sync-client/src/utils/data-structures/locks.test.ts +++ b/frontend/sync-client/src/utils/data-structures/locks.test.ts @@ -1,22 +1,24 @@ import { describe, it, beforeEach } from "node:test"; import assert from "node:assert"; import { Logger } from "../../tracing/logger"; -import type { RelativePath } from "../../persistence/database"; +import type { RelativePath } from "../../sync-operations/types"; import { Locks } from "./locks"; import { awaitAll } from "../await-all"; import { sleep } from "../sleep"; -import { SyncResetError } from "../../services/sync-reset-error"; +import { SyncResetError } from "../../errors/sync-reset-error"; describe("withLock", () => { const testPath: RelativePath = "test/document/path"; const testPath2: RelativePath = "test/document/path2"; + const testPath3: RelativePath = "test/document/path3"; + const logger = new Logger(); // eslint-disable-next-line @typescript-eslint/init-declarations let locks: Locks; beforeEach(() => { - locks = new Locks(logger); + locks = new Locks("locks-test", logger); }); it("should execute function with single key lock", async () => { @@ -56,22 +58,32 @@ describe("withLock", () => { it("should sort multiple keys to prevent deadlocks", async () => { const executionOrder: string[] = []; - // Start two concurrent operations with keys in different orders - const promise1 = locks.withLock([testPath2, testPath], async () => { - executionOrder.push("operation1-start"); - await sleep(50); - executionOrder.push("operation1-end"); - return "result1"; - }); + await locks.waitForLock(testPath); - const promise2 = locks.withLock([testPath, testPath2], async () => { - executionOrder.push("operation2-start"); - await sleep(50); - executionOrder.push("operation2-end"); - return "result2"; - }); + const promise = awaitAll([ + locks.withLock([testPath2, testPath3, testPath], async () => { + executionOrder.push("operation1-start"); + executionOrder.push("operation1-end"); + return "result1"; + }), - const [result1, result2] = await awaitAll([promise1, promise2]); + locks.withLock([testPath3, testPath, testPath2], async () => { + executionOrder.push("operation2-start"); + executionOrder.push("operation2-end"); + return "result2"; + }) + ]); + + locks.unlock(testPath); + + const [result1, result2] = await Promise.race([ + promise, + new Promise((_, reject) => { + setTimeout(() => { + reject(new Error("Deadlock detected")); + }, 1000); + }) + ]); assert.strictEqual(result1, "result1"); assert.strictEqual(result2, "result2"); @@ -234,13 +246,14 @@ describe("withLock", () => { describe("reset", () => { const testPath: RelativePath = "test/document/path"; + const testPath2: RelativePath = "test/document/path2"; const logger = new Logger(); // eslint-disable-next-line @typescript-eslint/init-declarations let locks: Locks; beforeEach(() => { - locks = new Locks(logger); + locks = new Locks("locks-test", logger); }); it("should reject pending waiters with SyncResetError while running operation completes", async () => { @@ -289,4 +302,38 @@ describe("reset", () => { const result = await locks.withLock(testPath, () => "success"); assert.strictEqual(result, "success"); }); + + it("should release partially acquired locks when reset interrupts multi-key acquisition", async () => { + // Hold testPath2 so multi-key acquisition will block on it + await locks.waitForLock(testPath2); + + // Start multi-key lock that will acquire testPath first, then block on testPath2 + const multiKeyPromise = locks.withLock( + [testPath, testPath2], + async () => "multi" + ); + void multiKeyPromise.catch(() => {}); // eslint-disable-line @typescript-eslint/no-empty-function + + // Wait for the multi-key operation to acquire testPath and start waiting on testPath2 + await sleep(10); + + // Reset should reject the waiting operation + locks.reset(); + + await assert.rejects(multiKeyPromise, (err: Error) => { + assert.ok(err instanceof SyncResetError); + return true; + }); + + // The key that was already acquired (testPath) should now be released + // This would hang/timeout if the lock was leaked + const result = await Promise.race([ + locks.withLock(testPath, () => "success"), + sleep(100).then(() => { + throw new Error("Lock was not released - deadlock detected"); + }) + ]); + + assert.strictEqual(result, "success"); + }); }); diff --git a/frontend/sync-client/src/utils/data-structures/locks.ts b/frontend/sync-client/src/utils/data-structures/locks.ts index e55c76b0..99c33075 100644 --- a/frontend/sync-client/src/utils/data-structures/locks.ts +++ b/frontend/sync-client/src/utils/data-structures/locks.ts @@ -1,6 +1,5 @@ -import { SyncResetError } from "../../services/sync-reset-error"; +import { SyncResetError } from "../../errors/sync-reset-error"; import type { Logger } from "../../tracing/logger"; -import { awaitAll } from "../await-all"; /** * Manages exclusive locks on items to prevent concurrent modifications. @@ -8,47 +7,53 @@ import { awaitAll } from "../await-all"; * * @template T The type of the key used for locking */ +/** Waiter entry with callbacks */ +interface WaiterEntry { + resolve: () => unknown; + reject: (err: unknown) => unknown; +} + export class Locks { /** Currently locked keys */ private readonly locked = new Set(); - /** Queue of resolve functions waiting for each key */ - private readonly waiters = new Map< - T, - [() => unknown, (err: unknown) => unknown][] - >(); + /** Queue of waiters for each key */ + private readonly waiters = new Map(); - public constructor(private readonly logger?: Logger) {} + public constructor( + private readonly name: string, + private readonly logger?: Logger + ) {} /** - * Executes a function while holding exclusive locks on one or more keys. - * - * This method ensures that the provided function runs with exclusive access to the - * specified key(s). Multiple keys are sorted to prevent deadlocks when different - * operations request the same keys in different orders. - * - * @template R The return type of the function to execute - * @param keyOrKeys A single key or array of keys to lock during function execution - * @param fn The function to execute while holding the lock(s). Can be sync or async. - * @returns A Promise that resolves to the return value of the executed function - * - * @example - * ```typescript - * // Lock a single key - * const result = await locks.withLock('file1', () => { - * // Critical section - only one operation can access 'file1' at a time - * return processFile('file1'); - * }); - * - * // Lock multiple keys (prevents deadlocks through consistent ordering) - * await locks.withLock(['file1', 'file2'], async () => { - * // Critical section - exclusive access to both files - * await moveFile('file1', 'file2'); - * }); - * ``` - * - * @throws Any error thrown by the provided function will be propagated after locks are released - */ + * Executes a function while holding exclusive locks on one or more keys. + * + * This method ensures that the provided function runs with exclusive access to the + * specified key(s). Multiple keys are sorted to prevent deadlocks when different + * operations request the same keys in different orders. + * + * @template R The return type of the function to execute + * @param keyOrKeys A single key or array of keys to lock during function execution + * @param fn The function to execute while holding the lock(s). Can be sync or async. + * @returns A Promise that resolves to the return value of the executed function + * + * @example + * ```typescript + * // Lock a single key + * const result = await locks.withLock('file1', () => { + * // Critical section - only one operation can access 'file1' at a time + * return processFile('file1'); + * }); + * + * // Lock multiple keys (prevents deadlocks through consistent ordering) + * await locks.withLock(['file1', 'file2'], async () => { + * // Critical section - exclusive access to both files + * await moveFile('file1', 'file2'); + * }); + * ``` + * + * @throws Any error thrown by the provided function will be propagated after locks are released + */ public async withLock( keyOrKeys: T | T[], fn: () => R | Promise @@ -59,12 +64,17 @@ export class Locks { const uniqueKeys = Array.from(new Set(keys)); uniqueKeys.sort((a, b) => String(a).localeCompare(String(b))); // Ensure consistent order to prevent deadlocks - await awaitAll(uniqueKeys.map(async (key) => this.waitForLock(key))); - + const lockedKeys = []; try { + for (const key of uniqueKeys) { + // Must acquire locks in-order (not concurrently) to prevent deadlocks + await this.waitForLock(key); + lockedKeys.push(key); + } + return await fn(); } finally { - uniqueKeys.forEach((key) => { + lockedKeys.forEach((key) => { this.unlock(key); }); } @@ -74,7 +84,7 @@ export class Locks { // Resolve all waiting promises before clearing to prevent deadlock // Any operation waiting for a lock will be granted access immediately for (const waiting of this.waiters.values()) { - for (const [_, reject] of waiting) { + for (const { reject } of waiting) { reject(new SyncResetError()); } } @@ -82,13 +92,17 @@ export class Locks { this.waiters.clear(); } + public isLocked(key: T): boolean { + return this.locked.has(key); + } + /** - * Attempts to acquire a lock immediately without waiting. - * Must call `unlock()` if successful. - * - * @param key The key to lock - * @returns `true` if lock acquired, `false` if already locked - */ + * Attempts to acquire a lock immediately without waiting. + * Must call `unlock()` if successful. + * + * @param key The key to lock + * @returns `true` if lock acquired, `false` if already locked + */ public tryLock(key: T): boolean { if (this.locked.has(key)) { return false; @@ -100,18 +114,18 @@ export class Locks { } /** - * Waits to acquire a lock, blocking until available. - * Operations are queued in FIFO order. Must call `unlock()` when done. - * - * @param key The key to wait for and lock - * @returns Promise that resolves when lock is acquired - */ + * Waits to acquire a lock, blocking until available. + * Operations are queued in FIFO order. Must call `unlock()` when done. + * + * @param key The key to wait for and lock + * @returns Promise that resolves when lock is acquired + */ public async waitForLock(key: T): Promise { if (this.tryLock(key)) { return Promise.resolve(); } - this.logger?.debug(`Waiting for lock on ${key}`); + this.logger?.debug(`Waiting for lock '${this.name}' on '${key}'`); return new Promise((resolve, reject) => { // DefaultDict behavior @@ -121,28 +135,36 @@ export class Locks { this.waiters.set(key, waiting); } - waiting.push([resolve, reject]); + waiting.push({ + resolve, + reject + }); }); } /** - * Releases a lock and grants access to the next waiting operation in FIFO order. - * Removes the key from locked set if no waiters. - * - * @param key The key to unlock - * @throws {Error} If key is not currently locked - */ + * Releases a lock and grants access to the next waiting operation in FIFO order. + * Removes the key from locked set if no waiters. + * + * @param key The key to unlock + * @throws {Error} If key is not currently locked + */ public unlock(key: T): void { if (!this.locked.has(key)) { + this.logger?.debug( + `Attempted to unlock '${this.name}' on '${key}' which is not locked` + ); return; } - // Remove first waiter to ensure FIFO order - const [resolveNextWaiting, _] = this.waiters.get(key)?.shift() ?? []; + this.logger?.debug(`Releasing lock '${this.name}' on '${key}'`); - if (resolveNextWaiting) { - this.logger?.debug(`Granted lock on ${key}`); - resolveNextWaiting(); + // Remove first waiter to ensure FIFO order + const nextWaiter = this.waiters.get(key)?.shift(); + + if (nextWaiter) { + this.logger?.debug(`Granted lock '${this.name}' on '${key}'`); + nextWaiter.resolve(); } else { this.locked.delete(key); } @@ -152,8 +174,8 @@ export class Locks { export class Lock { private readonly locks: Locks; - public constructor(logger?: Logger) { - this.locks = new Locks(logger); + public constructor(name: string, logger?: Logger) { + this.locks = new Locks(name, logger); } public async withLock(fn: () => R | Promise): Promise { diff --git a/frontend/sync-client/src/utils/data-structures/min-covered.test.ts b/frontend/sync-client/src/utils/data-structures/min-covered.test.ts index 7b7271d7..752227c0 100644 --- a/frontend/sync-client/src/utils/data-structures/min-covered.test.ts +++ b/frontend/sync-client/src/utils/data-structures/min-covered.test.ts @@ -1,15 +1,15 @@ import { describe, it } from "node:test"; import assert from "node:assert"; -import { CoveredValues } from "./min-covered"; +import { MinCovered } from "./min-covered"; -describe("CoveredValues", () => { +describe("MinCovered", () => { it("should initialize with the given min value", () => { - const covered = new CoveredValues(5); + const covered = new MinCovered(5); assert.strictEqual(covered.min, 5); }); it("should add values greater than min", () => { - const covered = new CoveredValues(0); + const covered = new MinCovered(0); covered.add(3); assert.strictEqual(covered.min, 0); covered.add(1); @@ -21,7 +21,7 @@ describe("CoveredValues", () => { }); it("should ignore duplicate values", () => { - const covered = new CoveredValues(0); + const covered = new MinCovered(0); covered.add(3); covered.add(3); covered.add(3); @@ -32,7 +32,7 @@ describe("CoveredValues", () => { }); it("should handle multiple consecutive values", () => { - const covered = new CoveredValues(132); + const covered = new MinCovered(132); for (let i = 250; i > 132; i--) { assert.strictEqual(covered.min, 132); covered.add(i); @@ -41,36 +41,32 @@ describe("CoveredValues", () => { }); it("should handle adding values lower than current min", () => { - const covered = new CoveredValues(5); + const covered = new MinCovered(5); covered.add(3); assert.strictEqual(covered.min, 5); covered.add(6); assert.strictEqual(covered.min, 6); }); - it("should auto-advance when setting min value", () => { - const covered = new CoveredValues(5); + it("should auto-advance when adding the value that fills the next gap", () => { + const covered = new MinCovered(5); covered.add(7); covered.add(8); covered.add(9); assert.strictEqual(covered.min, 5); - // Setting min to 6 should auto-advance through 7, 8, 9 - covered.min = 6; + // Adding 6 fills the gap and auto-advances through 7, 8, 9 + covered.add(6); assert.strictEqual(covered.min, 9); covered.add(10); assert.strictEqual(covered.min, 10); }); - it("should handle setting min value with no consecutive values", () => { - const covered = new CoveredValues(5); - covered.add(10); - covered.add(15); - assert.strictEqual(covered.min, 5); - // Setting min to 8 should not auto-advance (no consecutive values) - covered.min = 8; - assert.strictEqual(covered.min, 8); - // Add 9 to trigger auto-advance to 10 - covered.add(9); - assert.strictEqual(covered.min, 10); + it("should rewind when reset is called explicitly", () => { + const covered = new MinCovered(5); + covered.add(7); + covered.reset(3); + assert.strictEqual(covered.min, 3); + covered.add(4); + assert.strictEqual(covered.min, 4); }); }); diff --git a/frontend/sync-client/src/utils/data-structures/min-covered.ts b/frontend/sync-client/src/utils/data-structures/min-covered.ts index 8b38822f..ed0b9d2e 100644 --- a/frontend/sync-client/src/utils/data-structures/min-covered.ts +++ b/frontend/sync-client/src/utils/data-structures/min-covered.ts @@ -7,13 +7,13 @@ * * @example * ```typescript - * const covered = new CoveredValues(0); + * const covered = new MinCovered(0); * covered.add(2); // seenValues = [2], min = 0 * covered.add(1); // seenValues = [], min = 2 * covered.min; // returns 2 * ``` */ -export class CoveredValues { +export class MinCovered { private seenValues: number[] = []; public constructor(private minValue: number) {} @@ -22,12 +22,6 @@ export class CoveredValues { return this.minValue; } - public set min(value: number) { - this.minValue = Math.max(value, this.minValue); - this.seenValues = this.seenValues.filter((v) => v > this.minValue); - this.advanceMinWhilePossible(); - } - public add(value: number | undefined): void { if (value === undefined || value < this.minValue) { return; @@ -49,6 +43,11 @@ export class CoveredValues { this.advanceMinWhilePossible(); } + public reset(minValue?: number): void { + this.minValue = minValue ?? 0; + this.seenValues = []; + } + private advanceMinWhilePossible(): void { while ( this.seenValues.length > 0 && diff --git a/frontend/sync-client/src/utils/debugging/in-memory-file-system.ts b/frontend/sync-client/src/utils/debugging/in-memory-file-system.ts new file mode 100644 index 00000000..0d26b175 --- /dev/null +++ b/frontend/sync-client/src/utils/debugging/in-memory-file-system.ts @@ -0,0 +1,69 @@ +import type { RelativePath } from "../../sync-operations/types"; +import type { TextWithCursors } from "reconcile-text"; +import type { FileSystemOperations } from "../../file-operations/filesystem-operations"; + +export class InMemoryFileSystem implements FileSystemOperations { + protected readonly files = new Map(); + + public async listFilesRecursively( + _root: RelativePath | undefined = undefined // we don't use multi-level paths during tests + ): Promise { + return Array.from(this.files.keys()); + } + + public async read(path: RelativePath): Promise { + const file = this.files.get(path); + if (!file) { + throw new Error(`File ${path} does not exist`); + } + return file; + } + + public async write(path: RelativePath, content: Uint8Array): Promise { + this.files.set(path, content); + } + + public async atomicUpdateText( + path: RelativePath, + updater: (current: TextWithCursors) => TextWithCursors + ): Promise { + const file = this.files.get(path); + if (!file) { + throw new Error(`File ${path} does not exist`); + } + const currentContent = new TextDecoder().decode(file); + const newContent = updater({ text: currentContent, cursors: [] }).text; + this.files.set(path, new TextEncoder().encode(newContent)); + return newContent; + } + + public async getFileSize(path: RelativePath): Promise { + return (await this.read(path)).length; + } + + public async exists(path: RelativePath): Promise { + return this.files.has(path); + } + + public async createDirectory(_path: RelativePath): Promise { + // This doesn't mean anything in our virtual FS representation + } + + public async delete(path: RelativePath): Promise { + this.files.delete(path); + } + + public async rename( + oldPath: RelativePath, + newPath: RelativePath + ): Promise { + const file = this.files.get(oldPath); + if (!file) { + throw new Error(`File ${oldPath} does not exist`); + } + this.files.set(newPath, file); + if (oldPath !== newPath) { + this.files.delete(oldPath); + } + } +} diff --git a/frontend/sync-client/src/utils/debugging/log-to-console.ts b/frontend/sync-client/src/utils/debugging/log-to-console.ts index c47f18f6..def71400 100644 --- a/frontend/sync-client/src/utils/debugging/log-to-console.ts +++ b/frontend/sync-client/src/utils/debugging/log-to-console.ts @@ -1,10 +1,44 @@ -import type { SyncClient } from "../../sync-client"; -import type { LogLine } from "../../tracing/logger"; +/* eslint-disable no-console */ +import type { Logger, LogLine } from "../../tracing/logger"; import { LogLevel } from "../../tracing/logger"; -export function logToConsole(client: SyncClient): void { - client.logger.onLogEmitted.add((logLine: LogLine) => { - const formatted = `${logLine.timestamp.toISOString()} ${logLine.level} ${logLine.message}`; +const COLORS = { + reset: "\x1b[0m", + red: "\x1b[31m", + yellow: "\x1b[33m", + blue: "\x1b[34m", + gray: "\x1b[90m" +}; + +export function logToConsole( + logger: Logger, + { useColors = true }: { useColors?: boolean } = {} +): void { + logger.onLogEmitted.add((logLine: LogLine) => { + const timestamp = logLine.timestamp.toISOString(); + const { message } = logLine; + + let color = ""; + let reset = ""; + if (useColors) { + ({ reset } = COLORS); + switch (logLine.level) { + case LogLevel.ERROR: + color = COLORS.red; + break; + case LogLevel.WARNING: + color = COLORS.yellow; + break; + case LogLevel.INFO: + color = COLORS.blue; + break; + case LogLevel.DEBUG: + color = COLORS.gray; + break; + } + } + + const formatted = `${timestamp} ${color}${logLine.level}${reset} ${message}`; switch (logLine.level) { case LogLevel.ERROR: diff --git a/frontend/sync-client/src/utils/debugging/slow-web-socket-factory.ts b/frontend/sync-client/src/utils/debugging/slow-web-socket-factory.ts index c64bff18..b93460b5 100644 --- a/frontend/sync-client/src/utils/debugging/slow-web-socket-factory.ts +++ b/frontend/sync-client/src/utils/debugging/slow-web-socket-factory.ts @@ -11,7 +11,7 @@ export function slowWebSocketFactory( private static readonly RECEIVE_KEY = "websocket-receive"; private static readonly SEND_KEY = "websocket-send"; - private readonly locks = new Locks(logger); + private readonly locks = new Locks(FlakyWebSocket.name, logger); public set onopen(callback: ((event: Event) => void) | null) { super.onopen = async (event: Event): Promise => { diff --git a/frontend/sync-client/src/utils/find-matching-file.ts b/frontend/sync-client/src/utils/find-matching-file.ts index c3d323d3..1e0b352c 100644 --- a/frontend/sync-client/src/utils/find-matching-file.ts +++ b/frontend/sync-client/src/utils/find-matching-file.ts @@ -1,14 +1,17 @@ -import type { DocumentRecord } from "../persistence/database"; +import type { DocumentRecord } from "../sync-operations/types"; import { EMPTY_HASH } from "./hash"; // TODO: make this smarter so that offline files can be renamed & edited at the same time -export function findMatchingFile( +export async function findMatchingFile( contentHash: string, candidates: DocumentRecord[] -): DocumentRecord | undefined { - if (contentHash === EMPTY_HASH) { +): Promise { + if (contentHash === (await EMPTY_HASH)) { return undefined; } - return candidates.find(({ metadata }) => metadata?.hash === contentHash); + return candidates.find( + (record) => + record.remoteHash !== undefined && record.remoteHash === contentHash + ); } diff --git a/frontend/sync-client/src/utils/hash.ts b/frontend/sync-client/src/utils/hash.ts index 906b6fad..b9d23041 100644 --- a/frontend/sync-client/src/utils/hash.ts +++ b/frontend/sync-client/src/utils/hash.ts @@ -1,12 +1,14 @@ -// https://stackoverflow.com/questions/7616461/generate-a-hash-from-string-in-javascript -export function hash(content: Uint8Array): string { - let result = 0; - // eslint-disable-next-line @typescript-eslint/prefer-for-of - for (let i = 0; i < content.length; i++) { - result = (result << 5) - result + content[i]; - result |= 0; // Convert to 32bit integer - } - return Math.abs(result).toString(16).padStart(8, "0"); +export async function hash(content: Uint8Array): Promise { + // Re-wrap into a fresh Uint8Array so SubtleCrypto's + // BufferSource overload accepts it without an unsafe type assertion. + // The lib types require an ArrayBuffer-backed view; the source may + // be backed by SharedArrayBuffer in some runtimes. + const buffer = new ArrayBuffer(content.byteLength); + new Uint8Array(buffer).set(content); + const digest = await crypto.subtle.digest("SHA-256", buffer); + const bytes = new Uint8Array(digest); + return Array.from(bytes, (b) => b.toString(16).padStart(2, "0")).join(""); } -export const EMPTY_HASH = hash(new Uint8Array(0)); +// SHA-256 of empty content, computed once at import time +export const EMPTY_HASH: Promise = hash(new Uint8Array()); diff --git a/frontend/sync-client/src/utils/rate-limit.ts b/frontend/sync-client/src/utils/rate-limit.ts index 52cbbce7..acb86393 100644 --- a/frontend/sync-client/src/utils/rate-limit.ts +++ b/frontend/sync-client/src/utils/rate-limit.ts @@ -1,4 +1,4 @@ -import { createPromise } from "./create-promise"; +import { awaitAll } from "./await-all"; import { sleep } from "./sleep"; /** @@ -45,18 +45,16 @@ export function rateLimit< newArgs = undefined; } - const [promise, resolve] = createPromise(); - running = promise; - sleep( + // `running` must signal both "minimum interval has elapsed" *and* + // "fn() has finished" — otherwise an `fn` that takes longer than + // the interval would let a queued waiter fire a concurrent `fn` + const interval = typeof minIntervalMs === "function" ? minIntervalMs() - : minIntervalMs - ) - .then(resolve) - .catch(() => { - // sleep cannot fail - }); - return fn(...args); + : minIntervalMs; + const fnPromise = fn(...args); + running = awaitAll([fnPromise.catch(() => undefined), sleep(interval)]); + return fnPromise; }; return decoratedFn; diff --git a/sync-server/Cargo.lock b/sync-server/Cargo.lock index b3da1486..82a7ce92 100644 --- a/sync-server/Cargo.lock +++ b/sync-server/Cargo.lock @@ -337,10 +337,11 @@ checksum = "325918d6fe32f23b19878fe4b34794ae41fc19ddbe53b10571a4874d44ffd39b" [[package]] name = "cc" -version = "1.2.2" +version = "1.2.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f34d93e62b03caf570cccc334cbc6c2fceca82f39211051345108adcba3eebdc" +checksum = "7a0dd1ca384932ff3641c8718a02769f1698e7563dc6974ffd03346116310423" dependencies = [ + "find-msvc-tools", "shlex", ] @@ -456,6 +457,15 @@ version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" +[[package]] +name = "crossbeam-channel" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-queue" version = "0.3.11" @@ -533,6 +543,15 @@ dependencies = [ "zeroize", ] +[[package]] +name = "deranged" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" +dependencies = [ + "powerfmt", +] + [[package]] name = "digest" version = "0.10.7" @@ -624,6 +643,12 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "486f806e73c5707928240ddc295403b1b93c96a02038563881c4a2fd84b81ac4" +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + [[package]] name = "flume" version = "0.11.1" @@ -1272,6 +1297,16 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "mime_guess" +version = "2.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e" +dependencies = [ + "mime", + "unicase", +] + [[package]] name = "miniz_oxide" version = "0.8.0" @@ -1335,6 +1370,12 @@ dependencies = [ "zeroize", ] +[[package]] +name = "num-conv" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6673768db2d862beb9b39a78fdcb1a69439615d5794a1be50caa9bc92c81967" + [[package]] name = "num-integer" version = "0.1.46" @@ -1463,6 +1504,12 @@ version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2" +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "ppv-lite86" version = "0.2.20" @@ -1582,12 +1629,12 @@ dependencies = [ [[package]] name = "reconcile-text" -version = "0.8.0" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "599cf9539996a2a19e501110404c59ba62f4974009f8fb864a8b7151c15ee5a5" +checksum = "52e0cf361887ea64c479ca871c1170dda761f84e122f2616b5579906a38d7557" dependencies = [ "serde", - "thiserror 2.0.17", + "thiserror 2.0.18", ] [[package]] @@ -1648,6 +1695,40 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rust-embed" +version = "8.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04113cb9355a377d83f06ef1f0a45b8ab8cd7d8b1288160717d66df5c7988d27" +dependencies = [ + "rust-embed-impl", + "rust-embed-utils", + "walkdir", +] + +[[package]] +name = "rust-embed-impl" +version = "8.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da0902e4c7c8e997159ab384e6d0fc91c221375f6894346ae107f47dd0f3ccaa" +dependencies = [ + "proc-macro2", + "quote", + "rust-embed-utils", + "syn 2.0.90", + "walkdir", +] + +[[package]] +name = "rust-embed-utils" +version = "8.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5bcdef0be6fe7f6fa333b1073c949729274b05f123a0ad7efcb8efd878e5c3b1" +dependencies = [ + "sha2", + "walkdir", +] + [[package]] name = "rustc-demangle" version = "0.1.24" @@ -1679,6 +1760,15 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "sanitize-filename" version = "0.6.0" @@ -1916,7 +2006,7 @@ dependencies = [ "serde_json", "sha2", "smallvec", - "thiserror 2.0.17", + "thiserror 2.0.18", "tokio", "tokio-stream", "tracing", @@ -2000,7 +2090,7 @@ dependencies = [ "smallvec", "sqlx-core", "stringprep", - "thiserror 2.0.17", + "thiserror 2.0.18", "tracing", "uuid", "whoami", @@ -2039,7 +2129,7 @@ dependencies = [ "smallvec", "sqlx-core", "stringprep", - "thiserror 2.0.17", + "thiserror 2.0.18", "tracing", "uuid", "whoami", @@ -2065,7 +2155,7 @@ dependencies = [ "serde", "serde_urlencoded", "sqlx-core", - "thiserror 2.0.17", + "thiserror 2.0.18", "tracing", "url", "uuid", @@ -2100,6 +2190,12 @@ version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" +[[package]] +name = "symlink" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7973cce6668464ea31f176d85b13c7ab3bba2cb3b77a2ed26abd7801688010a" + [[package]] name = "syn" version = "1.0.109" @@ -2136,18 +2232,22 @@ dependencies = [ "futures", "humantime-serde", "log", + "mime_guess", "rand 0.9.0", "reconcile-text", "regex", + "rust-embed", "sanitize-filename", "serde", "serde_json", "serde_yaml", "sqlx", - "thiserror 2.0.17", + "subtle", + "thiserror 2.0.18", "tokio", "tower-http", "tracing", + "tracing-appender", "tracing-subscriber", "ts-rs", "uuid", @@ -2203,11 +2303,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.17" +version = "2.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f63587ca0f12b72a0600bcba1d40081f830876000bb46dd2337a3051618f4fc8" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" dependencies = [ - "thiserror-impl 2.0.17", + "thiserror-impl 2.0.18", ] [[package]] @@ -2223,9 +2323,9 @@ dependencies = [ [[package]] name = "thiserror-impl" -version = "2.0.17" +version = "2.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", @@ -2242,6 +2342,37 @@ dependencies = [ "once_cell", ] +[[package]] +name = "time" +version = "0.3.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" +dependencies = [ + "deranged", + "itoa", + "num-conv", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" + +[[package]] +name = "time-macros" +version = "0.2.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215" +dependencies = [ + "num-conv", + "time-core", +] + [[package]] name = "tinystr" version = "0.7.6" @@ -2276,7 +2407,6 @@ dependencies = [ "bytes", "libc", "mio", - "parking_lot", "pin-project-lite", "signal-hook-registry", "socket2", @@ -2376,6 +2506,19 @@ dependencies = [ "tracing-core", ] +[[package]] +name = "tracing-appender" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "050686193eb999b4bb3bc2acfa891a13da00f79734704c4b8b4ef1a10b368a3c" +dependencies = [ + "crossbeam-channel", + "symlink", + "thiserror 2.0.18", + "time", + "tracing-subscriber", +] + [[package]] name = "tracing-attributes" version = "0.1.28" @@ -2434,7 +2577,7 @@ checksum = "e640d9b0964e9d39df633548591090ab92f7a4567bc31d3891af23471a3365c6" dependencies = [ "chrono", "lazy_static", - "thiserror 2.0.17", + "thiserror 2.0.18", "ts-rs-macros", "uuid", ] @@ -2481,6 +2624,12 @@ version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f720def6ce1ee2fc44d40ac9ed6d3a59c361c80a75a7aa8e75bb9baed31cf2ea" +[[package]] +name = "unicase" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142" + [[package]] name = "unicode-bidi" version = "0.3.17" @@ -2577,6 +2726,16 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" diff --git a/sync-server/Cargo.toml b/sync-server/Cargo.toml index fac06efa..6de17653 100644 --- a/sync-server/Cargo.toml +++ b/sync-server/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sync_server" -rust-version = "1.89.0" +rust-version = "1.94.0" authors = ["Andras Schmelczer "] edition = "2024" license = "MIT" @@ -10,7 +10,7 @@ version = "0.14.0" [dependencies] serde = { version = "1.0.219", default-features = false, features = ["derive"] } thiserror = { version = "2.0.12", default-features = false } -tokio = { version = "1.48.0", features = ["full"]} +tokio = { version = "1.48.0", features = ["macros", "rt-multi-thread", "sync", "time", "net", "fs", "signal"]} uuid = { version = "1.16.0", features = ["v4", "serde"] } log = { version = "0.4.28" } anyhow = { version = "1.0.100", features = ["backtrace"] } @@ -20,6 +20,7 @@ axum_typed_multipart = "0.11.0" tower-http = { version = "0.6.1", features = ["cors", "trace", "limit", "timeout"] } tracing = "0.1.41" tracing-subscriber = { version = "0.3.20", features = ["fmt", "env-filter"]} +tracing-appender = "0.2.5" humantime-serde = "1.1.1" sqlx = { version = "0.8.6", features = ["sqlite", "runtime-tokio", "uuid", "chrono"] } chrono = { version = "0.4.41", features = ["serde"] } @@ -33,7 +34,10 @@ serde_json = "1.0.140" bimap = "0.6.3" ts-rs = { version = "10.1", features = ["uuid-impl", "chrono-impl"] } base64 = "0.22.1" -reconcile-text = { version = "0.8.0", features = ["serde"] } +reconcile-text = { version = "0.11.0", features = ["serde"] } +rust-embed = "8.5" +mime_guess = "2.0" +subtle = "2.6.1" [profile.release] codegen-units = 1 diff --git a/sync-server/build.rs b/sync-server/build.rs index d5068697..53bd111b 100644 --- a/sync-server/build.rs +++ b/sync-server/build.rs @@ -1,5 +1,16 @@ -// generated by `sqlx migrate build-script` fn main() { // trigger recompilation when a new migration is added println!("cargo:rerun-if-changed=migrations"); + + // Ensure the history-ui dist directory exists so rust-embed can compile + // even when the frontend hasn't been built yet. + let dist_path = std::path::Path::new("../frontend/history-ui/dist"); + if !dist_path.exists() { + std::fs::create_dir_all(dist_path).expect("Failed to create history-ui dist directory"); + std::fs::write( + dist_path.join("index.html"), + "

Run npm run build -w history-ui first.

", + ) + .expect("Failed to write placeholder index.html"); + } } diff --git a/sync-server/config-e2e.yml b/sync-server/config-e2e.yml index 1f235b01..03b860b7 100644 --- a/sync-server/config-e2e.yml +++ b/sync-server/config-e2e.yml @@ -1,32 +1,34 @@ database: - databases_directory_path: databases - max_connections_per_vault: 12 + databases_directory_path: /host/tmp/vaultlink-e2e-databases + max_connections_per_vault: 8 cursor_timeout: 1m server: host: 0.0.0.0 - port: 3000 + port: 3010 max_body_size_mb: 512 max_clients_per_vault: 256 + max_pending_websocket_connections: 4096 + broadcast_channel_capacity: 1024 response_timeout: 30m mergeable_file_extensions: - - md - - txt + - md + - txt users: user_configs: - - name: admin - token: test-token-change-me - vault_access: - type: allow_access_to_all - - name: other-admin - token: test-token-change-me2 - vault_access: - type: allow_access_to_all - - name: test - token: other-test-token - vault_access: - type: allow_list - allowed: - - default + - name: admin + token: test-token-change-me + vault_access: + type: allow_access_to_all + - name: other-admin + token: test-token-change-me2 + vault_access: + type: allow_access_to_all + - name: test + token: other-test-token + vault_access: + type: allow_list + allowed: + - default logging: log_directory: logs log_rotation: 7days diff --git a/sync-server/rust-toolchain.toml b/sync-server/rust-toolchain.toml index 010956cc..567721ef 100644 --- a/sync-server/rust-toolchain.toml +++ b/sync-server/rust-toolchain.toml @@ -1,5 +1,5 @@ [toolchain] -channel = "1.89.0" +channel = "1.94.0" targets = [ "x86_64-unknown-linux-gnu", "x86_64-unknown-linux-musl", diff --git a/sync-server/src/app_state.rs b/sync-server/src/app_state.rs index 2019e08e..1bd3222e 100644 --- a/sync-server/src/app_state.rs +++ b/sync-server/src/app_state.rs @@ -2,6 +2,8 @@ pub mod cursors; pub mod database; pub mod websocket; +use std::sync::{Arc, atomic::AtomicUsize}; + use anyhow::Result; use cursors::Cursors; use database::Database; @@ -15,21 +17,42 @@ pub struct AppState { pub database: Database, pub cursors: Cursors, pub broadcasts: Broadcasts, + /// Tracks WebSocket connections that have upgraded but not yet completed + /// the authentication handshake + pub pending_ws_connections: Arc, + /// Send on this channel to stop background tasks (cursor cleanup, + /// idle-pool cleanup) + shutdown_tx: Arc>, } impl AppState { pub async fn try_new(config: Config) -> Result { + let (shutdown_tx, shutdown_rx) = tokio::sync::watch::channel(()); + let broadcasts = Broadcasts::new(&config.server); - let database = Database::try_new(&config.database, &broadcasts).await?; + let database = + Database::try_new(&config.database, &broadcasts, shutdown_rx.clone()).await?; let cursors: Cursors = Cursors::new(&config.database, &broadcasts); - Cursors::start_background_task(cursors.clone()); + Cursors::start_background_task(cursors.clone(), shutdown_rx); Ok(Self { config, database, cursors, broadcasts, + pending_ws_connections: Arc::new(AtomicUsize::new(0)), + shutdown_tx: Arc::new(shutdown_tx), }) } + + /// Signal all background tasks (idle pool cleanup, cursor cleanup) to stop + pub fn shutdown(&self) { + let _ = self.shutdown_tx.send(()); + } + + /// Get a receiver to be notified when shutdown is triggered + pub fn subscribe_shutdown(&self) -> tokio::sync::watch::Receiver<()> { + self.shutdown_tx.subscribe() + } } diff --git a/sync-server/src/app_state/cursors.rs b/sync-server/src/app_state/cursors.rs index d083e1ac..e17fb4f7 100644 --- a/sync-server/src/app_state/cursors.rs +++ b/sync-server/src/app_state/cursors.rs @@ -42,7 +42,9 @@ impl Cursors { ) { let mut vault_to_cursors = self.vault_to_cursors.lock().await; - let all_device_cursors = vault_to_cursors.entry(vault_id).or_insert_with(Vec::new); + let all_device_cursors = vault_to_cursors + .entry(vault_id.clone()) + .or_insert_with(Vec::new); all_device_cursors.retain(|c| &c.client_cursors.device_id != device_id); all_device_cursors.push(ClientCursorsWithTimeToLive::new(ClientCursors { @@ -52,7 +54,7 @@ impl Cursors { })); drop(vault_to_cursors); // Explicitly drop the lock before broadcasting to avoid deadlock - self.broadcast_cursors().await; + self.broadcast_cursors_for_vault(&vault_id).await; } pub async fn get_cursors(&self, vault_id: &VaultId) -> Vec { @@ -69,45 +71,81 @@ impl Cursors { .unwrap_or_default() } - pub fn start_background_task(self) { + pub fn start_background_task(self, mut shutdown: tokio::sync::watch::Receiver<()>) { tokio::spawn(async move { loop { - self.remove_expired_cursors().await; - tokio::time::sleep(Duration::from_secs(1)).await; + tokio::select! { + () = tokio::time::sleep(Duration::from_secs(1)) => { + self.remove_expired_cursors().await; + } + Ok(()) = shutdown.changed() => break, + } } }); } async fn remove_expired_cursors(&self) { - let mut vault_to_cursors = self.vault_to_cursors.lock().await; + let changed_vaults: Vec = { + let mut vault_to_cursors = self.vault_to_cursors.lock().await; - for (_vault_id, cursors) in vault_to_cursors.iter_mut() { - cursors.retain(|cursor| !cursor.is_expired(self.config.cursor_timeout)); + let mut changed = Vec::new(); + for (vault_id, cursors) in vault_to_cursors.iter_mut() { + let before = cursors.len(); + cursors.retain(|cursor| !cursor.is_expired(self.config.cursor_timeout)); + if cursors.len() != before { + changed.push(vault_id.clone()); + } + } + + // Remove empty vault entries to prevent unbounded growth + vault_to_cursors.retain(|_, cursors| !cursors.is_empty()); + + changed + }; + + for vault_id in &changed_vaults { + self.broadcast_cursors_for_vault(vault_id).await; } } - async fn broadcast_cursors(&self) { - let vault_to_cursors = self.vault_to_cursors.lock().await; + async fn broadcast_cursors_for_vault(&self, vault_id: &VaultId) { + let client_cursors: Vec = { + let vault_to_cursors = self.vault_to_cursors.lock().await; + vault_to_cursors + .get(vault_id) + .map(|cursors| cursors.iter().map(|c| c.client_cursors.clone()).collect()) + .unwrap_or_default() + }; - for (vault_id, cursors) in vault_to_cursors.iter() { - self.broadcasts - .send_document_update( - vault_id.clone(), - WebSocketServerMessageWithOrigin::new(WebSocketServerMessage::CursorPositions( - CursorPositionFromServer { - clients: cursors.iter().map(|c| c.client_cursors.clone()).collect(), - }, - )), - ) - .await; - } + self.broadcasts.send_document_update( + vault_id.clone(), + WebSocketServerMessageWithOrigin::new(WebSocketServerMessage::CursorPositions( + CursorPositionFromServer { + clients: client_cursors, + }, + )), + ); } - pub async fn remove_cursors_of_device(&self, vault_id: &str, device_id: &str) { - let mut vault_to_cursors = self.vault_to_cursors.lock().await; + pub async fn remove_cursors_of_device(&self, vault_id: &VaultId, device_id: &DeviceId) { + let changed = { + let mut vault_to_cursors = self.vault_to_cursors.lock().await; - if let Some(cursors) = vault_to_cursors.get_mut(vault_id) { - cursors.retain(|c| c.client_cursors.device_id != device_id); + if let Some(cursors) = vault_to_cursors.get_mut(vault_id) { + let before = cursors.len(); + cursors.retain(|c| c.client_cursors.device_id != *device_id); + let changed = cursors.len() != before; + if cursors.is_empty() { + vault_to_cursors.remove(vault_id); + } + changed + } else { + false + } + }; + + if changed { + self.broadcast_cursors_for_vault(vault_id).await; } } } diff --git a/sync-server/src/app_state/database.rs b/sync-server/src/app_state/database.rs index 75ce6df4..28acde41 100644 --- a/sync-server/src/app_state/database.rs +++ b/sync-server/src/app_state/database.rs @@ -1,16 +1,29 @@ use core::time::Duration; -use std::{collections::HashMap, sync::Arc}; +use std::{ + collections::HashMap, + sync::Arc, + sync::atomic::{AtomicU64, Ordering}, +}; use anyhow::{Context as _, Result}; use log::info; use models::{ DocumentId, DocumentVersionWithoutContent, StoredDocumentVersion, VaultId, VaultUpdateId, }; -use sqlx::{ConnectOptions, sqlite::SqliteConnectOptions, types::chrono::Utc}; +use sqlx::{ConnectOptions, Connection, sqlite::SqliteConnectOptions, types::chrono::Utc}; pub mod models; -use sqlx::{Pool, Sqlite, sqlite::SqlitePoolOptions}; -use tokio::sync::Mutex; + +/// Sentinel error indicating the `SQLite` database is busy (`SQLITE_BUSY`). +/// Handlers can downcast to this to return 429 instead of 500. +#[derive(Debug, thiserror::Error)] +#[error("Database is busy")] +pub struct WriteBusyError; + +use sqlx::{ + Pool, Sqlite, pool::PoolConnection, sqlite::SqliteConnection, sqlite::SqlitePoolOptions, +}; +use tokio::sync::{Mutex, OnceCell}; use tokio::time::Instant; use uuid::fmt::Hyphenated; @@ -19,33 +32,200 @@ use super::websocket::{ models::{WebSocketServerMessage, WebSocketServerMessageWithOrigin, WebSocketVaultUpdate}, }; use crate::config::database_config::DatabaseConfig; +use crate::consts::IDLE_POOL_TIMEOUT; -#[derive(Clone)] -struct PoolWithTimestamp { - pool: Pool, - last_accessed: Instant, +/// Holds separate reader and writer pools for a single vault. +/// The writer pool has exactly 1 connection so writes never compete +/// with reads for pool slots. +#[derive(Debug, Clone)] +struct VaultPools { + reader: Pool, + writer: Pool, } -impl std::fmt::Debug for PoolWithTimestamp { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("PoolWithTimestamp") - .field("pool", &"Pool") - .field("last_accessed", &self.last_accessed) - .finish() - } +#[derive(Debug)] +struct VaultPool { + cell: Arc>, + /// Monotonic timestamp in milliseconds (from `Instant::now()` at server start) + last_accessed_ms: AtomicU64, } #[derive(Clone, Debug)] pub struct Database { config: DatabaseConfig, broadcasts: Broadcasts, - connection_pools: Arc>>, + connection_pools: Arc>>>, + /// Per-vault write serialization. `SQLite` allows only one writer at a + /// time; `BEGIN IMMEDIATE` on a second connection blocks until the first + /// commits (up to `busy_timeout`). Under concurrent load the blocked + /// connections consume the pool, starving even read-only requests. + /// This mutex moves the wait from the `SQLite` layer (where it holds a + /// pool connection) to the Tokio layer (where it holds nothing). + write_locks: Arc>>>>, + /// Monotonic epoch for lock-free `last_accessed_ms` timestamps + epoch: Instant, } -pub type Transaction<'a> = sqlx::Transaction<'a, Sqlite>; +/// A write transaction backed by a raw `BEGIN IMMEDIATE` instead of sqlx's +/// savepoint-based `Transaction`. This avoids the savepoint mismatch caused +/// by the old `END; BEGIN IMMEDIATE;` workaround. +/// +/// Holds an `OwnedMutexGuard` that serializes write transactions per vault +/// at the application level (see `Database::write_locks`). The guard is +/// released when the transaction is committed, rolled back, or dropped. +pub struct WriteTransaction { + conn: Option>, + _write_guard: tokio::sync::OwnedMutexGuard<()>, +} + +impl WriteTransaction { + async fn new( + pool: &Pool, + write_guard: tokio::sync::OwnedMutexGuard<()>, + ) -> Result { + let mut conn = pool + .acquire() + .await + .context("Cannot acquire connection for write transaction")?; + if let Err(e) = sqlx::query("BEGIN IMMEDIATE").execute(&mut *conn).await { + let is_busy = match &e { + sqlx::Error::Database(db_err) => { + // SQLITE_BUSY base code is 5. Extended codes share base 5. + let busy_by_code = db_err + .code() + .is_some_and(|c| c.parse::().is_ok_and(|n| n & 0xFF == 5)); + busy_by_code || db_err.message().contains("database is locked") + } + _ => false, + }; + if is_busy { + return Err(WriteBusyError.into()); + } + return Err(e).context("Cannot begin immediate transaction"); + } + Ok(Self { + conn: Some(conn), + _write_guard: write_guard, + }) + } + + pub async fn commit(mut self) -> Result<()> { + if let Some(mut conn) = self.conn.take() { + sqlx::query("COMMIT") + .execute(&mut *conn) + .await + .context("Failed to commit transaction")?; + } + Ok(()) + } + + pub async fn rollback(mut self) -> Result<()> { + if let Some(mut conn) = self.conn.take() { + sqlx::query("ROLLBACK") + .execute(&mut *conn) + .await + .context("Failed to rollback transaction")?; + } + Ok(()) + } +} + +impl Drop for WriteTransaction { + fn drop(&mut self) { + if self.conn.is_some() { + // The connection is returned to the pool with an open transaction. + // The pool's `before_acquire` hook issues a ROLLBACK before + // handing it to the next consumer, so no async work is needed + // here. If the pool is being shut down, SQLite itself rolls back + // uncommitted transactions when the connection closes. + log::warn!("WriteTransaction dropped without commit or rollback"); + } + } +} + +impl std::ops::Deref for WriteTransaction { + type Target = SqliteConnection; + fn deref(&self) -> &Self::Target { + self.conn + .as_ref() + .expect("BUG: WriteTransaction dereferenced after being consumed") + .deref() + } +} + +impl std::ops::DerefMut for WriteTransaction { + fn deref_mut(&mut self) -> &mut Self::Target { + self.conn + .as_mut() + .expect("BUG: WriteTransaction dereferenced after being consumed") + .deref_mut() + } +} + +/// Ensure the connection has no leftover open transaction (e.g. from a +/// `WriteTransaction` that was dropped without commit/rollback). ROLLBACK +/// is a harmless no-op if no transaction is active. +fn rollback_before_acquire( + conn: &mut SqliteConnection, + _meta: sqlx::pool::PoolConnectionMetadata, +) -> futures::future::BoxFuture<'_, Result> { + Box::pin(async move { + if let Err(e) = sqlx::query("ROLLBACK").execute(&mut *conn).await { + // "cannot rollback - no transaction is active" is the common + // case (connection returned cleanly). Only unexpected errors + // deserve attention. + log::debug!("before_acquire ROLLBACK failed: {e}"); + } + Ok(true) + }) +} impl Database { - pub async fn try_new(config: &DatabaseConfig, broadcasts: &Broadcasts) -> Result { + fn now_ms(&self) -> u64 { + self.epoch.elapsed().as_millis() as u64 + } + + /// Lists all vault IDs that exist on disk (have a `.sqlite` file). + pub async fn list_vaults(&self) -> Result> { + let mut vaults = Vec::new(); + let mut entries = tokio::fs::read_dir(&self.config.databases_directory_path) + .await + .context("Failed to read databases directory")?; + while let Some(entry) = entries.next_entry().await? { + let name = entry.file_name().to_string_lossy().to_string(); + if let Some(vault) = name.strip_suffix(".sqlite") { + vaults.push(vault.to_owned()); + } + } + vaults.sort(); + Ok(vaults) + } + + pub async fn get_vault_stats(&self, vault: &VaultId) -> Result { + let pool = self.get_connection_pool(vault).await?; + let row = sqlx::query!( + r#" + SELECT + (SELECT MIN(updated_date) FROM documents) + AS "created_at: chrono::DateTime", + (SELECT COUNT(DISTINCT document_id) FROM latest_document_versions + WHERE is_deleted = false) + AS "document_count!: u32" + "#, + ) + .fetch_one(&pool) + .await?; + Ok(models::VaultStats { + created_at: row.created_at, + document_count: row.document_count, + }) + } + + pub async fn try_new( + config: &DatabaseConfig, + broadcasts: &Broadcasts, + shutdown: tokio::sync::watch::Receiver<()>, + ) -> Result { tokio::fs::create_dir_all(&config.databases_directory_path) .await .with_context(|| { @@ -70,122 +250,207 @@ impl Database { .trim_end_matches(".sqlite") .to_owned(); - let pool = Self::create_vault_database(config, &vault).await?; + Self::validate_vault_id(&vault)?; + + let pools = Self::create_vault_database(config, &vault).await?; + let cell = Arc::new(OnceCell::new()); + cell.set(pools).expect("cell is new"); connection_pools.insert( vault.clone(), - PoolWithTimestamp { - pool, - last_accessed: Instant::now(), - }, + Arc::new(VaultPool { + cell, + last_accessed_ms: AtomicU64::new(0), + }), ); } + info!("Database migrations applied"); let database = Self { config: config.clone(), connection_pools: Arc::new(Mutex::new(connection_pools)), broadcasts: broadcasts.clone(), + write_locks: Arc::new(Mutex::new(HashMap::new())), + epoch: Instant::now(), }; - // Start background task to cleanup idle connection pools - database.start_idle_pool_cleanup(); + database.start_idle_pool_cleanup(shutdown); Ok(database) } - async fn create_vault_database( - config: &DatabaseConfig, - vault: &VaultId, - ) -> Result> { + async fn create_vault_database(config: &DatabaseConfig, vault: &VaultId) -> Result { let file_name = config .databases_directory_path .join(format!("{vault}.sqlite")); - let connection_options = SqliteConnectOptions::new() + // Database-level PRAGMAs (auto_vacuum, journal_mode) require a write + // lock and persist across connections. Set them once with a dedicated + // init connection so pool connections never need the write lock just to + // open. + let init_options = SqliteConnectOptions::new() .filename(file_name.clone()) .create_if_missing(true) - .auto_vacuum(sqlx::sqlite::SqliteAutoVacuum::Full) - .busy_timeout(Duration::from_secs(3600)) - .journal_mode(sqlx::sqlite::SqliteJournalMode::Wal) - .log_slow_statements(log::LevelFilter::Warn, Duration::from_secs(30)); + .auto_vacuum(sqlx::sqlite::SqliteAutoVacuum::Incremental) + .journal_mode(sqlx::sqlite::SqliteJournalMode::Wal); - let pool = SqlitePoolOptions::new() + // Run migrations on a dedicated connection, NOT through the pool. + // The pool's `before_acquire` hook issues ROLLBACK on every checkout, + // which can roll back the migration's bookkeeping transaction (the + // _sqlx_migrations INSERT) while the DDL (ALTER TABLE) has already + // auto-committed — leaving the migration in a dirty state. + // + // Uses `run_direct` instead of `run` because `run` takes + // `impl Acquire<'_>`, whose lifetime bound prevents the enclosing + // future from satisfying the `Send` requirement of axum handlers. + let mut init_conn = sqlx::SqliteConnection::connect_with(&init_options).await?; + sqlx::migrate!("src/app_state/database/migrations") + .run_direct(&mut init_conn) + .await + .context("Cannot run pending migrations")?; + drop(init_conn); + + // Per-connection PRAGMAs shared by both reader and writer pools. + // journal_mode = WAL is a no-op on an already-WAL database. + let base_options = SqliteConnectOptions::new() + .filename(file_name.clone()) + .journal_mode(sqlx::sqlite::SqliteJournalMode::Wal) + .busy_timeout(Duration::from_secs(30)) + .log_slow_statements(log::LevelFilter::Warn, Duration::from_secs(30)) + // In WAL mode, NORMAL is safe: data survives OS crashes, only the + // last transaction can be lost on power failure. The default FULL + // forces an extra fsync() per commit, roughly halving write throughput. + .pragma("synchronous", "NORMAL") + // 16 MB page cache per connection (negative = KiB). Reduces disk + // reads for the latest_document_versions GROUP BY view. + .pragma("cache_size", "-16384") + // Memory-mapped I/O avoids read() syscalls. SQLite falls back to + // regular I/O for writes and beyond the mapped region. 256 MB is + // conservative; the OS handles actual memory pressure. + .pragma("mmap_size", "268435456") + // Keep temp tables and sort spillovers in memory instead of temp files. + .pragma("temp_store", "MEMORY") + // Cap WAL file growth at 64 MB. Without this, the WAL can grow + // unbounded during heavy write bursts (e.g. E2E tests with many + // concurrent clients). SQLite truncates to this size on checkpoint. + .pragma("journal_size_limit", "67108864"); + + // Reader pool: multiple connections for concurrent reads. + let reader = SqlitePoolOptions::new() .max_connections(config.max_connections_per_vault) .acquire_slow_threshold(Duration::from_secs(30)) - .test_before_acquire(true) - .connect_with(connection_options) + // Disabled: the health-check query is subject to busy_timeout + // and blocks all connection checkouts when a write is active, + // starving the pool for up to 30s even for simple reads. + // The before_acquire ROLLBACK hook is sufficient for cleanup. + .test_before_acquire(false) + .before_acquire(rollback_before_acquire) + .connect_with(base_options.clone()) .await - .with_context(|| format!("Cannot open database at `{}`", file_name.display()))?; + .with_context(|| format!("Cannot open reader pool at `{}`", file_name.display()))?; - Self::run_migrations(&pool).await?; + // Writer pool: exactly 1 connection, dedicated to writes. + // Since the Tokio mutex already serializes writers per vault, this + // single connection is never contended. Separating it from the + // reader pool ensures writes never compete with reads for pool slots. + let writer = SqlitePoolOptions::new() + .max_connections(1) + .acquire_slow_threshold(Duration::from_secs(30)) + .test_before_acquire(false) + .before_acquire(rollback_before_acquire) + .connect_with(base_options) + .await + .with_context(|| format!("Cannot open writer pool at `{}`", file_name.display()))?; - Ok(pool) + Ok(VaultPools { reader, writer }) } - async fn run_migrations(pool: &Pool) -> Result<()> { - sqlx::migrate!("src/app_state/database/migrations") - .run(pool) - .await - .context("Cannot check for pending migrations") - } - - async fn get_connection_pool(&self, vault: &VaultId) -> Result> { - let mut pools = self.connection_pools.lock().await; - - if !pools.contains_key(vault) { - let pool = Self::create_vault_database(&self.config, vault).await?; - pools.insert( - vault.clone(), - PoolWithTimestamp { - pool, - last_accessed: Instant::now(), - }, + fn validate_vault_id(vault: &VaultId) -> Result<()> { + if vault.is_empty() { + anyhow::bail!("Vault ID must not be empty"); + } + if vault.contains('/') + || vault.contains('\\') + || vault.contains("..") + || vault.contains('\0') + { + anyhow::bail!( + "Invalid vault ID: must not contain path separators, '..', or null bytes" ); } - - let pool_with_timestamp = pools - .get_mut(vault) - .expect("Pool was just inserted or already exists"); - - // Update last accessed time - pool_with_timestamp.last_accessed = Instant::now(); - - Ok(pool_with_timestamp.pool.clone()) + Ok(()) } - /// Attempting to write from this transaction might result in a - /// database locked error. Use this transaction for read-only operations. - pub async fn create_readonly_transaction( - &self, - vault: &VaultId, - ) -> Result> { - self.get_connection_pool(vault) - .await? - .begin() - .await - .context("Cannot create transaction") - } + async fn get_vault_pools(&self, vault: &VaultId) -> Result { + Self::validate_vault_id(vault)?; - pub async fn create_write_transaction(&self, vault: &VaultId) -> Result> { - let mut transaction = self.create_readonly_transaction(vault).await?; + // Get or create the VaultPool entry. The global lock is held only + // long enough for a HashMap lookup/insert — never across + // create_vault_database. + let vault_pool = { + let mut pools = self.connection_pools.lock().await; + pools + .entry(vault.clone()) + .or_insert_with(|| { + Arc::new(VaultPool { + cell: Arc::new(OnceCell::new()), + last_accessed_ms: AtomicU64::new(self.now_ms()), + }) + }) + .clone() + }; - // sqlx doesn't support immediate transactions for sqlite: https://github.com/launchbadge/sqlx/issues/481 - sqlx::query!("END; BEGIN IMMEDIATE;") - .execute(&mut *transaction) + // OnceCell::get_or_try_init guarantees exactly-once + // initialization: concurrent callers for the same vault wait + // here; callers for other vaults are not blocked. + let config = self.config.clone(); + let vault_clone = vault.clone(); + let pools = vault_pool + .cell + .get_or_try_init(|| async { Self::create_vault_database(&config, &vault_clone).await }) .await?; - Ok(transaction) + vault_pool + .last_accessed_ms + .store(self.now_ms(), Ordering::Relaxed); + Ok(pools.clone()) } - /// Return the latest state of all documents in the vault + /// Return the reader pool for read-only queries. + async fn get_connection_pool(&self, vault: &VaultId) -> Result> { + Ok(self.get_vault_pools(vault).await?.reader) + } + + pub async fn create_write_transaction(&self, vault: &VaultId) -> Result { + let write_lock = { + let mut locks = self.write_locks.lock().await; + locks + .entry(vault.clone()) + .or_insert_with(|| Arc::new(tokio::sync::Mutex::new(()))) + .clone() + }; + let write_guard = write_lock.lock_owned().await; + let pools = self.get_vault_pools(vault).await?; + WriteTransaction::new(&pools.writer, write_guard).await + } + + /// Return the latest state of all documents in the vault, optionally + /// bounded above by `up_to_vault_update_id` so that the result is a + /// stable snapshot at exactly that cursor (commits past the cursor + /// will be delivered separately via the broadcast channel). pub async fn get_latest_documents( &self, vault: &VaultId, - transaction: Option<&mut Transaction<'_>>, + up_to_vault_update_id: Option, + connection: Option<&mut SqliteConnection>, ) -> Result> { + // `i64::MAX` makes the upper bound a no-op for callers that don't + // care about an exact snapshot (they pass `None`). + let upper = up_to_vault_update_id.unwrap_or(i64::MAX); let query = sqlx::query!( r#" select vault_update_id, + creation_vault_update_id, document_id as "document_id: Hyphenated", relative_path, updated_date as "updated_date: chrono::DateTime", @@ -194,12 +459,14 @@ impl Database { device_id, length(content) as "content_size: u64" from latest_document_versions + where vault_update_id <= ? order by vault_update_id "#, + upper, ); - if let Some(transaction) = transaction { - query.fetch_all(&mut **transaction).await + if let Some(conn) = connection { + query.fetch_all(&mut *conn).await } else { query .fetch_all(&self.get_connection_pool(vault).await?) @@ -216,42 +483,72 @@ impl Database { is_deleted: row.is_deleted, user_id: row.user_id, device_id: row.device_id, - content_size: row - .content_size - .expect("Content size can't be null but sqlx can't infer it"), + content_size: row.content_size.unwrap_or(0), + is_new_file: row.creation_vault_update_id == row.vault_update_id, }) .collect() }) } /// Return the latest state of all documents (including deleted) in the - /// vault which have changed since the given update id + /// vault which have changed since the given update id, bounded above + /// by `up_to_vault_update_id` so the catch-up result is a stable + /// snapshot at exactly that cursor. Commits past the cursor will be + /// delivered separately via the broadcast channel. pub async fn get_latest_documents_since( &self, vault: &VaultId, vault_update_id: VaultUpdateId, - transaction: Option<&mut Transaction<'_>>, + up_to_vault_update_id: Option, + connection: Option<&mut SqliteConnection>, ) -> Result> { + // `i64::MAX` makes the upper bound a no-op for callers that don't + // care about an exact snapshot (they pass `None`). + let upper = up_to_vault_update_id.unwrap_or(i64::MAX); + // Compute "latest version as of `upper`" per document — NOT + // global latest. The `latest_document_versions` view is keyed + // on global max, so a write that commits between the catch-up's + // cursor capture (under broadcast send-lock) and this query + // (which runs after drop-lock) would expose a `vault_update_id + // > cursor` row that the cursor filter then drops, removing + // the doc from the catch-up entirely. The post-cursor live + // broadcast then carries `is_new_file = false` (per real-time + // semantics it's an update of a previously-existing version), + // and the receiving client — which has no record of the doc — + // ignores it as stale, stranding the doc forever. Computing + // the snapshot from the documents table directly with the + // upper bound applied at the GROUP BY layer keeps the + // catch-up self-contained at exactly the cursor. let query = sqlx::query!( r#" select - vault_update_id, - document_id as "document_id: Hyphenated", - relative_path, - updated_date as "updated_date: chrono::DateTime", - is_deleted, - user_id, - device_id, - length(content) as "content_size: u64" - from latest_document_versions - where vault_update_id > ? - order by vault_update_id + d.vault_update_id, + d.creation_vault_update_id, + d.document_id as "document_id: Hyphenated", + d.relative_path, + d.updated_date as "updated_date: chrono::DateTime", + d.is_deleted, + d.user_id, + d.device_id, + length(d.content) as "content_size: u64" + from documents d + inner join ( + select document_id, max(vault_update_id) as max_vid + from documents + where vault_update_id <= ? + group by document_id + ) latest_at_cursor + on d.document_id = latest_at_cursor.document_id + and d.vault_update_id = latest_at_cursor.max_vid + where d.vault_update_id > ? + order by d.vault_update_id "#, - vault_update_id + upper, + vault_update_id, ); - if let Some(transaction) = transaction { - query.fetch_all(&mut **transaction).await + if let Some(conn) = connection { + query.fetch_all(&mut *conn).await } else { query .fetch_all(&self.get_connection_pool(vault).await?) @@ -270,9 +567,18 @@ impl Database { is_deleted: row.is_deleted, user_id: row.user_id, device_id: row.device_id, - content_size: row - .content_size - .expect("Content size can't be null but sqlx can't infer it"), + content_size: row.content_size.unwrap_or(0), + // For catch-up streams, "new file" means "new to this + // recipient" — the doc was created past the recipient's + // watermark. The catch-up only carries the doc's + // *latest* version (not its full history), so using + // `creation == latest` instead would mis-flag every + // doc that was created and then updated before the + // client reconnected, and the client's + // `processRemoteChange` would drop it as "stale + // RemoteChange for untracked, non-new document", + // silently leaking docs to clients catching up. + is_new_file: row.creation_vault_update_id > vault_update_id, }) .collect() }) @@ -281,7 +587,7 @@ impl Database { pub async fn get_max_update_id_in_vault( &self, vault: &VaultId, - transaction: Option<&mut Transaction<'_>>, + connection: Option<&mut SqliteConnection>, ) -> Result { let query = sqlx::query!( r#" @@ -290,8 +596,8 @@ impl Database { "#, ); - if let Some(transaction) = transaction { - query.fetch_one(&mut **transaction).await + if let Some(conn) = connection { + query.fetch_one(&mut *conn).await } else { query .fetch_one(&self.get_connection_pool(vault).await?) @@ -301,17 +607,18 @@ impl Database { .context("Cannot fetch max update id in vault") } - pub async fn get_latest_document_by_path( + pub async fn get_latest_non_deleted_document_by_path( &self, vault: &VaultId, relative_path: &str, - transaction: Option<&mut Transaction<'_>>, + connection: Option<&mut SqliteConnection>, ) -> Result> { let query = sqlx::query_as!( StoredDocumentVersion, r#" select vault_update_id, + creation_vault_update_id, document_id as "document_id: Hyphenated", relative_path, updated_date as "updated_date: chrono::DateTime", @@ -330,8 +637,8 @@ impl Database { relative_path ); - if let Some(transaction) = transaction { - query.fetch_optional(&mut **transaction).await + if let Some(conn) = connection { + query.fetch_optional(&mut *conn).await } else { query .fetch_optional(&self.get_connection_pool(vault).await?) @@ -340,11 +647,79 @@ impl Database { .context("Cannot fetch latest document version") } + /// Find a doc whose CREATE was authored by this device with + /// matching content, and whose creation the requesting client + /// hasn't observed yet (`creation_vault_update_id > last_seen`). + /// Used by `create_document` to recover from a "lost create" + /// race: this device's create response was discarded mid-flight, + /// so the retry comes in as a brand-new create — possibly at a + /// renamed path. Binding the retry to the existing doc avoids + /// duplicating the content under a deconflicted path. + /// + /// Matches against the doc's CREATION version (not the latest) + /// because a same-path concurrent create from another agent may + /// have merged into our doc since: the latest version's content + /// is the merge result, not what we originally sent. Joining on + /// `creation_vault_update_id` recovers the original bytes. + /// + /// The `device_id` + `creation > last_seen` combination scopes + /// the dedup to "we genuinely lost track of our own create"; + /// another agent's same-content doc won't match because of + /// `device_id`, and a doc this client already saw won't match + /// because of the watermark check. + pub async fn find_unseen_lost_create_by_device_and_content( + &self, + vault: &VaultId, + device_id: &str, + last_seen_vault_update_id: VaultUpdateId, + content: &[u8], + connection: Option<&mut SqliteConnection>, + ) -> Result> { + let query = sqlx::query_as!( + StoredDocumentVersion, + r#" + select + lv.vault_update_id, + lv.creation_vault_update_id, + lv.document_id as "document_id: Hyphenated", + lv.relative_path, + lv.updated_date as "updated_date: chrono::DateTime", + lv.content, + lv.is_deleted, + lv.user_id, + lv.device_id, + lv.has_been_merged + from latest_document_versions lv + inner join documents creation + on creation.document_id = lv.document_id + and creation.vault_update_id = lv.creation_vault_update_id + where creation.device_id = ? + and creation.content = ? + and lv.is_deleted = false + and lv.creation_vault_update_id > ? + order by lv.creation_vault_update_id desc + limit 1 + "#, + device_id, + content, + last_seen_vault_update_id, + ); + + if let Some(conn) = connection { + query.fetch_optional(&mut *conn).await + } else { + query + .fetch_optional(&self.get_connection_pool(vault).await?) + .await + } + .context("Cannot fetch lost-create candidate") + } + pub async fn get_latest_document( &self, vault: &VaultId, document_id: &DocumentId, - transaction: Option<&mut Transaction<'_>>, + connection: Option<&mut SqliteConnection>, ) -> Result> { let document_id = document_id.as_hyphenated(); let query = sqlx::query_as!( @@ -352,6 +727,7 @@ impl Database { r#" select vault_update_id, + creation_vault_update_id, document_id as "document_id: Hyphenated", relative_path, updated_date as "updated_date: chrono::DateTime", @@ -366,8 +742,8 @@ impl Database { document_id ); - if let Some(transaction) = transaction { - query.fetch_optional(&mut **transaction).await + if let Some(conn) = connection { + query.fetch_optional(&mut *conn).await } else { query .fetch_optional(&self.get_connection_pool(vault).await?) @@ -380,13 +756,14 @@ impl Database { &self, vault: &VaultId, vault_update_id: VaultUpdateId, - transaction: Option<&mut Transaction<'_>>, + connection: Option<&mut SqliteConnection>, ) -> Result> { let query = sqlx::query_as!( StoredDocumentVersion, r#" select vault_update_id, + creation_vault_update_id, document_id as "document_id: Hyphenated", relative_path, updated_date as "updated_date: chrono::DateTime", @@ -400,8 +777,8 @@ impl Database { vault_update_id ); - if let Some(transaction) = transaction { - query.fetch_optional(&mut **transaction).await + if let Some(conn) = connection { + query.fetch_optional(&mut *conn).await } else { query .fetch_optional(&self.get_connection_pool(vault).await?) @@ -410,105 +787,307 @@ impl Database { .context("Cannot fetch document version") } - // inserting the document must be the last step of the transaction if there's one + // inserting the document must be the last step of the transaction pub async fn insert_document_version( &self, vault_id: &VaultId, version: &StoredDocumentVersion, - transaction: Option>, + mut transaction: WriteTransaction, ) -> Result<()> { let document_id = version.document_id.as_hyphenated(); let query = sqlx::query!( r#" insert into documents ( vault_update_id, + creation_vault_update_id, document_id, relative_path, updated_date, content, is_deleted, user_id, - device_id + device_id, + has_been_merged ) - values (?, ?, ?, ?, ?, ?, ?, ?) + values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) "#, version.vault_update_id, + version.creation_vault_update_id, document_id, version.relative_path, version.updated_date, version.content, version.is_deleted, version.user_id, - version.device_id + version.device_id, + version.has_been_merged ); - if let Some(mut transaction) = transaction { - query - .execute(&mut *transaction) - .await - .context("Cannot insert document version")?; + // Acquire the broadcast send lock before the insert so that + // broadcasts are serialized in vault_update_id order even after + // the write transaction (and its per-vault lock) is released. + let _send_guard = self.broadcasts.acquire_send_lock(vault_id).await; - transaction - .commit() - .await - .context("Failed to commit transaction")?; + query + .execute(&mut *transaction) + .await + .context("Cannot insert document version")?; + + transaction + .commit() + .await + .context("Failed to commit transaction")?; + + // For non-delete writes the originating device already has + // authoritative state from its HTTP response, so we tag the + // broadcast with `origin_device_id` and the send task in + // `websocket.rs` filters it out for that device. Deletes are + // delivered to *every* connected client including the author — + // the originator only removes the document from its sync queue + // once it receives this receipt. + let envelope = WebSocketServerMessage::VaultUpdate(WebSocketVaultUpdate { + document: version.clone().into(), + }); + let with_origin = if version.is_deleted { + WebSocketServerMessageWithOrigin::new(envelope) } else { - query - .execute(&self.get_connection_pool(vault_id).await?) - .await - .context("Cannot insert document version")?; - } - + WebSocketServerMessageWithOrigin::with_origin(version.device_id.clone(), envelope) + }; self.broadcasts - .send_document_update( - vault_id.clone(), - WebSocketServerMessageWithOrigin::with_origin( - version.device_id.clone(), - WebSocketServerMessage::VaultUpdate(WebSocketVaultUpdate { - documents: vec![version.clone().into()], - is_initial_sync: false, - }), - ), - ) - .await; + .send_document_update(vault_id.clone(), with_origin); Ok(()) } + /// Return all versions (without content) of a specific document, ordered by `vault_update_id` + pub async fn get_document_versions( + &self, + vault: &VaultId, + document_id: &DocumentId, + connection: Option<&mut SqliteConnection>, + ) -> Result> { + let document_id = document_id.as_hyphenated(); + let query = sqlx::query!( + r#" + select + vault_update_id, + creation_vault_update_id, + document_id as "document_id: Hyphenated", + relative_path, + updated_date as "updated_date: chrono::DateTime", + is_deleted, + user_id, + device_id, + length(content) as "content_size: u64" + from documents + where document_id = ? + order by vault_update_id + "#, + document_id, + ); + + if let Some(conn) = connection { + query.fetch_all(&mut *conn).await + } else { + query + .fetch_all(&self.get_connection_pool(vault).await?) + .await + } + .with_context(|| format!("Cannot fetch document versions for document `{document_id}`")) + .map(|rows| { + rows.into_iter() + .map(|row| DocumentVersionWithoutContent { + vault_update_id: row.vault_update_id, + document_id: row.document_id.into(), + relative_path: row.relative_path, + updated_date: row.updated_date, + is_deleted: row.is_deleted, + user_id: row.user_id, + device_id: row.device_id, + content_size: row.content_size.unwrap_or(0), + is_new_file: row.creation_vault_update_id == row.vault_update_id, + }) + .collect() + }) + } + + /// Return all versions across all documents, paginated, ordered by `vault_update_id` DESC + pub async fn get_vault_history( + &self, + vault: &VaultId, + limit: i64, + before_update_id: Option, + connection: Option<&mut SqliteConnection>, + ) -> Result> { + let map_row = |row: models::VaultHistoryRow| DocumentVersionWithoutContent { + vault_update_id: row.vault_update_id, + document_id: row.document_id, + relative_path: row.relative_path, + updated_date: row.updated_date, + is_deleted: row.is_deleted, + user_id: row.user_id, + device_id: row.device_id, + content_size: row.content_size.unwrap_or(0), + is_new_file: row.creation_vault_update_id == row.vault_update_id, + }; + + if let Some(before) = before_update_id { + let query = sqlx::query_as!( + models::VaultHistoryRow, + r#" + select + vault_update_id, + creation_vault_update_id, + document_id as "document_id: Hyphenated", + relative_path, + updated_date as "updated_date: chrono::DateTime", + is_deleted, + user_id, + device_id, + length(content) as "content_size: u64" + from documents + where vault_update_id < ? + order by vault_update_id desc + limit ? + "#, + before, + limit, + ); + + let rows = if let Some(conn) = connection { + query.fetch_all(&mut *conn).await + } else { + query + .fetch_all(&self.get_connection_pool(vault).await?) + .await + } + .context("Cannot fetch vault history")?; + + Ok(rows.into_iter().map(map_row).collect()) + } else { + let query = sqlx::query_as!( + models::VaultHistoryRow, + r#" + select + vault_update_id, + creation_vault_update_id, + document_id as "document_id: Hyphenated", + relative_path, + updated_date as "updated_date: chrono::DateTime", + is_deleted, + user_id, + device_id, + length(content) as "content_size: u64" + from documents + order by vault_update_id desc + limit ? + "#, + limit, + ); + + let rows = if let Some(conn) = connection { + query.fetch_all(&mut *conn).await + } else { + query + .fetch_all(&self.get_connection_pool(vault).await?) + .await + } + .context("Cannot fetch vault history")?; + + Ok(rows.into_iter().map(map_row).collect()) + } + } + /// Cleanup idle connection pools that haven't been accessed in more than 5 minutes async fn cleanup_idle_pools(&self) { - let mut pools = self.connection_pools.lock().await; - let now = Instant::now(); - let idle_timeout = Duration::from_secs(5 * 60); // 5 minutes + // Collect idle vaults and remove them from the map while holding + // the lock briefly. Close pools OUTSIDE the lock so that + // pool.close().await doesn't block other get_connection_pool calls. + let idle_pools: Vec<(VaultId, Arc)> = { + let mut pools = self.connection_pools.lock().await; + let now_ms = self.now_ms(); + let idle_threshold_ms = IDLE_POOL_TIMEOUT.as_millis() as u64; - // Collect vaults to remove - let vaults_to_remove: Vec = pools - .iter() - .filter(|(_, pool_with_timestamp)| { - now.duration_since(pool_with_timestamp.last_accessed) > idle_timeout + let vaults_to_remove: Vec = pools + .iter() + .filter(|(_, vp)| { + let last = vp.last_accessed_ms.load(Ordering::Relaxed); + now_ms.saturating_sub(last) > idle_threshold_ms + }) + .map(|(vault_id, _)| vault_id.clone()) + .collect(); + + vaults_to_remove + .into_iter() + .filter_map(|id| pools.remove(&id).map(|vp| (id, vp))) + .collect() + }; + + // Close pools concurrently so cleanup doesn't serialise across vaults + let closures: Vec<_> = idle_pools + .into_iter() + .filter_map(|(vault_id, vault_pool)| { + vault_pool + .cell + .get() + .cloned() + .map(|pools| (vault_id, pools)) }) - .map(|(vault_id, _)| vault_id.clone()) .collect(); - // Close and remove idle pools - for vault_id in &vaults_to_remove { - if let Some(pool_with_timestamp) = pools.remove(vault_id) { - info!("Closing idle database connection pool for vault `{vault_id}`"); - pool_with_timestamp.pool.close().await; - } + let handles: Vec<_> = closures + .into_iter() + .map(|(vault_id, pools)| { + tokio::spawn(async move { + // Checkpoint the WAL before closing to reclaim disk space. + // Run on the blocking pool so disk I/O doesn't starve the runtime + let writer_clone = pools.writer.clone(); + let ckpt_result = tokio::task::spawn_blocking(move || { + futures::executor::block_on( + sqlx::query("PRAGMA wal_checkpoint(TRUNCATE)").execute(&writer_clone), + ) + }) + .await; + + match ckpt_result { + Ok(Err(e)) => { + log::warn!("WAL checkpoint failed for vault `{vault_id}`: {e}"); + } + Err(e) => { + log::warn!("WAL checkpoint task panicked for vault `{vault_id}`: {e}"); + } + _ => {} + } + + info!("Closing idle database connection pools for vault `{vault_id}`"); + pools.reader.close().await; + pools.writer.close().await; + }) + }) + .collect(); + + for handle in handles { + let _ = handle.await; } } /// Start a background task that periodically cleans up idle connection pools - fn start_idle_pool_cleanup(&self) { + fn start_idle_pool_cleanup(&self, mut shutdown: tokio::sync::watch::Receiver<()>) { let database = self.clone(); tokio::spawn(async move { let mut interval = tokio::time::interval(Duration::from_secs(60)); // Check every minute interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); loop { - interval.tick().await; - database.cleanup_idle_pools().await; + tokio::select! { + _ = interval.tick() => { + database.cleanup_idle_pools().await; + } + _ = shutdown.changed() => { + info!("Idle pool cleanup task shutting down"); + break; + } + } } }); } diff --git a/sync-server/src/app_state/database/migrations/20260314000000_add_idempotency_key.sql b/sync-server/src/app_state/database/migrations/20260314000000_add_idempotency_key.sql new file mode 100644 index 00000000..f3ee8dd3 --- /dev/null +++ b/sync-server/src/app_state/database/migrations/20260314000000_add_idempotency_key.sql @@ -0,0 +1,2 @@ +CREATE INDEX IF NOT EXISTS idx_documents_document_id +ON documents (document_id, vault_update_id); diff --git a/sync-server/src/app_state/database/migrations/20260421000000_add_creation_vault_update_id.sql b/sync-server/src/app_state/database/migrations/20260421000000_add_creation_vault_update_id.sql new file mode 100644 index 00000000..40dc85fb --- /dev/null +++ b/sync-server/src/app_state/database/migrations/20260421000000_add_creation_vault_update_id.sql @@ -0,0 +1,20 @@ +ALTER TABLE documents ADD COLUMN creation_vault_update_id INTEGER NOT NULL DEFAULT 0; + +UPDATE documents +SET creation_vault_update_id = ( + SELECT MIN(d2.vault_update_id) + FROM documents d2 + WHERE d2.document_id = documents.document_id +); + +DROP VIEW latest_document_versions; + +CREATE VIEW IF NOT EXISTS latest_document_versions AS --recreate view as it now includes one more field +SELECT d.* +FROM documents d +INNER JOIN ( + SELECT MAX(vault_update_id) AS max_version_id + FROM documents + GROUP BY document_id +) max_versions +ON d.vault_update_id = max_versions.max_version_id; diff --git a/sync-server/src/app_state/database/models.rs b/sync-server/src/app_state/database/models.rs index a216125a..89867067 100644 --- a/sync-server/src/app_state/database/models.rs +++ b/sync-server/src/app_state/database/models.rs @@ -13,6 +13,7 @@ pub type DeviceId = String; #[derive(Debug, Clone)] pub struct StoredDocumentVersion { pub vault_update_id: VaultUpdateId, + pub creation_vault_update_id: VaultUpdateId, pub document_id: DocumentId, pub relative_path: String, pub updated_date: DateTime, @@ -33,7 +34,7 @@ impl PartialEq for StoredDocumentVersion { #[derive(TS, Debug, Clone, Serialize)] #[serde(rename_all = "camelCase")] pub struct DocumentVersionWithoutContent { - #[ts(as = "i32")] + #[ts(type = "number")] pub vault_update_id: VaultUpdateId, pub document_id: DocumentId, @@ -43,12 +44,16 @@ pub struct DocumentVersionWithoutContent { pub user_id: UserId, pub device_id: DeviceId, - #[ts(as = "i32")] + #[ts(type = "number")] pub content_size: u64, + + /// True iff this is the first version of the document + pub is_new_file: bool, } impl From for DocumentVersionWithoutContent { fn from(value: StoredDocumentVersion) -> Self { + let is_new_file = value.creation_vault_update_id == value.vault_update_id; Self { vault_update_id: value.vault_update_id, document_id: value.document_id, @@ -58,6 +63,7 @@ impl From for DocumentVersionWithoutContent { user_id: value.user_id, device_id: value.device_id, content_size: value.content.len() as u64, + is_new_file, } } } @@ -65,7 +71,7 @@ impl From for DocumentVersionWithoutContent { #[derive(TS, Debug, Clone, Serialize)] #[serde(rename_all = "camelCase")] pub struct DocumentVersion { - #[ts(as = "i32")] + #[ts(type = "number")] pub vault_update_id: VaultUpdateId, pub document_id: DocumentId, @@ -77,6 +83,25 @@ pub struct DocumentVersion { pub device_id: DeviceId, } +/// Row struct for vault history queries (used by `sqlx::query_as!`) +#[derive(Debug)] +pub struct VaultHistoryRow { + pub vault_update_id: VaultUpdateId, + pub creation_vault_update_id: VaultUpdateId, + pub document_id: DocumentId, + pub relative_path: String, + pub updated_date: DateTime, + pub is_deleted: bool, + pub user_id: String, + pub device_id: String, + pub content_size: Option, +} + +pub struct VaultStats { + pub created_at: Option>, + pub document_count: u32, +} + impl From for DocumentVersion { fn from(value: StoredDocumentVersion) -> Self { Self { diff --git a/sync-server/src/app_state/websocket/broadcasts.rs b/sync-server/src/app_state/websocket/broadcasts.rs index 60ae0219..b9e2ea39 100644 --- a/sync-server/src/app_state/websocket/broadcasts.rs +++ b/sync-server/src/app_state/websocket/broadcasts.rs @@ -1,69 +1,147 @@ -use std::{collections::HashMap, sync::Arc}; +use std::{ + collections::HashMap, + sync::{Arc, Mutex as StdMutex}, +}; -use anyhow::Context; -use log::{debug, warn}; +use log::{debug, info, warn}; use tokio::sync::{Mutex, broadcast}; -use super::models::WebSocketServerMessageWithOrigin; -use crate::{ - app_state::database::models::VaultId, config::server_config::ServerConfig, errors::server_error, -}; +use super::models::{WebSocketServerMessage, WebSocketServerMessageWithOrigin}; +use crate::{app_state::database::models::VaultId, config::server_config::ServerConfig}; #[derive(Debug, Clone)] pub struct Broadcasts { - max_clients_per_vault: usize, - tx: Arc>>>, + broadcast_channel_capacity: usize, + // `tx` uses a blocking std::sync::Mutex because the critical section is + // a HashMap lookup plus a synchronous `broadcast::Sender::send`. Making + // this non-async lets `send_document_update` run without an `.await`, + // so an axum handler that is cancelled between `transaction.commit()` + // and the broadcast can never drop the notification mid-flight. + tx: Arc>>>, + send_locks: Arc>>>>, } +type TxMap = HashMap>; + impl Broadcasts { pub fn new(server_config: &ServerConfig) -> Self { Self { - max_clients_per_vault: server_config.max_clients_per_vault, - tx: Arc::new(Mutex::new(HashMap::new())), + broadcast_channel_capacity: server_config.broadcast_channel_capacity, + tx: Arc::new(StdMutex::new(HashMap::new())), + send_locks: Arc::new(Mutex::new(HashMap::new())), } } - pub async fn get_receiver( + /// Acquire a per-vault lock that serializes broadcasts in commit order. + /// Must be acquired before the insert, held through commit and broadcast. + pub async fn acquire_send_lock(&self, vault: &VaultId) -> tokio::sync::OwnedMutexGuard<()> { + let lock = { + let mut locks = self.send_locks.lock().await; + locks + .entry(vault.clone()) + .or_insert_with(|| Arc::new(tokio::sync::Mutex::new(()))) + .clone() + }; + lock.lock_owned().await + } + + /// Remove senders for vaults with no active receivers + fn prune_inactive_vaults(tx_map: &mut TxMap) -> Vec { + let mut pruned = Vec::new(); + tx_map.retain(|vault, sender| { + let alive = sender.receiver_count() > 0; + if !alive { + pruned.push(vault.clone()); + } + alive + }); + pruned + } + + pub fn get_receiver( &self, vault: VaultId, - ) -> broadcast::Receiver { - let tx = self.get_or_create(vault).await; + max_clients: usize, + ) -> Result, crate::errors::SyncServerError> + { + let mut tx_map = self + .tx + .lock() + .expect("broadcasts.tx mutex poisoned — a previous holder panicked"); - tx.subscribe() + let count_before_prune = tx_map + .get(&vault) + .map_or(0, tokio::sync::broadcast::Sender::receiver_count); + let pruned = Self::prune_inactive_vaults(&mut tx_map); + let pruned_self = pruned.contains(&vault); + + let sender = tx_map + .entry(vault.clone()) + .or_insert_with(|| broadcast::channel(self.broadcast_channel_capacity).0); + + // Hold the lock across the count check *and* the subscribe so the + // `max_clients` cap is atomic: two concurrent callers can't both + // observe `receiver_count() < max_clients` and both subscribe. + if sender.receiver_count() >= max_clients { + return Err(crate::errors::client_error(anyhow::anyhow!( + "Vault has reached the maximum number of clients ({max_clients})" + ))); + } + + let receiver = sender.subscribe(); + let count_after = sender.receiver_count(); + info!( + "[BCAST] get_receiver vault={vault} count_before_prune={count_before_prune} pruned_self={pruned_self} pruned_total={} count_after_subscribe={count_after}", + pruned.len() + ); + Ok(receiver) } /// Notify all clients (who are subscribed to the vault) about an update. - /// We only log failures and don't propagate them. - pub async fn send_document_update( - &self, - vault: VaultId, - document: WebSocketServerMessageWithOrigin, - ) { - let tx = self.get_or_create(vault.clone()).await; + /// Synchronous: safe to invoke from a handler between `commit()` and + /// function return without worrying about task cancellation dropping + /// the broadcast mid-flight. Failures are logged, never propagated. + pub fn send_document_update(&self, vault: VaultId, document: WebSocketServerMessageWithOrigin) { + let vault_update_id = match &document.message { + WebSocketServerMessage::VaultUpdate(u) => Some(u.document.vault_update_id), + WebSocketServerMessage::CursorPositions(_) => None, + }; + let is_deleted = match &document.message { + WebSocketServerMessage::VaultUpdate(u) => Some(u.document.is_deleted), + WebSocketServerMessage::CursorPositions(_) => None, + }; + let mut tx_map = self + .tx + .lock() + .expect("broadcasts.tx mutex poisoned — a previous holder panicked"); + let count_before_prune = tx_map + .get(&vault) + .map_or(0, tokio::sync::broadcast::Sender::receiver_count); + let pruned = Self::prune_inactive_vaults(&mut tx_map); + let pruned_self = pruned.contains(&vault); - if tx.receiver_count() == 0 { + let sender = tx_map + .entry(vault.clone()) + .or_insert_with(|| broadcast::channel(self.broadcast_channel_capacity).0); + + let count_before_send = sender.receiver_count(); + + if count_before_send == 0 { + info!( + "[BCAST] send_document_update vault={vault} vuid={vault_update_id:?} is_deleted={is_deleted:?} count_before_prune={count_before_prune} pruned_self={pruned_self} count_before_send=0 SKIPPED" + ); debug!("Skipping broadcast, no clients connected for vault `{vault}`"); return; } - let result = tx - .send(document) - .context("Cannot broadcast server message to websocket listeners") - .map_err(server_error); - - if result.is_err() { - warn!("Failed to send message: {result:?}"); + let send_result = sender.send(document); + match &send_result { + Ok(n) => info!( + "[BCAST] send_document_update vault={vault} vuid={vault_update_id:?} is_deleted={is_deleted:?} count_before_prune={count_before_prune} pruned_self={pruned_self} count_before_send={count_before_send} SENT delivered_to={n}" + ), + Err(e) => warn!( + "[BCAST] send_document_update vault={vault} vuid={vault_update_id:?} is_deleted={is_deleted:?} count_before_prune={count_before_prune} pruned_self={pruned_self} count_before_send={count_before_send} FAILED err={e}" + ), } } - - async fn get_or_create( - &self, - vault: VaultId, - ) -> broadcast::Sender { - let mut tx = self.tx.lock().await; - - tx.entry(vault) - .or_insert_with(|| broadcast::channel(self.max_clients_per_vault).0.clone()) - .clone() - } } diff --git a/sync-server/src/app_state/websocket/models.rs b/sync-server/src/app_state/websocket/models.rs index e037fb7e..eb6c956a 100644 --- a/sync-server/src/app_state/websocket/models.rs +++ b/sync-server/src/app_state/websocket/models.rs @@ -11,7 +11,7 @@ pub struct WebSocketHandshake { pub token: String, pub device_id: DeviceId, - #[ts(as = "Option")] + #[ts(type = "number | null")] pub last_seen_vault_update_id: Option, } @@ -22,13 +22,14 @@ pub struct CursorPositionFromClient { } #[derive(TS, Serialize, Deserialize, Clone, Debug)] +#[serde(rename_all = "camelCase")] pub struct DocumentWithCursors { // It's None in case the document is dirty. // We still want to sync the cursor to mark // that it exists and can be client-side // interpolated. However, the actual // position is meaningless. - #[ts(as = "Option")] + #[ts(type = "number | null")] pub vault_update_id: Option, pub document_id: DocumentId, @@ -57,11 +58,19 @@ pub struct CursorPositionFromServer { pub clients: Vec, } +// One committed version. Non-delete updates are broadcast to every +// connected client *except* the device that authored them — that +// device already has the new state via its HTTP response. Deletes are +// broadcast to every client including the author: the author keeps +// the document in its sync queue until this receipt arrives so a late +// remote update can't sneak in between the HTTP response and the +// queue cleanup. The server also emits these one-at-a-time to catch +// up a freshly-connected client on versions committed while it was +// offline, in ascending `vault_update_id` order. #[derive(TS, Serialize, Clone, Debug)] #[serde(rename_all = "camelCase")] pub struct WebSocketVaultUpdate { - pub documents: Vec, - pub is_initial_sync: bool, + pub document: DocumentVersionWithoutContent, } #[derive(TS, Deserialize, Clone, Debug)] @@ -80,6 +89,10 @@ pub enum WebSocketServerMessage { CursorPositions(CursorPositionFromServer), } +/// Broadcast envelope carrying the message plus the device that produced +/// it. The per-recipient send task compares `origin_device_id` against +/// its own device id to fill in `originates_from_self` before the message +/// is serialized on the wire. #[derive(Clone, Debug)] pub struct WebSocketServerMessageWithOrigin { pub origin_device_id: Option, diff --git a/sync-server/src/app_state/websocket/utils.rs b/sync-server/src/app_state/websocket/utils.rs index 1e0dd243..d78360de 100644 --- a/sync-server/src/app_state/websocket/utils.rs +++ b/sync-server/src/app_state/websocket/utils.rs @@ -9,7 +9,7 @@ use crate::{ database::models::{DocumentVersionWithoutContent, VaultId, VaultUpdateId}, }, config::user_config::User, - errors::{SyncServerError, server_error, unauthenticated_error}, + errors::{SyncServerError, client_error, server_error, unauthenticated_error}, server::auth::auth, }; @@ -26,7 +26,7 @@ pub fn get_authenticated_handshake( if let Some(Message::Text(message)) = message { let message: WebSocketClientMessage = serde_json::from_str(&message) .context("Failed to parse message") - .map_err(server_error)?; + .map_err(client_error)?; match message { WebSocketClientMessage::Handshake(handshake) => { @@ -44,21 +44,29 @@ pub fn get_authenticated_handshake( } } +/// Stream the documents the client missed while offline, bounded above +/// by `up_to_vault_update_id` so the catch-up is a stable snapshot at +/// exactly that cursor. The WebSocket handshake atomically subscribes +/// to the broadcast channel and snapshots this cursor under the per- +/// vault send lock; commits past the cursor are then delivered solely +/// through the broadcast channel (filtered by the same cursor on the +/// receive side), so every committed update is delivered exactly once. pub async fn get_unseen_documents( state: &AppState, vault_id: &VaultId, last_seen_vault_update_id: Option, + up_to_vault_update_id: VaultUpdateId, ) -> Result, SyncServerError> { if let Some(update_id) = last_seen_vault_update_id { state .database - .get_latest_documents_since(vault_id, update_id, None) + .get_latest_documents_since(vault_id, update_id, Some(up_to_vault_update_id), None) .await .map_err(server_error) } else { state .database - .get_latest_documents(vault_id, None) + .get_latest_documents(vault_id, Some(up_to_vault_update_id), None) .await .map_err(server_error) } diff --git a/sync-server/src/config.rs b/sync-server/src/config.rs index 6a003d2e..26b11a4c 100644 --- a/sync-server/src/config.rs +++ b/sync-server/src/config.rs @@ -27,24 +27,34 @@ pub struct Config { } impl Config { + pub fn validate(&self) -> Result<()> { + self.server + .validate() + .context("Invalid server configuration")?; + self.logging + .validate() + .context("Invalid logging configuration")?; + self.database + .validate() + .context("Invalid database configuration")?; + Ok(()) + } + pub async fn read_or_create(path: &Path) -> Result { - let config = if path.exists() { - info!( - "Loading configuration from `{}`", - path.canonicalize().unwrap().display() - ); - Self::load_from_file(path).await? + let display_path = path.canonicalize().unwrap_or_else(|_| path.to_path_buf()); + + if path.exists() { + info!("Loading configuration from `{}`", display_path.display()); + Self::load_from_file(path).await } else { - Self::default() - }; - - config.write(path).await?; - info!( - "Updated configuration at `{}`", - path.canonicalize().unwrap().display() - ); - - Ok(config) + let config = Self::default(); + config.write(path).await?; + info!( + "Created default configuration at `{}`", + display_path.display() + ); + Ok(config) + } } pub async fn load_from_file(path: &Path) -> Result { diff --git a/sync-server/src/config/database_config.rs b/sync-server/src/config/database_config.rs index 20a9a21e..a6f57e1f 100644 --- a/sync-server/src/config/database_config.rs +++ b/sync-server/src/config/database_config.rs @@ -1,5 +1,6 @@ use std::{path::PathBuf, time::Duration}; +use anyhow::{Result, ensure}; use log::debug; use serde::{Deserialize, Serialize}; @@ -34,6 +35,24 @@ fn default_cursor_timeout() -> Duration { DEFAULT_CURSOR_TIMEOUT } +impl DatabaseConfig { + pub fn validate(&self) -> Result<()> { + ensure!( + !self.databases_directory_path.as_os_str().is_empty(), + "databases_directory_path must not be empty" + ); + ensure!( + self.max_connections_per_vault > 0, + "max_connections_per_vault must be greater than 0" + ); + ensure!( + !self.cursor_timeout.is_zero(), + "cursor_timeout must be greater than 0" + ); + Ok(()) + } +} + impl Default for DatabaseConfig { fn default() -> Self { Self { diff --git a/sync-server/src/config/logging_config.rs b/sync-server/src/config/logging_config.rs index ad449d1a..dae67288 100644 --- a/sync-server/src/config/logging_config.rs +++ b/sync-server/src/config/logging_config.rs @@ -1,10 +1,13 @@ use std::time::Duration; +use anyhow::{Result, ensure}; use log::debug; use serde::{Deserialize, Serialize}; use crate::{ - consts::{DEFAULT_LOG_DIRECTORY, DEFAULT_LOG_LEVEL, DEFAULT_LOG_ROTATION_INTERVAL}, + consts::{ + DEFAULT_LOG_DIRECTORY, DEFAULT_LOG_LEVEL, DEFAULT_LOG_ROTATION_INTERVAL, DURATION_ZERO, + }, utils::log_level::LogLevel, }; @@ -20,6 +23,20 @@ pub struct LoggingConfig { pub log_level: LogLevel, } +impl LoggingConfig { + pub fn validate(&self) -> Result<()> { + ensure!( + !self.log_directory.is_empty(), + "log_directory must not be an empty string" + ); + ensure!( + self.log_rotation > DURATION_ZERO, + "log_rotation must be greater than 0" + ); + Ok(()) + } +} + impl Default for LoggingConfig { fn default() -> Self { Self { diff --git a/sync-server/src/config/server_config.rs b/sync-server/src/config/server_config.rs index 4a9da0f4..715d216c 100644 --- a/sync-server/src/config/server_config.rs +++ b/sync-server/src/config/server_config.rs @@ -1,10 +1,13 @@ +use anyhow::{Result, ensure}; use log::debug; use serde::{Deserialize, Serialize}; use std::time::Duration; use crate::consts::{ - DEFAULT_HOST, DEFAULT_MAX_BODY_SIZE_MB, DEFAULT_MAX_CLIENTS_PER_VAULT, - DEFAULT_MERGEABLE_FILE_EXTENSIONS, DEFAULT_PORT, DEFAULT_RESPONSE_TIMEOUT_SECONDS, + DEFAULT_ALLOWED_ORIGINS, DEFAULT_BROADCAST_CHANNEL_CAPACITY, DEFAULT_HOST, + DEFAULT_MAX_BODY_SIZE_MB, DEFAULT_MAX_CLIENTS_PER_VAULT, DEFAULT_MAX_PENDING_WS_CONNECTIONS, + DEFAULT_MERGEABLE_FILE_EXTENSIONS, DEFAULT_PORT, DEFAULT_RATE_LIMIT_PER_USER_PER_SECOND, + DEFAULT_RESPONSE_TIMEOUT_SECONDS, DURATION_ZERO, }; #[derive(Debug, Deserialize, Serialize, Clone, Default)] @@ -21,11 +24,56 @@ pub struct ServerConfig { #[serde(default = "default_max_clients_per_vault")] pub max_clients_per_vault: usize, + #[serde(default = "default_broadcast_channel_capacity")] + pub broadcast_channel_capacity: usize, + #[serde(default = "default_response_timeout", with = "humantime_serde")] pub response_timeout: Duration, #[serde(default = "default_mergeable_file_extensions")] pub mergeable_file_extensions: Vec, + + /// Per-user maximum requests per second (keyed by bearer token). + /// `None` disables rate limiting. + #[serde(default = "default_rate_limit_per_user_per_second")] + pub rate_limit_per_user_per_second: Option, + + /// Allowed CORS origins. Default: `["*"]` (allow all). + #[serde(default = "default_allowed_origins")] + pub allowed_origins: Vec, + + /// Maximum concurrent unauthenticated WebSocket connections waiting for + /// handshake. Limits resource consumption from clients that connect but + /// never authenticate. + #[serde(default = "default_max_pending_websocket_connections")] + pub max_pending_websocket_connections: usize, +} + +impl ServerConfig { + pub fn validate(&self) -> Result<()> { + ensure!( + self.response_timeout > DURATION_ZERO, + "response_timeout must be greater than 0" + ); + ensure!( + self.max_body_size_mb > 0, + "max_body_size_mb must be greater than 0" + ); + ensure!( + self.max_clients_per_vault > 0, + "max_clients_per_vault must be greater than 0" + ); + ensure!( + self.broadcast_channel_capacity > 0, + "broadcast_channel_capacity must be greater than 0" + ); + ensure!( + self.max_pending_websocket_connections > 0, + "max_pending_websocket_connections must be greater than 0" + ); + + Ok(()) + } } fn default_host() -> String { @@ -48,6 +96,11 @@ fn default_max_clients_per_vault() -> usize { DEFAULT_MAX_CLIENTS_PER_VAULT } +fn default_broadcast_channel_capacity() -> usize { + debug!("Using default broadcast channel capacity: {DEFAULT_BROADCAST_CHANNEL_CAPACITY}"); + DEFAULT_BROADCAST_CHANNEL_CAPACITY +} + fn default_response_timeout() -> Duration { debug!("Using default response timeout: {DEFAULT_RESPONSE_TIMEOUT_SECONDS:?}"); DEFAULT_RESPONSE_TIMEOUT_SECONDS @@ -60,3 +113,21 @@ fn default_mergeable_file_extensions() -> Vec { .map(|s| (*s).to_owned()) .collect() } + +fn default_rate_limit_per_user_per_second() -> Option { + debug!("Using default rate limit per second: {DEFAULT_RATE_LIMIT_PER_USER_PER_SECOND:?}"); + DEFAULT_RATE_LIMIT_PER_USER_PER_SECOND +} + +fn default_allowed_origins() -> Vec { + debug!("Using default allowed origins: {DEFAULT_ALLOWED_ORIGINS:?}"); + DEFAULT_ALLOWED_ORIGINS + .iter() + .map(|s| (*s).to_owned()) + .collect() +} + +fn default_max_pending_websocket_connections() -> usize { + debug!("Using default max pending WebSocket connections: {DEFAULT_MAX_PENDING_WS_CONNECTIONS}"); + DEFAULT_MAX_PENDING_WS_CONNECTIONS +} diff --git a/sync-server/src/config/user_config.rs b/sync-server/src/config/user_config.rs index 8b2537f0..fd824f39 100644 --- a/sync-server/src/config/user_config.rs +++ b/sync-server/src/config/user_config.rs @@ -1,6 +1,7 @@ use bimap::BiHashMap; use rand::{Rng, distr::Alphanumeric, rng}; use serde::{Deserialize, Deserializer, Serialize, de::Error}; +use subtle::ConstantTimeEq; use crate::app_state::database::models::VaultId; @@ -19,10 +20,19 @@ where let mut user_token_map = BiHashMap::new(); for user in &users { if let Some(existing_name) = user_token_map.get_by_right(&user.token) { + let redacted = if user.token.len() > 6 { + format!( + "{}...{}", + &user.token[..3], + &user.token[user.token.len() - 3..] + ) + } else { + "***".to_owned() + }; return Err(D::Error::custom(format!( - "Duplicate user token found: `{}` for users `{}` and `{}`. User tokens must be \ - unique.", - user.token, existing_name, user.name + "Duplicate user token found: `{redacted}` for users `{}` and `{}`. User tokens \ + must be unique.", + existing_name, user.name ))); } @@ -41,7 +51,9 @@ where impl UserConfig { pub fn get_user(&self, token: &str) -> Option<&User> { - self.user_configs.iter().find(|u| u.token == token) + self.user_configs + .iter() + .find(|u| u.token.as_bytes().ct_eq(token.as_bytes()).into()) } } diff --git a/sync-server/src/consts.rs b/sync-server/src/consts.rs index 98ed1c1f..e03b848f 100644 --- a/sync-server/src/consts.rs +++ b/sync-server/src/consts.rs @@ -2,22 +2,36 @@ use std::time::Duration; use crate::utils::log_level::LogLevel; +pub const DURATION_ZERO: Duration = Duration::from_secs(0); + pub const DEFAULT_CONFIG_PATH: &str = "config.yml"; pub const DEFAULT_DATABASES_DIRECTORY_PATH: &str = "databases"; -pub const DEFAULT_MAX_CONNECTIONS_PER_VAULT: u32 = 12; +pub const DEFAULT_MAX_CONNECTIONS_PER_VAULT: u32 = 6; pub const DEFAULT_CURSOR_TIMEOUT: Duration = Duration::from_secs(60); pub const DEFAULT_HOST: &str = "127.0.0.1"; pub const DEFAULT_PORT: u16 = 3000; pub const DEFAULT_MAX_BODY_SIZE_MB: usize = 4096; -pub const DEFAULT_RESPONSE_TIMEOUT_SECONDS: Duration = Duration::from_secs(1800); +pub const DEFAULT_RESPONSE_TIMEOUT_SECONDS: Duration = Duration::from_mins(30); pub const DEFAULT_MAX_CLIENTS_PER_VAULT: usize = 256; +pub const DEFAULT_BROADCAST_CHANNEL_CAPACITY: usize = 4096; +pub const DEFAULT_MAX_PENDING_WS_CONNECTIONS: usize = 128; pub const DEFAULT_LOG_DIRECTORY: &str = "logs"; -pub const DEFAULT_LOG_ROTATION_INTERVAL: Duration = Duration::from_secs(60 * 60 * 24); // 1 day +pub const DEFAULT_LOG_ROTATION_INTERVAL: Duration = Duration::from_hours(24); +pub const IDLE_POOL_TIMEOUT: Duration = Duration::from_mins(5); +pub const GRACEFUL_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(10); +pub const HANDSHAKE_TIMEOUT: Duration = Duration::from_secs(10); + +pub const MAX_CURSOR_DOCUMENTS: usize = 1000; +pub const MAX_CURSORS_PER_DOCUMENT: usize = 100; +pub const MAX_RELATIVE_PATH_LEN: usize = 4096; + pub const DEFAULT_LOG_LEVEL: LogLevel = LogLevel::Info; pub const DEFAULT_MERGEABLE_FILE_EXTENSIONS: &[&str] = &["md", "txt"]; -pub const SUPPORTED_API_VERSION: u32 = 2; +pub const DEFAULT_RATE_LIMIT_PER_USER_PER_SECOND: Option = None; +pub const DEFAULT_ALLOWED_ORIGINS: &[&str] = &["*"]; +pub const SUPPORTED_API_VERSION: u32 = 3; diff --git a/sync-server/src/errors.rs b/sync-server/src/errors.rs index 831b0e86..892db36f 100644 --- a/sync-server/src/errors.rs +++ b/sync-server/src/errors.rs @@ -5,7 +5,7 @@ use axum::{ http::StatusCode, response::{IntoResponse, Response}, }; -use log::{debug, error}; +use log::{debug, error, warn}; use serde::Serialize; use thiserror::Error; use ts_rs::TS; @@ -29,6 +29,9 @@ pub enum SyncServerError { #[error("Permission denied error: {0}")] PermissionDeniedError(#[source] anyhow::Error), + + #[error("Too many requests: {0}")] + TooManyRequests(#[source] anyhow::Error), } impl SyncServerError { @@ -39,7 +42,8 @@ impl SyncServerError { | Self::ServerError(error) | Self::NotFound(error) | Self::Unauthenticated(error) - | Self::PermissionDeniedError(error) => error.into(), + | Self::PermissionDeniedError(error) + | Self::TooManyRequests(error) => error.into(), } } } @@ -69,7 +73,22 @@ impl Display for SerializedError { impl IntoResponse for SyncServerError { fn into_response(self) -> Response { - let body = Json(self.serialize()); + let serialized = self.serialize(); + + match &self { + Self::InitError(_) | Self::ServerError(_) => { + error!("{serialized}"); + } + Self::ClientError(_) | Self::NotFound(_) => { + warn!("{serialized}"); + } + Self::TooManyRequests(_) => { + warn!("{serialized}"); + } + Self::Unauthenticated(_) | Self::PermissionDeniedError(_) => {} + } + + let body = Json(serialized); match self { Self::InitError(_) | Self::ServerError(_) => { @@ -79,6 +98,7 @@ impl IntoResponse for SyncServerError { Self::NotFound(_) => (StatusCode::NOT_FOUND, body).into_response(), Self::Unauthenticated(_) => (StatusCode::UNAUTHORIZED, body).into_response(), Self::PermissionDeniedError(_) => (StatusCode::FORBIDDEN, body).into_response(), + Self::TooManyRequests(_) => (StatusCode::TOO_MANY_REQUESTS, body).into_response(), } } } @@ -102,6 +122,7 @@ impl From<&anyhow::Error> for SerializedError { SyncServerError::NotFound(_) => "NotFound", SyncServerError::Unauthenticated(_) => "Unauthenticated", SyncServerError::PermissionDeniedError(_) => "PermissionDeniedError", + SyncServerError::TooManyRequests(_) => "TooManyRequests", }, ), message: error.to_string(), @@ -139,3 +160,21 @@ pub fn permission_denied_error(error: anyhow::Error) -> SyncServerError { debug!("Permission denied: {error:?}"); SyncServerError::PermissionDeniedError(error) } + +pub fn too_many_requests_error(error: anyhow::Error) -> SyncServerError { + debug!("Too many requests: {error:?}"); + SyncServerError::TooManyRequests(error) +} + +/// Maps a `create_write_transaction` error to 429 if the database is busy, +/// or 500 for all other failures. +pub fn write_transaction_error(error: anyhow::Error) -> SyncServerError { + if error + .downcast_ref::() + .is_some() + { + too_many_requests_error(error) + } else { + server_error(error) + } +} diff --git a/sync-server/src/main.rs b/sync-server/src/main.rs index 1285ed7b..dc00d4d5 100644 --- a/sync-server/src/main.rs +++ b/sync-server/src/main.rs @@ -16,6 +16,7 @@ use consts::DEFAULT_CONFIG_PATH; use errors::{SyncServerError, init_error}; use log::info; use server::create_server; +use tracing_appender::non_blocking::WorkerGuard; use tracing_subscriber::{EnvFilter, fmt::format, layer::SubscriberExt, util::SubscriberInitExt}; use utils::rotating_file_writer::RotatingFileWriter; @@ -41,11 +42,14 @@ async fn main() -> ExitCode { } }; - let mut result = set_up_logging(&args, &config.logging); - - if result.is_ok() { - result = start_server(config).await; + let result = async { + config.validate().map_err(init_error)?; + // Hold the non-blocking writer guards until shutdown so the + // dedicated writer threads stay alive and flush queued log lines. + let _log_guards = set_up_logging(&args, &config.logging)?; + start_server(config).await } + .await; match result { Ok(()) => ExitCode::SUCCESS, @@ -59,7 +63,7 @@ async fn main() -> ExitCode { fn set_up_logging( args: &Args, logging_config: &config::logging_config::LoggingConfig, -) -> Result<(), SyncServerError> { +) -> Result<[WorkerGuard; 2], SyncServerError> { let level_filter = logging_config.log_level.as_tracing_level(); let env_filter = EnvFilter::builder() @@ -80,6 +84,14 @@ fn set_up_logging( .context("Failed to create rotating file writer") .map_err(init_error)?; + // Decouple log emission from disk/stderr I/O. Without this, a tokio + // worker that holds the writer's std::sync::Mutex while a `write(2)` + // is throttled by the kernel (e.g. btrfs writeback) cascades the + // stall to every other worker that tries to log, freezing the whole + // runtime. The guards must outlive every emitter. + let (file_writer, file_guard) = tracing_appender::non_blocking(file_appender); + let (stderr_writer, stderr_guard) = tracing_appender::non_blocking(std::io::stderr()); + let format = format() .with_target(is_debug_mode) .with_line_number(is_debug_mode) @@ -87,12 +99,12 @@ fn set_up_logging( let stderr_layer = tracing_subscriber::fmt::layer() .with_ansi(use_colors) - .with_writer(std::io::stderr) + .with_writer(stderr_writer) .event_format(format.clone()); let file_layer = tracing_subscriber::fmt::layer() .with_ansi(false) - .with_writer(file_appender) + .with_writer(file_writer) .event_format(format); tracing_subscriber::registry() @@ -103,7 +115,7 @@ fn set_up_logging( .context("Failed to initialise tracing") .map_err(init_error)?; - Ok(()) + Ok([file_guard, stderr_guard]) } async fn start_server(config: Config) -> Result<(), SyncServerError> { diff --git a/sync-server/src/server.rs b/sync-server/src/server.rs index 01b09cf6..934e9428 100644 --- a/sync-server/src/server.rs +++ b/sync-server/src/server.rs @@ -4,27 +4,30 @@ mod delete_document; mod device_id_header; mod fetch_document_version; mod fetch_document_version_content; +mod fetch_document_versions; mod fetch_latest_document_version; mod fetch_latest_documents; +mod fetch_vault_history; mod index; +mod list_vaults; mod ping; +mod rate_limit; mod requests; mod responses; mod update_document; mod websocket; -use anyhow::{Context as _, Result, anyhow}; +use anyhow::{Context as _, Result}; use auth::auth_middleware; use axum::{ Router, extract::{DefaultBodyLimit, Request}, http::{self, HeaderValue, Method}, middleware, - response::IntoResponse, routing::{IntoMakeService, delete, get, post, put}, }; use device_id_header::DEVICE_ID_HEADER_NAME; -use log::info; +use log::{info, warn}; use tokio::signal; use tower_http::{ LatencyUnit, @@ -41,7 +44,7 @@ use tracing::{Level, info_span}; use crate::{ app_state::AppState, config::{Config, server_config::ServerConfig}, - errors::{client_error, not_found_error}, + consts::GRACEFUL_SHUTDOWN_TIMEOUT, }; pub async fn create_server(config: Config) -> Result<()> { @@ -51,26 +54,33 @@ pub async fn create_server(config: Config) -> Result<()> { let server_config = app_state.config.server.clone(); - let app = Router::new() + let mut app = Router::new() .nest("/", get_authed_routes(app_state.clone())) .route("/", get(index::index)) + .route("/assets/*path", get(index::spa_assets)) + .route("/vaults", get(list_vaults::list_vaults)) .route("/vaults/:vault_id/ping", get(ping::ping)) .route("/vaults/:vault_id/ws", get(websocket::websocket_handler)) + .fallback(index::spa_fallback); + + let cors_layer = build_cors_layer(&server_config).context("Invalid CORS configuration")?; + + if let Some(rate_limit) = server_config.rate_limit_per_user_per_second { + info!("Rate limiting enabled: {rate_limit} requests/second per user"); + let limiter = rate_limit::RateLimiter::new(rate_limit); + app = app.layer(middleware::from_fn_with_state( + limiter, + rate_limit::rate_limit_middleware, + )); + } + + let app = app .layer(DefaultBodyLimit::disable()) .layer(RequestBodyLimitLayer::new( app_state.config.server.max_body_size_mb * 1024 * 1024, )) .layer(TimeoutLayer::new(server_config.response_timeout)) - .layer( - CorsLayer::new() - .allow_origin("*".parse::().expect("Failed to parse origin")) - .allow_headers([ - http::header::CONTENT_TYPE, - http::header::AUTHORIZATION, - DEVICE_ID_HEADER_NAME.clone(), - ]) - .allow_methods([Method::GET, Method::POST, Method::PUT, Method::DELETE]), - ) + .layer(cors_layer) .layer( TraceLayer::new_for_http() .make_span_with(|request: &Request<_>| { @@ -90,12 +100,39 @@ pub async fn create_server(config: Config) -> Result<()> { .on_eos(DefaultOnEos::new()) .on_failure(DefaultOnFailure::new().level(Level::ERROR)), ) - .with_state(app_state) - .fallback(handle_404) - .fallback(handle_405) + .with_state(app_state.clone()) .into_make_service(); - start_server(app, &server_config).await + start_server(app, &server_config, app_state).await +} + +fn build_cors_layer(server_config: &ServerConfig) -> Result { + let origins = &server_config.allowed_origins; + + let cors = if origins.len() == 1 && origins[0] == "*" { + info!("CORS: allowing all origins"); + let header: HeaderValue = "*" + .parse() + .context("Failed to parse wildcard CORS origin")?; + CorsLayer::new().allow_origin(header) + } else { + let parsed: Vec = origins + .iter() + .map(|o| { + o.parse::() + .with_context(|| format!("Failed to parse CORS origin: `{o}`")) + }) + .collect::>>()?; + CorsLayer::new().allow_origin(parsed) + }; + + Ok(cors + .allow_headers([ + http::header::CONTENT_TYPE, + http::header::AUTHORIZATION, + DEVICE_ID_HEADER_NAME.clone(), + ]) + .allow_methods([Method::GET, Method::POST, Method::PUT, Method::DELETE])) } fn get_authed_routes(app_state: AppState) -> Router { @@ -120,6 +157,10 @@ fn get_authed_routes(app_state: AppState) -> Router { "/vaults/:vault_id/documents/:document_id/text", put(update_document::update_text), ) + .route( + "/vaults/:vault_id/documents/:document_id/versions", + get(fetch_document_versions::fetch_document_versions), + ) .route( "/vaults/:vault_id/documents/:document_id/versions/:vault_update_id", get(fetch_document_version::fetch_document_version), @@ -132,10 +173,18 @@ fn get_authed_routes(app_state: AppState) -> Router { "/vaults/:vault_id/documents/:document_id", delete(delete_document::delete_document), ) + .route( + "/vaults/:vault_id/history", + get(fetch_vault_history::fetch_vault_history), + ) .layer(middleware::from_fn_with_state(app_state, auth_middleware)) } -async fn start_server(app: IntoMakeService, config: &ServerConfig) -> Result<()> { +async fn start_server( + app: IntoMakeService, + config: &ServerConfig, + app_state: AppState, +) -> Result<()> { let address = format!("{}:{}", config.host, config.port); let listener = tokio::net::TcpListener::bind(address.clone()) .await @@ -148,26 +197,46 @@ async fn start_server(app: IntoMakeService, config: &ServerConfig) .context("Failed to get local address")? ); - axum::serve(listener, app) - .with_graceful_shutdown(shutdown_signal()) - .tcp_nodelay(true) - .await - .context("Failed to start server") + let mut shutdown_rx = app_state.subscribe_shutdown(); + + let server = axum::serve(listener, app) + .with_graceful_shutdown(async move { + shutdown_signal().await; + app_state.shutdown(); + }) + .tcp_nodelay(true); + + tokio::select! { + result = server => result.context("Failed to start server"), + () = async { + let _ = shutdown_rx.changed().await; + info!( + "Shutdown signal received, waiting up to {}s for in-flight requests to complete...", + GRACEFUL_SHUTDOWN_TIMEOUT.as_secs() + ); + tokio::time::sleep(GRACEFUL_SHUTDOWN_TIMEOUT).await; + warn!("Graceful shutdown timed out, forcing exit"); + } => Ok(()), + } } async fn shutdown_signal() { let ctrl_c = async { - signal::ctrl_c() - .await - .expect("failed to install Ctrl+C handler"); + if let Err(e) = signal::ctrl_c().await { + log::error!("Failed to install Ctrl+C handler: {e}"); + } }; #[cfg(unix)] let terminate = async { - signal::unix::signal(signal::unix::SignalKind::terminate()) - .expect("failed to install signal handler") - .recv() - .await; + match signal::unix::signal(signal::unix::SignalKind::terminate()) { + Ok(mut signal) => { + signal.recv().await; + } + Err(e) => { + log::error!("Failed to install SIGTERM handler: {e}"); + } + } }; #[cfg(not(unix))] @@ -178,11 +247,3 @@ async fn shutdown_signal() { () = terminate => {}, } } - -async fn handle_404() -> impl IntoResponse { - not_found_error(anyhow!("Page not found")) -} - -async fn handle_405() -> impl IntoResponse { - client_error(anyhow!("Method not allowed")) -} diff --git a/sync-server/src/server/auth.rs b/sync-server/src/server/auth.rs index e56f4acc..7fa45abd 100644 --- a/sync-server/src/server/auth.rs +++ b/sync-server/src/server/auth.rs @@ -9,7 +9,7 @@ use axum_extra::{ TypedHeader, headers::{Authorization, authorization::Bearer}, }; -use log::info; +use log::{debug, info}; use crate::{ app_state::{AppState, database::models::VaultId}, @@ -21,10 +21,12 @@ use crate::{ pub async fn auth_middleware( State(state): State, Path(path_params): Path>, - TypedHeader(auth_header): TypedHeader>, + auth_header: Option>>, mut req: Request, next: Next, ) -> Result { + let auth_header = auth_header + .ok_or_else(|| unauthenticated_error(anyhow::anyhow!("Missing Authorization header")))?; let token = auth_header.token().trim(); let vault_id = normalize_string( path_params @@ -39,20 +41,24 @@ pub async fn auth_middleware( Ok(next.run(req).await) } -pub fn auth(state: &AppState, token: &str, vault_id: &VaultId) -> Result { - let user = state +pub fn authenticate(state: &AppState, token: &str) -> Result { + state .config .users .get_user(token) .cloned() - .ok_or_else(|| unauthenticated_error(anyhow::anyhow!("Invalid token")))?; + .ok_or_else(|| unauthenticated_error(anyhow::anyhow!("Invalid token"))) +} + +pub fn auth(state: &AppState, token: &str, vault_id: &VaultId) -> Result { + let user = authenticate(state, token)?; if match user.vault_access { VaultAccess::AllowAccessToAll => true, VaultAccess::AllowList(AllowListedVaults { ref allowed }) => allowed.contains(vault_id), } { - info!( - "User `{}` is authenticated and is authorised to access to vault `{vault_id}`", + debug!( + "User `{}` is authenticated and is authorised to access vault `{vault_id}`", user.name ); diff --git a/sync-server/src/server/create_document.rs b/sync-server/src/server/create_document.rs index 859c0db4..d772e16a 100644 --- a/sync-server/src/server/create_document.rs +++ b/sync-server/src/server/create_document.rs @@ -11,12 +11,14 @@ use super::{device_id_header::DeviceIdHeader, requests::CreateDocumentVersion}; use crate::{ app_state::{ AppState, - database::models::{DocumentVersionWithoutContent, StoredDocumentVersion, VaultId}, + database::models::{StoredDocumentVersion, VaultId}, }, config::user_config::User, - errors::{SyncServerError, client_error, server_error}, + errors::{SyncServerError, client_error, server_error, write_transaction_error}, + server::{responses::DocumentUpdateResponse, update_document}, utils::{ - find_first_available_path::find_first_available_path, normalize::normalize, + find_first_available_path::find_first_available_path, is_binary::is_binary, + is_file_type_mergable::is_file_type_mergable, normalize::normalize, sanitize_path::sanitize_path, }, }; @@ -30,48 +32,137 @@ pub struct CreateDocumentPathParams { /// Create a new document in case a document with the same doesn't exist /// already. If a document with the same path exists, a new version is created /// with their content merged. +/// +/// Text content must be UTF-8 encoded. Clients are responsible for +/// transcoding other encodings (e.g. UTF-16) to UTF-8 before sending. #[axum::debug_handler] +#[allow(clippy::too_many_lines)] pub async fn create_document( Path(CreateDocumentPathParams { vault_id }): Path, Extension(user): Extension, TypedHeader(device_id): TypedHeader, State(state): State, TypedMultipart(request): TypedMultipart, -) -> Result, SyncServerError> { +) -> Result, SyncServerError> { debug!("Creating document in vault `{vault_id}`"); let mut transaction = state .database .create_write_transaction(&vault_id) .await - .map_err(server_error)?; + .map_err(write_transaction_error)?; - let document_id = match request.document_id { - Some(document_id) => { - let existing_version = state - .database - .get_latest_document(&vault_id, &document_id, Some(&mut transaction)) - .await - .map_err(server_error)?; + let sanitized_relative_path = sanitize_path(&request.relative_path).map_err(client_error)?; + let new_content = request.content.contents.to_vec(); - if existing_version.is_some() { - return Err(client_error(anyhow::anyhow!( - "Document with the same ID `{document_id}` already exists" - ))); - } - - document_id - } - None => uuid::Uuid::new_v4(), - }; - - let last_update_id = state + let latest_version = state .database - .get_max_update_id_in_vault(&vault_id, Some(&mut transaction)) + .get_latest_non_deleted_document_by_path( + &vault_id, + &sanitized_relative_path, + Some(&mut *transaction), + ) + .await + .map_err(server_error)?; + + if let Some(latest_version) = latest_version { + // Only merge with an existing document the client couldn't have + // known about: its creation is newer than the client's last seen + // vault update to avoid creating cycles by merging two documents into one. + // This could happen if both clients know of document A at path P1, + // but client 2 moves it to P2 while client 1 creates a new document at P2, + // then client 1 would merge its new document with the moved version of A at P2 + // that client 2 resulting in two files (P1 and P2) with the same doc id (A). + if latest_version.creation_vault_update_id > request.last_seen_vault_update_id + && latest_version.creation_vault_update_id == latest_version.vault_update_id + // can't allow merging with a moved document as that could create a cycle + { + let is_mergeable_text = is_file_type_mergable( + &sanitized_relative_path, + &state.config.server.mergeable_file_extensions, + ) && !is_binary(&latest_version.content) + && !is_binary(&new_content); + + if is_mergeable_text || new_content == latest_version.content { + return update_document::update_document( + &sanitized_relative_path, + Vec::new(), + vault_id, + latest_version.document_id, + Some(&request.relative_path), + new_content, + user, + device_id, + state, + transaction, + ) + .await; + } + + // For non-mergeable (binary) files with different content, don't + // merge, create a separate document at a deconflicted path so + // neither client's data is silently overwritten. + } + } + + // Lost-create + local rename recovery. If this device has a doc + // the requesting client hasn't seen yet (its create succeeded + // server-side but the response was discarded — e.g. a sync + // reset mid-flight) and the new request carries the same content + // at a different path (the user renamed the file before the + // retry), bind the retry to that existing doc instead of + // creating a duplicate. The dedup is scoped tightly: + // - same `device_id` (only this client's own lost create), + // - `creation_vault_update_id > last_seen` (client never saw + // this doc, so it can't be deliberately creating another + // copy with matching content), + // - `creation == latest` (the doc has only its create version, + // nobody else has touched it; safe to relocate), + // - exact content match. + // Outside that window we fall through to the normal deconflict + // path, so legitimate "this device created a duplicate of an + // already-acknowledged file" flows still produce a new doc. + if let Some(lost_create) = state + .database + .find_unseen_lost_create_by_device_and_content( + &vault_id, + &device_id.0, + request.last_seen_vault_update_id, + &new_content, + Some(&mut *transaction), + ) + .await + .map_err(server_error)? + { + info!( + "Lost-create recovery: binding retry at `{sanitized_relative_path}` to existing doc {} (was at `{}`) in vault `{vault_id}` for device `{}`", + lost_create.document_id, + lost_create.relative_path, + device_id.0 + ); + return update_document::update_document( + &sanitized_relative_path, + Vec::new(), + vault_id, + lost_create.document_id, + Some(&request.relative_path), + new_content, + user, + device_id, + state, + transaction, + ) + .await; + } + + let document_id = uuid::Uuid::new_v4(); + + let last_update_id = state + .database + .get_max_update_id_in_vault(&vault_id, Some(&mut *transaction)) .await .map_err(server_error)?; - let sanitized_relative_path = sanitize_path(&request.relative_path); let deduped_path = find_first_available_path( &vault_id, &sanitized_relative_path, @@ -87,11 +178,13 @@ pub async fn create_document( ); } + let new_vault_update_id = last_update_id + 1; let new_version = StoredDocumentVersion { - vault_update_id: last_update_id + 1, + vault_update_id: new_vault_update_id, + creation_vault_update_id: new_vault_update_id, document_id, relative_path: deduped_path, - content: request.content.contents.to_vec(), + content: new_content, updated_date: chrono::Utc::now(), is_deleted: false, user_id: user.name, @@ -101,9 +194,11 @@ pub async fn create_document( state .database - .insert_document_version(&vault_id, &new_version, Some(transaction)) + .insert_document_version(&vault_id, &new_version, transaction) .await .map_err(server_error)?; - Ok(Json(new_version.into())) + Ok(Json(DocumentUpdateResponse::FastForwardUpdate( + new_version.into(), + ))) } diff --git a/sync-server/src/server/delete_document.rs b/sync-server/src/server/delete_document.rs index e126d6b5..2ee6eac3 100644 --- a/sync-server/src/server/delete_document.rs +++ b/sync-server/src/server/delete_document.rs @@ -1,4 +1,4 @@ -use anyhow::Context; +use anyhow::{Context, anyhow}; use axum::{ Extension, Json, extract::{Path, State}, @@ -7,7 +7,7 @@ use axum_extra::TypedHeader; use log::{debug, info}; use serde::Deserialize; -use super::{device_id_header::DeviceIdHeader, requests::DeleteDocumentVersion}; +use super::device_id_header::DeviceIdHeader; use crate::{ app_state::{ AppState, @@ -16,8 +16,8 @@ use crate::{ }, }, config::user_config::User, - errors::{SyncServerError, server_error}, - utils::{normalize::normalize, sanitize_path::sanitize_path}, + errors::{SyncServerError, not_found_error, server_error, write_transaction_error}, + utils::normalize::normalize, }; #[derive(Deserialize)] @@ -37,7 +37,6 @@ pub async fn delete_document( Extension(user): Extension, TypedHeader(device_id): TypedHeader, State(state): State, - Json(request): Json, ) -> Result, SyncServerError> { debug!("Deleting document `{document_id}` in vault `{vault_id}`"); @@ -45,7 +44,7 @@ pub async fn delete_document( .database .create_write_transaction(&vault_id) .await - .map_err(server_error)?; + .map_err(write_transaction_error)?; let last_update_id = state .database @@ -59,9 +58,18 @@ pub async fn delete_document( .await .map_err(server_error)?; - if let Some(latest_version) = &latest_version - && latest_version.is_deleted - { + let Some(latest_version) = latest_version else { + transaction + .rollback() + .await + .context("Failed to roll back transaction") + .map_err(server_error)?; + return Err(not_found_error(anyhow!( + "Document `{document_id}` not found in vault `{vault_id}`" + ))); + }; + + if latest_version.is_deleted { transaction .rollback() .await @@ -69,15 +77,19 @@ pub async fn delete_document( .map_err(server_error)?; info!("Document `{document_id}` has already been deleted",); - return Ok(Json(latest_version.clone().into())); + return Ok(Json(latest_version.into())); } - let latest_content = latest_version.map_or_else(Vec::new, |version| version.content); // in case the document has never existed before deleting it + let new_vault_update_id = last_update_id + 1; + let latest_relative_path = latest_version.relative_path; + let latest_content = latest_version.content; + let creation_vault_update_id = latest_version.creation_vault_update_id; let new_version = StoredDocumentVersion { - vault_update_id: last_update_id + 1, + vault_update_id: new_vault_update_id, + creation_vault_update_id, document_id, - relative_path: sanitize_path(&request.relative_path), + relative_path: latest_relative_path, content: latest_content, // copy the content from the latest version updated_date: chrono::Utc::now(), is_deleted: true, @@ -88,7 +100,7 @@ pub async fn delete_document( state .database - .insert_document_version(&vault_id, &new_version, Some(transaction)) + .insert_document_version(&vault_id, &new_version, transaction) .await .map_err(server_error)?; diff --git a/sync-server/src/server/device_id_header.rs b/sync-server/src/server/device_id_header.rs index af9d6413..13bd17a8 100644 --- a/sync-server/src/server/device_id_header.rs +++ b/sync-server/src/server/device_id_header.rs @@ -16,20 +16,31 @@ impl Header for DeviceIdHeader { { let value = values.next().ok_or_else(headers::Error::invalid)?; - Ok(DeviceIdHeader( - value - .to_str() - .map_err(|_| headers::Error::invalid())? - .to_owned(), - )) + let s = value.to_str().map_err(|_| headers::Error::invalid())?; + + if s.is_empty() || s.len() > 256 { + return Err(headers::Error::invalid()); + } + + // Only allow safe characters to prevent log injection and similar attacks. + // Covers UUIDs, user-agent strings like "vault-link/1.0 (12345; linux)", + // and human-readable device names. + if !s + .chars() + .all(|c| c.is_ascii_alphanumeric() || "-_./ ();:@+,".contains(c)) + { + return Err(headers::Error::invalid()); + } + + Ok(DeviceIdHeader(s.to_owned())) } fn encode(&self, values: &mut E) where E: Extend, { - let value = HeaderValue::from_static(Box::leak(self.0.clone().into_boxed_str())); - - values.extend(std::iter::once(value)); + if let Ok(value) = HeaderValue::from_str(&self.0) { + values.extend(std::iter::once(value)); + } } } diff --git a/sync-server/src/server/fetch_document_version.rs b/sync-server/src/server/fetch_document_version.rs index c30f1d76..159cad3a 100644 --- a/sync-server/src/server/fetch_document_version.rs +++ b/sync-server/src/server/fetch_document_version.rs @@ -11,7 +11,7 @@ use crate::{ AppState, database::models::{DocumentId, DocumentVersion, VaultId, VaultUpdateId}, }, - errors::{SyncServerError, not_found_error, server_error}, + errors::{SyncServerError, client_error, not_found_error, server_error}, utils::normalize::normalize, }; @@ -52,7 +52,7 @@ pub async fn fetch_document_version( )?; if result.document_id != document_id { - return Err(not_found_error(anyhow!( + return Err(client_error(anyhow!( "Document with document id `{document_id}` does not have a version with id \ `{vault_update_id}`", ))); diff --git a/sync-server/src/server/fetch_document_version_content.rs b/sync-server/src/server/fetch_document_version_content.rs index 9fdd0ad8..a163b036 100644 --- a/sync-server/src/server/fetch_document_version_content.rs +++ b/sync-server/src/server/fetch_document_version_content.rs @@ -11,7 +11,7 @@ use crate::{ AppState, database::models::{DocumentId, VaultId, VaultUpdateId}, }, - errors::{SyncServerError, not_found_error, server_error}, + errors::{SyncServerError, client_error, not_found_error, server_error}, utils::normalize::normalize, }; @@ -52,7 +52,7 @@ pub async fn fetch_document_version_content( )?; if result.document_id != document_id { - return Err(not_found_error(anyhow!( + return Err(client_error(anyhow!( "Document with document id `{document_id}` does not have a version with id \ `{vault_update_id}`", ))); diff --git a/sync-server/src/server/fetch_document_versions.rs b/sync-server/src/server/fetch_document_versions.rs new file mode 100644 index 00000000..46d0e073 --- /dev/null +++ b/sync-server/src/server/fetch_document_versions.rs @@ -0,0 +1,42 @@ +use axum::{ + Json, + extract::{Path, State}, +}; +use log::debug; +use serde::Deserialize; + +use crate::{ + app_state::{ + AppState, + database::models::{DocumentId, DocumentVersionWithoutContent, VaultId}, + }, + errors::{SyncServerError, server_error}, + utils::normalize::normalize, +}; + +#[derive(Deserialize)] +pub struct FetchDocumentVersionsPathParams { + #[serde(deserialize_with = "normalize")] + vault_id: VaultId, + + document_id: DocumentId, +} + +#[axum::debug_handler] +pub async fn fetch_document_versions( + Path(FetchDocumentVersionsPathParams { + vault_id, + document_id, + }): Path, + State(state): State, +) -> Result>, SyncServerError> { + debug!("Fetching all versions for document `{document_id}` in vault `{vault_id}`"); + + let versions = state + .database + .get_document_versions(&vault_id, &document_id, None) + .await + .map_err(server_error)?; + + Ok(Json(versions)) +} diff --git a/sync-server/src/server/fetch_latest_documents.rs b/sync-server/src/server/fetch_latest_documents.rs index 209374ce..f1ca702d 100644 --- a/sync-server/src/server/fetch_latest_documents.rs +++ b/sync-server/src/server/fetch_latest_documents.rs @@ -37,13 +37,13 @@ pub async fn fetch_latest_documents( let documents = if let Some(since_update_id) = since_update_id { state .database - .get_latest_documents_since(&vault_id, since_update_id, None) + .get_latest_documents_since(&vault_id, since_update_id, None, None) .await .map_err(server_error) } else { state .database - .get_latest_documents(&vault_id, None) + .get_latest_documents(&vault_id, None, None) .await .map_err(server_error) }?; diff --git a/sync-server/src/server/fetch_vault_history.rs b/sync-server/src/server/fetch_vault_history.rs new file mode 100644 index 00000000..42cceaa6 --- /dev/null +++ b/sync-server/src/server/fetch_vault_history.rs @@ -0,0 +1,70 @@ +use axum::{ + Json, + extract::{Path, Query, State}, +}; +use log::debug; +use serde::Deserialize; + +use super::responses::VaultHistoryResponse; +use crate::{ + app_state::{ + AppState, + database::models::{VaultId, VaultUpdateId}, + }, + errors::{SyncServerError, client_error, server_error}, + utils::normalize::normalize, +}; + +const DEFAULT_LIMIT: i64 = 50; +const MAX_LIMIT: i64 = 500; + +#[derive(Deserialize)] +pub struct FetchVaultHistoryPathParams { + #[serde(deserialize_with = "normalize")] + vault_id: VaultId, +} + +#[derive(Deserialize)] +pub struct QueryParams { + limit: Option, + before_update_id: Option, +} + +#[axum::debug_handler] +pub async fn fetch_vault_history( + Path(FetchVaultHistoryPathParams { vault_id }): Path, + Query(QueryParams { + limit, + before_update_id, + }): Query, + State(state): State, +) -> Result, SyncServerError> { + if let Some(id) = before_update_id + && id <= 0 + { + return Err(client_error(anyhow::anyhow!( + "before_update_id must be a positive integer" + ))); + } + + let limit = limit.unwrap_or(DEFAULT_LIMIT).clamp(1, MAX_LIMIT); + + debug!( + "Fetching vault history for vault `{vault_id}` (limit={limit}, before={before_update_id:?})" + ); + + // Fetch one extra row to determine if there are more results + let mut versions = state + .database + .get_vault_history(&vault_id, limit + 1, before_update_id, None) + .await + .map_err(server_error)?; + + #[allow(clippy::cast_sign_loss)] // limit is clamped to [1, 500] above + let has_more = versions.len() > limit as usize; + if has_more { + versions.pop(); + } + + Ok(Json(VaultHistoryResponse { versions, has_more })) +} diff --git a/sync-server/src/server/index.rs b/sync-server/src/server/index.rs index 64b053f7..ca8f38ff 100644 --- a/sync-server/src/server/index.rs +++ b/sync-server/src/server/index.rs @@ -1,7 +1,77 @@ -use axum::response::{Html, IntoResponse}; +use axum::{ + body::Body, + extract::{Path, State}, + http::{StatusCode, header}, + response::{Html, IntoResponse, Response}, +}; +use log::warn; +use rust_embed::Embed; -pub async fn index() -> impl IntoResponse { - const HTML_CONTENT: &str = include_str!("./assets/index.html"); - let html_content = HTML_CONTENT; - Html(html_content) +use crate::app_state::AppState; + +#[derive(Embed)] +#[folder = "../frontend/history-ui/dist/"] +struct HistoryUiAssets; + +pub async fn index(State(_state): State) -> impl IntoResponse { + if let Some(content) = HistoryUiAssets::get("index.html") { + Html( + std::str::from_utf8(content.data.as_ref()) + .inspect_err(|e| warn!("Embedded index.html is not valid UTF-8: {e}")) + .unwrap_or("

VaultLink

") + .to_owned(), + ) + .into_response() + } else { + warn!("No embedded index.html found — history UI may not have been built"); + Html("

VaultLink server

".to_owned()).into_response() + } +} + +pub async fn spa_assets(Path(path): Path) -> impl IntoResponse { + // The route is /assets/*path so path is relative to assets/. + // The embedded files include the assets/ prefix from the dist directory. + let full_path = format!("assets/{path}"); + if let Some(content) = HistoryUiAssets::get(&full_path) { + let mime = mime_guess::from_path(&full_path).first_or_octet_stream(); + return Response::builder() + .status(StatusCode::OK) + .header(header::CONTENT_TYPE, mime.as_ref()) + .body(Body::from(content.data.to_vec())) + .unwrap_or_else(|_| { + Response::builder() + .status(StatusCode::INTERNAL_SERVER_ERROR) + .body(Body::empty()) + .unwrap_or_else(|_| Response::new(Body::empty())) + }); + } + + // Asset paths must match an embedded file — no SPA fallback. + // Serving index.html here would return 200 with text/html for missing + // .css/.js files, causing the browser to silently ignore the content. + Response::builder() + .status(StatusCode::NOT_FOUND) + .body(Body::from("Not found")) + .unwrap_or_else(|_| Response::new(Body::from("Not found"))) +} + +/// SPA fallback for production: serves index.html for client-side routes +/// (e.g. `/documents/123`). +pub async fn spa_fallback() -> impl IntoResponse { + match HistoryUiAssets::get("index.html") { + Some(content) => Response::builder() + .status(StatusCode::OK) + .header(header::CONTENT_TYPE, "text/html") + .body(Body::from(content.data.to_vec())) + .unwrap_or_else(|_| { + Response::builder() + .status(StatusCode::INTERNAL_SERVER_ERROR) + .body(Body::empty()) + .unwrap_or_else(|_| Response::new(Body::empty())) + }), + None => Response::builder() + .status(StatusCode::NOT_FOUND) + .body(Body::from("Not found")) + .unwrap_or_else(|_| Response::new(Body::from("Not found"))), + } } diff --git a/sync-server/src/server/list_vaults.rs b/sync-server/src/server/list_vaults.rs new file mode 100644 index 00000000..7ef23405 --- /dev/null +++ b/sync-server/src/server/list_vaults.rs @@ -0,0 +1,82 @@ +use axum::{ + Json, + extract::{Query, State}, +}; +use axum_extra::{ + TypedHeader, + headers::{Authorization, authorization::Bearer}, +}; +use log::debug; +use serde::Deserialize; + +use super::{ + auth::authenticate, + responses::{ListVaultsResponse, VaultInfo}, +}; +use crate::{ + app_state::AppState, + config::user_config::{AllowListedVaults, VaultAccess}, + errors::{SyncServerError, server_error, unauthenticated_error}, +}; + +const DEFAULT_LIMIT: usize = 50; +const MAX_LIMIT: usize = 200; + +#[derive(Deserialize)] +pub struct QueryParams { + limit: Option, + after: Option, +} + +#[axum::debug_handler] +pub async fn list_vaults( + auth_header: Option>>, + Query(QueryParams { limit, after }): Query, + State(state): State, +) -> Result, SyncServerError> { + let auth_header = auth_header + .ok_or_else(|| unauthenticated_error(anyhow::anyhow!("Missing Authorization header")))?; + + let user = authenticate(&state, auth_header.token().trim())?; + + debug!("User `{}` listing accessible vaults", user.name); + + let existing_vaults = state.database.list_vaults().await.map_err(server_error)?; + + let mut accessible: Vec = match user.vault_access { + VaultAccess::AllowAccessToAll => existing_vaults, + VaultAccess::AllowList(AllowListedVaults { ref allowed }) => existing_vaults + .into_iter() + .filter(|v| allowed.contains(v)) + .collect(), + }; + + // Cursor-based pagination: skip vaults up to and including `after` + if let Some(ref cursor) = after { + accessible.retain(|v| v.as_str() > cursor.as_str()); + } + + let limit = limit.unwrap_or(DEFAULT_LIMIT).clamp(1, MAX_LIMIT); + let has_more = accessible.len() > limit; + accessible.truncate(limit); + + let mut vaults = Vec::with_capacity(accessible.len()); + for name in accessible { + let stats = state + .database + .get_vault_stats(&name) + .await + .map_err(server_error)?; + vaults.push(VaultInfo { + name, + document_count: stats.document_count, + created_at: stats.created_at, + }); + } + + Ok(Json(ListVaultsResponse { + vaults, + has_more, + user_name: user.name, + })) +} diff --git a/sync-server/src/server/rate_limit.rs b/sync-server/src/server/rate_limit.rs new file mode 100644 index 00000000..7792a814 --- /dev/null +++ b/sync-server/src/server/rate_limit.rs @@ -0,0 +1,102 @@ +use std::{ + collections::HashMap, + sync::{Arc, Mutex}, + time::Instant, +}; + +use axum::{extract::Request, http::StatusCode, middleware::Next, response::Response}; +use axum_extra::{ + TypedHeader, + headers::{Authorization, authorization::Bearer}, +}; + +/// Per-user token-bucket rate limiter. Each bearer token gets its own bucket +/// that refills to `max_per_second` tokens every second. +#[derive(Clone, Debug)] +pub struct RateLimiter { + max_per_second: u64, + buckets: Arc>>>, +} + +#[derive(Debug)] +struct TokenBucket { + state: Mutex, + max_tokens: u64, +} + +#[derive(Debug)] +struct BucketState { + tokens: u64, + last_refill: Instant, +} + +impl RateLimiter { + /// Create a new per-user rate limiter. + /// + /// # Panics + /// + /// Panics if `max_per_second` is 0. + pub fn new(max_per_second: u64) -> Self { + assert!( + max_per_second > 0, + "max_per_second must be > 0 (set rate_limit_per_user_per_second to null in config to disable)" + ); + + Self { + max_per_second, + buckets: Arc::new(Mutex::new(HashMap::new())), + } + } + + fn get_or_create_bucket(&self, token: &str) -> Arc { + self.buckets + .lock() + .expect("rate limiter lock poisoned") + .entry(token.to_owned()) + .or_insert_with(|| { + Arc::new(TokenBucket { + state: Mutex::new(BucketState { + tokens: self.max_per_second, + last_refill: Instant::now(), + }), + max_tokens: self.max_per_second, + }) + }) + .clone() + } +} + +impl TokenBucket { + fn try_acquire(&self) -> bool { + let mut state = self.state.lock().expect("token bucket lock poisoned"); + let now = Instant::now(); + if now.duration_since(state.last_refill).as_secs() >= 1 { + state.tokens = self.max_tokens; + state.last_refill = now; + } + if state.tokens > 0 { + state.tokens -= 1; + true + } else { + false + } + } +} + +pub async fn rate_limit_middleware( + axum::extract::State(limiter): axum::extract::State, + auth_header: Option>>, + req: Request, + next: Next, +) -> Result { + let Some(TypedHeader(auth)) = auth_header else { + return Ok(next.run(req).await); + }; + + let bucket = limiter.get_or_create_bucket(auth.token()); + if bucket.try_acquire() { + Ok(next.run(req).await) + } else { + Err(StatusCode::TOO_MANY_REQUESTS) + } +} diff --git a/sync-server/src/server/requests.rs b/sync-server/src/server/requests.rs index 119ad467..232e514d 100644 --- a/sync-server/src/server/requests.rs +++ b/sync-server/src/server/requests.rs @@ -4,18 +4,16 @@ use reconcile_text::NumberOrText; use serde::{self, Deserialize}; use ts_rs::TS; -use crate::app_state::database::models::{DocumentId, VaultUpdateId}; +use crate::app_state::database::models::VaultUpdateId; #[derive(TS, Debug, TryFromMultipart)] #[ts(export)] pub struct CreateDocumentVersion { - /// The client can decide the document id (if it wishes to) in order - /// to help with syncing. If the client does not provide a document id, - /// the server will generate one. If the client provides a document id - /// it must not already exist in the database. - pub document_id: Option, pub relative_path: String, + #[ts(type = "number")] + pub last_seen_vault_update_id: VaultUpdateId, + #[ts(as = "Vec")] #[form_data(limit = "unlimited")] pub content: FieldData, @@ -24,7 +22,9 @@ pub struct CreateDocumentVersion { #[derive(Debug, TryFromMultipart)] pub struct UpdateBinaryDocumentVersion { pub parent_version_id: VaultUpdateId, - pub relative_path: String, + // None on a content-only edit; Some on a user rename. When None, + // the server keeps the document at its current path. + pub relative_path: Option, #[form_data(limit = "unlimited")] pub content: FieldData, @@ -34,18 +34,13 @@ pub struct UpdateBinaryDocumentVersion { #[serde(rename_all = "camelCase")] #[ts(export)] pub struct UpdateTextDocumentVersion { - #[ts(as = "i32")] + #[ts(type = "number")] pub parent_version_id: VaultUpdateId, - pub relative_path: String, + // None on a content-only edit; Some on a user rename. When None, + // the server keeps the document at its current path. + pub relative_path: Option, #[ts(type = "Array")] pub content: Vec, } - -#[derive(TS, Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -#[ts(export)] -pub struct DeleteDocumentVersion { - pub relative_path: String, -} diff --git a/sync-server/src/server/responses.rs b/sync-server/src/server/responses.rs index a8b3fcd7..f5b30782 100644 --- a/sync-server/src/server/responses.rs +++ b/sync-server/src/server/responses.rs @@ -1,3 +1,4 @@ +use chrono::{DateTime, Utc}; use serde::{self, Serialize}; use ts_rs::TS; @@ -36,7 +37,36 @@ pub struct FetchLatestDocumentsResponse { pub last_update_id: VaultUpdateId, } -/// Response to an update document request. +/// Response to a vault history request (paginated). +#[derive(TS, Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +#[ts(export)] +pub struct VaultHistoryResponse { + pub versions: Vec, + pub has_more: bool, +} + +/// Summary of a single vault returned by the list-vaults endpoint. +#[derive(TS, Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +#[ts(export)] +pub struct VaultInfo { + pub name: String, + pub document_count: u32, + pub created_at: Option>, +} + +/// Response to listing vaults accessible to the authenticated user. +#[derive(TS, Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +#[ts(export)] +pub struct ListVaultsResponse { + pub vaults: Vec, + pub has_more: bool, + pub user_name: String, +} + +/// Response to a create/update document request. #[derive(TS, Debug, Clone, Serialize)] #[serde(tag = "type")] #[ts(export)] diff --git a/sync-server/src/server/update_document.rs b/sync-server/src/server/update_document.rs index 00fbd008..0145288c 100644 --- a/sync-server/src/server/update_document.rs +++ b/sync-server/src/server/update_document.rs @@ -16,10 +16,15 @@ use super::{ use crate::{ app_state::{ AppState, - database::models::{DocumentId, StoredDocumentVersion, VaultId, VaultUpdateId}, + database::{ + WriteTransaction, + models::{DocumentId, StoredDocumentVersion, VaultId, VaultUpdateId}, + }, }, config::user_config::User, - errors::{SyncServerError, client_error, not_found_error, server_error}, + errors::{ + SyncServerError, client_error, not_found_error, server_error, write_transaction_error, + }, server::requests::UpdateBinaryDocumentVersion, utils::{ find_first_available_path::find_first_available_path, is_binary::is_binary, @@ -46,18 +51,27 @@ pub async fn update_binary( State(state): State, TypedMultipart(request): TypedMultipart, ) -> Result, SyncServerError> { - let parent_document = get_parent_document(&state, &vault_id, request.parent_version_id).await?; + let parent_document = + get_parent_document(&state, &vault_id, &document_id, request.parent_version_id).await?; let content = request.content.contents.to_vec(); + let transaction = state + .database + .create_write_transaction(&vault_id) + .await + .map_err(write_transaction_error)?; + update_document( - parent_document, + &parent_document.relative_path, + parent_document.content, vault_id, document_id, + request.relative_path.as_deref(), + content, user, device_id, state, - &request.relative_path, - content, + transaction, ) .await } @@ -74,28 +88,36 @@ pub async fn update_text( State(state): State, Json(request): Json, ) -> Result, SyncServerError> { - let parent_document = get_parent_document(&state, &vault_id, request.parent_version_id).await?; + let parent_document = + get_parent_document(&state, &vault_id, &document_id, request.parent_version_id).await?; - let edited_text = EditedText::from_diff( - str::from_utf8(&parent_document.content) - .expect("parent must be valid UTF-8 because it's a text document"), - request.content, - &*BuiltinTokenizer::Word, - ) - .context("Failed to apply given diff to parent document") - .map_err(client_error)?; + let parent_text = str::from_utf8(&parent_document.content) + .context("Parent version contains binary content; use putBinary instead of putText") + .map_err(client_error)?; + + let edited_text = EditedText::from_diff(parent_text, request.content, &*BuiltinTokenizer::Word) + .context("Failed to apply given diff to parent document") + .map_err(client_error)?; let content = edited_text.apply().text().into_bytes(); + let transaction = state + .database + .create_write_transaction(&vault_id) + .await + .map_err(write_transaction_error)?; + update_document( - parent_document, + &parent_document.relative_path, + parent_document.content, vault_id, document_id, + request.relative_path.as_deref(), + content, user, device_id, state, - &request.relative_path, - content, + transaction, ) .await } @@ -103,9 +125,10 @@ pub async fn update_text( async fn get_parent_document( state: &AppState, vault_id: &VaultId, + document_id: &DocumentId, parent_version_id: VaultUpdateId, ) -> Result { - state + let parent = state .database .get_document_version(vault_id, parent_version_id, None) .await @@ -117,29 +140,36 @@ async fn get_parent_document( ))) }, Ok, - ) + )?; + + if &parent.document_id != document_id { + return Err(client_error(anyhow!( + "Parent version `{parent_version_id}` does not belong to document `{document_id}`" + ))); + } + + Ok(parent) } #[allow(clippy::too_many_lines, clippy::too_many_arguments)] -async fn update_document( - parent_document: StoredDocumentVersion, +pub async fn update_document( + parent_relative_path: &str, + parent_content: Vec, vault_id: VaultId, document_id: DocumentId, + relative_path: Option<&str>, + content: Vec, user: User, device_id: DeviceIdHeader, state: AppState, - relative_path: &str, - content: Vec, + mut transaction: WriteTransaction, ) -> Result, SyncServerError> { debug!("Updating document `{document_id}` in vault `{vault_id}`"); - let sanitized_relative_path = sanitize_path(relative_path); - - let mut transaction = state - .database - .create_write_transaction(&vault_id) - .await - .map_err(server_error)?; + let sanitized_relative_path = relative_path + .map(sanitize_path) + .transpose() + .map_err(client_error)?; let last_update_id = state .database @@ -175,9 +205,12 @@ async fn update_document( } // Return the latest version if the content and path are the same as the latest - // version - if content == latest_version.content && sanitized_relative_path == latest_version.relative_path - { + // version. A missing relative_path means "keep current path", so the path + // is implicitly unchanged. + let path_unchanged = sanitized_relative_path + .as_deref() + .is_none_or(|p| p == latest_version.relative_path); + if content == latest_version.content && path_unchanged { info!( "Document content is the same as the latest version for `{document_id}`, skipping update" ); @@ -192,62 +225,89 @@ async fn update_document( ))); } + // For mergability, use whichever path the new version will live at — the + // requested rename target if the client sent one, otherwise the existing + // server-side path. + let mergable_check_path = sanitized_relative_path + .as_deref() + .unwrap_or(&latest_version.relative_path); let are_all_participants_mergable = is_file_type_mergable( - &sanitized_relative_path, + mergable_check_path, &state.config.server.mergeable_file_extensions, - ) && !is_binary(&parent_document.content) + ) && !is_binary(&parent_content) && !is_binary(&latest_version.content) && !is_binary(&content); - let merged_content = if are_all_participants_mergable { + let (merged_content, is_different_from_request_content) = if are_all_participants_mergable { info!("Merging changes for document `{document_id}` in vault `{vault_id}`"); - reconcile( - str::from_utf8(&parent_document.content) - .expect("parent must be valid UTF-8 because it's not binary"), - &str::from_utf8(&latest_version.content) - .expect("latest_version must be valid UTF-8 because it's not binary") - .into(), - &str::from_utf8(&content) - .expect("content must be valid UTF-8 because it's not binary") - .into(), - &*BuiltinTokenizer::Word, - ) - .apply() - .text() - .into_bytes() + let parent_text = str::from_utf8(&parent_content) + .context("Parent document content is not valid UTF-8") + .map_err(client_error)?; + let latest_text = str::from_utf8(&latest_version.content) + .context("Latest version content is not valid UTF-8") + .map_err(client_error)?; + let new_text = str::from_utf8(&content) + .context("New content is not valid UTF-8") + .map_err(client_error)?; + let parent_owned = parent_text.to_owned(); + let latest_owned = latest_text.to_owned(); + let new_owned = new_text.to_owned(); + let content_clone = content.clone(); + + let (merged, is_different) = tokio::task::spawn_blocking(move || { + let merged = reconcile( + &parent_owned, + &latest_owned.into(), + &new_owned.into(), + &*BuiltinTokenizer::Word, + ) + .apply() + .text() + .into_bytes(); + let is_different = merged != content_clone; + (merged, is_different) + }) + .await + .map_err(|e| server_error(anyhow::anyhow!("Reconcile task failed: {e}")))?; + + (merged, is_different) } else { - content.clone() + (content, false) // false means that the client doesn't need to refetch the file as we can ensure the remote and local versions are the same as LWW is the merging method for binary files }; - let is_different_from_request_content = merged_content != content; + // Rename resolution: only apply the client's rename if (a) the client + // requested one (`sanitized_relative_path` is `Some`) and (b) the + // document's path hasn't changed since this client's parent version. + // If the parent and latest paths differ, another client already renamed + // the document — keep the latest path (first rename wins). Content + // changes from both clients are still merged correctly via the 3-way + // reconcile above, independent of which rename wins. A missing + // relative_path means "keep current path" (content-only edit). + let new_relative_path = match sanitized_relative_path.as_deref() { + Some(requested) + if parent_relative_path == latest_version.relative_path + && requested != latest_version.relative_path => + { + let new_path = + find_first_available_path(&vault_id, requested, &state.database, &mut transaction) + .await + .map_err(server_error)?; - // We can only update the relative path if we're the first one to do so - let new_relative_path = if parent_document.relative_path == latest_version.relative_path - && latest_version.relative_path != sanitized_relative_path - { - let new_path = find_first_available_path( - &vault_id, - &sanitized_relative_path, - &state.database, - &mut transaction, - ) - .await - .map_err(server_error)?; + if new_path != requested { + info!( + "Document already exists at new location: `{requested}` when trying to update it in vault `{vault_id}`, deconflicting by creating at `{new_path}`" + ); + } - if new_path != sanitized_relative_path { - info!( - "Document already exists at new location: `{sanitized_relative_path}` when trying to update it in vault `{vault_id}`, deconflicting by creating at `{new_path}`" - ); + new_path } - - new_path - } else { - latest_version.relative_path.clone() + _ => latest_version.relative_path.clone(), }; let new_version = StoredDocumentVersion { document_id, vault_update_id: last_update_id + 1, + creation_vault_update_id: latest_version.creation_vault_update_id, relative_path: new_relative_path, content: merged_content, updated_date: chrono::Utc::now(), @@ -259,7 +319,7 @@ async fn update_document( state .database - .insert_document_version(&vault_id, &new_version, Some(transaction)) + .insert_document_version(&vault_id, &new_version, transaction) .await .map_err(server_error)?; diff --git a/sync-server/src/server/websocket.rs b/sync-server/src/server/websocket.rs index bb10b49f..6e1af0ba 100644 --- a/sync-server/src/server/websocket.rs +++ b/sync-server/src/server/websocket.rs @@ -1,15 +1,3 @@ -use anyhow::Context; -use axum::{ - extract::{ - Path, State, - ws::{Message, WebSocket, WebSocketUpgrade}, - }, - response::Response, -}; -use futures::stream::StreamExt; -use log::{debug, info}; -use serde::Deserialize; - use crate::{ app_state::{ AppState, @@ -24,9 +12,35 @@ use crate::{ }, }, }, + consts::{ + HANDSHAKE_TIMEOUT, MAX_CURSOR_DOCUMENTS, MAX_CURSORS_PER_DOCUMENT, MAX_RELATIVE_PATH_LEN, + }, errors::{SyncServerError, client_error, server_error}, utils::normalize::normalize, }; +use anyhow::Context; +use axum::{ + extract::{ + Path, State, + ws::{Message, WebSocket, WebSocketUpgrade}, + }, + response::Response, +}; +use futures::sink::SinkExt; +use futures::stream::StreamExt; +use log::{debug, info, warn}; +use serde::Deserialize; + +/// Tracks a pending (not yet authenticated) WebSocket connection. +/// Decrements the counter when dropped, ensuring cleanup even if +/// the upgrade never completes or auth fails. +struct PendingWsGuard(std::sync::Arc); + +impl Drop for PendingWsGuard { + fn drop(&mut self) { + self.0.fetch_sub(1, std::sync::atomic::Ordering::Relaxed); + } +} #[derive(Deserialize)] pub struct WebSocketPathParams { @@ -39,13 +53,31 @@ pub async fn websocket_handler( Path(WebSocketPathParams { vault_id }): Path, State(state): State, ) -> Result { - Ok(ws.on_upgrade(move |socket| websocket_wrapped(state, socket, vault_id))) + let current = state + .pending_ws_connections + .fetch_add(1, std::sync::atomic::Ordering::Relaxed); + if current >= state.config.server.max_pending_websocket_connections { + state + .pending_ws_connections + .fetch_sub(1, std::sync::atomic::Ordering::Relaxed); + return Err(client_error(anyhow::anyhow!( + "Too many pending WebSocket connections" + ))); + } + + let guard = PendingWsGuard(state.pending_ws_connections.clone()); + Ok(ws.on_upgrade(move |socket| websocket_wrapped(state, socket, vault_id, guard))) } -async fn websocket_wrapped(state: AppState, stream: WebSocket, vault_id: VaultId) { +async fn websocket_wrapped( + state: AppState, + stream: WebSocket, + vault_id: VaultId, + pending_guard: PendingWsGuard, +) { info!("WebSocket connection opened on vault `{vault_id}`"); - let result = websocket(state, stream, vault_id.clone()).await; + let result = websocket(state, stream, vault_id.clone(), pending_guard).await; if let Err(err) = result { debug!("WebSocket connection error on vault `{vault_id}`: {err}"); @@ -57,39 +89,112 @@ async fn websocket( state: AppState, stream: WebSocket, vault_id: VaultId, + pending_guard: PendingWsGuard, ) -> Result<(), SyncServerError> { let (mut sender, mut websocket_receiver) = stream.split(); - let authed_handshake = get_authenticated_handshake( - &state, - &vault_id, - websocket_receiver - .next() - .await - .transpose() - .unwrap_or_default(), - )?; + let handshake_msg = tokio::time::timeout(HANDSHAKE_TIMEOUT, websocket_receiver.next()) + .await + .map_err(|_| client_error(anyhow::anyhow!("WebSocket handshake timed out")))? + .transpose() + .map_err(|e| client_error(anyhow::anyhow!("WebSocket error during handshake: {e}")))?; + + let authed_handshake = get_authenticated_handshake(&state, &vault_id, handshake_msg)?; info!( "WebSocket handshake successful for vault `{vault_id}` for `{}`", authed_handshake.handshake.device_id ); - let mut broadcast_receiver = state.broadcasts.get_receiver(vault_id.clone()).await; + // Auth complete — no longer a pending connection. + drop(pending_guard); - send_update_over_websocket( - &WebSocketServerMessage::VaultUpdate(WebSocketVaultUpdate { - documents: get_unseen_documents( - &state, - &vault_id, - authed_handshake.handshake.last_seen_vault_update_id, - ) - .await?, - is_initial_sync: true, - }), - &mut sender, + let max_clients = state.config.server.max_clients_per_vault; + + // Atomic subscribe + cursor snapshot, serialized against in-flight + // broadcasts: + // + // 1. Acquire the per-vault broadcast send lock. While we hold it, + // no `send_document_update` can run, so no broadcast can fire + // between our subscribe and our cursor snapshot. + // 2. Subscribe to the broadcast channel (now we'll see every + // broadcast that fires after we drop the send guard). + // 3. Snapshot `cursor = max committed vault_update_id`. Because + // `insert_document_version` holds the same send lock from + // *before* the commit through *after* the broadcast, every doc + // visible at this cursor has either (a) already had its + // broadcast delivered to all then-existing subscribers — and we + // weren't one of them, so we'll catch it via the snapshot — or + // (b) had its broadcast contend on the lock we're holding, and + // will be delivered to us as soon as we drop the guard, with + // `vault_update_id > cursor`. + // 4. Drop the send guard so writers can resume broadcasting. + // 5. Stream the catch-up bounded by the cursor — i.e. only docs + // with `vault_update_id <= cursor` — exactly once. + // 6. The send task forwards broadcasts but filters to + // `vault_update_id > cursor`, so a doc that's both in the + // catch-up and in a contended-then-released broadcast is + // delivered exactly once (via the catch-up). + let send_guard = state.broadcasts.acquire_send_lock(&vault_id).await; + let mut broadcast_receiver = match state.broadcasts.get_receiver(vault_id.clone(), max_clients) + { + Ok(receiver) => receiver, + Err(err) => { + drop(send_guard); + warn!( + "Vault `{vault_id}` has reached the maximum number of clients ({max_clients}), rejecting connection from `{}`", + authed_handshake.handshake.device_id + ); + if let Err(e) = sender + .send(Message::Close(Some(axum::extract::ws::CloseFrame { + code: 4000, + reason: format!( + "Vault has reached the maximum number of clients ({max_clients})" + ) + .into(), + }))) + .await + { + warn!("Failed to send WebSocket close frame: {e}"); + } + return Err(err); + } + }; + let cursor = state + .database + .get_max_update_id_in_vault(&vault_id, None) + .await + .map_err(server_error)?; + drop(send_guard); + + // Catch-up on versions committed while this client was offline, + // streamed one-at-a-time in ascending `vault_update_id` order, up + // to the snapshot cursor. + let unseen_documents = get_unseen_documents( + &state, + &vault_id, + authed_handshake.handshake.last_seen_vault_update_id, + cursor, ) .await?; + let unseen_summary: Vec<(i64, bool, String)> = unseen_documents + .iter() + .map(|d| (d.vault_update_id, d.is_deleted, d.relative_path.clone())) + .collect(); + info!( + "[CATCHUP] vault={vault_id} device={} last_seen={:?} cursor={cursor} unseen_count={} unseen={:?}", + authed_handshake.handshake.device_id, + authed_handshake.handshake.last_seen_vault_update_id, + unseen_summary.len(), + unseen_summary + ); + for document in unseen_documents { + send_update_over_websocket( + &WebSocketServerMessage::VaultUpdate(WebSocketVaultUpdate { document }), + &mut sender, + ) + .await?; + } send_update_over_websocket( &WebSocketServerMessage::CursorPositions(CursorPositionFromServer { @@ -101,24 +206,57 @@ async fn websocket( let device_id = authed_handshake.handshake.device_id.clone(); let mut send_task = tokio::spawn(async move { - while let Ok(update) = broadcast_receiver.recv().await { - if Some(&device_id) == update.origin_device_id.as_ref() { - continue; - } + loop { + match broadcast_receiver.recv().await { + Ok(update) => { + // Drop messages this device authored because the HTTP + // response already carried authoritative state back. + // Delete broadcasts are sent without an origin so the + // author also receives them — that's the receipt the + // client needs to drop the doc from its sync queue. + if Some(&device_id) == update.origin_device_id.as_ref() { + continue; + } - let message = match update.message { - WebSocketServerMessage::CursorPositions(CursorPositionFromServer { clients }) => { - WebSocketServerMessage::CursorPositions(CursorPositionFromServer { - clients: clients - .into_iter() - .filter(|client| client.device_id != device_id) - .collect(), - }) + // Filter out vault updates already covered by the + // catch-up snapshot. The handshake atomically + // subscribed and snapshotted `cursor` under the + // broadcast send lock, so any broadcast with + // `vault_update_id <= cursor` is one that contended + // on the lock during our subscribe — its row is + // already in the catch-up stream and re-delivering + // it via this channel would duplicate the message. + // Cursor messages aren't versioned and are always + // forwarded. + if let WebSocketServerMessage::VaultUpdate(WebSocketVaultUpdate { document }) = + &update.message + && document.vault_update_id <= cursor + { + continue; + } + + let message = match update.message { + WebSocketServerMessage::CursorPositions(CursorPositionFromServer { + clients, + }) => WebSocketServerMessage::CursorPositions(CursorPositionFromServer { + clients: clients + .into_iter() + .filter(|client| client.device_id != device_id) + .collect(), + }), + WebSocketServerMessage::VaultUpdate(_) => update.message, + }; + + send_update_over_websocket(&message, &mut sender).await?; } - WebSocketServerMessage::VaultUpdate(_) => update.message, - }; - - send_update_over_websocket(&message, &mut sender).await?; + Err(tokio::sync::broadcast::error::RecvError::Lagged(n)) => { + warn!( + "WebSocket receiver lagged, dropped {n} messages — disconnecting client to force full resync" + ); + break; + } + Err(tokio::sync::broadcast::error::RecvError::Closed) => break, + } } Ok::<(), SyncServerError>(()) @@ -128,26 +266,59 @@ async fn websocket( let vault_id_clone = vault_id.clone(); let cursor_manager = state.cursors.clone(); let mut receive_task = tokio::spawn(async move { - while let Some(Ok(Message::Text(message))) = websocket_receiver.next().await { - let message: WebSocketClientMessage = serde_json::from_str(&message) - .context("Failed to parse WebSocket message from client") - .map_err(server_error)?; + while let Some(msg) = websocket_receiver.next().await { + match msg { + Ok(Message::Text(message)) => { + let message: WebSocketClientMessage = serde_json::from_str(&message) + .context("Failed to parse WebSocket message from client") + .map_err(client_error)?; - match message { - WebSocketClientMessage::Handshake(_) => { - return Err(client_error(anyhow::anyhow!( - "Unexpected handshake message" - ))); + match message { + WebSocketClientMessage::Handshake(_) => { + return Err(client_error(anyhow::anyhow!( + "Unexpected handshake message" + ))); + } + WebSocketClientMessage::CursorPositions(cursors) => { + let docs = cursors.documents_with_cursors; + if docs.len() > MAX_CURSOR_DOCUMENTS { + warn!( + "Cursor update rejected: {} documents exceeds limit of {MAX_CURSOR_DOCUMENTS}", + docs.len() + ); + continue; + } + + let valid = docs.iter().all(|doc| { + doc.cursors.len() <= MAX_CURSORS_PER_DOCUMENT + && doc.relative_path.len() <= MAX_RELATIVE_PATH_LEN + }); + if !valid { + warn!( + "Cursor update rejected: a document exceeds cursor or path length limits" + ); + continue; + } + + cursor_manager + .update_cursors( + vault_id_clone.clone(), + authed_handshake.user.name.clone(), + &device_id, + docs, + ) + .await; + } + } } - WebSocketClientMessage::CursorPositions(cursors) => { - cursor_manager - .update_cursors( - vault_id_clone.clone(), - authed_handshake.user.name.clone(), - &device_id, - cursors.documents_with_cursors, - ) - .await; + Ok(Message::Close(_)) => break, + Ok(Message::Binary(_)) => { + warn!("Received unexpected binary WebSocket message, ignoring"); + } + Ok(_) => {} // Ping/Pong frames handled by axum + Err(e) => { + debug!("WebSocket receive error: {e}"); + break; } } } @@ -155,38 +326,47 @@ async fn websocket( Ok::<(), SyncServerError>(()) }); - tokio::select! { - _ = &mut send_task => receive_task.abort(), - _ = &mut receive_task => send_task.abort(), + let result: Result<(), SyncServerError> = tokio::select! { + send_result = &mut send_task => { + receive_task.abort(); + let _ = receive_task.await; + match send_result { + Err(e) => Err(server_error( + anyhow::Error::from(e).context("WebSocket send task failed"), + )), + Ok(inner) => inner, + } + }, + receive_result = &mut receive_task => { + send_task.abort(); + let _ = send_task.await; + match receive_result { + Err(e) => Err(server_error( + anyhow::Error::from(e).context("WebSocket receive task failed"), + )), + Ok(inner) => inner, + } + }, }; - let result: Result<(), SyncServerError> = (async { - send_task - .await - .context("WebSocket send task failed") - .map_err(client_error) - .and_then(|err| err)?; - - receive_task - .await - .context("WebSocket receive task failed") - .map_err(client_error) - .and_then(|err| err)?; - - Ok(()) - }) - .await; - state .cursors .remove_cursors_of_device(&vault_id, &authed_handshake.handshake.device_id) .await; - if result.is_err() { - info!( - "WebSocket disconnected on vault `{vault_id}` for `{}`", - authed_handshake.handshake.device_id - ); + match &result { + Ok(()) => { + info!( + "WebSocket disconnected on vault `{vault_id}` for `{}`", + authed_handshake.handshake.device_id + ); + } + Err(err) => { + warn!( + "WebSocket error on vault `{vault_id}` for `{}`: {err}", + authed_handshake.handshake.device_id + ); + } } result diff --git a/sync-server/src/utils/dedup_paths.rs b/sync-server/src/utils/dedup_paths.rs index bc687f6a..0baf8ba8 100644 --- a/sync-server/src/utils/dedup_paths.rs +++ b/sync-server/src/utils/dedup_paths.rs @@ -1,8 +1,17 @@ +use std::sync::LazyLock; + use regex::Regex; +static DEDUP_SUFFIX_REGEX: LazyLock = + LazyLock::new(|| Regex::new(r" \((\d+)\)$").expect("invalid regex")); + pub fn dedup_paths(path: &str) -> impl Iterator { let mut path_parts = path.split('/').collect::>(); - let file_name = path_parts.pop().unwrap().to_owned(); + let file_name = path_parts + .pop() + .filter(|s| !s.is_empty()) + .unwrap_or(path) + .to_owned(); let mut directory = path_parts.join("/"); if !directory.is_empty() { @@ -29,14 +38,13 @@ pub fn dedup_paths(path: &str) -> impl Iterator { } }; - let regex = Regex::new(r" \((\d+)\)$").unwrap(); - let start_number = regex + let start_number = DEDUP_SUFFIX_REGEX .captures(&stem) .and_then(|caps| caps.get(1)) .and_then(|m| m.as_str().parse::().ok()) .unwrap_or(0); - let clean_stem = regex.replace(&stem, "").to_string(); + let clean_stem = DEDUP_SUFFIX_REGEX.replace(&stem, "").to_string(); (start_number..).map(move |dedup_number| { if dedup_number == 0 { diff --git a/sync-server/src/utils/find_first_available_path.rs b/sync-server/src/utils/find_first_available_path.rs index 7629d8f1..eddd81d2 100644 --- a/sync-server/src/utils/find_first_available_path.rs +++ b/sync-server/src/utils/find_first_available_path.rs @@ -1,25 +1,30 @@ use crate::app_state::database::models::VaultId; -use crate::{app_state::database::Transaction, utils::dedup_paths::dedup_paths}; +use crate::utils::dedup_paths::dedup_paths; use anyhow::Result; use log::{debug, info}; +use sqlx::sqlite::SqliteConnection; pub async fn find_first_available_path( vault_id: &VaultId, sanitized_relative_path: &str, database: &crate::app_state::database::Database, - transaction: &mut Transaction<'_>, + connection: &mut SqliteConnection, ) -> Result { info!("Finding first available path for `{sanitized_relative_path}` in vault `{vault_id}`"); for candidate in dedup_paths(sanitized_relative_path) { debug!("Checking candidate path for deconflicting names: `{candidate}`"); if database - .get_latest_document_by_path(vault_id, &candidate, Some(transaction)) + .get_latest_non_deleted_document_by_path(vault_id, &candidate, Some(connection)) .await? .is_none() { info!("Selected available path: `{candidate}`"); return Ok(candidate); } + + info!( + "Finding first available path for `{sanitized_relative_path}` in vault `{vault_id}` as `{candidate}` is already taken" + ); } unreachable!("dedup_paths produces infinite paths"); diff --git a/sync-server/src/utils/rotating_file_writer.rs b/sync-server/src/utils/rotating_file_writer.rs index f04f9ba9..1c5c86c5 100644 --- a/sync-server/src/utils/rotating_file_writer.rs +++ b/sync-server/src/utils/rotating_file_writer.rs @@ -6,7 +6,7 @@ use std::{ time::{Duration, SystemTime, UNIX_EPOCH}, }; -use chrono::{Local, NaiveDateTime}; +use chrono::NaiveDateTime; use tracing_subscriber::fmt::MakeWriter; #[derive(Clone)] @@ -55,7 +55,7 @@ impl RotatingFileWriter { let timestamp_str = filename.get(prefix_len..filename.len().checked_sub(4)?)?; let dt = NaiveDateTime::parse_from_str(timestamp_str, "%Y-%m-%d_%H-%M-%S").ok()?; - let timestamp = dt.and_local_timezone(Local).single()?; + let timestamp = dt.and_utc(); let secs: u64 = timestamp.timestamp().try_into().ok()?; Some(UNIX_EPOCH + Duration::from_secs(secs)) @@ -114,7 +114,7 @@ impl RotatingFileWriter { } fn rotate(inner: &mut RotatingFileWriterInner) -> io::Result<()> { - let timestamp = Local::now().format("%Y-%m-%d_%H-%M-%S"); + let timestamp = chrono::Utc::now().format("%Y-%m-%d_%H-%M-%S"); let filename = format!("{}.{}.log", inner.file_prefix, timestamp); let filepath = inner.directory.join(filename); @@ -132,8 +132,14 @@ impl RotatingFileWriter { impl Write for RotatingFileWriter { fn write(&mut self, buf: &[u8]) -> io::Result { - let mut inner = self.inner.lock().unwrap(); + let mut inner = self.inner.lock().unwrap_or_else(|poisoned| { + eprintln!("RotatingFileWriter mutex was poisoned, recovering"); + poisoned.into_inner() + }); + // Reset file handle after poison recovery so the next branch + // re-opens a valid file rather than writing to a potentially + // half-closed handle. if inner.current_file.is_none() { Self::open_or_create_log_file(&mut inner)?; } else if Self::should_rotate(&inner) { @@ -148,7 +154,10 @@ impl Write for RotatingFileWriter { } fn flush(&mut self) -> io::Result<()> { - let mut inner = self.inner.lock().unwrap(); + let mut inner = self.inner.lock().unwrap_or_else(|poisoned| { + eprintln!("RotatingFileWriter mutex was poisoned, recovering"); + poisoned.into_inner() + }); if let Some(ref mut file) = inner.current_file { file.flush() } else { @@ -267,7 +276,7 @@ mod tests { // Parse the expected time let expected_dt = NaiveDateTime::parse_from_str(timestamp_str, "%Y-%m-%d_%H-%M-%S").unwrap(); - let expected_timestamp = expected_dt.and_local_timezone(Local).single().unwrap(); + let expected_timestamp = expected_dt.and_utc(); let expected_duration = Duration::from_secs(expected_timestamp.timestamp().try_into().unwrap()); let expected_next = UNIX_EPOCH + expected_duration + rotation_duration; @@ -306,7 +315,7 @@ mod tests { // Should use the latest file (2025-10-26_14-00-00) let expected_dt = NaiveDateTime::parse_from_str("2025-10-26_14-00-00", "%Y-%m-%d_%H-%M-%S").unwrap(); - let expected_timestamp = expected_dt.and_local_timezone(Local).single().unwrap(); + let expected_timestamp = expected_dt.and_utc(); let expected_duration = Duration::from_secs(expected_timestamp.timestamp().try_into().unwrap()); let expected_next = UNIX_EPOCH + expected_duration + rotation_duration; diff --git a/sync-server/src/utils/sanitize_path.rs b/sync-server/src/utils/sanitize_path.rs index 9703225c..05100f68 100644 --- a/sync-server/src/utils/sanitize_path.rs +++ b/sync-server/src/utils/sanitize_path.rs @@ -1,14 +1,28 @@ +use anyhow::{Result, ensure}; + +use crate::consts::MAX_RELATIVE_PATH_LEN; + /// Sanitize the document's path to allow all clients to create the same path in /// their filesystem. If we didn't do this server-side, client's would need to /// deal with mapping invalid names to valid ones and then back. -pub fn sanitize_path(path: &str) -> String { +pub fn sanitize_path(path: &str) -> Result { + // Enforce the length cap at the single chokepoint every create/update + // handler goes through, so clients can't blow up axum's JSON/multipart + // parser with a 1 MB `relative_path` before the handler ever runs. + // The WebSocket cursor handler enforces this separately. + ensure!( + path.len() <= MAX_RELATIVE_PATH_LEN, + "Relative path exceeds the maximum length of {MAX_RELATIVE_PATH_LEN} bytes" + ); + let options = sanitize_filename::Options { truncate: true, windows: true, // Windows is the lowest common denominator replacement: "", }; - path.split('/') + let result = path + .split('/') .map(|part| { let proposal = sanitize_filename::sanitize_with_options(part, options.clone()); if !part.is_empty() && proposal.is_empty() { @@ -18,7 +32,13 @@ pub fn sanitize_path(path: &str) -> String { } }) .collect::>() - .join("/") + .join("/"); + + ensure!( + !result.is_empty(), + "Relative path is empty after sanitization" + ); + Ok(result) } #[cfg(test)] @@ -27,8 +47,32 @@ mod test { #[test] fn test_sanitize_path() { - assert_eq!(sanitize_path("/my/path/what?"), "/my/path/what"); - assert_eq!(sanitize_path("file (1).md"), "file (1).md"); - assert_eq!(sanitize_path("/my/path/\\\\:?"), "/my/path/_"); + assert_eq!(sanitize_path("/my/path/what?").unwrap(), "/my/path/what"); + assert_eq!(sanitize_path("file (1).md").unwrap(), "file (1).md"); + assert_eq!(sanitize_path("/my/path/\\\\:?").unwrap(), "/my/path/_"); + } + + #[test] + fn test_sanitize_path_empty() { + assert!(sanitize_path("").is_err()); + } + + #[test] + fn test_sanitize_path_idempotent_simple() { + let mut result = sanitize_path("notes/my file.md").unwrap(); + for _ in 0..5 { + result = sanitize_path(&result).unwrap(); + } + assert_eq!(result, "notes/my file.md"); + } + + #[test] + fn test_sanitize_path_idempotent_special_chars() { + let first = sanitize_path("/my/path/what?/file:name<>.md").unwrap(); + let mut result = first.clone(); + for _ in 0..5 { + result = sanitize_path(&result).unwrap(); + } + assert_eq!(result, first); } }