diff --git a/frontend/deterministic-tests/README.md b/frontend/deterministic-tests/README.md new file mode 100644 index 00000000..71578ed1 --- /dev/null +++ b/frontend/deterministic-tests/README.md @@ -0,0 +1,81 @@ +# Deterministic Tests + +Scripted multi-client (with an in-memory filesystem) sync tests that run against a real server. Each test defines a sequence of file operations, sync/server controls, and assertions to exercise a specific conflict or edge case. + +Complements the fuzz-based E2E tests (`test-client`): fuzz tests discover bugs through random operations; deterministic tests pin down exact reproduction sequences for known scenarios. + +## How it works + +Each test is a `TestDefinition`: a name, a client count, and an ordered list of steps. The `TestRunner` spins up N `DeterministicAgent` instances (each wrapping a real `SyncClient` with an `InMemoryFileSystem`) pointed at a shared vault on the server, then executes steps one by one. + +Tests that don't pause the server share a single server process (vault-name isolation). Tests that use `pause-server`/`resume-server` (SIGSTOP/SIGCONT) each get a dedicated server, since SIGSTOP freezes the entire process. + +All tests run in parallel up to a concurrency limit. + +## Step types + +Clients always start with syincing being disabled. + +**File operations** (per-client, fire-and-forget — sync is enqueued but not awaited): +- `create`, `update`, `rename`, `delete` + +**Sync control:** +- `sync` — wait for a specific client or all clients to finish pending operations +- `barrier` — retry until all clients converge to identical file state (60s timeout) +- `enable-sync` / `disable-sync` — simulate going online/offline + +**Server control:** +- `pause-server` / `resume-server` — SIGSTOP/SIGCONT the server process +- `wait` — sleep for N milliseconds + +**Assertions:** +- `assert-content`, `assert-exists`, `assert-not-exists` +- `assert-consistent` — all clients have identical files; optionally takes a custom verify function + +## Running + +```sh +# Build server first +cd sync-server && cargo build --release + +# Run all tests +cd frontend && npm run test -w deterministic-tests + +# Filter by name +npm run test -w deterministic-tests -- --filter=rename + +# Control parallelism (default: number of CPU cores) +npm run test -w deterministic-tests -- -j 4 +``` + +## Adding a test + +1. Create `src/tests/my-scenario.test.ts`: + +```typescript +import type { TestDefinition } from "../test-definition"; + +export const myScenarioTest: TestDefinition = { + name: "My Scenario", + description: "What this test verifies", + clients: 2, + steps: [ + { type: "create", client: 0, path: "A.md", content: "hello" }, + { type: "sync" }, + { type: "barrier" }, + { type: "assert-consistent" } + ] +}; +``` + +2. Register it in `src/test-registry.ts`: + +```typescript +import { myScenarioTest } from "./tests/my-scenario.test"; + +const TESTS = { + // ... + "my-scenario": myScenarioTest +}; +``` + diff --git a/frontend/deterministic-tests/package.json b/frontend/deterministic-tests/package.json new file mode 100644 index 00000000..e1c1b276 --- /dev/null +++ b/frontend/deterministic-tests/package.json @@ -0,0 +1,22 @@ +{ + "name": "deterministic-tests", + "version": "0.14.0", + "private": true, + "bin": { + "deterministic-tests": "./dist/cli.js" + }, + "scripts": { + "dev": "webpack watch --mode development", + "build": "webpack --mode production", + "test": "npm run build && node dist/cli.js" + }, + "devDependencies": { + "@types/node": "^25.0.2", + "sync-client": "file:../sync-client", + "ts-loader": "^9.5.4", + "tslib": "2.8.1", + "typescript": "5.9.3", + "webpack": "^5.103.0", + "webpack-cli": "^6.0.1" + } +} diff --git a/frontend/deterministic-tests/src/cli.ts b/frontend/deterministic-tests/src/cli.ts new file mode 100644 index 00000000..2815abae --- /dev/null +++ b/frontend/deterministic-tests/src/cli.ts @@ -0,0 +1,228 @@ +import { TestRunner } from "./test-runner"; +import { ServerControl } from "./server-control"; +import { ServerManager } from "./server-manager"; +import { PrefixedLogger } from "./prefixed-logger"; +import { TESTS } from "./test-registry"; +import type { TestDefinition, TestResult } from "./test-definition"; +import { parseConcurrency } from "./parse-concurrency"; +import { runWithConcurrency } from "./run-with-concurrency"; +import { TOKEN, SERVER_BINARY_PATH, CONFIG_PATH } from "./consts"; +import * as path from "node:path"; +import * as fs from "node:fs"; +import { debugging, Logger } from "sync-client"; + +const logger = new Logger(); +debugging.logToConsole(logger, { useColors: true }); + +process.on("unhandledRejection", (reason) => { + logger.error(`Unhandled Rejection: ${reason}`); + process.exit(1); +}); + +process.on("uncaughtException", (error) => { + logger.error(`Uncaught Exception: ${error}`); + process.exit(1); +}); + +const serverManager = new ServerManager(logger); +serverManager.installSignalHandlers(); + +function testUsesPauseServer(test: TestDefinition): boolean { + return test.steps.some( + (step) => step.type === "pause-server" || step.type === "resume-server" + ); +} + +interface NamedTestResult { + test: TestDefinition; + result: TestResult; +} + + +async function main(): Promise { + const cwd = process.cwd(); + let projectRoot = cwd; + + if (cwd.endsWith("frontend/deterministic-tests")) { + projectRoot = path.resolve(cwd, "../.."); + } else if (cwd.endsWith("frontend")) { + projectRoot = path.resolve(cwd, ".."); + } + + const serverPath = path.join(projectRoot, SERVER_BINARY_PATH); + if (!fs.existsSync(serverPath)) { + logger.error(`Server binary not found at: ${serverPath}`); + process.exit(1); + } + + const configPath = path.join(projectRoot, CONFIG_PATH); + if (!fs.existsSync(configPath)) { + logger.error(`Config file not found at: ${configPath}`); + process.exit(1); + } + + const filterArg = process.argv.find((a) => a.startsWith("--filter=")); + const filter = filterArg?.slice("--filter=".length); + + const testsToRun: TestDefinition[] = []; + for (const [key, test] of Object.entries(TESTS)) { + if (test) { + if (filter && !key.includes(filter) && !test.name.toLowerCase().includes(filter.toLowerCase())) { + continue; + } + testsToRun.push(test); + } + } + + if (testsToRun.length === 0) { + logger.error( + filter + ? `No tests matched filter "${filter}"` + : "No tests found" + ); + process.exit(1); + } + + const concurrency = parseConcurrency(); + const regularTests = testsToRun.filter((t) => !testUsesPauseServer(t)); + const pauseTests = testsToRun.filter((t) => testUsesPauseServer(t)); + + logger.info(`Server: ${serverPath}`); + logger.info(`Config: ${configPath}`); + logger.info( + `Tests: ${testsToRun.length} total (${regularTests.length} regular, ${pauseTests.length} server-pause)` + ); + logger.info(`Concurrency: ${concurrency}`); + + const allResults: NamedTestResult[] = []; + + if (regularTests.length > 0) { + logger.info( + `\n--- Running ${regularTests.length} regular tests (shared server, concurrency ${concurrency}) ---` + ); + const sharedServer = new ServerControl( + serverPath, + configPath, + logger + ); + serverManager.track(sharedServer); + + try { + await sharedServer.start(); + + const results = await runWithConcurrency( + regularTests, + concurrency, + async (test) => runSharedServerTest(test, sharedServer) + ); + + allResults.push(...results); + } finally { + try { + await sharedServer.stop(); + } catch (error) { + logger.warn( + `Error stopping shared server: ${error instanceof Error ? error.message : String(error)}` + ); + } + serverManager.untrack(sharedServer); + } + } + + if (pauseTests.length > 0) { + logger.info( + `\n--- Running ${pauseTests.length} server-pause tests (dedicated servers, concurrency ${concurrency}) ---` + ); + + const results = await runWithConcurrency( + pauseTests, + concurrency, + async (test) => runDedicatedServerTest(test, serverPath, configPath) + ); + + allResults.push(...results); + } + + const passed = allResults.filter((r) => r.result.success); + const failed = allResults.filter((r) => !r.result.success); + + logger.info(`\n--- Results: ${passed.length}/${allResults.length} passed ---`); + + if (failed.length > 0) { + for (const { test, result } of failed) { + logger.error(` FAILED: ${test.name}: ${result.error}`); + } + process.exit(1); + } else { + logger.info("All tests passed!"); + process.exit(0); + } +} + +main().catch((err: unknown) => { + logger.error(`Unexpected error: ${err}`); + process.exit(1); +}); + + +/** + * Run a test on a shared server (for tests that don't use pause-server). + */ +async function runSharedServerTest( + test: TestDefinition, + sharedServer: ServerControl +): Promise { + const testLogger = new PrefixedLogger(logger, test.name); + const runner = new TestRunner( + sharedServer, + testLogger, + TOKEN, + sharedServer.remoteUri + ); + const result = await runner.runTest(test); + if (result.success) { + logger.info(`PASSED: ${test.name} (${result.duration}ms)`); + } else { + logger.error(`FAILED: ${test.name} - ${result.error}`); + } + return { test, result }; +} + +/** + * Run a test with its own dedicated server (for tests that use pause-server). + * SIGSTOP/SIGCONT affects the entire server process, so these tests need + * isolated servers to avoid interfering with other tests. + */ +async function runDedicatedServerTest( + test: TestDefinition, + serverPath: string, + configPath: string +): Promise { + const testLogger = new PrefixedLogger(logger, test.name); + const server = new ServerControl(serverPath, configPath, testLogger); + serverManager.track(server); + + try { + await server.start(); + const runner = new TestRunner( + server, + testLogger, + TOKEN, + server.remoteUri + ); + const result = await runner.runTest(test); + if (result.success) { + logger.info(`PASSED: ${test.name} (${result.duration}ms)`); + } else { + logger.error(`FAILED: ${test.name} - ${result.error}`); + } + return { test, result }; + } finally { + try { + await server.stop(); + } catch { + // best-effort cleanup + } + serverManager.untrack(server); + } +} diff --git a/frontend/deterministic-tests/src/consts.ts b/frontend/deterministic-tests/src/consts.ts new file mode 100644 index 00000000..32c03efa --- /dev/null +++ b/frontend/deterministic-tests/src/consts.ts @@ -0,0 +1,13 @@ +export const TOKEN = "test-token-change-me"; +export const SERVER_BINARY_PATH = "sync-server/target/release/sync_server"; +export const CONFIG_PATH = "sync-server/config-e2e.yml"; + +export const STOP_TIMEOUT_MS = 5_000; +export const CONVERGENCE_TIMEOUT_MS = 60_000; +export const CONVERGENCE_RETRY_DELAY_MS = 500; +export const AGENT_INIT_TIMEOUT_MS = 30_000; +export const IS_SYNC_ENABLED_DEFAULT = false; + +export const WAIT_TIMEOUT_MS = 60_000; +export const WEBSOCKET_CONNECT_TIMEOUT_MS = 10_000; +export const WEBSOCKET_POLL_INTERVAL_MS = 50; diff --git a/frontend/deterministic-tests/src/deterministic-agent.ts b/frontend/deterministic-tests/src/deterministic-agent.ts new file mode 100644 index 00000000..3f4631b2 --- /dev/null +++ b/frontend/deterministic-tests/src/deterministic-agent.ts @@ -0,0 +1,280 @@ +import type { StoredDatabase, SyncSettings, RelativePath } from "sync-client"; +import { SyncClient, debugging, LogLevel } from "sync-client"; +import { assert } from "./utils/assert"; +import { sleep } from "./utils/sleep"; +import { withTimeout } from "./utils/with-timeout"; +import { IS_SYNC_ENABLED_DEFAULT, WAIT_TIMEOUT_MS, WEBSOCKET_CONNECT_TIMEOUT_MS, WEBSOCKET_POLL_INTERVAL_MS } from "./consts"; + + + +export class DeterministicAgent extends debugging.InMemoryFileSystem { + public readonly clientId: number; + private readonly logger: (msg: string) => void; + private client!: SyncClient; + private data: Partial<{ + settings: Partial; + database: Partial; + }> = {}; + private isSyncEnabled = IS_SYNC_ENABLED_DEFAULT; + + public constructor( + clientId: number, + initialSettings: Partial, + logger: (msg: string) => void + ) { + super(); + this.clientId = clientId; + this.logger = logger; + this.data.settings = { ...initialSettings }; + } + + public async init( + fetchImplementation: typeof globalThis.fetch, + webSocketImplementation: typeof globalThis.WebSocket + ): Promise { + this.client = await SyncClient.create({ + fs: this, + persistence: { + load: async () => this.data, + save: async (data) => void (this.data = data) + }, + fetch: fetchImplementation, + webSocket: webSocketImplementation + }); + + this.client.logger.onLogEmitted.add((line) => { + const prefix = `[Client ${this.clientId}]`; + switch (line.level) { + case LogLevel.ERROR: + this.logger(`${prefix} ERROR: ${line.message}`); + break; + case LogLevel.WARNING: + this.logger(`${prefix} WARN: ${line.message}`); + break; + case LogLevel.INFO: + this.logger(`${prefix} ${line.message}`); + break; + case LogLevel.DEBUG: + // Skip debug logs to reduce noise + break; + } + }); + + await this.client.start(); + + const connectionCheck = await this.client.checkConnection(); + assert( + connectionCheck.isSuccessful, + `Client ${this.clientId} connection check failed` + ); + + if (this.isSyncEnabled) { + await this.waitForWebSocket(); + } + } + + public async createFile(path: string, content: string): Promise { + this.log(`Creating file ${path} with content: ${content}`); + if (this.files.has(path)) { + throw new Error(`File ${path} already exists`); + } + const contentBytes = new TextEncoder().encode(content); + this.files.set(path, contentBytes); + + this.enqueueSync(async () => + this.client.syncLocallyCreatedFile(path) + ); + } + + public async updateFile(path: string, content: string): Promise { + this.log(`Updating file ${path} with content: ${content}`); + if (!this.files.has(path)) { + throw new Error( + `File ${path} does not exist on client ${this.clientId}` + ); + } + const contentBytes = new TextEncoder().encode(content); + this.files.set(path, contentBytes); + + this.enqueueSync(async () => + this.client.syncLocallyUpdatedFile({ relativePath: path }) + ); + } + + public async renameFile(oldPath: string, newPath: string): Promise { + this.log(`Renaming file ${oldPath} to ${newPath}`); + const file = this.files.get(oldPath); + if (!file) { + throw new Error( + `File ${oldPath} does not exist on client ${this.clientId}` + ); + } + if (oldPath !== newPath && this.files.has(newPath)) { + this.log( + `Target path ${newPath} already exists, will be overwritten (ensureClearPath)` + ); + } + this.files.set(newPath, file); + if (oldPath !== newPath) { + this.files.delete(oldPath); + } + if (this.isSyncEnabled) { + this.enqueueSync(async () => + this.client.syncLocallyUpdatedFile({ + oldPath, + relativePath: newPath + }) + ); + } + } + + public async deleteFile(path: string): Promise { + this.log(`Deleting file ${path}`); + this.files.delete(path); + if (this.isSyncEnabled) { + this.enqueueSync(async () => + this.client.syncLocallyDeletedFile(path) + ); + } + } + + public async waitForSync(): Promise { + this.log("Waiting for sync to complete..."); + await withTimeout( + this.client.waitUntilFinished(), + WAIT_TIMEOUT_MS, + `Client ${this.clientId} waitForSync timed out after ${WAIT_TIMEOUT_MS}ms` + ); + this.log("Sync complete"); + } + + public async disableSync(): Promise { + this.log("Disabling sync"); + await this.client.setSetting("isSyncEnabled", false); + this.isSyncEnabled = false; + } + + public async enableSync(): Promise { + this.log("Enabling sync"); + await this.client.setSetting("isSyncEnabled", true); + this.isSyncEnabled = true; + await this.waitForWebSocket(); + } + + public async assertContent( + path: string, + expectedContent: string + ): Promise { + this.log(`Asserting content of ${path} equals "${expectedContent}"`); + const actualBytes = await this.read(path).catch(() => { + throw new Error( + `File ${path} does not exist on client ${this.clientId}` + ); + }); + const actualContent = new TextDecoder().decode(actualBytes); + assert( + actualContent === expectedContent, + `Content mismatch on client ${this.clientId} for ${path}:\nExpected: "${expectedContent}"\nActual: "${actualContent}"` + ); + this.log(`✓ Content assertion passed for ${path}`); + } + + public async assertExists(path: string): Promise { + this.log(`Asserting ${path} exists`); + const exists = await this.exists(path); + assert( + exists, + `File ${path} does not exist on client ${this.clientId}` + ); + this.log(`✓ File ${path} exists`); + } + + public async assertNotExists(path: string): Promise { + this.log(`Asserting ${path} does not exist`); + const exists = await this.exists(path); + assert( + !exists, + `File ${path} exists on client ${this.clientId} but should not` + ); + this.log(`✓ File ${path} does not exist`); + } + + public async getFiles(): Promise { + return this.listFilesRecursively(); + } + + public async getFileContent(path: string): Promise { + const bytes = await this.read(path); + return new TextDecoder().decode(bytes); + } + + public async cleanup(): Promise { + this.log("Cleaning up..."); + // Guard against uninitialized client (init() failed partway) + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition + if (!this.client) { + this.log("Client not initialized, nothing to clean up"); + return; + } + try { + await withTimeout( + this.client.waitUntilFinished(), + WAIT_TIMEOUT_MS, + `Client ${this.clientId} cleanup waitUntilFinished timed out` + ); + } catch (error) { + if (error instanceof Error && error.name === "SyncResetError") { + this.log(`Cleanup interrupted by reset (expected): ${error}`); + } else { + this.log(`Cleanup waitUntilFinished failed: ${error}`); + } + } + await this.client.destroy(); + this.log("Cleanup complete"); + } + + private async waitForWebSocket(): Promise { + const deadline = Date.now() + WEBSOCKET_CONNECT_TIMEOUT_MS; + while (!this.client.isWebSocketConnected && Date.now() < deadline) { + await sleep(WEBSOCKET_POLL_INTERVAL_MS); + } + assert( + this.client.isWebSocketConnected, + `Client ${this.clientId} WebSocket failed to connect within ${WEBSOCKET_CONNECT_TIMEOUT_MS}ms` + ); + } + + private enqueueSync(operation: () => Promise): void { + void this.executeSyncOperation(operation).catch((error) => { + this.log( + `Background sync failed (will retry on reconnect): ${error}` + ); + }); + } + + private async executeSyncOperation( + operation: () => Promise + ): Promise { + try { + await operation(); + } catch (error) { + if (error instanceof Error && error.name === "SyncResetError") { + this.log(`Sync operation interrupted by reset: ${error}`); + return; + } + if ( + error instanceof Error && + error.message.includes("has been destroyed") + ) { + this.log(`Sync operation interrupted by destroy: ${error}`); + return; + } + + throw error; + } + } + + private log(message: string): void { + this.logger(`[Client ${this.clientId}] ${message}`); + } +} diff --git a/frontend/deterministic-tests/src/parse-concurrency.ts b/frontend/deterministic-tests/src/parse-concurrency.ts new file mode 100644 index 00000000..a6622a04 --- /dev/null +++ b/frontend/deterministic-tests/src/parse-concurrency.ts @@ -0,0 +1,15 @@ +import * as os from "node:os"; + +export function parseConcurrency(): number { + const args = process.argv.slice(2); + for (let i = 0; i < args.length; i++) { + if ( + (args[i] === "--concurrency" || args[i] === "-j") && + i + 1 < args.length + ) { + const n = parseInt(args[i + 1], 10); + if (!isNaN(n) && n > 0) return n; + } + } + return os.cpus().length; +} diff --git a/frontend/deterministic-tests/src/prefixed-logger.ts b/frontend/deterministic-tests/src/prefixed-logger.ts new file mode 100644 index 00000000..769d7545 --- /dev/null +++ b/frontend/deterministic-tests/src/prefixed-logger.ts @@ -0,0 +1,28 @@ +import { Logger } from "sync-client"; + +export class PrefixedLogger extends Logger { + private readonly base: Logger; + private readonly prefix: string; + + public constructor(base: Logger, prefix: string) { + super(); + this.base = base; + this.prefix = prefix; + } + + public override debug(message: string): void { + this.base.debug(`[${this.prefix}] ${message}`); + } + + public override info(message: string): void { + this.base.info(`[${this.prefix}] ${message}`); + } + + public override warn(message: string): void { + this.base.warn(`[${this.prefix}] ${message}`); + } + + public override error(message: string): void { + this.base.error(`[${this.prefix}] ${message}`); + } +} diff --git a/frontend/deterministic-tests/src/run-with-concurrency.ts b/frontend/deterministic-tests/src/run-with-concurrency.ts new file mode 100644 index 00000000..f5bcf745 --- /dev/null +++ b/frontend/deterministic-tests/src/run-with-concurrency.ts @@ -0,0 +1,33 @@ +export async function runWithConcurrency( + items: T[], + concurrency: number, + fn: (item: T) => Promise +): Promise { + const results: R[] = []; + const errors: unknown[] = []; + const executing = new Set>(); + + for (let i = 0; i < items.length; i++) { + const index = i; + const p = fn(items[index]) + .then((result) => { + results[index] = result; + }) + .catch((error: unknown) => { + errors.push(error); + }) + .finally(() => executing.delete(p)); + executing.add(p); + if (executing.size >= concurrency) { + await Promise.race(executing); + } + } + + // eslint-disable-next-line no-restricted-properties + await Promise.all(executing); + + if (errors.length > 0) { + throw errors[0]; + } + return results; +} diff --git a/frontend/deterministic-tests/src/server-control.ts b/frontend/deterministic-tests/src/server-control.ts new file mode 100644 index 00000000..5c8aff17 --- /dev/null +++ b/frontend/deterministic-tests/src/server-control.ts @@ -0,0 +1,236 @@ +import { spawn, type ChildProcess } from "node:child_process"; +import * as fs from "node:fs"; +import * as os from "node:os"; +import * as path from "node:path"; +import { sleep } from "./utils/sleep"; +import { findFreePort } from "./utils/find-free-port"; +import type { Logger } from "sync-client"; +import { STOP_TIMEOUT_MS } from "./consts"; + +export class ServerControl { + private process: ChildProcess | null = null; + private readonly serverPath: string; + private readonly baseConfigPath: string; + private readonly logger: Logger; + private _port: number | undefined; + private tempDir: string | undefined; + private _isPaused = false; + + public constructor(serverPath: string, configPath: string, logger: Logger) { + this.serverPath = serverPath; + this.baseConfigPath = configPath; + this.logger = logger; + } + + public get port(): number { + if (this._port === undefined) { + throw new Error("Server has not been started yet"); + } + return this._port; + } + + public get remoteUri(): string { + return `http://localhost:${this.port}`; + } + + public async start(): Promise { + if (this.process !== null) { + throw new Error("Server is already running"); + } + + const reservation = await findFreePort(); + this._port = reservation.port; + this.tempDir = fs.mkdtempSync( + path.join(os.tmpdir(), "vault-link-test-") + ); + const tempConfigPath = path.join(this.tempDir, "config.yml"); + const dbDir = path.join(this.tempDir, "databases"); + + this.writeConfigFile(tempConfigPath, dbDir); + + this.logger.info( + `Starting server: ${this.serverPath} (port ${this._port})` + ); + + // Release the port reservation right before spawning to minimize + // the TOCTOU window between port discovery and server binding. + reservation.release(); + + this.process = spawn(this.serverPath, [tempConfigPath], { + stdio: ["ignore", "pipe", "pipe"], + detached: false + }); + + this.process.stdout?.on("data", (data: Buffer) => { + this.logger.info(`[SERVER] ${data.toString().trim()}`); + }); + + this.process.stderr?.on("data", (data: Buffer) => { + this.logger.info(`[SERVER] ${data.toString().trim()}`); + }); + + this.process.on("error", (err) => { + this.logger.error(`[SERVER] Process error: ${err.message}`); + }); + + const currentProcess = this.process; + currentProcess.on("exit", (code, signal) => { + this.logger.info( + `Server exited with code ${code}, signal ${signal}` + ); + // Only clear state if this handler is for the current process. + // A fast stop→start cycle could create a new process before this + // handler fires — clearing state here would corrupt the new one. + if (this.process === currentProcess) { + this.process = null; + this._isPaused = false; + } + }); + + try { + await this.waitForReady(); + } catch (error) { + // Kill the spawned process if it failed to become ready, + // preventing a zombie process from lingering. + try { + await this.stop(); + } catch { + // Best-effort cleanup + } + throw error; + } + } + + public async waitForReady(maxAttempts = 50): Promise { + const pingUrl = `${this.remoteUri}/vaults/test/ping`; + for (let i = 0; i < maxAttempts; i++) { + if (this.process === null || this.process.exitCode !== null) { + throw new Error( + "Server process died while waiting for it to become ready" + ); + } + try { + const response = await fetch(pingUrl); + if (response.ok) { + this.logger.info("[SERVER] Ready"); + return; + } + } catch { + // Server not ready yet, continue polling + } + await sleep(100); + } + throw new Error("Server failed to start within timeout"); + } + + public pause(): void { + if (this.process?.pid === undefined) { + throw new Error("Server is not running"); + } + if (this._isPaused) { + this.logger.warn("Server is already paused, skipping double-pause"); + return; + } + this.logger.info("Server pausing..."); + try { + process.kill(this.process.pid, "SIGSTOP"); + this._isPaused = true; + this.logger.info("Server paused (SIGSTOP sent)"); + } catch (error) { + throw new Error( + `Failed to pause server (pid ${this.process.pid}): ${error instanceof Error ? error.message : String(error)}` + ); + } + } + + public resume(): void { + if (this.process?.pid === undefined) { + throw new Error("Server is not running"); + } + if (!this._isPaused) { + return; + } + this.logger.info("Server resuming..."); + try { + process.kill(this.process.pid, "SIGCONT"); + this._isPaused = false; + this.logger.info("Server resumed (SIGCONT sent)"); + } catch (error) { + throw new Error( + `Failed to resume server (pid ${this.process.pid}): ${error instanceof Error ? error.message : String(error)}` + ); + } + } + + public async stop(): Promise { + const proc = this.process; + if (proc?.pid === undefined) { + this.cleanupTempDir(); + return; + } + + // Resume if paused — a SIGSTOP'd process ignores SIGKILL + if (this._isPaused) { + try { + process.kill(proc.pid, "SIGCONT"); + } catch { + // Process may already be gone + } + this._isPaused = false; + } + + this.logger.info("Server stopping..."); + + // Set up a promise that resolves when the process actually exits. + const exitPromise = new Promise((resolve) => { + if (proc.exitCode !== null) { + resolve(); + return; + } + proc.on("exit", () => { + resolve(); + }); + }); + + try { + process.kill(proc.pid, "SIGKILL"); + } catch { + // Process already gone + } + + // Wait for the process to actually exit before cleaning up, + // with a 5s safety timeout to avoid hanging forever. + await Promise.race([exitPromise, sleep(STOP_TIMEOUT_MS)]); + + this.process = null; + this._isPaused = false; + this.cleanupTempDir(); + } + + public isRunning(): boolean { + return this.process?.pid !== undefined; + } + + private writeConfigFile(destPath: string, dbDir: string): void { + const baseConfig = fs.readFileSync(this.baseConfigPath, "utf-8"); + const config = baseConfig + .replace(/^\s*port:\s*\d+/m, ` port: ${this._port}`) + .replace( + /^\s*databases_directory_path:\s*.+/m, + ` databases_directory_path: ${dbDir}` + ); + fs.writeFileSync(destPath, config); + } + + private cleanupTempDir(): void { + if (this.tempDir) { + try { + fs.rmSync(this.tempDir, { recursive: true, force: true }); + } catch { + // Best-effort cleanup + } + this.tempDir = undefined; + } + } + +} diff --git a/frontend/deterministic-tests/src/server-manager.ts b/frontend/deterministic-tests/src/server-manager.ts new file mode 100644 index 00000000..51e162ee --- /dev/null +++ b/frontend/deterministic-tests/src/server-manager.ts @@ -0,0 +1,53 @@ +import { ServerControl } from "./server-control"; +import type { Logger } from "sync-client"; + +export class ServerManager { + private readonly activeServers = new Set(); + private readonly logger: Logger; + private isShuttingDown = false; + + public constructor(logger: Logger) { + this.logger = logger; + } + + public track(server: ServerControl): void { + this.activeServers.add(server); + } + + public untrack(server: ServerControl): void { + this.activeServers.delete(server); + } + + public async stopAll(): Promise { + if (this.isShuttingDown) return; + this.isShuttingDown = true; + + const servers = Array.from(this.activeServers); + // eslint-disable-next-line no-restricted-properties + await Promise.all( + servers.map(async (server) => { + try { + await server.stop(); + } catch { + // Best-effort cleanup during shutdown + } + }) + ); + } + + public installSignalHandlers(): void { + process.on("SIGINT", () => { + this.logger.info("Received SIGINT, shutting down..."); + void this.stopAll() + .catch(() => {}) + .then(() => process.exit(130)); + }); + + process.on("SIGTERM", () => { + this.logger.info("Received SIGTERM, shutting down..."); + void this.stopAll() + .catch(() => {}) + .then(() => process.exit(143)); + }); + } +} diff --git a/frontend/deterministic-tests/src/test-definition.ts b/frontend/deterministic-tests/src/test-definition.ts new file mode 100644 index 00000000..453a5d01 --- /dev/null +++ b/frontend/deterministic-tests/src/test-definition.ts @@ -0,0 +1,32 @@ +export interface ClientState { + files: Map; +} + +export type TestStep = + | { type: "create"; client: number; path: string; content: string } + | { type: "update"; client: number; path: string; content: string } + | { type: "rename"; client: number; oldPath: string; newPath: string } + | { type: "delete"; client: number; path: string } + | { type: "sync"; client?: number } + | { type: "disable-sync"; client: number } + | { type: "enable-sync"; client: number } + | { type: "pause-server" } + | { type: "resume-server" } + | { type: "barrier" } + | { type: "assert-content"; client: number; path: string; content: string } + | { type: "assert-exists"; client: number; path: string } + | { type: "assert-not-exists"; client: number; path: string } + | { type: "assert-consistent"; verify?: (state: ClientState) => void }; + +export interface TestDefinition { + name: string; + description?: string; + clients: number; + steps: TestStep[]; +} + +export interface TestResult { + success: boolean; + error?: string; + duration?: number; +} diff --git a/frontend/deterministic-tests/src/test-registry.ts b/frontend/deterministic-tests/src/test-registry.ts new file mode 100644 index 00000000..6ff5c9d3 --- /dev/null +++ b/frontend/deterministic-tests/src/test-registry.ts @@ -0,0 +1,226 @@ +import type { TestDefinition } from "./test-definition"; +import { writeWriteConflictTest } from "./tests/write-write-conflict.test"; +import { renameCreateConflictTest } from "./tests/rename-create-conflict.test"; +import { createDeleteNoopTest } from "./tests/create-delete-noop.test"; +import { renameChainTest } from "./tests/rename-chain.test"; +import { serverPauseResumeTest } from "./tests/server-pause-resume.test"; +import { createMergeDeleteTest } from "./tests/create-merge-delete.test"; +import { renameUpdateConflictTest } from "./tests/rename-update-conflict.test"; +import { deleteRenameConflictTest } from "./tests/delete-rename-conflict.test"; +import { multiFileOperationsTest } from "./tests/multi-file-operations.test"; +import { duplicateContentFilesTest } from "./tests/duplicate-content-files.test"; +import { deleteRecreateSamePathTest } from "./tests/delete-recreate-same-path.test"; +import { rapidSyncToggleTest } from "./tests/rapid-sync-toggle.test"; +import { concurrentDeleteUpdateTest } from "./tests/concurrent-delete-update.test"; +import { offlineRenameAndEditTest } from "./tests/offline-rename-and-edit.test"; +import { threeClientConvergenceTest } from "./tests/three-client-convergence.test"; +import { updateDuringServerPauseTest } from "./tests/update-during-server-pause.test"; +import { emptyFileSyncTest } from "./tests/empty-file-sync.test"; +import { renameToExistingPathTest } from "./tests/rename-to-existing-path.test"; +import { concurrentRenameSameTargetTest } from "./tests/concurrent-rename-same-target.test"; +import { multipleUpdatesCoalesceTest } from "./tests/multiple-updates-coalesce.test"; +import { deleteNonexistentFileTest } from "./tests/delete-nonexistent-file.test"; +import { createWhileServerPausedTest } from "./tests/create-while-server-paused.test"; +import { interleavedOperationsTest } from "./tests/interleaved-operations.test"; +import { simultaneousCreateDeleteSamePathTest } from "./tests/simultaneous-create-delete-same-path.test"; +import { largeFileCountTest } from "./tests/large-file-count.test"; +import { offlineOperationsBothClientsTest } from "./tests/offline-operations-both-clients.test"; +import { updateThenRenameTest } from "./tests/update-then-rename.test"; +import { idempotencyAfterServerPauseTest } from "./tests/idempotency-after-server-pause.test"; +import { concurrentCreateSamePathMergeTest } from "./tests/concurrent-create-same-path-merge.test"; +import { sequentialCreateDuplicateContentTest } from "./tests/sequential-create-duplicate-content.test"; +import { offlineMultiUpdateCatchupTest } from "./tests/offline-multi-update-catchup.test"; +import { mcThreeClientRenameOfflineUpdateTest } from "./tests/mc-three-client-rename-offline-update.test"; +import { mcMultiDeleteOfflineRenameTest } from "./tests/mc-multi-delete-offline-rename.test"; +import { mcCrossCreateRenameSameTargetTest } from "./tests/mc-cross-create-rename-same-target.test"; +import { mcDeleteThenOfflineRenameTest } from "./tests/mc-delete-then-offline-rename.test"; +import { offlineMixedOperationsTest } from "./tests/offline-mixed-operations.test"; +import { offlineCreateRenameCreateTest } from "./tests/offline-create-rename-create.test"; +import { offlineConcurrentRenamesTest } from "./tests/offline-concurrent-renames.test"; +import { offlineMultipleEditsTest } from "./tests/offline-multiple-edits.test"; +import { serverPauseBothClientsCreateTest } from "./tests/server-pause-both-clients-create.test"; +import { serverPauseRenameTest } from "./tests/server-pause-rename-propagation.test"; +import { serverPauseConcurrentCreatesTest } from "./tests/server-pause-concurrent-creates.test"; +import { serverPauseUpdateAndCreateTest } from "./tests/server-pause-update-and-create.test"; +import { renameSwapTest } from "./tests/rename-swap.test"; +import { renameCircularTest } from "./tests/rename-circular.test"; +import { renameNestedPathTest } from "./tests/rename-nested-path.test"; +import { renameRoundtripTest } from "./tests/rename-roundtrip.test"; +import { offlineRenameRemoteCreateOldPathTest } from "./tests/offline-rename-remote-create-old-path.test"; +import { offlineEditRemoteRenameTest } from "./tests/offline-edit-remote-rename.test"; +import { renameChainThenDeleteTest } from "./tests/rename-chain-then-delete.test"; +import { offlineDeleteRemoteRenameTest } from "./tests/offline-delete-remote-rename.test"; +import { renameToRecentlyDeletedPathTest } from "./tests/rename-to-recently-deleted-path.test"; +import { createUpdateCoalesceServerPauseTest } from "./tests/create-update-coalesce-server-pause.test"; +import { overlappingEditsSameSectionTest } from "./tests/overlapping-edits-same-section.test"; +import { rapidUpdatesAfterMergeTest } from "./tests/rapid-updates-after-merge.test"; +import { offlineRenamePendingCreateTest } from "./tests/offline-rename-pending-create.test"; +import { deleteRecreateConcurrentUpdateTest } from "./tests/delete-recreate-concurrent-update.test"; +import { moveAndConcurrentRemoteUpdateTest } from "./tests/move-and-concurrent-remote-update.test"; +import { offlineDeleteVsRemoteUpdateTest } from "./tests/offline-delete-vs-remote-update.test"; +import { doubleOfflineCycleTest } from "./tests/double-offline-cycle.test"; +import { createRenameCreateSamePathTest } from "./tests/create-rename-create-same-path.test"; +import { concurrentEditExactSamePositionTest } from "./tests/concurrent-edit-exact-same-position.test"; +import { serverPauseRenameEditResumeTest } from "./tests/server-pause-rename-edit-resume.test"; +import { renameTrackedToOccupiedPendingPathTest } from "./tests/rename-tracked-to-occupied-pending-path.test"; +import { offlineUpdateBothThenDeleteOneTest } from "./tests/offline-update-both-then-delete-one.test"; +import { moveIdenticalContentAmbiguityTest } from "./tests/move-identical-content-ambiguity.test"; +import { coalesceUpdateRemoteUpdateDataLossTest } from "./tests/coalesce-update-remote-update-data-loss.test"; +import { offlineCreateSamePathMergeableTest } from "./tests/offline-create-same-path-binary-conflict.test"; +import { deleteDuringPendingCreateTest } from "./tests/delete-during-pending-create.test"; +import { threeClientRenameCreateDeleteTest } from "./tests/three-client-rename-create-delete.test"; +import { keyMigrationEventDropTest } from "./tests/key-migration-event-drop.test"; +import { renameToPathOfUnconfirmedDeleteTest } from "./tests/rename-to-path-of-unconfirmed-delete.test"; +import { offlineEditThenMoveSameContentTest } from "./tests/offline-edit-then-move-same-content.test"; +import { concurrentRenameAndCreateAtTargetTest } from "./tests/concurrent-rename-and-create-at-target.test"; +import { createRenameCreateSamePathOfflineTest } from "./tests/create-rename-create-same-path-offline.test"; +import { rapidCreateUpdateDeleteCycleTest } from "./tests/rapid-create-update-delete-cycle.test"; +import { serverPauseBothEditSameFileTest } from "./tests/server-pause-both-edit-same-file.test"; +import { reconcilePendingAtOccupiedPathTest } from "./tests/reconcile-pending-at-occupied-path.test"; +import { offlineRenameBothClientsSameSourceTest } from "./tests/offline-rename-both-clients-same-source.test"; +import { createDuringReconciliationTest } from "./tests/create-during-reconciliation.test"; +import { deleteRecreateDifferentContentTest } from "./tests/delete-recreate-different-content.test"; +import { moveChainThreeFilesTest } from "./tests/move-chain-three-files.test"; +import { updateDuringCreateProcessingTest } from "./tests/update-during-create-processing.test"; +import { offlineMoveThenRemoteDeleteTest } from "./tests/offline-move-then-remote-delete.test"; +import { resetClearsRecentlyDeletedResurrectionTest } from "./tests/reset-clears-recently-deleted-resurrection.test"; +import { moveThenDeleteStalePathTest } from "./tests/move-then-delete-stale-path.test"; +import { interruptedDeleteRetryTest } from "./tests/interrupted-delete-retry.test"; +import { updateSurvivesRemoteDeleteTest } from "./tests/update-survives-remote-delete.test"; +import { movePreservesRemoteUpdateTest } from "./tests/move-preserves-remote-update.test"; +import { recentlyDeletedClearedOnReconnectTest } from "./tests/recently-deleted-cleared-on-reconnect.test"; +import { migrateKeyPreservesExistingTest } from "./tests/migrate-key-preserves-existing.test"; +import { userParenthesizedFileNotDeletedTest } from "./tests/user-parenthesized-file-not-deleted.test"; +import { concurrentUpdateDiffConsistencyTest } from "./tests/concurrent-update-diff-consistency.test"; +import { concurrentDeleteDuringRemoteUpdateTest } from "./tests/concurrent-delete-during-remote-update.test"; +import { binaryPendingCreateNotDisplacedTest } from "./tests/binary-pending-create-not-displaced.test"; +import { failedVfsMoveFallsBackTest } from "./tests/failed-vfs-move-falls-back.test"; +import { watermarkAdvancesOnSkipTest } from "./tests/watermark-advances-on-skip.test"; +import { remoteDeleteCoalesceLosesLocalUpdateTest } from "./tests/remote-delete-coalesce-loses-local-update.test"; +import { updateVsRemoteDeleteDataLossTest } from "./tests/update-vs-remote-delete-data-loss.test"; +import { watermarkGapRemoteUpdateNotRecordedTest } from "./tests/watermark-gap-remote-update-not-recorded.test"; +import { renameEmptyFileLosesIdentityTest } from "./tests/rename-empty-file-loses-identity.test"; +import { queueResetLosesCoalescedLocalEditTest } from "./tests/queue-reset-loses-coalesced-local-edit.test"; +import { renameToPendingPathFallbackTest } from "./tests/rename-to-pending-path-fallback.test"; +import { coalescedRemoteUpdateWatermarkLossTest } from "./tests/coalesced-remote-update-watermark-loss.test"; +import { moveRemoteUpdateRevertsRenameTest } from "./tests/move-remote-update-reverts-rename.test"; +import { createMergePreservesRenamedUpdateTest } from "./tests/create-merge-preserves-renamed-update.test"; +import { localEditLostDuringCreateMergeTest } from "./tests/local-edit-lost-during-create-merge.test"; +import { concurrentBinaryCreateDeconflictionTest } from "./tests/concurrent-binary-create-deconfliction.test"; +import { renamePendingCreateBeforeResponseTest } from "./tests/rename-pending-create-before-response.test"; +import { createRenameResponseSkipsFileTest } from "./tests/create-rename-response-skips-file.test"; +import { staleDocOrphanDuplicateContentTest } from "./tests/stale-doc-orphan-duplicate-content.test"; + +export const TESTS: Partial> = { + "write-write-conflict": writeWriteConflictTest, + "rename-create-conflict": renameCreateConflictTest, + "create-delete-noop": createDeleteNoopTest, + "rename-chain": renameChainTest, + "server-pause-resume": serverPauseResumeTest, + "create-merge-delete": createMergeDeleteTest, + "rename-update-conflict": renameUpdateConflictTest, + "delete-rename-conflict": deleteRenameConflictTest, + "multi-file-operations": multiFileOperationsTest, + "duplicate-content-files": duplicateContentFilesTest, + "delete-recreate-same-path": deleteRecreateSamePathTest, + "rapid-sync-toggle": rapidSyncToggleTest, + "concurrent-delete-update": concurrentDeleteUpdateTest, + "offline-rename-and-edit": offlineRenameAndEditTest, + "three-client-convergence": threeClientConvergenceTest, + "update-during-server-pause": updateDuringServerPauseTest, + "empty-file-sync": emptyFileSyncTest, + "rename-to-existing-path": renameToExistingPathTest, + "concurrent-rename-same-target": concurrentRenameSameTargetTest, + "multiple-updates-coalesce": multipleUpdatesCoalesceTest, + "delete-nonexistent-file": deleteNonexistentFileTest, + "create-while-server-paused": createWhileServerPausedTest, + "interleaved-operations": interleavedOperationsTest, + "simultaneous-create-delete-same-path": simultaneousCreateDeleteSamePathTest, + "large-file-count": largeFileCountTest, + "offline-operations-both-clients": offlineOperationsBothClientsTest, + "update-then-rename": updateThenRenameTest, + "idempotency-after-server-pause": idempotencyAfterServerPauseTest, + "concurrent-create-same-path-merge": concurrentCreateSamePathMergeTest, + "sequential-create-duplicate-content": sequentialCreateDuplicateContentTest, + "offline-multi-update-catchup": offlineMultiUpdateCatchupTest, + "mc-three-client-rename-offline-update": mcThreeClientRenameOfflineUpdateTest, + "mc-multi-delete-offline-rename": mcMultiDeleteOfflineRenameTest, + "mc-cross-create-rename-same-target": mcCrossCreateRenameSameTargetTest, + "mc-delete-then-offline-rename": mcDeleteThenOfflineRenameTest, + "offline-mixed-operations": offlineMixedOperationsTest, + "offline-create-rename-create": offlineCreateRenameCreateTest, + "offline-concurrent-renames": offlineConcurrentRenamesTest, + "offline-multiple-edits": offlineMultipleEditsTest, + "server-pause-both-clients-create": serverPauseBothClientsCreateTest, + "server-pause-rename-propagation": serverPauseRenameTest, + "server-pause-concurrent-creates": serverPauseConcurrentCreatesTest, + "server-pause-update-and-create": serverPauseUpdateAndCreateTest, + "rename-swap": renameSwapTest, + "rename-circular": renameCircularTest, + "rename-nested-path": renameNestedPathTest, + "rename-roundtrip": renameRoundtripTest, + "offline-rename-remote-create-old-path": offlineRenameRemoteCreateOldPathTest, + "offline-edit-remote-rename": offlineEditRemoteRenameTest, + "rename-chain-then-delete": renameChainThenDeleteTest, + "offline-delete-remote-rename": offlineDeleteRemoteRenameTest, + "rename-to-recently-deleted-path": renameToRecentlyDeletedPathTest, + "create-update-coalesce-server-pause": createUpdateCoalesceServerPauseTest, + "overlapping-edits-same-section": overlappingEditsSameSectionTest, + "rapid-updates-after-merge": rapidUpdatesAfterMergeTest, + "offline-rename-pending-create": offlineRenamePendingCreateTest, + "delete-recreate-concurrent-update": deleteRecreateConcurrentUpdateTest, + "move-and-concurrent-remote-update": moveAndConcurrentRemoteUpdateTest, + "double-offline-cycle": doubleOfflineCycleTest, + "create-rename-create-same-path": createRenameCreateSamePathTest, + "concurrent-edit-exact-same-position": concurrentEditExactSamePositionTest, + "server-pause-rename-edit-resume": serverPauseRenameEditResumeTest, + "rename-tracked-to-occupied-pending-path": renameTrackedToOccupiedPendingPathTest, + "offline-update-both-then-delete-one": offlineUpdateBothThenDeleteOneTest, + "move-identical-content-ambiguity": moveIdenticalContentAmbiguityTest, + "coalesce-update-remote-update-data-loss": coalesceUpdateRemoteUpdateDataLossTest, + "offline-create-same-path-mergeable": offlineCreateSamePathMergeableTest, + "delete-during-pending-create": deleteDuringPendingCreateTest, + "three-client-rename-create-delete": threeClientRenameCreateDeleteTest, + "key-migration-event-drop": keyMigrationEventDropTest, + "rename-to-path-of-unconfirmed-delete": renameToPathOfUnconfirmedDeleteTest, + "offline-edit-then-move-same-content": offlineEditThenMoveSameContentTest, + "concurrent-rename-and-create-at-target": concurrentRenameAndCreateAtTargetTest, + "create-rename-create-same-path-offline": createRenameCreateSamePathOfflineTest, + "rapid-create-update-delete-cycle": rapidCreateUpdateDeleteCycleTest, + "server-pause-both-edit-same-file": serverPauseBothEditSameFileTest, + "reconcile-pending-at-occupied-path": reconcilePendingAtOccupiedPathTest, + "offline-rename-both-clients-same-source": offlineRenameBothClientsSameSourceTest, + "create-during-reconciliation": createDuringReconciliationTest, + "delete-recreate-different-content": deleteRecreateDifferentContentTest, + "move-chain-three-files": moveChainThreeFilesTest, + "update-during-create-processing": updateDuringCreateProcessingTest, + "offline-move-then-remote-delete": offlineMoveThenRemoteDeleteTest, + "reset-clears-recently-deleted-resurrection": resetClearsRecentlyDeletedResurrectionTest, + "move-then-delete-stale-path": moveThenDeleteStalePathTest, + "offline-delete-vs-remote-update": offlineDeleteVsRemoteUpdateTest, + "interrupted-delete-retry": interruptedDeleteRetryTest, + "update-survives-remote-delete": updateSurvivesRemoteDeleteTest, + "move-preserves-remote-update": movePreservesRemoteUpdateTest, + "recently-deleted-cleared-on-reconnect": recentlyDeletedClearedOnReconnectTest, + "migrate-key-preserves-existing": migrateKeyPreservesExistingTest, + "user-parenthesized-file-not-deleted": userParenthesizedFileNotDeletedTest, + "concurrent-update-diff-consistency": concurrentUpdateDiffConsistencyTest, + "concurrent-delete-during-remote-update": concurrentDeleteDuringRemoteUpdateTest, + "binary-pending-create-not-displaced": binaryPendingCreateNotDisplacedTest, + "failed-vfs-move-falls-back": failedVfsMoveFallsBackTest, + "watermark-advances-on-skip": watermarkAdvancesOnSkipTest, + "remote-delete-coalesce-loses-local-update": remoteDeleteCoalesceLosesLocalUpdateTest, + "update-vs-remote-delete-data-loss": updateVsRemoteDeleteDataLossTest, + "watermark-gap-remote-update-not-recorded": watermarkGapRemoteUpdateNotRecordedTest, + "rename-empty-file-loses-identity": renameEmptyFileLosesIdentityTest, + "queue-reset-loses-coalesced-local-edit": queueResetLosesCoalescedLocalEditTest, + "rename-to-pending-path-fallback": renameToPendingPathFallbackTest, + "coalesced-remote-update-watermark-loss": coalescedRemoteUpdateWatermarkLossTest, + "move-remote-update-reverts-rename": moveRemoteUpdateRevertsRenameTest, + "create-merge-preserves-renamed-update": createMergePreservesRenamedUpdateTest, + "local-edit-lost-during-create-merge": localEditLostDuringCreateMergeTest, + "concurrent-binary-create-deconfliction": concurrentBinaryCreateDeconflictionTest, + "rename-pending-create-before-response": renamePendingCreateBeforeResponseTest, + "create-rename-response-skips-file": createRenameResponseSkipsFileTest, + "stale-doc-orphan-duplicate-content": staleDocOrphanDuplicateContentTest +}; diff --git a/frontend/deterministic-tests/src/test-runner.ts b/frontend/deterministic-tests/src/test-runner.ts new file mode 100644 index 00000000..d9a42fa0 --- /dev/null +++ b/frontend/deterministic-tests/src/test-runner.ts @@ -0,0 +1,372 @@ +import type { + TestDefinition, + TestResult, + TestStep, + ClientState +} from "./test-definition"; +import { DeterministicAgent } from "./deterministic-agent"; +import type { ServerControl } from "./server-control"; +import type { SyncSettings, Logger } from "sync-client"; +import { assert } from "./utils/assert"; +import { sleep } from "./utils/sleep"; +import { withTimeout } from "./utils/with-timeout"; +import { + CONVERGENCE_TIMEOUT_MS, + CONVERGENCE_RETRY_DELAY_MS, + AGENT_INIT_TIMEOUT_MS, + IS_SYNC_ENABLED_DEFAULT +} from "./consts"; +import { randomUUID } from "node:crypto"; + +export class TestRunner { + private agents: DeterministicAgent[] = []; + private readonly serverControl: ServerControl; + private readonly token: string; + private readonly remoteUri: string; + private readonly logger: Logger; + + public constructor( + serverControl: ServerControl, + logger: Logger, + token: string, + remoteUri: string + ) { + this.serverControl = serverControl; + this.logger = logger; + this.token = token; + this.remoteUri = remoteUri; + } + + public async runTest(test: TestDefinition): Promise { + const startTime = Date.now(); + this.logger.info(`Running test: ${test.name}`); + if (test.description !== undefined && test.description !== "") { + this.logger.info(`Description: ${test.description}`); + } + this.logger.info(`Clients: ${test.clients}`); + this.logger.info(`Steps: ${test.steps.length}`); + + try { + assert( + this.serverControl.isRunning(), + "Server is not running before test start" + ); + + await this.initializeAgents(test.clients); + + for (let i = 0; i < test.steps.length; i++) { + const step = test.steps[i]; + this.logger.info( + `Step ${i + 1}/${test.steps.length}: ${JSON.stringify(step)}` + ); + await this.executeStep(step); + } + + await this.cleanup(); + + const duration = Date.now() - startTime; + this.logger.info(`\n✓ Test passed: ${test.name} (${duration}ms)`); + + return { + success: true, + duration + }; + } catch (error) { + const duration = Date.now() - startTime; + const errorMessage = + error instanceof Error ? error.message : String(error); + this.logger.info(`\n✗ Test failed: ${test.name}`); + this.logger.info(`Error: ${errorMessage}`); + + await this.cleanup(); + + return { + success: false, + error: errorMessage, + duration + }; + } + } + + private async initializeAgents(count: number): Promise { + assert(count > 0, `Client count must be positive, got ${count}`); + const vaultName = `test-${randomUUID()}`; + this.logger.info( + `Initializing ${count} agents with vault: ${vaultName}` + ); + + for (let i = 0; i < count; i++) { + const settings: Partial = { + isSyncEnabled: IS_SYNC_ENABLED_DEFAULT, + token: this.token, + vaultName, + remoteUri: this.remoteUri + }; + + const agent = new DeterministicAgent(i, settings, (msg) => { + this.logger.info(msg); + }); + + // Push before init so cleanup() handles this agent if init fails + this.agents.push(agent); + await withTimeout( + agent.init( + fetch, + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion + WebSocket as unknown as typeof globalThis.WebSocket + ), + AGENT_INIT_TIMEOUT_MS, + `Client ${i} init timed out after ${AGENT_INIT_TIMEOUT_MS}ms` + ); + this.logger.info(`Initialized client ${i}`); + } + + this.logger.info("All agents initialized"); + } + + private getAgent(index: number): DeterministicAgent { + assert( + index >= 0 && index < this.agents.length, + `Client index ${index} out of bounds (have ${this.agents.length} agents)` + ); + return this.agents[index]; + } + + private async executeStep(step: TestStep): Promise { + switch (step.type) { + case "create": + await this.getAgent(step.client).createFile( + step.path, + step.content + ); + break; + + case "update": + await this.getAgent(step.client).updateFile( + step.path, + step.content + ); + break; + + case "rename": + await this.getAgent(step.client).renameFile( + step.oldPath, + step.newPath + ); + break; + + case "delete": + await this.getAgent(step.client).deleteFile(step.path); + break; + + case "sync": + if (step.client !== undefined) { + await this.getAgent(step.client).waitForSync(); + } else { + for (const agent of this.agents) { + await agent.waitForSync(); + } + } + break; + + case "disable-sync": + await this.getAgent(step.client).disableSync(); + break; + + case "enable-sync": + await this.getAgent(step.client).enableSync(); + break; + + case "pause-server": + this.serverControl.pause(); + break; + + case "resume-server": + this.serverControl.resume(); + // Verify the server is actually responsive before proceeding. + // This replaces relying solely on hardcoded waits. + await this.serverControl.waitForReady(); + break; + + case "barrier": + await this.waitForConvergence(); + break; + + case "assert-content": + await this.getAgent(step.client).assertContent( + step.path, + step.content + ); + break; + + case "assert-exists": + await this.getAgent(step.client).assertExists(step.path); + break; + + case "assert-not-exists": + await this.getAgent(step.client).assertNotExists(step.path); + break; + + case "assert-consistent": + await this.assertConsistent(step.verify); + break; + + default: { + const unknownStep = step as { type: string }; + throw new Error(`Unknown step type: ${unknownStep.type}`); + } + } + } + + /** + * Wait for all agents to reach a consistent state. + * + * Waiting for agents is done in two full rounds: the first round + * drains in-flight operations, but completing those operations can + * trigger new work on OTHER agents via server broadcasts. The second + * round waits for that cascading work to settle. Deeper cascades + * are handled by the outer retry loop. + */ + private async waitForConvergence(): Promise { + this.logger.info("Barrier: waiting for convergence..."); + + const deadline = Date.now() + CONVERGENCE_TIMEOUT_MS; + let lastError: Error | undefined = undefined; + + while (Date.now() < deadline) { + await this.waitAllAgentsSettled(); + + try { + await this.assertConsistent(); + this.logger.info("Barrier complete: all clients converged"); + return; + } catch (error) { + lastError = + error instanceof Error ? error : new Error(String(error)); + this.logger.info("Barrier: not yet converged, retrying..."); + await sleep(CONVERGENCE_RETRY_DELAY_MS); + } + } + + // Final attempt — let the error propagate + await this.waitAllAgentsSettled(); + + try { + await this.assertConsistent(); + this.logger.info("Barrier complete: all clients converged"); + } catch (error) { + throw new Error( + `Convergence timed out after ${CONVERGENCE_TIMEOUT_MS}ms: ${error instanceof Error ? error.message : String(error)}`, + { cause: lastError } + ); + } + } + + /** + * Wait for all agents to be simultaneously idle. Two full rounds are + * needed because completing work on agent A can trigger a server + * broadcast that enqueues new work on agent B, and vice versa. + * + * However, the 2nd sync may result in merges which can trigger another + * round of syncs, so this function should be called in a loop with a + * timeout to ensure true convergence rather than just waiting for the + * current round of syncs to complete. + */ + private async waitAllAgentsSettled(): Promise { + for (let round = 0; round < 2; round++) { + for (const agent of this.agents) { + await agent.waitForSync(); + } + } + } + + private async assertConsistent( + verify?: (state: ClientState) => void + ): Promise { + this.logger.info("Asserting all clients are consistent..."); + assert(this.agents.length >= 2, "Need at least 2 agents for consistency check"); + + const [referenceAgent] = this.agents; + const referenceFiles = (await referenceAgent.getFiles()).sort(); + const referenceState: ClientState = { files: new Map() }; + + for (const file of referenceFiles) { + const content = await referenceAgent.getFileContent(file); + referenceState.files.set(file, content); + } + + this.logger.info( + `Reference client has ${referenceFiles.length} files: ${referenceFiles.join(", ")}` + ); + + for (let i = 1; i < this.agents.length; i++) { + const agent = this.agents[i]; + const files = (await agent.getFiles()).sort(); + + this.logger.info( + `Client ${i} has ${files.length} files: ${files.join(", ")}` + ); + + assert( + files.length === referenceFiles.length, + `File count mismatch: client 0 has ${referenceFiles.length} files, client ${i} has ${files.length} files` + ); + + for (let j = 0; j < files.length; j++) { + assert( + files[j] === referenceFiles[j], + `File list mismatch at index ${j}: client 0 has "${referenceFiles[j]}", client ${i} has "${files[j]}"` + ); + } + + for (const file of referenceFiles) { + const referenceContent = referenceState.files.get(file); + const agentContent = await agent.getFileContent(file); + + assert( + referenceContent === agentContent, + `Content mismatch for ${file}:\nClient 0: "${referenceContent}"\nClient ${i}: "${agentContent}"` + ); + } + } + + this.logger.info("✓ All clients are consistent"); + + if (verify) { + this.logger.info("Running custom verification..."); + try { + verify(referenceState); + } catch (error) { + const msg = + error instanceof Error ? error.message : String(error); + throw new Error(`Custom verification failed: ${msg}`); + } + this.logger.info("✓ Custom verification passed"); + } + } + + private async cleanup(): Promise { + // Always resume the server in case a test paused it and then + // failed before reaching the resume step. Without this, all + // subsequent tests would hang because the server process is + // frozen (SIGSTOP) and can't respond to HTTP or WebSocket. + try { + this.serverControl.resume(); + } catch { + // Server wasn't paused or isn't running — safe to ignore + } + + this.logger.info("\nCleaning up agents..."); + for (const agent of this.agents) { + try { + await agent.cleanup(); + } catch (error) { + this.logger.warn( + `Agent cleanup error: ${error instanceof Error ? error.message : String(error)}` + ); + } + } + this.agents = []; + this.logger.info("Cleanup complete"); + } +} diff --git a/frontend/deterministic-tests/src/tests/binary-pending-create-not-displaced.test.ts b/frontend/deterministic-tests/src/tests/binary-pending-create-not-displaced.test.ts new file mode 100644 index 00000000..61f9be82 --- /dev/null +++ b/frontend/deterministic-tests/src/tests/binary-pending-create-not-displaced.test.ts @@ -0,0 +1,67 @@ +import type { ClientState, TestDefinition } from "../test-definition"; +import { assert } from "../utils/assert"; + + +function verifyBothFilesExist(state: ClientState): void { + assert( + state.files.size === 2, + `Expected 2 files, got ${state.files.size}: ${[...state.files.keys()].join(", ")}` + ); + assert( + state.files.has("data.bin"), + "Expected data.bin to exist" + ); + assert( + state.files.has("data (1).bin"), + "Expected data (1).bin to exist" + ); + + const contents = new Set(state.files.values()); + assert( + contents.has("binary data from client 0"), + `Expected one file to contain "binary data from client 0"` + ); + assert( + contents.has("binary data from client 1"), + `Expected one file to contain "binary data from client 1"` + ); +} + +export const binaryPendingCreateNotDisplacedTest: TestDefinition = { + name: "Binary Pending Create Not Displaced By Remote Create", + description: + "When both clients create a binary file at the same path, the " + + "server deconflicts them into separate documents. Both files " + + "should exist on both clients after sync.", + clients: 2, + steps: [ + { type: "enable-sync", client: 0 }, + { type: "enable-sync", client: 1 }, + + // Both go offline + { type: "disable-sync", client: 0 }, + { type: "disable-sync", client: 1 }, + + // Both create binary file at same path (use .bin extension) + { + type: "create", + client: 0, + path: "data.bin", + content: "binary data from client 0" + }, + { + type: "create", + client: 1, + path: "data.bin", + content: "binary data from client 1" + }, + + // Both come online + { type: "enable-sync", client: 0 }, + { type: "enable-sync", client: 1 }, + { type: "barrier" }, + + // Both files should exist (server deconflicted them) + { type: "assert-consistent", verify: verifyBothFilesExist } + ] +}; diff --git a/frontend/deterministic-tests/src/utils/assert.ts b/frontend/deterministic-tests/src/utils/assert.ts new file mode 100644 index 00000000..4e709060 --- /dev/null +++ b/frontend/deterministic-tests/src/utils/assert.ts @@ -0,0 +1,5 @@ +export function assert(value: boolean, message: string): asserts value { + if (!value) { + throw new Error(message); + } +} diff --git a/frontend/deterministic-tests/src/utils/find-free-port.ts b/frontend/deterministic-tests/src/utils/find-free-port.ts new file mode 100644 index 00000000..3c965049 --- /dev/null +++ b/frontend/deterministic-tests/src/utils/find-free-port.ts @@ -0,0 +1,29 @@ +import * as net from "node:net"; + +export interface PortReservation { + port: number; + release: () => void; +} + +/** + * Find a free port and keep it reserved until the caller explicitly releases it. + */ +export async function findFreePort(): Promise { + return new Promise((resolve, reject) => { + const server = net.createServer(); + server.listen(0, "127.0.0.1", () => { + const addr = server.address(); + if (addr === null || typeof addr === "string") { + server.close(); + reject(new Error("Failed to get port from server")); + return; + } + const { port } = addr; + resolve({ + port, + release: () => server.close() + }); + }); + server.on("error", reject); + }); +} diff --git a/frontend/deterministic-tests/src/utils/sleep.ts b/frontend/deterministic-tests/src/utils/sleep.ts new file mode 100644 index 00000000..ff474799 --- /dev/null +++ b/frontend/deterministic-tests/src/utils/sleep.ts @@ -0,0 +1,3 @@ +export async function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} diff --git a/frontend/deterministic-tests/src/utils/with-timeout.ts b/frontend/deterministic-tests/src/utils/with-timeout.ts new file mode 100644 index 00000000..14ee3f27 --- /dev/null +++ b/frontend/deterministic-tests/src/utils/with-timeout.ts @@ -0,0 +1,15 @@ +export async function withTimeout( + promise: Promise, + timeoutMs: number, + message: string +): Promise { + let timeoutId: ReturnType | undefined = undefined; + const timeoutPromise = new Promise((_resolve, reject) => { + timeoutId = setTimeout(() => { + reject(new Error(message)); + }, timeoutMs); + }); + return Promise.race([promise, timeoutPromise]).finally(() => { + clearTimeout(timeoutId); + }); +} diff --git a/frontend/deterministic-tests/tsconfig.json b/frontend/deterministic-tests/tsconfig.json new file mode 100644 index 00000000..7558871d --- /dev/null +++ b/frontend/deterministic-tests/tsconfig.json @@ -0,0 +1,12 @@ +{ + "compilerOptions": { + "baseUrl": ".", + "strict": true, + "target": "ES2022", + "module": "CommonJS", + "esModuleInterop": true, + "lib": ["DOM", "ES2024"], + "moduleResolution": "node" + }, + "exclude": ["./dist"] +} diff --git a/frontend/deterministic-tests/webpack.config.js b/frontend/deterministic-tests/webpack.config.js new file mode 100644 index 00000000..6aee1547 --- /dev/null +++ b/frontend/deterministic-tests/webpack.config.js @@ -0,0 +1,30 @@ +const path = require("path"); +const webpack = require("webpack"); + +module.exports = { + entry: "./src/cli.ts", + target: "node", + mode: "production", + optimization: { + minimize: false + }, + module: { + rules: [ + { + test: /\.ts$/, + use: "ts-loader" + } + ] + }, + resolve: { + extensions: [".ts", ".js"] + }, + output: { + globalObject: "this", + filename: "cli.js", + path: path.resolve(__dirname, "dist") + }, + plugins: [ + new webpack.BannerPlugin({ banner: "#!/usr/bin/env node", raw: true }) + ] +};