diff --git a/frontend/deterministic-tests/README.md b/frontend/deterministic-tests/README.md index c422406d..6fa2848c 100644 --- a/frontend/deterministic-tests/README.md +++ b/frontend/deterministic-tests/README.md @@ -10,7 +10,7 @@ Each test is a `TestDefinition`: a client count and an ordered list of steps. Th Tests that don't pause the server share a single server process (vault-name isolation). Tests that use `pause-server`/`resume-server` (SIGSTOP/SIGCONT) each get a dedicated server, since SIGSTOP freezes the entire process. -All tests run in parallel up to a concurrency limit. +The runner executes two sequential phases: regular tests on the shared server, then pause-server tests on dedicated servers. Within each phase tests run in parallel up to a concurrency limit. ## Step types @@ -19,12 +19,15 @@ Clients always start with syncing disabled. **File operations** (per-client, fire-and-forget — sync is enqueued but not awaited): - `create`, `update`, `rename`, `delete` +- `rename-next-write` — arm a deferred rename that fires the next time the given path is written. Lets a test race a user-rename against an in-flight remote create that's about to land at the same path. **Sync control:** - `sync` — wait for a specific client or all clients to finish pending operations - `barrier` — retry until all clients converge to identical file state (60s timeout) - `enable-sync` / `disable-sync` — simulate going online/offline +- `reset` — reset a client's tracked sync state (keeps disk files); equivalent to a forced re-handshake on next enable +- `sleep` — wall-clock pause; use sparingly, prefer `barrier` / `sync` **WebSocket control** (per-client): @@ -33,6 +36,12 @@ Clients always start with syncing disabled. **Server control:** - `pause-server` / `resume-server` — SIGSTOP/SIGCONT the server process +- `resume-server-until-history-then-pause` — resume the server, wait until a specific client observes a matching history entry (`CREATE`/`UPDATE`/`DELETE` for a path), then re-pause. Used to land exactly one operation across the wire. + +**Fault injection** (per-client): + +- `drop-next-create-response` — arm a one-shot interceptor that lets the next `POST /documents` reach the server (commit happens) but throws `SyncResetError` before the client sees the response, simulating connection loss after server commit. +- `wait-for-dropped-create-response` — wait until the armed drop has fired. **Assertions:** @@ -72,7 +81,9 @@ export const myScenarioTest: TestDefinition = { { type: "barrier" }, { type: "assert-consistent", - verify: (s) => s.assertFileCount(1).assertContent("A.md", "hello") + verify: (s) => { + s.assertFileCount(1).assertContent("A.md", "hello"); + } } ] }; @@ -81,14 +92,18 @@ export const myScenarioTest: TestDefinition = { The `verify` callback receives an `AssertableState` object with chainable assertion methods: ```typescript -s.assertFileCount(n) // exact file count -s.assertFileExists("path") // file must exist -s.assertFileNotExists("path") // file must not exist -s.assertContent("path", "expected") // exact content match -s.assertContains("path", "a", "b") // all substrings present -s.assertAnyFileContains("text") // substring in any file -s.assertContentInAtMostOneFile("text") // no duplicate content -s.ifFileExists("path", (s) => ...) // conditional assertion +s.assertFileCount(n); // exact file count +s.assertFileExists("path"); // file must exist +s.assertFileNotExists("path"); // file must not exist +s.assertContent("path", "expected"); // exact content match +s.assertContains("path", "a", "b"); // all substrings present in file +s.assertContainsAny("path", "a", "b"); // at least one substring present +s.assertAnyFileContains("text"); // substring present in some file +s.assertNoFileContains("text"); // substring absent from every file +s.assertSubstringCount("path", "x", 3); // substring appears exactly N times +s.assertContentInAtMostOneFile("text"); // no duplicate content +s.ifFileExists("path", (s) => { /* … */ }); // conditional block +s.getContent("path"); // raw content (or "" if missing) ``` 2. Register it in `src/test-registry.ts`: diff --git a/frontend/deterministic-tests/package.json b/frontend/deterministic-tests/package.json index e1c1b276..4bd82c74 100644 --- a/frontend/deterministic-tests/package.json +++ b/frontend/deterministic-tests/package.json @@ -11,6 +11,7 @@ "test": "npm run build && node dist/cli.js" }, "devDependencies": { + "commander": "^14.0.2", "@types/node": "^25.0.2", "sync-client": "file:../sync-client", "ts-loader": "^9.5.4", diff --git a/frontend/deterministic-tests/src/cli.ts b/frontend/deterministic-tests/src/cli.ts index 6e0e764f..6e15cac0 100644 --- a/frontend/deterministic-tests/src/cli.ts +++ b/frontend/deterministic-tests/src/cli.ts @@ -4,7 +4,7 @@ import { ServerManager } from "./server-manager"; import { PrefixedLogger } from "./prefixed-logger"; import { TESTS } from "./test-registry"; import type { TestDefinition, TestResult } from "./test-definition"; -import { parseConcurrency } from "./parse-concurrency"; +import { parseArgs } from "./parse-args"; import { runWithConcurrency } from "./run-with-concurrency"; import { TOKEN, SERVER_BINARY_PATH, CONFIG_PATH } from "./consts"; import * as path from "node:path"; @@ -29,7 +29,31 @@ serverManager.installSignalHandlers(); function testUsesPauseServer(test: TestDefinition): boolean { return test.steps.some( - (step) => step.type === "pause-server" || step.type === "resume-server" + (step) => + step.type === "pause-server" || + step.type === "resume-server" || + step.type === "resume-server-until-history-then-pause" + ); +} + +/** + * Walk up from the CLI binary's location until we find a directory + * containing `sync-server/` and `frontend/`. + */ +function findProjectRoot(): string { + let dir = path.dirname(__filename); + const root = path.parse(dir).root; + while (dir !== root) { + if ( + fs.existsSync(path.join(dir, "sync-server")) && + fs.existsSync(path.join(dir, "frontend")) + ) { + return dir; + } + dir = path.dirname(dir); + } + throw new Error( + `Could not locate project root (no ancestor of ${__filename} contains both 'sync-server' and 'frontend')` ); } @@ -100,15 +124,7 @@ async function runDedicatedServerTest( } async function main(): Promise { - const cwd = process.cwd(); - let projectRoot = cwd; - - if (cwd.endsWith("frontend/deterministic-tests")) { - projectRoot = path.resolve(cwd, "../.."); - } else if (cwd.endsWith("frontend")) { - projectRoot = path.resolve(cwd, ".."); - } - + const projectRoot = findProjectRoot(); const serverPath = path.join(projectRoot, SERVER_BINARY_PATH); if (!fs.existsSync(serverPath)) { logger.error(`Server binary not found at: ${serverPath}`); @@ -121,8 +137,7 @@ async function main(): Promise { process.exit(1); } - const filterArg = process.argv.find((a) => a.startsWith("--filter=")); - const filter = filterArg?.slice("--filter=".length); + const { filter, concurrency } = parseArgs(process.argv); const testsToRun: [string, TestDefinition][] = []; for (const [key, test] of Object.entries(TESTS)) { @@ -147,7 +162,6 @@ async function main(): Promise { process.exit(1); } - const concurrency = parseConcurrency(); const regularTests = testsToRun.filter(([, t]) => !testUsesPauseServer(t)); const pauseTests = testsToRun.filter(([, t]) => testUsesPauseServer(t)); diff --git a/frontend/deterministic-tests/src/consts.ts b/frontend/deterministic-tests/src/consts.ts index a04c9b61..d9a2498f 100644 --- a/frontend/deterministic-tests/src/consts.ts +++ b/frontend/deterministic-tests/src/consts.ts @@ -11,3 +11,7 @@ export const IS_SYNC_ENABLED_BY_DEFAULT = false; export const WAIT_TIMEOUT_MS = 60_000; export const WEBSOCKET_CONNECT_TIMEOUT_MS = 10_000; export const WEBSOCKET_POLL_INTERVAL_MS = 50; + +export const SERVER_READY_POLL_INTERVAL_MS = 100; +export const SERVER_READY_MAX_ATTEMPTS = 50; +export const SERVER_START_MAX_ATTEMPTS = 5; diff --git a/frontend/deterministic-tests/src/deterministic-agent.ts b/frontend/deterministic-tests/src/deterministic-agent.ts index 74ec2b8d..b32b01c2 100644 --- a/frontend/deterministic-tests/src/deterministic-agent.ts +++ b/frontend/deterministic-tests/src/deterministic-agent.ts @@ -37,15 +37,15 @@ export class DeterministicAgent extends debugging.InMemoryFileSystem { private readonly wsFactory = new ManagedWebSocketFactory(); private nextWriteRename: | { - oldPath: RelativePath; - newPath: RelativePath; - } + oldPath: RelativePath; + newPath: RelativePath; + } | undefined; private nextCreateResponseDrop: | { - dropped: Promise; - resolveDropped: () => void; - } + dropped: Promise; + resolveDropped: () => void; + } | undefined; public constructor( @@ -82,10 +82,10 @@ export class DeterministicAgent extends debugging.InMemoryFileSystem { this.logger(`${prefix} WARN: ${line.message}`); break; case LogLevel.INFO: - this.logger(`${prefix} ${line.message}`); + this.logger(`${prefix} INFO: ${line.message}`); break; case LogLevel.DEBUG: - // Skip debug logs to reduce noise + this.logger(`${prefix} DEBUG: ${line.message}`); break; } }); @@ -271,8 +271,18 @@ export class DeterministicAgent extends debugging.InMemoryFileSystem { this.log(`Cleanup waitUntilFinished failed: ${error}`); } } + // Surface any background sync errors that arrived after the last + // waitForSync (e.g. between the final assert-consistent and here). + // Without this, regressions that fault the engine during the very + // last step of a test would be silently swallowed. + const pendingErrors = this.syncErrors.splice(0); await this.client.destroy(); this.log("Cleanup complete"); + if (pendingErrors.length > 0) { + throw new Error( + `Client ${this.clientId} had ${pendingErrors.length} background sync error(s) during cleanup:\n${pendingErrors.map((e) => e.message).join("\n")}` + ); + } } public override async read(path: RelativePath): Promise { @@ -312,6 +322,10 @@ export class DeterministicAgent extends debugging.InMemoryFileSystem { }); }); } + // The rename consumed `path`. Skip the post-update enqueue below + // — it would send a syncLocallyUpdatedFile for a path that no + // longer exists. + return; } if (!this.isSyncEnabled) { @@ -435,6 +449,11 @@ export class DeterministicAgent extends debugging.InMemoryFileSystem { DeterministicAgent.isCreateDocumentRequest(input, init) ) { this.nextCreateResponseDrop = undefined; + try { + await response.body?.cancel(); + } catch { + // Best-effort — body may already be consumed/closed. + } drop.resolveDropped(); throw new SyncResetError(); } diff --git a/frontend/deterministic-tests/src/managed-websocket.ts b/frontend/deterministic-tests/src/managed-websocket.ts index 421561fd..c759891b 100644 --- a/frontend/deterministic-tests/src/managed-websocket.ts +++ b/frontend/deterministic-tests/src/managed-websocket.ts @@ -139,11 +139,21 @@ class ManagedWebSocket implements WebSocket { } public resume(): void { - this.paused = false; - const messages = this.bufferedMessages.splice(0); - for (const msg of messages) { - this.externalOnMessage?.(msg); + // Drain buffered messages BEFORE flipping `paused` to false. + // If `externalOnMessage` is async (its return type is `unknown`), + // dispatch yields control between buffered messages, and a fresh + // live `ws.onmessage` event firing during that yield would jump + // ahead of unprocessed buffered messages — silently reordering + // events relative to the wire. Keeping `paused = true` during the + // drain forces the live handler to keep buffering, so we splice + // those late arrivals onto the tail and dispatch them in order. + while (this.bufferedMessages.length > 0) { + const messages = this.bufferedMessages.splice(0); + for (const msg of messages) { + this.externalOnMessage?.(msg); + } } + this.paused = false; } public send(data: string | ArrayBufferLike | Blob | ArrayBufferView): void { @@ -157,6 +167,17 @@ class ManagedWebSocket implements WebSocket { public addEventListener( ...args: Parameters ): void { + // Only the `.onmessage` setter routes through the pause buffer. + // If sync-client ever attaches "message" listeners via + // addEventListener instead, those messages would bypass pause/resume + // and deterministic tests would silently lose their fault injection. + if (args[0] === "message") { + throw new Error( + "ManagedWebSocket: addEventListener('message') bypasses the " + + "pause buffer. Use the .onmessage setter instead, or " + + "extend ManagedWebSocket to route message listeners." + ); + } this.ws.addEventListener(...args); } @@ -176,6 +197,11 @@ class ManagedWebSocket implements WebSocket { * for pause/resume control from the test harness */ export class ManagedWebSocketFactory { + // Append-only: closed sockets stay tracked. Bounded per test (one + // factory per agent, each test discards its agents on cleanup), so + // not a real leak — but iterating over closed instances on + // pause/resume is a deliberate no-op since their `.onmessage` is + // already detached. private readonly instances: ManagedWebSocket[] = []; // Sticky pause state: applied to current instances on `pause()` AND // to any new instance created later (e.g. WS reconnect after a diff --git a/frontend/deterministic-tests/src/parse-args.ts b/frontend/deterministic-tests/src/parse-args.ts new file mode 100644 index 00000000..11c56f19 --- /dev/null +++ b/frontend/deterministic-tests/src/parse-args.ts @@ -0,0 +1,43 @@ +import * as os from "node:os"; +import { Command, InvalidArgumentError } from "commander"; + +export interface CliArgs { + filter: string | undefined; + concurrency: number; +} + +function parsePositiveInt(value: string): number { + const n = parseInt(value, 10); + if (isNaN(n) || n <= 0) { + throw new InvalidArgumentError("must be a positive integer"); + } + return n; +} + +export function parseArgs(argv: string[]): CliArgs { + const program = new Command(); + + program + .name("deterministic-tests") + .description("Scripted multi-client sync tests against a real server") + .option( + "-f, --filter ", + "Run only tests whose name contains this substring" + ) + .option( + "-j, --concurrency ", + "Number of tests to run in parallel", + parsePositiveInt, + os.cpus().length + ); + + program.parse(argv); + + /* eslint-disable @typescript-eslint/no-unsafe-type-assertion */ + const opts = program.opts(); + const filter = opts.filter as string | undefined; + const concurrency = opts.concurrency as number; + /* eslint-enable @typescript-eslint/no-unsafe-type-assertion */ + + return { filter, concurrency }; +} diff --git a/frontend/deterministic-tests/src/parse-concurrency.ts b/frontend/deterministic-tests/src/parse-concurrency.ts deleted file mode 100644 index f926d1fa..00000000 --- a/frontend/deterministic-tests/src/parse-concurrency.ts +++ /dev/null @@ -1,17 +0,0 @@ -import * as os from "node:os"; - -export function parseConcurrency(): number { - const args = process.argv.slice(2); - for (let i = 0; i < args.length; i++) { - if ( - (args[i] === "--concurrency" || args[i] === "-j") && - i + 1 < args.length - ) { - const n = parseInt(args[i + 1], 10); - if (!isNaN(n) && n > 0) { - return n; - } - } - } - return os.cpus().length; -} diff --git a/frontend/deterministic-tests/src/server-control.ts b/frontend/deterministic-tests/src/server-control.ts index f903cc4c..9cb4cde0 100644 --- a/frontend/deterministic-tests/src/server-control.ts +++ b/frontend/deterministic-tests/src/server-control.ts @@ -5,7 +5,12 @@ import * as path from "node:path"; import { sleep } from "./utils/sleep"; import { findFreePort } from "./utils/find-free-port"; import type { Logger } from "sync-client"; -import { STOP_TIMEOUT_MS } from "./consts"; +import { + STOP_TIMEOUT_MS, + SERVER_READY_POLL_INTERVAL_MS, + SERVER_READY_MAX_ATTEMPTS, + SERVER_START_MAX_ATTEMPTS +} from "./consts"; export class ServerControl { private process: ChildProcess | null = null; @@ -38,10 +43,32 @@ export class ServerControl { throw new Error("Server is already running"); } + // Retry on bind failure: findFreePort closes its probe before we + // spawn, so under heavy parallelism another process can grab the + // same port. Each attempt picks a fresh port. + let lastError: unknown; + for (let attempt = 1; attempt <= SERVER_START_MAX_ATTEMPTS; attempt++) { + try { + await this.startOnce(); + return; + } catch (error) { + lastError = error; + this.logger.warn( + `Server start attempt ${attempt}/${SERVER_START_MAX_ATTEMPTS} failed: ${error instanceof Error ? error.message : String(error)}` + ); + // startOnce already cleaned up its child + tempdir on failure. + } + } + throw new Error( + `Server failed to start after ${SERVER_START_MAX_ATTEMPTS} attempts: ${lastError instanceof Error ? lastError.message : String(lastError)}`, + { cause: lastError instanceof Error ? lastError : undefined } + ); + } + + private async startOnce(): Promise { const reservation = await findFreePort(); this._port = reservation.port; - // Prefer tmpfs (/host/tmp) over disk-backed /tmp for faster SQLite I/O - const tmpBase = fs.existsSync("/host/tmp") ? "/host/tmp" : os.tmpdir(); + const tmpBase = os.tmpdir(); this.tempDir = fs.mkdtempSync(path.join(tmpBase, "vault-link-test-")); const tempConfigPath = path.join(this.tempDir, "config.yml"); const dbDir = path.join(this.tempDir, "databases"); @@ -101,7 +128,9 @@ export class ServerControl { } } - public async waitForReady(maxAttempts = 50): Promise { + public async waitForReady( + maxAttempts: number = SERVER_READY_MAX_ATTEMPTS + ): Promise { const pingUrl = `${this.remoteUri}/vaults/test/ping`; for (let i = 0; i < maxAttempts; i++) { if (this.process?.exitCode !== null) { @@ -118,7 +147,7 @@ export class ServerControl { } catch { // Server not ready yet, continue polling } - await sleep(100); + await sleep(SERVER_READY_POLL_INTERVAL_MS); } throw new Error("Server failed to start within timeout"); } @@ -208,10 +237,42 @@ export class ServerControl { } public isRunning(): boolean { - return this.process?.pid !== undefined; + const proc = this.process; + return ( + proc !== null && + proc.pid !== undefined && + proc.exitCode === null && + proc.signalCode === null + ); + } + + /** + * Synchronously SIGCONT-then-SIGKILL the child process. Safe to call + * from a `process.on("exit", ...)` handler, where async work cannot + * run. Used as a last-resort cleanup so a SIGSTOP'd server doesn't + * outlive the test runner and wedge the next CI invocation. + */ + public forceKillSync(): void { + const proc = this.process; + if (proc?.pid === undefined) { + return; + } + try { + process.kill(proc.pid, "SIGCONT"); + } catch { + // Process may already be gone or never paused. + } + try { + process.kill(proc.pid, "SIGKILL"); + } catch { + // Process already gone. + } } private writeConfigFile(destPath: string, dbDir: string): void { + // Assumes config-e2e.yml has exactly one 2-space-indented `port:` and + // one `databases_directory_path:` (under `server:` and `database:` + // respectively) const baseConfig = fs.readFileSync(this.baseConfigPath, "utf-8"); const config = baseConfig .replace(/^\s*port:\s*\d+/m, ` port: ${this._port}`) diff --git a/frontend/deterministic-tests/src/server-manager.ts b/frontend/deterministic-tests/src/server-manager.ts index a9697eb0..76c624f7 100644 --- a/frontend/deterministic-tests/src/server-manager.ts +++ b/frontend/deterministic-tests/src/server-manager.ts @@ -55,5 +55,17 @@ export class ServerManager { }) .then(() => process.exit(143)); }); + + // Last-resort synchronous cleanup. Runs even when the process is + // exiting via process.exit() from unhandledRejection / + // uncaughtException — paths where async stopAll() cannot complete. + // SIGSTOP'd servers MUST receive SIGCONT before SIGKILL or the + // kernel keeps them as zombies holding the test's tmpdir, and the + // next CI run can't reuse the port. + process.on("exit", () => { + for (const server of this.activeServers) { + server.forceKillSync(); + } + }); } } diff --git a/frontend/deterministic-tests/src/test-registry.ts b/frontend/deterministic-tests/src/test-registry.ts index ed4fe026..1a07b411 100644 --- a/frontend/deterministic-tests/src/test-registry.ts +++ b/frontend/deterministic-tests/src/test-registry.ts @@ -33,10 +33,9 @@ import { offlineDeleteVsRemoteUpdateTest } from "./tests/offline-delete-vs-remot import { doubleOfflineCycleTest } from "./tests/double-offline-cycle.test"; import { serverPauseRenameEditResumeTest } from "./tests/server-pause-rename-edit-resume.test"; import { offlineUpdateBothThenDeleteOneTest } from "./tests/offline-update-both-then-delete-one.test"; -import { offlineCreateSamePathMergeableTest } from "./tests/offline-create-same-path-binary-conflict.test"; +import { offlineCreateSamePathMergeableTest } from "./tests/offline-create-same-path-mergeable.test"; import { deleteDuringPendingCreateTest } from "./tests/delete-during-pending-create.test"; import { threeClientRenameCreateDeleteTest } from "./tests/three-client-rename-create-delete.test"; -import { keyMigrationEventDropTest } from "./tests/key-migration-event-drop.test"; import { renameToPathOfUnconfirmedDeleteTest } from "./tests/rename-to-path-of-unconfirmed-delete.test"; import { offlineEditThenMoveSameContentTest } from "./tests/offline-edit-then-move-same-content.test"; import { rapidCreateUpdateDeleteCycleTest } from "./tests/rapid-create-update-delete-cycle.test"; @@ -47,10 +46,9 @@ import { offlineMoveThenRemoteDeleteTest } from "./tests/offline-move-then-remot import { resetClearsRecentlyDeletedResurrectionTest } from "./tests/reset-clears-recently-deleted-resurrection.test"; import { moveThenDeleteStalePathTest } from "./tests/move-then-delete-stale-path.test"; import { interruptedDeleteRetryTest } from "./tests/interrupted-delete-retry.test"; -import { updateDoesNotSurvivesRemoteDeleteTest } from "./tests/update-survives-remote-delete.test"; +import { updateDoesNotSurviveRemoteDeleteTest } from "./tests/update-does-not-survive-remote-delete.test"; import { movePreservesRemoteUpdateTest } from "./tests/move-preserves-remote-update.test"; import { recentlyDeletedClearedOnReconnectTest } from "./tests/recently-deleted-cleared-on-reconnect.test"; -import { migrateKeyPreservesExistingTest } from "./tests/migrate-key-preserves-existing.test"; import { watermarkAdvancesOnSkipTest } from "./tests/watermark-advances-on-skip.test"; import { watermarkGapRemoteUpdateNotRecordedTest } from "./tests/watermark-gap-remote-update-not-recorded.test"; import { queueResetLosesCoalescedLocalEditTest } from "./tests/queue-reset-loses-coalesced-local-edit.test"; @@ -62,25 +60,25 @@ import { createRenameResponseSkipsFileTest } from "./tests/create-rename-respons import { onlineCreateRenameConcurrentCreateOrphanTest } from "./tests/online-create-rename-concurrent-create-orphan.test"; import { concurrentRenameFirstWinsTest } from "./tests/concurrent-rename-first-wins.test"; import { binaryToTextTransitionTest } from "./tests/binary-to-text-transition.test"; -import { textPendingCreateNotDisplacedTest } from "./tests/1-text-pending-create-not-displaced.test"; -import { binaryPendingCreateNotDisplacedTest } from "./tests/2-binary-pending-create-not-displaced.test"; -import { coalesceUpdateRemoteUpdateDataLossTest } from "./tests/3-coalesce-update-remote-update-data-loss.test"; -import { coalescedRemoteUpdateWatermarkLossTest } from "./tests/4-coalesced-remote-update-watermark-loss.test"; -import { concurrentDeleteDuringRemoteUpdateTest } from "./tests/5-concurrent-delete-during-remote-update.test"; -import { concurrentEditExactSamePositionTest } from "./tests/6-concurrent-edit-exact-same-position.test"; -import { concurrentRenameAndCreateAtTargetTest as concurrentRenameAndCreateAtTargetRenameFirstTest } from "./tests/7-concurrent-rename-and-create-at-target.test"; -import { concurrentRenameAndCreateAtTargetTest as concurrentRenameAndCreateAtTargetCreateFirstTest } from "./tests/8-concurrent-rename-and-create-at-target.test"; -import { concurrentRenameSameTargetTest } from "./tests/9-concurrent-rename-same-target.test"; -import { concurrentUpdateDiffConsistencyTest } from "./tests/10-concurrent-update-diff-consistency.test"; -import { userParenthesizedFileNotDeletedTest } from "./tests/10-user-parenthesized-file-not-deleted.test"; -import { createDeleteNoopTest } from "./tests/11-create-delete-noop.test"; -import { createMergeDeleteTest } from "./tests/12-create-merge-delete.test"; -import { moveIdenticalContentAmbiguityTest } from "./tests/13-move-identical-content-ambiguity.test"; -import { createUpdateCoalesceServerPauseTest } from "./tests/15-create-update-coalesce-server-pause.test"; -import { createDuringReconciliationTest } from "./tests/16-create-during-reconciliation.test"; -import { createMergePreservesRenamedUpdateTest } from "./tests/17-create-merge-preserves-renamed-update.test"; -import { createRenameCreateSamePathTest } from "./tests/18-create-rename-create-same-path.test"; -import { moveChainThreeFilesTest } from "./tests/19-move-chain-three-files.test"; +import { textPendingCreateNotDisplacedTest } from "./tests/text-pending-create-not-displaced.test"; +import { binaryPendingCreateNotDisplacedTest } from "./tests/binary-pending-create-not-displaced.test"; +import { coalesceUpdateRemoteUpdateDataLossTest } from "./tests/coalesce-update-remote-update-data-loss.test"; +import { coalescedRemoteUpdateWatermarkLossTest } from "./tests/coalesced-remote-update-watermark-loss.test"; +import { concurrentDeleteDuringRemoteUpdateTest } from "./tests/concurrent-delete-during-remote-update.test"; +import { concurrentEditExactSamePositionTest } from "./tests/concurrent-edit-exact-same-position.test"; +import { concurrentRenameAndCreateAtTargetRenameFirstTest } from "./tests/concurrent-rename-and-create-at-target-rename-first.test"; +import { concurrentRenameAndCreateAtTargetCreateFirstTest } from "./tests/concurrent-rename-and-create-at-target-create-first.test"; +import { concurrentRenameSameTargetTest } from "./tests/concurrent-rename-same-target.test"; +import { concurrentUpdateDiffConsistencyTest } from "./tests/concurrent-update-diff-consistency.test"; +import { userParenthesizedFileNotDeletedTest } from "./tests/user-parenthesized-file-not-deleted.test"; +import { createDeleteNoopTest } from "./tests/create-delete-noop.test"; +import { createMergeDeleteTest } from "./tests/create-merge-delete.test"; +import { moveIdenticalContentAmbiguityTest } from "./tests/move-identical-content-ambiguity.test"; +import { createUpdateCoalesceServerPauseTest } from "./tests/create-update-coalesce-server-pause.test"; +import { createDuringReconciliationTest } from "./tests/create-during-reconciliation.test"; +import { createMergePreservesRenamedUpdateTest } from "./tests/create-merge-preserves-renamed-update.test"; +import { createRenameCreateSamePathTest } from "./tests/create-rename-create-same-path.test"; +import { moveChainThreeFilesTest } from "./tests/move-chain-three-files.test"; import { deleteByOtherClientThenRecreateTest } from "./tests/delete-by-other-client-then-recreate.test"; import { onlineDeleteRecreateRapidCycleTest } from "./tests/online-delete-recreate-rapid-cycle.test"; import { onlineEditVsDeleteConvergenceTest } from "./tests/online-edit-vs-delete-convergence.test"; @@ -147,7 +145,6 @@ export const TESTS: Partial> = { "offline-create-same-path-mergeable": offlineCreateSamePathMergeableTest, "delete-during-pending-create": deleteDuringPendingCreateTest, "three-client-rename-create-delete": threeClientRenameCreateDeleteTest, - "key-migration-event-drop": keyMigrationEventDropTest, "rename-to-path-of-unconfirmed-delete": renameToPathOfUnconfirmedDeleteTest, "offline-edit-then-move-same-content": offlineEditThenMoveSameContentTest, "rapid-create-update-delete-cycle": rapidCreateUpdateDeleteCycleTest, @@ -160,11 +157,10 @@ export const TESTS: Partial> = { "move-then-delete-stale-path": moveThenDeleteStalePathTest, "offline-delete-vs-remote-update": offlineDeleteVsRemoteUpdateTest, "interrupted-delete-retry": interruptedDeleteRetryTest, - "update-survives-remote-delete": updateDoesNotSurvivesRemoteDeleteTest, + "update-does-not-survive-remote-delete": updateDoesNotSurviveRemoteDeleteTest, "move-preserves-remote-update": movePreservesRemoteUpdateTest, "recently-deleted-cleared-on-reconnect": recentlyDeletedClearedOnReconnectTest, - "migrate-key-preserves-existing": migrateKeyPreservesExistingTest, "watermark-advances-on-skip": watermarkAdvancesOnSkipTest, "watermark-gap-remote-update-not-recorded": watermarkGapRemoteUpdateNotRecordedTest, diff --git a/frontend/deterministic-tests/src/test-runner.ts b/frontend/deterministic-tests/src/test-runner.ts index c8cbadd0..411e9b08 100644 --- a/frontend/deterministic-tests/src/test-runner.ts +++ b/frontend/deterministic-tests/src/test-runner.ts @@ -266,18 +266,10 @@ export class TestRunner { } } - // Final attempt — let the error propagate - await this.waitAllAgentsSettled(); - - try { - await this.assertConsistent(); - this.logger.info("Barrier complete: all clients converged"); - } catch (error) { - throw new Error( - `Convergence timed out after ${CONVERGENCE_TIMEOUT_MS}ms: ${error instanceof Error ? error.message : String(error)}`, - { cause: lastError } - ); - } + throw new Error( + `Convergence timed out after ${CONVERGENCE_TIMEOUT_MS}ms: ${lastError?.message ?? "no consistency check ran"}`, + { cause: lastError } + ); } /** diff --git a/frontend/deterministic-tests/src/tests/2-binary-pending-create-not-displaced.test.ts b/frontend/deterministic-tests/src/tests/binary-pending-create-not-displaced.test.ts similarity index 100% rename from frontend/deterministic-tests/src/tests/2-binary-pending-create-not-displaced.test.ts rename to frontend/deterministic-tests/src/tests/binary-pending-create-not-displaced.test.ts diff --git a/frontend/deterministic-tests/src/tests/3-coalesce-update-remote-update-data-loss.test.ts b/frontend/deterministic-tests/src/tests/coalesce-update-remote-update-data-loss.test.ts similarity index 70% rename from frontend/deterministic-tests/src/tests/3-coalesce-update-remote-update-data-loss.test.ts rename to frontend/deterministic-tests/src/tests/coalesce-update-remote-update-data-loss.test.ts index 69a5ff10..1972526a 100644 --- a/frontend/deterministic-tests/src/tests/3-coalesce-update-remote-update-data-loss.test.ts +++ b/frontend/deterministic-tests/src/tests/coalesce-update-remote-update-data-loss.test.ts @@ -3,8 +3,14 @@ import type { TestDefinition } from "../test-definition"; export const coalesceUpdateRemoteUpdateDataLossTest: TestDefinition = { description: - "Client 0 edits a file while client 1 is offline. Client 1 reconnects " + - "and immediately edits the same file. Both edits should be preserved.", + "Divergent offline edits with text-merge expectation. Client 0's " + + "remote update fully lands before Client 1 reconnects (`sync`-after " + + "the c0 update enforces this), so Client 1's offline edit merges " + + "against a server-known version, not a coalesced batch. Both " + + "additions must survive in the final merged content. (Filename's " + + "'coalesce' framing is aspirational — a true update-coalesce test " + + "would skip the c0 sync and queue overlapping local + remote " + + "updates against the same parent version.)", clients: 2, steps: [ { diff --git a/frontend/deterministic-tests/src/tests/4-coalesced-remote-update-watermark-loss.test.ts b/frontend/deterministic-tests/src/tests/coalesced-remote-update-watermark-loss.test.ts similarity index 100% rename from frontend/deterministic-tests/src/tests/4-coalesced-remote-update-watermark-loss.test.ts rename to frontend/deterministic-tests/src/tests/coalesced-remote-update-watermark-loss.test.ts diff --git a/frontend/deterministic-tests/src/tests/5-concurrent-delete-during-remote-update.test.ts b/frontend/deterministic-tests/src/tests/concurrent-delete-during-remote-update.test.ts similarity index 100% rename from frontend/deterministic-tests/src/tests/5-concurrent-delete-during-remote-update.test.ts rename to frontend/deterministic-tests/src/tests/concurrent-delete-during-remote-update.test.ts diff --git a/frontend/deterministic-tests/src/tests/6-concurrent-edit-exact-same-position.test.ts b/frontend/deterministic-tests/src/tests/concurrent-edit-exact-same-position.test.ts similarity index 100% rename from frontend/deterministic-tests/src/tests/6-concurrent-edit-exact-same-position.test.ts rename to frontend/deterministic-tests/src/tests/concurrent-edit-exact-same-position.test.ts diff --git a/frontend/deterministic-tests/src/tests/8-concurrent-rename-and-create-at-target.test.ts b/frontend/deterministic-tests/src/tests/concurrent-rename-and-create-at-target-create-first.test.ts similarity index 94% rename from frontend/deterministic-tests/src/tests/8-concurrent-rename-and-create-at-target.test.ts rename to frontend/deterministic-tests/src/tests/concurrent-rename-and-create-at-target-create-first.test.ts index a6f34102..cd8046ce 100644 --- a/frontend/deterministic-tests/src/tests/8-concurrent-rename-and-create-at-target.test.ts +++ b/frontend/deterministic-tests/src/tests/concurrent-rename-and-create-at-target-create-first.test.ts @@ -1,7 +1,7 @@ import type { AssertableState } from "../utils/assertable-state"; import type { TestDefinition } from "../test-definition"; -export const concurrentRenameAndCreateAtTargetTest: TestDefinition = { +export const concurrentRenameAndCreateAtTargetCreateFirstTest: TestDefinition = { description: "One client renames X to Y while another creates a new file at Y, " + "both offline. After syncing, Y should contain merged content from " + diff --git a/frontend/deterministic-tests/src/tests/7-concurrent-rename-and-create-at-target.test.ts b/frontend/deterministic-tests/src/tests/concurrent-rename-and-create-at-target-rename-first.test.ts similarity index 94% rename from frontend/deterministic-tests/src/tests/7-concurrent-rename-and-create-at-target.test.ts rename to frontend/deterministic-tests/src/tests/concurrent-rename-and-create-at-target-rename-first.test.ts index 63dee0db..0ac0b721 100644 --- a/frontend/deterministic-tests/src/tests/7-concurrent-rename-and-create-at-target.test.ts +++ b/frontend/deterministic-tests/src/tests/concurrent-rename-and-create-at-target-rename-first.test.ts @@ -1,7 +1,7 @@ import type { AssertableState } from "../utils/assertable-state"; import type { TestDefinition } from "../test-definition"; -export const concurrentRenameAndCreateAtTargetTest: TestDefinition = { +export const concurrentRenameAndCreateAtTargetRenameFirstTest: TestDefinition = { description: "One client renames X to Y while another creates a new file at Y, " + "both offline. We can't merge the create because it would result in a cycle", diff --git a/frontend/deterministic-tests/src/tests/9-concurrent-rename-same-target.test.ts b/frontend/deterministic-tests/src/tests/concurrent-rename-same-target.test.ts similarity index 100% rename from frontend/deterministic-tests/src/tests/9-concurrent-rename-same-target.test.ts rename to frontend/deterministic-tests/src/tests/concurrent-rename-same-target.test.ts diff --git a/frontend/deterministic-tests/src/tests/10-concurrent-update-diff-consistency.test.ts b/frontend/deterministic-tests/src/tests/concurrent-update-diff-consistency.test.ts similarity index 100% rename from frontend/deterministic-tests/src/tests/10-concurrent-update-diff-consistency.test.ts rename to frontend/deterministic-tests/src/tests/concurrent-update-diff-consistency.test.ts diff --git a/frontend/deterministic-tests/src/tests/11-create-delete-noop.test.ts b/frontend/deterministic-tests/src/tests/create-delete-noop.test.ts similarity index 100% rename from frontend/deterministic-tests/src/tests/11-create-delete-noop.test.ts rename to frontend/deterministic-tests/src/tests/create-delete-noop.test.ts diff --git a/frontend/deterministic-tests/src/tests/16-create-during-reconciliation.test.ts b/frontend/deterministic-tests/src/tests/create-during-reconciliation.test.ts similarity index 100% rename from frontend/deterministic-tests/src/tests/16-create-during-reconciliation.test.ts rename to frontend/deterministic-tests/src/tests/create-during-reconciliation.test.ts diff --git a/frontend/deterministic-tests/src/tests/12-create-merge-delete.test.ts b/frontend/deterministic-tests/src/tests/create-merge-delete.test.ts similarity index 100% rename from frontend/deterministic-tests/src/tests/12-create-merge-delete.test.ts rename to frontend/deterministic-tests/src/tests/create-merge-delete.test.ts diff --git a/frontend/deterministic-tests/src/tests/17-create-merge-preserves-renamed-update.test.ts b/frontend/deterministic-tests/src/tests/create-merge-preserves-renamed-update.test.ts similarity index 100% rename from frontend/deterministic-tests/src/tests/17-create-merge-preserves-renamed-update.test.ts rename to frontend/deterministic-tests/src/tests/create-merge-preserves-renamed-update.test.ts diff --git a/frontend/deterministic-tests/src/tests/18-create-rename-create-same-path.test.ts b/frontend/deterministic-tests/src/tests/create-rename-create-same-path.test.ts similarity index 100% rename from frontend/deterministic-tests/src/tests/18-create-rename-create-same-path.test.ts rename to frontend/deterministic-tests/src/tests/create-rename-create-same-path.test.ts diff --git a/frontend/deterministic-tests/src/tests/15-create-update-coalesce-server-pause.test.ts b/frontend/deterministic-tests/src/tests/create-update-coalesce-server-pause.test.ts similarity index 100% rename from frontend/deterministic-tests/src/tests/15-create-update-coalesce-server-pause.test.ts rename to frontend/deterministic-tests/src/tests/create-update-coalesce-server-pause.test.ts diff --git a/frontend/deterministic-tests/src/tests/key-migration-event-drop.test.ts b/frontend/deterministic-tests/src/tests/key-migration-event-drop.test.ts deleted file mode 100644 index cc40e6b0..00000000 --- a/frontend/deterministic-tests/src/tests/key-migration-event-drop.test.ts +++ /dev/null @@ -1,39 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const keyMigrationEventDropTest: TestDefinition = { - description: - "Client 0 creates a file and immediately updates it while the server is paused. " + - "After resume, both clients should have the updated content.", - clients: 2, - steps: [ - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - - { type: "pause-server" }, - - { - type: "create", - client: 0, - path: "A.md", - content: "initial content" - }, - { - type: "update", - client: 0, - path: "A.md", - content: "updated content" - }, - - { type: "resume-server" }, - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertFileCount(1).assertContent("A.md", "updated content"); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/migrate-key-preserves-existing.test.ts b/frontend/deterministic-tests/src/tests/migrate-key-preserves-existing.test.ts deleted file mode 100644 index bb669e45..00000000 --- a/frontend/deterministic-tests/src/tests/migrate-key-preserves-existing.test.ts +++ /dev/null @@ -1,37 +0,0 @@ -import type { AssertableState } from "../utils/assertable-state"; -import type { TestDefinition } from "../test-definition"; - -export const migrateKeyPreservesExistingTest: TestDefinition = { - description: - "Client 0 creates a file and immediately updates it while the server is paused. " + - "After resume, the update must not be lost.", - clients: 2, - steps: [ - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - - { type: "pause-server" }, - - { type: "create", client: 0, path: "A.md", content: "initial" }, - { - type: "update", - client: 0, - path: "A.md", - content: "updated by client 0" - }, - - { type: "resume-server" }, - { type: "barrier" }, - - { - type: "assert-consistent", - verify: (s: AssertableState): void => { - s.assertFileCount(1).assertContains( - "A.md", - "updated by client 0" - ); - } - } - ] -}; diff --git a/frontend/deterministic-tests/src/tests/19-move-chain-three-files.test.ts b/frontend/deterministic-tests/src/tests/move-chain-three-files.test.ts similarity index 100% rename from frontend/deterministic-tests/src/tests/19-move-chain-three-files.test.ts rename to frontend/deterministic-tests/src/tests/move-chain-three-files.test.ts diff --git a/frontend/deterministic-tests/src/tests/13-move-identical-content-ambiguity.test.ts b/frontend/deterministic-tests/src/tests/move-identical-content-ambiguity.test.ts similarity index 100% rename from frontend/deterministic-tests/src/tests/13-move-identical-content-ambiguity.test.ts rename to frontend/deterministic-tests/src/tests/move-identical-content-ambiguity.test.ts diff --git a/frontend/deterministic-tests/src/tests/offline-create-same-path-binary-conflict.test.ts b/frontend/deterministic-tests/src/tests/offline-create-same-path-mergeable.test.ts similarity index 100% rename from frontend/deterministic-tests/src/tests/offline-create-same-path-binary-conflict.test.ts rename to frontend/deterministic-tests/src/tests/offline-create-same-path-mergeable.test.ts diff --git a/frontend/deterministic-tests/src/tests/1-text-pending-create-not-displaced.test.ts b/frontend/deterministic-tests/src/tests/text-pending-create-not-displaced.test.ts similarity index 100% rename from frontend/deterministic-tests/src/tests/1-text-pending-create-not-displaced.test.ts rename to frontend/deterministic-tests/src/tests/text-pending-create-not-displaced.test.ts diff --git a/frontend/deterministic-tests/src/tests/update-survives-remote-delete.test.ts b/frontend/deterministic-tests/src/tests/update-does-not-survive-remote-delete.test.ts similarity index 100% rename from frontend/deterministic-tests/src/tests/update-survives-remote-delete.test.ts rename to frontend/deterministic-tests/src/tests/update-does-not-survive-remote-delete.test.ts diff --git a/frontend/deterministic-tests/src/tests/10-user-parenthesized-file-not-deleted.test.ts b/frontend/deterministic-tests/src/tests/user-parenthesized-file-not-deleted.test.ts similarity index 100% rename from frontend/deterministic-tests/src/tests/10-user-parenthesized-file-not-deleted.test.ts rename to frontend/deterministic-tests/src/tests/user-parenthesized-file-not-deleted.test.ts