Improve tests

frontend/deterministic-tests/README.md

@@ -10,7 +10,7 @@ Each test is a `TestDefinition`: a client count and an ordered list of steps. Th
 
 Tests that don't pause the server share a single server process (vault-name isolation). Tests that use `pause-server`/`resume-server` (SIGSTOP/SIGCONT) each get a dedicated server, since SIGSTOP freezes the entire process.
 
-All tests run in parallel up to a concurrency limit.
+The runner executes two sequential phases: regular tests on the shared server, then pause-server tests on dedicated servers. Within each phase tests run in parallel up to a concurrency limit.
 
 ## Step types
 
@@ -19,12 +19,15 @@ Clients always start with syncing disabled.
 **File operations** (per-client, fire-and-forget — sync is enqueued but not awaited):
 
 - `create`, `update`, `rename`, `delete`
+- `rename-next-write` — arm a deferred rename that fires the next time the given path is written. Lets a test race a user-rename against an in-flight remote create that's about to land at the same path.
 
 **Sync control:**
 
 - `sync` — wait for a specific client or all clients to finish pending operations
 - `barrier` — retry until all clients converge to identical file state (60s timeout)
 - `enable-sync` / `disable-sync` — simulate going online/offline
+- `reset` — reset a client's tracked sync state (keeps disk files); equivalent to a forced re-handshake on next enable
+- `sleep` — wall-clock pause; use sparingly, prefer `barrier` / `sync`
 
 **WebSocket control** (per-client):
 
@@ -33,6 +36,12 @@ Clients always start with syncing disabled.
 **Server control:**
 
 - `pause-server` / `resume-server` — SIGSTOP/SIGCONT the server process
+- `resume-server-until-history-then-pause` — resume the server, wait until a specific client observes a matching history entry (`CREATE`/`UPDATE`/`DELETE` for a path), then re-pause. Used to land exactly one operation across the wire.
+
+**Fault injection** (per-client):
+
+- `drop-next-create-response` — arm a one-shot interceptor that lets the next `POST /documents` reach the server (commit happens) but throws `SyncResetError` before the client sees the response, simulating connection loss after server commit.
+- `wait-for-dropped-create-response` — wait until the armed drop has fired.
 
 **Assertions:**
 
@@ -72,7 +81,9 @@ export const myScenarioTest: TestDefinition = {
     { type: "barrier" },
     {
       type: "assert-consistent",
-      verify: (s) => s.assertFileCount(1).assertContent("A.md", "hello")
+      verify: (s) => {
+        s.assertFileCount(1).assertContent("A.md", "hello");
+      }
     }
   ]
 };
@@ -81,14 +92,18 @@ export const myScenarioTest: TestDefinition = {
 The `verify` callback receives an `AssertableState` object with chainable assertion methods:
 
 ```typescript
-s.assertFileCount(n)                   // exact file count
+s.assertFileCount(n);                       // exact file count
-s.assertFileExists("path")             // file must exist
+s.assertFileExists("path");                 // file must exist
-s.assertFileNotExists("path")          // file must not exist
+s.assertFileNotExists("path");              // file must not exist
-s.assertContent("path", "expected")    // exact content match
+s.assertContent("path", "expected");        // exact content match
-s.assertContains("path", "a", "b")    // all substrings present
+s.assertContains("path", "a", "b");         // all substrings present in file
-s.assertAnyFileContains("text")        // substring in any file
+s.assertContainsAny("path", "a", "b");      // at least one substring present
-s.assertContentInAtMostOneFile("text") // no duplicate content
+s.assertAnyFileContains("text");            // substring present in some file
-s.ifFileExists("path", (s) => ...)    // conditional assertion
+s.assertNoFileContains("text");             // substring absent from every file
+s.assertSubstringCount("path", "x", 3);     // substring appears exactly N times
+s.assertContentInAtMostOneFile("text");     // no duplicate content
+s.ifFileExists("path", (s) => { /* … */ }); // conditional block
+s.getContent("path");                       // raw content (or "" if missing)
 ```
 
 2. Register it in `src/test-registry.ts`:

frontend/deterministic-tests/src/cli.ts

@@ -33,6 +33,27 @@ function testUsesPauseServer(test: TestDefinition): boolean {
     );
 }
 
+/**
+ * Walk up from the CLI binary's location until we find a directory
+ * containing `sync-server/` and `frontend/`.
+ */
+function findProjectRoot(): string {
+    let dir = path.dirname(__filename);
+    const root = path.parse(dir).root;
+    while (dir !== root) {
+        if (
+            fs.existsSync(path.join(dir, "sync-server")) &&
+            fs.existsSync(path.join(dir, "frontend"))
+        ) {
+            return dir;
+        }
+        dir = path.dirname(dir);
+    }
+    throw new Error(
+        `Could not locate project root (no ancestor of ${__filename} contains both 'sync-server' and 'frontend')`
+    );
+}
+
 interface NamedTestResult {
     name: string;
     result: TestResult;
@@ -100,15 +121,7 @@ async function runDedicatedServerTest(
 }
 
 async function main(): Promise<void> {
-    const cwd = process.cwd();
+    const projectRoot = findProjectRoot();
-    let projectRoot = cwd;
-
-    if (cwd.endsWith("frontend/deterministic-tests")) {
-        projectRoot = path.resolve(cwd, "../..");
-    } else if (cwd.endsWith("frontend")) {
-        projectRoot = path.resolve(cwd, "..");
-    }
-
     const serverPath = path.join(projectRoot, SERVER_BINARY_PATH);
     if (!fs.existsSync(serverPath)) {
         logger.error(`Server binary not found at: ${serverPath}`);

frontend/deterministic-tests/src/consts.ts

@@ -11,3 +11,6 @@ export const IS_SYNC_ENABLED_BY_DEFAULT = false;
 export const WAIT_TIMEOUT_MS = 60_000;
 export const WEBSOCKET_CONNECT_TIMEOUT_MS = 10_000;
 export const WEBSOCKET_POLL_INTERVAL_MS = 50;
+
+export const SERVER_READY_POLL_INTERVAL_MS = 100;
+export const SERVER_READY_MAX_ATTEMPTS = 50;

frontend/deterministic-tests/src/deterministic-agent.ts

@@ -312,6 +312,10 @@ export class DeterministicAgent extends debugging.InMemoryFileSystem {
                     });
                 });
             }
+            // The rename consumed `path`. Skip the post-update enqueue below
+            // — it would send a syncLocallyUpdatedFile for a path that no
+            // longer exists.
+            return;
         }
 
         if (!this.isSyncEnabled) {
@@ -435,6 +439,13 @@ export class DeterministicAgent extends debugging.InMemoryFileSystem {
                 DeterministicAgent.isCreateDocumentRequest(input, init)
             ) {
                 this.nextCreateResponseDrop = undefined;
+                // Release the underlying socket: an unread body keeps the
+                // undici connection open until GC.
+                try {
+                    await response.body?.cancel();
+                } catch {
+                    // Best-effort — body may already be consumed/closed.
+                }
                 drop.resolveDropped();
                 throw new SyncResetError();
             }

frontend/deterministic-tests/src/managed-websocket.ts

@@ -157,6 +157,17 @@ class ManagedWebSocket implements WebSocket {
     public addEventListener(
         ...args: Parameters<WebSocket["addEventListener"]>
     ): void {
+        // Only the `.onmessage` setter routes through the pause buffer.
+        // If sync-client ever attaches "message" listeners via
+        // addEventListener instead, those messages would bypass pause/resume
+        // and deterministic tests would silently lose their fault injection.
+        if (args[0] === "message") {
+            throw new Error(
+                "ManagedWebSocket: addEventListener('message') bypasses the " +
+                    "pause buffer. Use the .onmessage setter instead, or " +
+                    "extend ManagedWebSocket to route message listeners."
+            );
+        }
         this.ws.addEventListener(...args);
     }
 
@@ -176,6 +187,11 @@ class ManagedWebSocket implements WebSocket {
  * for pause/resume control from the test harness
  */
 export class ManagedWebSocketFactory {
+    // Append-only: closed sockets stay tracked. Bounded per test (one
+    // factory per agent, each test discards its agents on cleanup), so
+    // not a real leak — but iterating over closed instances on
+    // pause/resume is a deliberate no-op since their `.onmessage` is
+    // already detached.
     private readonly instances: ManagedWebSocket[] = [];
     // Sticky pause state: applied to current instances on `pause()` AND
     // to any new instance created later (e.g. WS reconnect after a

frontend/deterministic-tests/src/server-control.ts

@@ -5,7 +5,11 @@ import * as path from "node:path";
 import { sleep } from "./utils/sleep";
 import { findFreePort } from "./utils/find-free-port";
 import type { Logger } from "sync-client";
-import { STOP_TIMEOUT_MS } from "./consts";
+import {
+    STOP_TIMEOUT_MS,
+    SERVER_READY_POLL_INTERVAL_MS,
+    SERVER_READY_MAX_ATTEMPTS
+} from "./consts";
 
 export class ServerControl {
     private process: ChildProcess | null = null;
@@ -101,7 +105,9 @@ export class ServerControl {
         }
     }
 
-    public async waitForReady(maxAttempts = 50): Promise<void> {
+    public async waitForReady(
+        maxAttempts: number = SERVER_READY_MAX_ATTEMPTS
+    ): Promise<void> {
         const pingUrl = `${this.remoteUri}/vaults/test/ping`;
         for (let i = 0; i < maxAttempts; i++) {
             if (this.process?.exitCode !== null) {
@@ -118,7 +124,7 @@ export class ServerControl {
             } catch {
                 // Server not ready yet, continue polling
             }
-            await sleep(100);
+            await sleep(SERVER_READY_POLL_INTERVAL_MS);
         }
         throw new Error("Server failed to start within timeout");
     }
@@ -212,6 +218,9 @@ export class ServerControl {
     }
 
     private writeConfigFile(destPath: string, dbDir: string): void {
+        // Assumes config-e2e.yml has exactly one 2-space-indented `port:` and
+        // one `databases_directory_path:` (under `server:` and `database:`
+        // respectively)
         const baseConfig = fs.readFileSync(this.baseConfigPath, "utf-8");
         const config = baseConfig
             .replace(/^\s*port:\s*\d+/m, `  port: ${this._port}`)

frontend/deterministic-tests/src/test-registry.ts

@@ -33,7 +33,7 @@ import { offlineDeleteVsRemoteUpdateTest } from "./tests/offline-delete-vs-remot
 import { doubleOfflineCycleTest } from "./tests/double-offline-cycle.test";
 import { serverPauseRenameEditResumeTest } from "./tests/server-pause-rename-edit-resume.test";
 import { offlineUpdateBothThenDeleteOneTest } from "./tests/offline-update-both-then-delete-one.test";
-import { offlineCreateSamePathMergeableTest } from "./tests/offline-create-same-path-binary-conflict.test";
+import { offlineCreateSamePathMergeableTest } from "./tests/offline-create-same-path-mergeable.test";
 import { deleteDuringPendingCreateTest } from "./tests/delete-during-pending-create.test";
 import { threeClientRenameCreateDeleteTest } from "./tests/three-client-rename-create-delete.test";
 import { keyMigrationEventDropTest } from "./tests/key-migration-event-drop.test";
@@ -62,25 +62,25 @@ import { createRenameResponseSkipsFileTest } from "./tests/create-rename-respons
 import { onlineCreateRenameConcurrentCreateOrphanTest } from "./tests/online-create-rename-concurrent-create-orphan.test";
 import { concurrentRenameFirstWinsTest } from "./tests/concurrent-rename-first-wins.test";
 import { binaryToTextTransitionTest } from "./tests/binary-to-text-transition.test";
-import { textPendingCreateNotDisplacedTest } from "./tests/1-text-pending-create-not-displaced.test";
+import { textPendingCreateNotDisplacedTest } from "./tests/text-pending-create-not-displaced.test";
-import { binaryPendingCreateNotDisplacedTest } from "./tests/2-binary-pending-create-not-displaced.test";
+import { binaryPendingCreateNotDisplacedTest } from "./tests/binary-pending-create-not-displaced.test";
-import { coalesceUpdateRemoteUpdateDataLossTest } from "./tests/3-coalesce-update-remote-update-data-loss.test";
+import { coalesceUpdateRemoteUpdateDataLossTest } from "./tests/coalesce-update-remote-update-data-loss.test";
-import { coalescedRemoteUpdateWatermarkLossTest } from "./tests/4-coalesced-remote-update-watermark-loss.test";
+import { coalescedRemoteUpdateWatermarkLossTest } from "./tests/coalesced-remote-update-watermark-loss.test";
-import { concurrentDeleteDuringRemoteUpdateTest } from "./tests/5-concurrent-delete-during-remote-update.test";
+import { concurrentDeleteDuringRemoteUpdateTest } from "./tests/concurrent-delete-during-remote-update.test";
-import { concurrentEditExactSamePositionTest } from "./tests/6-concurrent-edit-exact-same-position.test";
+import { concurrentEditExactSamePositionTest } from "./tests/concurrent-edit-exact-same-position.test";
-import { concurrentRenameAndCreateAtTargetTest as concurrentRenameAndCreateAtTargetRenameFirstTest } from "./tests/7-concurrent-rename-and-create-at-target.test";
+import { concurrentRenameAndCreateAtTargetRenameFirstTest } from "./tests/concurrent-rename-and-create-at-target-rename-first.test";
-import { concurrentRenameAndCreateAtTargetTest as concurrentRenameAndCreateAtTargetCreateFirstTest } from "./tests/8-concurrent-rename-and-create-at-target.test";
+import { concurrentRenameAndCreateAtTargetCreateFirstTest } from "./tests/concurrent-rename-and-create-at-target-create-first.test";
-import { concurrentRenameSameTargetTest } from "./tests/9-concurrent-rename-same-target.test";
+import { concurrentRenameSameTargetTest } from "./tests/concurrent-rename-same-target.test";
-import { concurrentUpdateDiffConsistencyTest } from "./tests/10-concurrent-update-diff-consistency.test";
+import { concurrentUpdateDiffConsistencyTest } from "./tests/concurrent-update-diff-consistency.test";
-import { userParenthesizedFileNotDeletedTest } from "./tests/10-user-parenthesized-file-not-deleted.test";
+import { userParenthesizedFileNotDeletedTest } from "./tests/user-parenthesized-file-not-deleted.test";
-import { createDeleteNoopTest } from "./tests/11-create-delete-noop.test";
+import { createDeleteNoopTest } from "./tests/create-delete-noop.test";
-import { createMergeDeleteTest } from "./tests/12-create-merge-delete.test";
+import { createMergeDeleteTest } from "./tests/create-merge-delete.test";
-import { moveIdenticalContentAmbiguityTest } from "./tests/13-move-identical-content-ambiguity.test";
+import { moveIdenticalContentAmbiguityTest } from "./tests/move-identical-content-ambiguity.test";
-import { createUpdateCoalesceServerPauseTest } from "./tests/15-create-update-coalesce-server-pause.test";
+import { createUpdateCoalesceServerPauseTest } from "./tests/create-update-coalesce-server-pause.test";
-import { createDuringReconciliationTest } from "./tests/16-create-during-reconciliation.test";
+import { createDuringReconciliationTest } from "./tests/create-during-reconciliation.test";
-import { createMergePreservesRenamedUpdateTest } from "./tests/17-create-merge-preserves-renamed-update.test";
+import { createMergePreservesRenamedUpdateTest } from "./tests/create-merge-preserves-renamed-update.test";
-import { createRenameCreateSamePathTest } from "./tests/18-create-rename-create-same-path.test";
+import { createRenameCreateSamePathTest } from "./tests/create-rename-create-same-path.test";
-import { moveChainThreeFilesTest } from "./tests/19-move-chain-three-files.test";
+import { moveChainThreeFilesTest } from "./tests/move-chain-three-files.test";
 import { deleteByOtherClientThenRecreateTest } from "./tests/delete-by-other-client-then-recreate.test";
 import { onlineDeleteRecreateRapidCycleTest } from "./tests/online-delete-recreate-rapid-cycle.test";
 import { onlineEditVsDeleteConvergenceTest } from "./tests/online-edit-vs-delete-convergence.test";

frontend/deterministic-tests/src/test-runner.ts

@@ -266,18 +266,10 @@ export class TestRunner {
             }
         }
 
-        // Final attempt — let the error propagate
+        throw new Error(
-        await this.waitAllAgentsSettled();
+            `Convergence timed out after ${CONVERGENCE_TIMEOUT_MS}ms: ${lastError?.message ?? "no consistency check ran"}`,
-
+            { cause: lastError }
-        try {
+        );
-            await this.assertConsistent();
-            this.logger.info("Barrier complete: all clients converged");
-        } catch (error) {
-            throw new Error(
-                `Convergence timed out after ${CONVERGENCE_TIMEOUT_MS}ms: ${error instanceof Error ? error.message : String(error)}`,
-                { cause: lastError }
-            );
-        }
     }
 
     /**

frontend/deterministic-tests/src/tests/concurrent-rename-and-create-at-target-create-first.test.ts

@@ -1,7 +1,7 @@
 import type { AssertableState } from "../utils/assertable-state";
 import type { TestDefinition } from "../test-definition";
 
-export const concurrentRenameAndCreateAtTargetTest: TestDefinition = {
+export const concurrentRenameAndCreateAtTargetCreateFirstTest: TestDefinition = {
     description:
         "One client renames X to Y while another creates a new file at Y, " +
         "both offline. After syncing, Y should contain merged content from " +

frontend/deterministic-tests/src/tests/concurrent-rename-and-create-at-target-rename-first.test.ts

@@ -1,7 +1,7 @@
 import type { AssertableState } from "../utils/assertable-state";
 import type { TestDefinition } from "../test-definition";
 
-export const concurrentRenameAndCreateAtTargetTest: TestDefinition = {
+export const concurrentRenameAndCreateAtTargetRenameFirstTest: TestDefinition = {
     description:
         "One client renames X to Y while another creates a new file at Y, " +
         "both offline. We can't merge the create because it would result in a cycle",