andras/vault-link
1
0
Fork 0
Code Issues Pull requests 18 Projects Releases 55 Packages Wiki Activity Actions 1

Add deterministic tests

Browse source
This commit is contained in:
Andras Schmelczer 2026-03-25 21:34:57 +00:00
parent 6fbbd1e12f
commit 0ce82353e0
20 changed files with 1780 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

81
frontend/deterministic-tests/README.md Normal file
View file

@ -0,0 +1,81 @@
# Deterministic Tests
Scripted multi-client (with an in-memory filesystem) sync tests that run against a real server. Each test defines a sequence of file operations, sync/server controls, and assertions to exercise a specific conflict or edge case.
Complements the fuzz-based E2E tests (`test-client`): fuzz tests discover bugs through random operations; deterministic tests pin down exact reproduction sequences for known scenarios.
## How it works
Each test is a `TestDefinition`: a name, a client count, and an ordered list of steps. The `TestRunner` spins up N `DeterministicAgent` instances (each wrapping a real `SyncClient` with an `InMemoryFileSystem`) pointed at a shared vault on the server, then executes steps one by one.
Tests that don't pause the server share a single server process (vault-name isolation). Tests that use `pause-server`/`resume-server` (SIGSTOP/SIGCONT) each get a dedicated server, since SIGSTOP freezes the entire process.
All tests run in parallel up to a concurrency limit.
## Step types
Clients always start with syincing being disabled.
**File operations** (per-client, fire-and-forget — sync is enqueued but not awaited):
- `create`, `update`, `rename`, `delete`
**Sync control:**
- `sync` — wait for a specific client or all clients to finish pending operations
- `barrier` — retry until all clients converge to identical file state (60s timeout)
- `enable-sync` / `disable-sync` — simulate going online/offline
**Server control:**
- `pause-server` / `resume-server` — SIGSTOP/SIGCONT the server process
- `wait` — sleep for N milliseconds
**Assertions:**
- `assert-content`, `assert-exists`, `assert-not-exists`
- `assert-consistent` — all clients have identical files; optionally takes a custom verify function
## Running
```sh
# Build server first
cd sync-server && cargo build --release
# Run all tests
cd frontend && npm run test -w deterministic-tests
# Filter by name
npm run test -w deterministic-tests -- --filter=rename
# Control parallelism (default: number of CPU cores)
npm run test -w deterministic-tests -- -j 4
```
## Adding a test
1. Create `src/tests/my-scenario.test.ts`:
```typescript
import type { TestDefinition } from "../test-definition";
export const myScenarioTest: TestDefinition = {
name: "My Scenario",
description: "What this test verifies",
clients: 2,
steps: [
{ type: "create", client: 0, path: "A.md", content: "hello" },
{ type: "sync" },
{ type: "barrier" },
{ type: "assert-consistent" }
]
};
```
2. Register it in `src/test-registry.ts`:
```typescript
import { myScenarioTest } from "./tests/my-scenario.test";
const TESTS = {
// ...
"my-scenario": myScenarioTest
};
```

22
frontend/deterministic-tests/package.json Normal file
View file

@ -0,0 +1,22 @@
{
"name": "deterministic-tests",
"version": "0.14.0",
"private": true,
"bin": {
"deterministic-tests": "./dist/cli.js"
},
"scripts": {
"dev": "webpack watch --mode development",
"build": "webpack --mode production",
"test": "npm run build && node dist/cli.js"
},
"devDependencies": {
"@types/node": "^25.0.2",
"sync-client": "file:../sync-client",
"ts-loader": "^9.5.4",
"tslib": "2.8.1",
"typescript": "5.9.3",
"webpack": "^5.103.0",
"webpack-cli": "^6.0.1"
}
}

228
frontend/deterministic-tests/src/cli.ts Normal file
View file

@ -0,0 +1,228 @@
import { TestRunner } from "./test-runner";
import { ServerControl } from "./server-control";
import { ServerManager } from "./server-manager";
import { PrefixedLogger } from "./prefixed-logger";
import { TESTS } from "./test-registry";
import type { TestDefinition, TestResult } from "./test-definition";
import { parseConcurrency } from "./parse-concurrency";
import { runWithConcurrency } from "./run-with-concurrency";
import { TOKEN, SERVER_BINARY_PATH, CONFIG_PATH } from "./consts";
import * as path from "node:path";
import * as fs from "node:fs";
import { debugging, Logger } from "sync-client";
const logger = new Logger();
debugging.logToConsole(logger, { useColors: true });
process.on("unhandledRejection", (reason) => {
logger.error(`Unhandled Rejection: ${reason}`);
process.exit(1);
});
process.on("uncaughtException", (error) => {
logger.error(`Uncaught Exception: ${error}`);
process.exit(1);
});
const serverManager = new ServerManager(logger);
serverManager.installSignalHandlers();
function testUsesPauseServer(test: TestDefinition): boolean {
return test.steps.some(
(step) => step.type === "pause-server" || step.type === "resume-server"
);
}
interface NamedTestResult {
test: TestDefinition;
result: TestResult;
}
async function main(): Promise<void> {
const cwd = process.cwd();
let projectRoot = cwd;
if (cwd.endsWith("frontend/deterministic-tests")) {
projectRoot = path.resolve(cwd, "../..");
} else if (cwd.endsWith("frontend")) {
projectRoot = path.resolve(cwd, "..");
}
const serverPath = path.join(projectRoot, SERVER_BINARY_PATH);
if (!fs.existsSync(serverPath)) {
logger.error(`Server binary not found at: ${serverPath}`);
process.exit(1);
}
const configPath = path.join(projectRoot, CONFIG_PATH);
if (!fs.existsSync(configPath)) {
logger.error(`Config file not found at: ${configPath}`);
process.exit(1);
}
const filterArg = process.argv.find((a) => a.startsWith("--filter="));
const filter = filterArg?.slice("--filter=".length);
const testsToRun: TestDefinition[] = [];
for (const [key, test] of Object.entries(TESTS)) {
if (test) {
if (filter && !key.includes(filter) && !test.name.toLowerCase().includes(filter.toLowerCase())) {
continue;
}
testsToRun.push(test);
}
}
if (testsToRun.length === 0) {
logger.error(
filter
? `No tests matched filter "${filter}"`
: "No tests found"
);
process.exit(1);
}
const concurrency = parseConcurrency();
const regularTests = testsToRun.filter((t) => !testUsesPauseServer(t));
const pauseTests = testsToRun.filter((t) => testUsesPauseServer(t));
logger.info(`Server: ${serverPath}`);
logger.info(`Config: ${configPath}`);
logger.info(
`Tests: ${testsToRun.length} total (${regularTests.length} regular, ${pauseTests.length} server-pause)`
);
logger.info(`Concurrency: ${concurrency}`);
const allResults: NamedTestResult[] = [];
if (regularTests.length > 0) {
logger.info(
`\n--- Running ${regularTests.length} regular tests (shared server, concurrency ${concurrency}) ---`
);
const sharedServer = new ServerControl(
serverPath,
configPath,
logger
);
serverManager.track(sharedServer);
try {
await sharedServer.start();
const results = await runWithConcurrency(
regularTests,
concurrency,
async (test) => runSharedServerTest(test, sharedServer)
);
allResults.push(...results);
} finally {
try {
await sharedServer.stop();
} catch (error) {
logger.warn(
`Error stopping shared server: ${error instanceof Error ? error.message : String(error)}`
);
}
serverManager.untrack(sharedServer);
}
}
if (pauseTests.length > 0) {
logger.info(
`\n--- Running ${pauseTests.length} server-pause tests (dedicated servers, concurrency ${concurrency}) ---`
);
const results = await runWithConcurrency(
pauseTests,
concurrency,
async (test) => runDedicatedServerTest(test, serverPath, configPath)
);
allResults.push(...results);
}
const passed = allResults.filter((r) => r.result.success);
const failed = allResults.filter((r) => !r.result.success);
logger.info(`\n--- Results: ${passed.length}/${allResults.length} passed ---`);
if (failed.length > 0) {
for (const { test, result } of failed) {
logger.error(` FAILED: ${test.name}: ${result.error}`);
}
process.exit(1);
} else {
logger.info("All tests passed!");
process.exit(0);
}
}
main().catch((err: unknown) => {
logger.error(`Unexpected error: ${err}`);
process.exit(1);
});
/**
* Run a test on a shared server (for tests that don't use pause-server).
*/
async function runSharedServerTest(
test: TestDefinition,
sharedServer: ServerControl
): Promise<NamedTestResult> {
const testLogger = new PrefixedLogger(logger, test.name);
const runner = new TestRunner(
sharedServer,
testLogger,
TOKEN,
sharedServer.remoteUri
);
const result = await runner.runTest(test);
if (result.success) {
logger.info(`PASSED: ${test.name} (${result.duration}ms)`);
} else {
logger.error(`FAILED: ${test.name} - ${result.error}`);
}
return { test, result };
}
/**
* Run a test with its own dedicated server (for tests that use pause-server).
* SIGSTOP/SIGCONT affects the entire server process, so these tests need
* isolated servers to avoid interfering with other tests.
*/
async function runDedicatedServerTest(
test: TestDefinition,
serverPath: string,
configPath: string
): Promise<NamedTestResult> {
const testLogger = new PrefixedLogger(logger, test.name);
const server = new ServerControl(serverPath, configPath, testLogger);
serverManager.track(server);
try {
await server.start();
const runner = new TestRunner(
server,
testLogger,
TOKEN,
server.remoteUri
);
const result = await runner.runTest(test);
if (result.success) {
logger.info(`PASSED: ${test.name} (${result.duration}ms)`);
} else {
logger.error(`FAILED: ${test.name} - ${result.error}`);
}
return { test, result };
} finally {
try {
await server.stop();
} catch {
// best-effort cleanup
}
serverManager.untrack(server);
}
}

13
frontend/deterministic-tests/src/consts.ts Normal file
View file

@ -0,0 +1,13 @@
export const TOKEN = "test-token-change-me";
export const SERVER_BINARY_PATH = "sync-server/target/release/sync_server";
export const CONFIG_PATH = "sync-server/config-e2e.yml";
export const STOP_TIMEOUT_MS = 5_000;
export const CONVERGENCE_TIMEOUT_MS = 60_000;
export const CONVERGENCE_RETRY_DELAY_MS = 500;
export const AGENT_INIT_TIMEOUT_MS = 30_000;
export const IS_SYNC_ENABLED_DEFAULT = false;
export const WAIT_TIMEOUT_MS = 60_000;
export const WEBSOCKET_CONNECT_TIMEOUT_MS = 10_000;
export const WEBSOCKET_POLL_INTERVAL_MS = 50;

280
frontend/deterministic-tests/src/deterministic-agent.ts Normal file
View file

@ -0,0 +1,280 @@
import type { StoredDatabase, SyncSettings, RelativePath } from "sync-client";
import { SyncClient, debugging, LogLevel } from "sync-client";
import { assert } from "./utils/assert";
import { sleep } from "./utils/sleep";
import { withTimeout } from "./utils/with-timeout";
import { IS_SYNC_ENABLED_DEFAULT, WAIT_TIMEOUT_MS, WEBSOCKET_CONNECT_TIMEOUT_MS, WEBSOCKET_POLL_INTERVAL_MS } from "./consts";
export class DeterministicAgent extends debugging.InMemoryFileSystem {
public readonly clientId: number;
private readonly logger: (msg: string) => void;
private client!: SyncClient;
private data: Partial<{
settings: Partial<SyncSettings>;
database: Partial<StoredDatabase>;
}> = {};
private isSyncEnabled = IS_SYNC_ENABLED_DEFAULT;
public constructor(
clientId: number,
initialSettings: Partial<SyncSettings>,
logger: (msg: string) => void
) {
super();
this.clientId = clientId;
this.logger = logger;
this.data.settings = { ...initialSettings };
}
public async init(
fetchImplementation: typeof globalThis.fetch,
webSocketImplementation: typeof globalThis.WebSocket
): Promise<void> {
this.client = await SyncClient.create({
fs: this,
persistence: {
load: async () => this.data,
save: async (data) => void (this.data = data)
},
fetch: fetchImplementation,
webSocket: webSocketImplementation
});
this.client.logger.onLogEmitted.add((line) => {
const prefix = `[Client ${this.clientId}]`;
switch (line.level) {
case LogLevel.ERROR:
this.logger(`${prefix} ERROR: ${line.message}`);
break;
case LogLevel.WARNING:
this.logger(`${prefix} WARN: ${line.message}`);
break;
case LogLevel.INFO:
this.logger(`${prefix} ${line.message}`);
break;
case LogLevel.DEBUG:
// Skip debug logs to reduce noise
break;
}
});
await this.client.start();
const connectionCheck = await this.client.checkConnection();
assert(
connectionCheck.isSuccessful,
`Client ${this.clientId} connection check failed`
);
if (this.isSyncEnabled) {
await this.waitForWebSocket();
}
}
public async createFile(path: string, content: string): Promise<void> {
this.log(`Creating file ${path} with content: ${content}`);
if (this.files.has(path)) {
throw new Error(`File ${path} already exists`);
}
const contentBytes = new TextEncoder().encode(content);
this.files.set(path, contentBytes);
this.enqueueSync(async () =>
this.client.syncLocallyCreatedFile(path)
);
}
public async updateFile(path: string, content: string): Promise<void> {
this.log(`Updating file ${path} with content: ${content}`);
if (!this.files.has(path)) {
throw new Error(
`File ${path} does not exist on client ${this.clientId}`
);
}
const contentBytes = new TextEncoder().encode(content);
this.files.set(path, contentBytes);
this.enqueueSync(async () =>
this.client.syncLocallyUpdatedFile({ relativePath: path })
);
}
public async renameFile(oldPath: string, newPath: string): Promise<void> {
this.log(`Renaming file ${oldPath} to ${newPath}`);
const file = this.files.get(oldPath);
if (!file) {
throw new Error(
`File ${oldPath} does not exist on client ${this.clientId}`
);
}
if (oldPath !== newPath && this.files.has(newPath)) {
this.log(
`Target path ${newPath} already exists, will be overwritten (ensureClearPath)`
);
}
this.files.set(newPath, file);
if (oldPath !== newPath) {
this.files.delete(oldPath);
}
if (this.isSyncEnabled) {
this.enqueueSync(async () =>
this.client.syncLocallyUpdatedFile({
oldPath,
relativePath: newPath
})
);
}
}
public async deleteFile(path: string): Promise<void> {
this.log(`Deleting file ${path}`);
this.files.delete(path);
if (this.isSyncEnabled) {
this.enqueueSync(async () =>
this.client.syncLocallyDeletedFile(path)
);
}
}
public async waitForSync(): Promise<void> {
this.log("Waiting for sync to complete...");
await withTimeout(
this.client.waitUntilFinished(),
WAIT_TIMEOUT_MS,
`Client ${this.clientId} waitForSync timed out after ${WAIT_TIMEOUT_MS}ms`
);
this.log("Sync complete");
}
public async disableSync(): Promise<void> {
this.log("Disabling sync");
await this.client.setSetting("isSyncEnabled", false);
this.isSyncEnabled = false;
}
public async enableSync(): Promise<void> {
this.log("Enabling sync");
await this.client.setSetting("isSyncEnabled", true);
this.isSyncEnabled = true;
await this.waitForWebSocket();
}
public async assertContent(
path: string,
expectedContent: string
): Promise<void> {
this.log(`Asserting content of ${path} equals "${expectedContent}"`);
const actualBytes = await this.read(path).catch(() => {
throw new Error(
`File ${path} does not exist on client ${this.clientId}`
);
});
const actualContent = new TextDecoder().decode(actualBytes);
assert(
actualContent === expectedContent,
`Content mismatch on client ${this.clientId} for ${path}:\nExpected: "${expectedContent}"\nActual: "${actualContent}"`
);
this.log(`✓ Content assertion passed for ${path}`);
}
public async assertExists(path: string): Promise<void> {
this.log(`Asserting ${path} exists`);
const exists = await this.exists(path);
assert(
exists,
`File ${path} does not exist on client ${this.clientId}`
);
this.log(`✓ File ${path} exists`);
}
public async assertNotExists(path: string): Promise<void> {
this.log(`Asserting ${path} does not exist`);
const exists = await this.exists(path);
assert(
!exists,
`File ${path} exists on client ${this.clientId} but should not`
);
this.log(`✓ File ${path} does not exist`);
}
public async getFiles(): Promise<RelativePath[]> {
return this.listFilesRecursively();
}
public async getFileContent(path: string): Promise<string> {
const bytes = await this.read(path);
return new TextDecoder().decode(bytes);
}
public async cleanup(): Promise<void> {
this.log("Cleaning up...");
// Guard against uninitialized client (init() failed partway)
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
if (!this.client) {
this.log("Client not initialized, nothing to clean up");
return;
}
try {
await withTimeout(
this.client.waitUntilFinished(),
WAIT_TIMEOUT_MS,
`Client ${this.clientId} cleanup waitUntilFinished timed out`
);
} catch (error) {
if (error instanceof Error && error.name === "SyncResetError") {
this.log(`Cleanup interrupted by reset (expected): ${error}`);
} else {
this.log(`Cleanup waitUntilFinished failed: ${error}`);
}
}
await this.client.destroy();
this.log("Cleanup complete");
}
private async waitForWebSocket(): Promise<void> {
const deadline = Date.now() + WEBSOCKET_CONNECT_TIMEOUT_MS;
while (!this.client.isWebSocketConnected && Date.now() < deadline) {
await sleep(WEBSOCKET_POLL_INTERVAL_MS);
}
assert(
this.client.isWebSocketConnected,
`Client ${this.clientId} WebSocket failed to connect within ${WEBSOCKET_CONNECT_TIMEOUT_MS}ms`
);
}
private enqueueSync(operation: () => Promise<void>): void {
void this.executeSyncOperation(operation).catch((error) => {
this.log(
`Background sync failed (will retry on reconnect): ${error}`
);
});
}
private async executeSyncOperation(
operation: () => Promise<void>
): Promise<void> {
try {
await operation();
} catch (error) {
if (error instanceof Error && error.name === "SyncResetError") {
this.log(`Sync operation interrupted by reset: ${error}`);
return;
}
if (
error instanceof Error &&
error.message.includes("has been destroyed")
) {
this.log(`Sync operation interrupted by destroy: ${error}`);
return;
}
throw error;
}
}
private log(message: string): void {
this.logger(`[Client ${this.clientId}] ${message}`);
}
}

15
frontend/deterministic-tests/src/parse-concurrency.ts Normal file
View file

@ -0,0 +1,15 @@
import * as os from "node:os";
export function parseConcurrency(): number {
const args = process.argv.slice(2);
for (let i = 0; i < args.length; i++) {
if (
(args[i] === "--concurrency" || args[i] === "-j") &&
i + 1 < args.length
) {
const n = parseInt(args[i + 1], 10);
if (!isNaN(n) && n > 0) return n;
}
}
return os.cpus().length;
}

28
frontend/deterministic-tests/src/prefixed-logger.ts Normal file
View file

@ -0,0 +1,28 @@
import { Logger } from "sync-client";
export class PrefixedLogger extends Logger {
private readonly base: Logger;
private readonly prefix: string;
public constructor(base: Logger, prefix: string) {
super();
this.base = base;
this.prefix = prefix;
}
public override debug(message: string): void {
this.base.debug(`[${this.prefix}] ${message}`);
}
public override info(message: string): void {
this.base.info(`[${this.prefix}] ${message}`);
}
public override warn(message: string): void {
this.base.warn(`[${this.prefix}] ${message}`);
}
public override error(message: string): void {
this.base.error(`[${this.prefix}] ${message}`);
}
}

33
frontend/deterministic-tests/src/run-with-concurrency.ts Normal file
View file

@ -0,0 +1,33 @@
export async function runWithConcurrency<T, R>(
items: T[],
concurrency: number,
fn: (item: T) => Promise<R>
): Promise<R[]> {
const results: R[] = [];
const errors: unknown[] = [];
const executing = new Set<Promise<void>>();
for (let i = 0; i < items.length; i++) {
const index = i;
const p = fn(items[index])
.then((result) => {
results[index] = result;
})
.catch((error: unknown) => {
errors.push(error);
})
.finally(() => executing.delete(p));
executing.add(p);
if (executing.size >= concurrency) {
await Promise.race(executing);
}
}
// eslint-disable-next-line no-restricted-properties
await Promise.all(executing);
if (errors.length > 0) {
throw errors[0];
}
return results;
}

236
frontend/deterministic-tests/src/server-control.ts Normal file
View file

@ -0,0 +1,236 @@
import { spawn, type ChildProcess } from "node:child_process";
import * as fs from "node:fs";
import * as os from "node:os";
import * as path from "node:path";
import { sleep } from "./utils/sleep";
import { findFreePort } from "./utils/find-free-port";
import type { Logger } from "sync-client";
import { STOP_TIMEOUT_MS } from "./consts";
export class ServerControl {
private process: ChildProcess | null = null;
private readonly serverPath: string;
private readonly baseConfigPath: string;
private readonly logger: Logger;
private _port: number | undefined;
private tempDir: string | undefined;
private _isPaused = false;
public constructor(serverPath: string, configPath: string, logger: Logger) {
this.serverPath = serverPath;
this.baseConfigPath = configPath;
this.logger = logger;
}
public get port(): number {
if (this._port === undefined) {
throw new Error("Server has not been started yet");
}
return this._port;
}
public get remoteUri(): string {
return `http://localhost:${this.port}`;
}
public async start(): Promise<void> {
if (this.process !== null) {
throw new Error("Server is already running");
}
const reservation = await findFreePort();
this._port = reservation.port;
this.tempDir = fs.mkdtempSync(
path.join(os.tmpdir(), "vault-link-test-")
);
const tempConfigPath = path.join(this.tempDir, "config.yml");
const dbDir = path.join(this.tempDir, "databases");
this.writeConfigFile(tempConfigPath, dbDir);
this.logger.info(
`Starting server: ${this.serverPath} (port ${this._port})`
);
// Release the port reservation right before spawning to minimize
// the TOCTOU window between port discovery and server binding.
reservation.release();
this.process = spawn(this.serverPath, [tempConfigPath], {
stdio: ["ignore", "pipe", "pipe"],
detached: false
});
this.process.stdout?.on("data", (data: Buffer) => {
this.logger.info(`[SERVER] ${data.toString().trim()}`);
});
this.process.stderr?.on("data", (data: Buffer) => {
this.logger.info(`[SERVER] ${data.toString().trim()}`);
});
this.process.on("error", (err) => {
this.logger.error(`[SERVER] Process error: ${err.message}`);
});
const currentProcess = this.process;
currentProcess.on("exit", (code, signal) => {
this.logger.info(
`Server exited with code ${code}, signal ${signal}`
);
// Only clear state if this handler is for the current process.
// A fast stop→start cycle could create a new process before this
// handler fires — clearing state here would corrupt the new one.
if (this.process === currentProcess) {
this.process = null;
this._isPaused = false;
}
});
try {
await this.waitForReady();
} catch (error) {
// Kill the spawned process if it failed to become ready,
// preventing a zombie process from lingering.
try {
await this.stop();
} catch {
// Best-effort cleanup
}
throw error;
}
}
public async waitForReady(maxAttempts = 50): Promise<void> {
const pingUrl = `${this.remoteUri}/vaults/test/ping`;
for (let i = 0; i < maxAttempts; i++) {
if (this.process === null || this.process.exitCode !== null) {
throw new Error(
"Server process died while waiting for it to become ready"
);
}
try {
const response = await fetch(pingUrl);
if (response.ok) {
this.logger.info("[SERVER] Ready");
return;
}
} catch {
// Server not ready yet, continue polling
}
await sleep(100);
}
throw new Error("Server failed to start within timeout");
}
public pause(): void {
if (this.process?.pid === undefined) {
throw new Error("Server is not running");
}
if (this._isPaused) {
this.logger.warn("Server is already paused, skipping double-pause");
return;
}
this.logger.info("Server pausing...");
try {
process.kill(this.process.pid, "SIGSTOP");
this._isPaused = true;
this.logger.info("Server paused (SIGSTOP sent)");
} catch (error) {
throw new Error(
`Failed to pause server (pid ${this.process.pid}): ${error instanceof Error ? error.message : String(error)}`
);
}
}
public resume(): void {
if (this.process?.pid === undefined) {
throw new Error("Server is not running");
}
if (!this._isPaused) {
return;
}
this.logger.info("Server resuming...");
try {
process.kill(this.process.pid, "SIGCONT");
this._isPaused = false;
this.logger.info("Server resumed (SIGCONT sent)");
} catch (error) {
throw new Error(
`Failed to resume server (pid ${this.process.pid}): ${error instanceof Error ? error.message : String(error)}`
);
}
}
public async stop(): Promise<void> {
const proc = this.process;
if (proc?.pid === undefined) {
this.cleanupTempDir();
return;
}
// Resume if paused — a SIGSTOP'd process ignores SIGKILL
if (this._isPaused) {
try {
process.kill(proc.pid, "SIGCONT");
} catch {
// Process may already be gone
}
this._isPaused = false;
}
this.logger.info("Server stopping...");
// Set up a promise that resolves when the process actually exits.
const exitPromise = new Promise<void>((resolve) => {
if (proc.exitCode !== null) {
resolve();
return;
}
proc.on("exit", () => {
resolve();
});
});
try {
process.kill(proc.pid, "SIGKILL");
} catch {
// Process already gone
}
// Wait for the process to actually exit before cleaning up,
// with a 5s safety timeout to avoid hanging forever.
await Promise.race([exitPromise, sleep(STOP_TIMEOUT_MS)]);
this.process = null;
this._isPaused = false;
this.cleanupTempDir();
}
public isRunning(): boolean {
return this.process?.pid !== undefined;
}
private writeConfigFile(destPath: string, dbDir: string): void {
const baseConfig = fs.readFileSync(this.baseConfigPath, "utf-8");
const config = baseConfig
.replace(/^\s*port:\s*\d+/m, ` port: ${this._port}`)
.replace(
/^\s*databases_directory_path:\s*.+/m,
` databases_directory_path: ${dbDir}`
);
fs.writeFileSync(destPath, config);
}
private cleanupTempDir(): void {
if (this.tempDir) {
try {
fs.rmSync(this.tempDir, { recursive: true, force: true });
} catch {
// Best-effort cleanup
}
this.tempDir = undefined;
}
}
}

53
frontend/deterministic-tests/src/server-manager.ts Normal file
View file

@ -0,0 +1,53 @@
import { ServerControl } from "./server-control";
import type { Logger } from "sync-client";
export class ServerManager {
private readonly activeServers = new Set<ServerControl>();
private readonly logger: Logger;
private isShuttingDown = false;
public constructor(logger: Logger) {
this.logger = logger;
}
public track(server: ServerControl): void {
this.activeServers.add(server);
}
public untrack(server: ServerControl): void {
this.activeServers.delete(server);
}
public async stopAll(): Promise<void> {
if (this.isShuttingDown) return;
this.isShuttingDown = true;
const servers = Array.from(this.activeServers);
// eslint-disable-next-line no-restricted-properties
await Promise.all(
servers.map(async (server) => {
try {
await server.stop();
} catch {
// Best-effort cleanup during shutdown
}
})
);
}
public installSignalHandlers(): void {
process.on("SIGINT", () => {
this.logger.info("Received SIGINT, shutting down...");
void this.stopAll()
.catch(() => {})
.then(() => process.exit(130));
});
process.on("SIGTERM", () => {
this.logger.info("Received SIGTERM, shutting down...");
void this.stopAll()
.catch(() => {})
.then(() => process.exit(143));
});
}
}

32
frontend/deterministic-tests/src/test-definition.ts Normal file
View file

@ -0,0 +1,32 @@
export interface ClientState {
files: Map<string, string>;
}
export type TestStep =
| { type: "create"; client: number; path: string; content: string }
| { type: "update"; client: number; path: string; content: string }
| { type: "rename"; client: number; oldPath: string; newPath: string }
| { type: "delete"; client: number; path: string }
| { type: "sync"; client?: number }
| { type: "disable-sync"; client: number }
| { type: "enable-sync"; client: number }
| { type: "pause-server" }
| { type: "resume-server" }
| { type: "barrier" }
| { type: "assert-content"; client: number; path: string; content: string }
| { type: "assert-exists"; client: number; path: string }
| { type: "assert-not-exists"; client: number; path: string }
| { type: "assert-consistent"; verify?: (state: ClientState) => void };
export interface TestDefinition {
name: string;
description?: string;
clients: number;
steps: TestStep[];
}
export interface TestResult {
success: boolean;
error?: string;
duration?: number;
}

226
frontend/deterministic-tests/src/test-registry.ts Normal file
View file

@ -0,0 +1,226 @@
import type { TestDefinition } from "./test-definition";
import { writeWriteConflictTest } from "./tests/write-write-conflict.test";
import { renameCreateConflictTest } from "./tests/rename-create-conflict.test";
import { createDeleteNoopTest } from "./tests/create-delete-noop.test";
import { renameChainTest } from "./tests/rename-chain.test";
import { serverPauseResumeTest } from "./tests/server-pause-resume.test";
import { createMergeDeleteTest } from "./tests/create-merge-delete.test";
import { renameUpdateConflictTest } from "./tests/rename-update-conflict.test";
import { deleteRenameConflictTest } from "./tests/delete-rename-conflict.test";
import { multiFileOperationsTest } from "./tests/multi-file-operations.test";
import { duplicateContentFilesTest } from "./tests/duplicate-content-files.test";
import { deleteRecreateSamePathTest } from "./tests/delete-recreate-same-path.test";
import { rapidSyncToggleTest } from "./tests/rapid-sync-toggle.test";
import { concurrentDeleteUpdateTest } from "./tests/concurrent-delete-update.test";
import { offlineRenameAndEditTest } from "./tests/offline-rename-and-edit.test";
import { threeClientConvergenceTest } from "./tests/three-client-convergence.test";
import { updateDuringServerPauseTest } from "./tests/update-during-server-pause.test";
import { emptyFileSyncTest } from "./tests/empty-file-sync.test";
import { renameToExistingPathTest } from "./tests/rename-to-existing-path.test";
import { concurrentRenameSameTargetTest } from "./tests/concurrent-rename-same-target.test";
import { multipleUpdatesCoalesceTest } from "./tests/multiple-updates-coalesce.test";
import { deleteNonexistentFileTest } from "./tests/delete-nonexistent-file.test";
import { createWhileServerPausedTest } from "./tests/create-while-server-paused.test";
import { interleavedOperationsTest } from "./tests/interleaved-operations.test";
import { simultaneousCreateDeleteSamePathTest } from "./tests/simultaneous-create-delete-same-path.test";
import { largeFileCountTest } from "./tests/large-file-count.test";
import { offlineOperationsBothClientsTest } from "./tests/offline-operations-both-clients.test";
import { updateThenRenameTest } from "./tests/update-then-rename.test";
import { idempotencyAfterServerPauseTest } from "./tests/idempotency-after-server-pause.test";
import { concurrentCreateSamePathMergeTest } from "./tests/concurrent-create-same-path-merge.test";
import { sequentialCreateDuplicateContentTest } from "./tests/sequential-create-duplicate-content.test";
import { offlineMultiUpdateCatchupTest } from "./tests/offline-multi-update-catchup.test";
import { mcThreeClientRenameOfflineUpdateTest } from "./tests/mc-three-client-rename-offline-update.test";
import { mcMultiDeleteOfflineRenameTest } from "./tests/mc-multi-delete-offline-rename.test";
import { mcCrossCreateRenameSameTargetTest } from "./tests/mc-cross-create-rename-same-target.test";
import { mcDeleteThenOfflineRenameTest } from "./tests/mc-delete-then-offline-rename.test";
import { offlineMixedOperationsTest } from "./tests/offline-mixed-operations.test";
import { offlineCreateRenameCreateTest } from "./tests/offline-create-rename-create.test";
import { offlineConcurrentRenamesTest } from "./tests/offline-concurrent-renames.test";
import { offlineMultipleEditsTest } from "./tests/offline-multiple-edits.test";
import { serverPauseBothClientsCreateTest } from "./tests/server-pause-both-clients-create.test";
import { serverPauseRenameTest } from "./tests/server-pause-rename-propagation.test";
import { serverPauseConcurrentCreatesTest } from "./tests/server-pause-concurrent-creates.test";
import { serverPauseUpdateAndCreateTest } from "./tests/server-pause-update-and-create.test";
import { renameSwapTest } from "./tests/rename-swap.test";
import { renameCircularTest } from "./tests/rename-circular.test";
import { renameNestedPathTest } from "./tests/rename-nested-path.test";
import { renameRoundtripTest } from "./tests/rename-roundtrip.test";
import { offlineRenameRemoteCreateOldPathTest } from "./tests/offline-rename-remote-create-old-path.test";
import { offlineEditRemoteRenameTest } from "./tests/offline-edit-remote-rename.test";
import { renameChainThenDeleteTest } from "./tests/rename-chain-then-delete.test";
import { offlineDeleteRemoteRenameTest } from "./tests/offline-delete-remote-rename.test";
import { renameToRecentlyDeletedPathTest } from "./tests/rename-to-recently-deleted-path.test";
import { createUpdateCoalesceServerPauseTest } from "./tests/create-update-coalesce-server-pause.test";
import { overlappingEditsSameSectionTest } from "./tests/overlapping-edits-same-section.test";
import { rapidUpdatesAfterMergeTest } from "./tests/rapid-updates-after-merge.test";
import { offlineRenamePendingCreateTest } from "./tests/offline-rename-pending-create.test";
import { deleteRecreateConcurrentUpdateTest } from "./tests/delete-recreate-concurrent-update.test";
import { moveAndConcurrentRemoteUpdateTest } from "./tests/move-and-concurrent-remote-update.test";
import { offlineDeleteVsRemoteUpdateTest } from "./tests/offline-delete-vs-remote-update.test";
import { doubleOfflineCycleTest } from "./tests/double-offline-cycle.test";
import { createRenameCreateSamePathTest } from "./tests/create-rename-create-same-path.test";
import { concurrentEditExactSamePositionTest } from "./tests/concurrent-edit-exact-same-position.test";
import { serverPauseRenameEditResumeTest } from "./tests/server-pause-rename-edit-resume.test";
import { renameTrackedToOccupiedPendingPathTest } from "./tests/rename-tracked-to-occupied-pending-path.test";
import { offlineUpdateBothThenDeleteOneTest } from "./tests/offline-update-both-then-delete-one.test";
import { moveIdenticalContentAmbiguityTest } from "./tests/move-identical-content-ambiguity.test";
import { coalesceUpdateRemoteUpdateDataLossTest } from "./tests/coalesce-update-remote-update-data-loss.test";
import { offlineCreateSamePathMergeableTest } from "./tests/offline-create-same-path-binary-conflict.test";
import { deleteDuringPendingCreateTest } from "./tests/delete-during-pending-create.test";
import { threeClientRenameCreateDeleteTest } from "./tests/three-client-rename-create-delete.test";
import { keyMigrationEventDropTest } from "./tests/key-migration-event-drop.test";
import { renameToPathOfUnconfirmedDeleteTest } from "./tests/rename-to-path-of-unconfirmed-delete.test";
import { offlineEditThenMoveSameContentTest } from "./tests/offline-edit-then-move-same-content.test";
import { concurrentRenameAndCreateAtTargetTest } from "./tests/concurrent-rename-and-create-at-target.test";
import { createRenameCreateSamePathOfflineTest } from "./tests/create-rename-create-same-path-offline.test";
import { rapidCreateUpdateDeleteCycleTest } from "./tests/rapid-create-update-delete-cycle.test";
import { serverPauseBothEditSameFileTest } from "./tests/server-pause-both-edit-same-file.test";
import { reconcilePendingAtOccupiedPathTest } from "./tests/reconcile-pending-at-occupied-path.test";
import { offlineRenameBothClientsSameSourceTest } from "./tests/offline-rename-both-clients-same-source.test";
import { createDuringReconciliationTest } from "./tests/create-during-reconciliation.test";
import { deleteRecreateDifferentContentTest } from "./tests/delete-recreate-different-content.test";
import { moveChainThreeFilesTest } from "./tests/move-chain-three-files.test";
import { updateDuringCreateProcessingTest } from "./tests/update-during-create-processing.test";
import { offlineMoveThenRemoteDeleteTest } from "./tests/offline-move-then-remote-delete.test";
import { resetClearsRecentlyDeletedResurrectionTest } from "./tests/reset-clears-recently-deleted-resurrection.test";
import { moveThenDeleteStalePathTest } from "./tests/move-then-delete-stale-path.test";
import { interruptedDeleteRetryTest } from "./tests/interrupted-delete-retry.test";
import { updateSurvivesRemoteDeleteTest } from "./tests/update-survives-remote-delete.test";
import { movePreservesRemoteUpdateTest } from "./tests/move-preserves-remote-update.test";
import { recentlyDeletedClearedOnReconnectTest } from "./tests/recently-deleted-cleared-on-reconnect.test";
import { migrateKeyPreservesExistingTest } from "./tests/migrate-key-preserves-existing.test";
import { userParenthesizedFileNotDeletedTest } from "./tests/user-parenthesized-file-not-deleted.test";
import { concurrentUpdateDiffConsistencyTest } from "./tests/concurrent-update-diff-consistency.test";
import { concurrentDeleteDuringRemoteUpdateTest } from "./tests/concurrent-delete-during-remote-update.test";
import { binaryPendingCreateNotDisplacedTest } from "./tests/binary-pending-create-not-displaced.test";
import { failedVfsMoveFallsBackTest } from "./tests/failed-vfs-move-falls-back.test";
import { watermarkAdvancesOnSkipTest } from "./tests/watermark-advances-on-skip.test";
import { remoteDeleteCoalesceLosesLocalUpdateTest } from "./tests/remote-delete-coalesce-loses-local-update.test";
import { updateVsRemoteDeleteDataLossTest } from "./tests/update-vs-remote-delete-data-loss.test";
import { watermarkGapRemoteUpdateNotRecordedTest } from "./tests/watermark-gap-remote-update-not-recorded.test";
import { renameEmptyFileLosesIdentityTest } from "./tests/rename-empty-file-loses-identity.test";
import { queueResetLosesCoalescedLocalEditTest } from "./tests/queue-reset-loses-coalesced-local-edit.test";
import { renameToPendingPathFallbackTest } from "./tests/rename-to-pending-path-fallback.test";
import { coalescedRemoteUpdateWatermarkLossTest } from "./tests/coalesced-remote-update-watermark-loss.test";
import { moveRemoteUpdateRevertsRenameTest } from "./tests/move-remote-update-reverts-rename.test";
import { createMergePreservesRenamedUpdateTest } from "./tests/create-merge-preserves-renamed-update.test";
import { localEditLostDuringCreateMergeTest } from "./tests/local-edit-lost-during-create-merge.test";
import { concurrentBinaryCreateDeconflictionTest } from "./tests/concurrent-binary-create-deconfliction.test";
import { renamePendingCreateBeforeResponseTest } from "./tests/rename-pending-create-before-response.test";
import { createRenameResponseSkipsFileTest } from "./tests/create-rename-response-skips-file.test";
import { staleDocOrphanDuplicateContentTest } from "./tests/stale-doc-orphan-duplicate-content.test";
export const TESTS: Partial<Record<string, TestDefinition>> = {
"write-write-conflict": writeWriteConflictTest,
"rename-create-conflict": renameCreateConflictTest,
"create-delete-noop": createDeleteNoopTest,
"rename-chain": renameChainTest,
"server-pause-resume": serverPauseResumeTest,
"create-merge-delete": createMergeDeleteTest,
"rename-update-conflict": renameUpdateConflictTest,
"delete-rename-conflict": deleteRenameConflictTest,
"multi-file-operations": multiFileOperationsTest,
"duplicate-content-files": duplicateContentFilesTest,
"delete-recreate-same-path": deleteRecreateSamePathTest,
"rapid-sync-toggle": rapidSyncToggleTest,
"concurrent-delete-update": concurrentDeleteUpdateTest,
"offline-rename-and-edit": offlineRenameAndEditTest,
"three-client-convergence": threeClientConvergenceTest,
"update-during-server-pause": updateDuringServerPauseTest,
"empty-file-sync": emptyFileSyncTest,
"rename-to-existing-path": renameToExistingPathTest,
"concurrent-rename-same-target": concurrentRenameSameTargetTest,
"multiple-updates-coalesce": multipleUpdatesCoalesceTest,
"delete-nonexistent-file": deleteNonexistentFileTest,
"create-while-server-paused": createWhileServerPausedTest,
"interleaved-operations": interleavedOperationsTest,
"simultaneous-create-delete-same-path": simultaneousCreateDeleteSamePathTest,
"large-file-count": largeFileCountTest,
"offline-operations-both-clients": offlineOperationsBothClientsTest,
"update-then-rename": updateThenRenameTest,
"idempotency-after-server-pause": idempotencyAfterServerPauseTest,
"concurrent-create-same-path-merge": concurrentCreateSamePathMergeTest,
"sequential-create-duplicate-content": sequentialCreateDuplicateContentTest,
"offline-multi-update-catchup": offlineMultiUpdateCatchupTest,
"mc-three-client-rename-offline-update": mcThreeClientRenameOfflineUpdateTest,
"mc-multi-delete-offline-rename": mcMultiDeleteOfflineRenameTest,
"mc-cross-create-rename-same-target": mcCrossCreateRenameSameTargetTest,
"mc-delete-then-offline-rename": mcDeleteThenOfflineRenameTest,
"offline-mixed-operations": offlineMixedOperationsTest,
"offline-create-rename-create": offlineCreateRenameCreateTest,
"offline-concurrent-renames": offlineConcurrentRenamesTest,
"offline-multiple-edits": offlineMultipleEditsTest,
"server-pause-both-clients-create": serverPauseBothClientsCreateTest,
"server-pause-rename-propagation": serverPauseRenameTest,
"server-pause-concurrent-creates": serverPauseConcurrentCreatesTest,
"server-pause-update-and-create": serverPauseUpdateAndCreateTest,
"rename-swap": renameSwapTest,
"rename-circular": renameCircularTest,
"rename-nested-path": renameNestedPathTest,
"rename-roundtrip": renameRoundtripTest,
"offline-rename-remote-create-old-path": offlineRenameRemoteCreateOldPathTest,
"offline-edit-remote-rename": offlineEditRemoteRenameTest,
"rename-chain-then-delete": renameChainThenDeleteTest,
"offline-delete-remote-rename": offlineDeleteRemoteRenameTest,
"rename-to-recently-deleted-path": renameToRecentlyDeletedPathTest,
"create-update-coalesce-server-pause": createUpdateCoalesceServerPauseTest,
"overlapping-edits-same-section": overlappingEditsSameSectionTest,
"rapid-updates-after-merge": rapidUpdatesAfterMergeTest,
"offline-rename-pending-create": offlineRenamePendingCreateTest,
"delete-recreate-concurrent-update": deleteRecreateConcurrentUpdateTest,
"move-and-concurrent-remote-update": moveAndConcurrentRemoteUpdateTest,
"double-offline-cycle": doubleOfflineCycleTest,
"create-rename-create-same-path": createRenameCreateSamePathTest,
"concurrent-edit-exact-same-position": concurrentEditExactSamePositionTest,
"server-pause-rename-edit-resume": serverPauseRenameEditResumeTest,
"rename-tracked-to-occupied-pending-path": renameTrackedToOccupiedPendingPathTest,
"offline-update-both-then-delete-one": offlineUpdateBothThenDeleteOneTest,
"move-identical-content-ambiguity": moveIdenticalContentAmbiguityTest,
"coalesce-update-remote-update-data-loss": coalesceUpdateRemoteUpdateDataLossTest,
"offline-create-same-path-mergeable": offlineCreateSamePathMergeableTest,
"delete-during-pending-create": deleteDuringPendingCreateTest,
"three-client-rename-create-delete": threeClientRenameCreateDeleteTest,
"key-migration-event-drop": keyMigrationEventDropTest,
"rename-to-path-of-unconfirmed-delete": renameToPathOfUnconfirmedDeleteTest,
"offline-edit-then-move-same-content": offlineEditThenMoveSameContentTest,
"concurrent-rename-and-create-at-target": concurrentRenameAndCreateAtTargetTest,
"create-rename-create-same-path-offline": createRenameCreateSamePathOfflineTest,
"rapid-create-update-delete-cycle": rapidCreateUpdateDeleteCycleTest,
"server-pause-both-edit-same-file": serverPauseBothEditSameFileTest,
"reconcile-pending-at-occupied-path": reconcilePendingAtOccupiedPathTest,
"offline-rename-both-clients-same-source": offlineRenameBothClientsSameSourceTest,
"create-during-reconciliation": createDuringReconciliationTest,
"delete-recreate-different-content": deleteRecreateDifferentContentTest,
"move-chain-three-files": moveChainThreeFilesTest,
"update-during-create-processing": updateDuringCreateProcessingTest,
"offline-move-then-remote-delete": offlineMoveThenRemoteDeleteTest,
"reset-clears-recently-deleted-resurrection": resetClearsRecentlyDeletedResurrectionTest,
"move-then-delete-stale-path": moveThenDeleteStalePathTest,
"offline-delete-vs-remote-update": offlineDeleteVsRemoteUpdateTest,
"interrupted-delete-retry": interruptedDeleteRetryTest,
"update-survives-remote-delete": updateSurvivesRemoteDeleteTest,
"move-preserves-remote-update": movePreservesRemoteUpdateTest,
"recently-deleted-cleared-on-reconnect": recentlyDeletedClearedOnReconnectTest,
"migrate-key-preserves-existing": migrateKeyPreservesExistingTest,
"user-parenthesized-file-not-deleted": userParenthesizedFileNotDeletedTest,
"concurrent-update-diff-consistency": concurrentUpdateDiffConsistencyTest,
"concurrent-delete-during-remote-update": concurrentDeleteDuringRemoteUpdateTest,
"binary-pending-create-not-displaced": binaryPendingCreateNotDisplacedTest,
"failed-vfs-move-falls-back": failedVfsMoveFallsBackTest,
"watermark-advances-on-skip": watermarkAdvancesOnSkipTest,
"remote-delete-coalesce-loses-local-update": remoteDeleteCoalesceLosesLocalUpdateTest,
"update-vs-remote-delete-data-loss": updateVsRemoteDeleteDataLossTest,
"watermark-gap-remote-update-not-recorded": watermarkGapRemoteUpdateNotRecordedTest,
"rename-empty-file-loses-identity": renameEmptyFileLosesIdentityTest,
"queue-reset-loses-coalesced-local-edit": queueResetLosesCoalescedLocalEditTest,
"rename-to-pending-path-fallback": renameToPendingPathFallbackTest,
"coalesced-remote-update-watermark-loss": coalescedRemoteUpdateWatermarkLossTest,
"move-remote-update-reverts-rename": moveRemoteUpdateRevertsRenameTest,
"create-merge-preserves-renamed-update": createMergePreservesRenamedUpdateTest,
"local-edit-lost-during-create-merge": localEditLostDuringCreateMergeTest,
"concurrent-binary-create-deconfliction": concurrentBinaryCreateDeconflictionTest,
"rename-pending-create-before-response": renamePendingCreateBeforeResponseTest,
"create-rename-response-skips-file": createRenameResponseSkipsFileTest,
"stale-doc-orphan-duplicate-content": staleDocOrphanDuplicateContentTest
};

372
frontend/deterministic-tests/src/test-runner.ts Normal file
View file

@ -0,0 +1,372 @@
import type {
TestDefinition,
TestResult,
TestStep,
ClientState
} from "./test-definition";
import { DeterministicAgent } from "./deterministic-agent";
import type { ServerControl } from "./server-control";
import type { SyncSettings, Logger } from "sync-client";
import { assert } from "./utils/assert";
import { sleep } from "./utils/sleep";
import { withTimeout } from "./utils/with-timeout";
import {
CONVERGENCE_TIMEOUT_MS,
CONVERGENCE_RETRY_DELAY_MS,
AGENT_INIT_TIMEOUT_MS,
IS_SYNC_ENABLED_DEFAULT
} from "./consts";
import { randomUUID } from "node:crypto";
export class TestRunner {
private agents: DeterministicAgent[] = [];
private readonly serverControl: ServerControl;
private readonly token: string;
private readonly remoteUri: string;
private readonly logger: Logger;
public constructor(
serverControl: ServerControl,
logger: Logger,
token: string,
remoteUri: string
) {
this.serverControl = serverControl;
this.logger = logger;
this.token = token;
this.remoteUri = remoteUri;
}
public async runTest(test: TestDefinition): Promise<TestResult> {
const startTime = Date.now();
this.logger.info(`Running test: ${test.name}`);
if (test.description !== undefined && test.description !== "") {
this.logger.info(`Description: ${test.description}`);
}
this.logger.info(`Clients: ${test.clients}`);
this.logger.info(`Steps: ${test.steps.length}`);
try {
assert(
this.serverControl.isRunning(),
"Server is not running before test start"
);
await this.initializeAgents(test.clients);
for (let i = 0; i < test.steps.length; i++) {
const step = test.steps[i];
this.logger.info(
`Step ${i + 1}/${test.steps.length}: ${JSON.stringify(step)}`
);
await this.executeStep(step);
}
await this.cleanup();
const duration = Date.now() - startTime;
this.logger.info(`\n✓ Test passed: ${test.name} (${duration}ms)`);
return {
success: true,
duration
};
} catch (error) {
const duration = Date.now() - startTime;
const errorMessage =
error instanceof Error ? error.message : String(error);
this.logger.info(`\n✗ Test failed: ${test.name}`);
this.logger.info(`Error: ${errorMessage}`);
await this.cleanup();
return {
success: false,
error: errorMessage,
duration
};
}
}
private async initializeAgents(count: number): Promise<void> {
assert(count > 0, `Client count must be positive, got ${count}`);
const vaultName = `test-${randomUUID()}`;
this.logger.info(
`Initializing ${count} agents with vault: ${vaultName}`
);
for (let i = 0; i < count; i++) {
const settings: Partial<SyncSettings> = {
isSyncEnabled: IS_SYNC_ENABLED_DEFAULT,
token: this.token,
vaultName,
remoteUri: this.remoteUri
};
const agent = new DeterministicAgent(i, settings, (msg) => {
this.logger.info(msg);
});
// Push before init so cleanup() handles this agent if init fails
this.agents.push(agent);
await withTimeout(
agent.init(
fetch,
// eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion
WebSocket as unknown as typeof globalThis.WebSocket
),
AGENT_INIT_TIMEOUT_MS,
`Client ${i} init timed out after ${AGENT_INIT_TIMEOUT_MS}ms`
);
this.logger.info(`Initialized client ${i}`);
}
this.logger.info("All agents initialized");
}
private getAgent(index: number): DeterministicAgent {
assert(
index >= 0 && index < this.agents.length,
`Client index ${index} out of bounds (have ${this.agents.length} agents)`
);
return this.agents[index];
}
private async executeStep(step: TestStep): Promise<void> {
switch (step.type) {
case "create":
await this.getAgent(step.client).createFile(
step.path,
step.content
);
break;
case "update":
await this.getAgent(step.client).updateFile(
step.path,
step.content
);
break;
case "rename":
await this.getAgent(step.client).renameFile(
step.oldPath,
step.newPath
);
break;
case "delete":
await this.getAgent(step.client).deleteFile(step.path);
break;
case "sync":
if (step.client !== undefined) {
await this.getAgent(step.client).waitForSync();
} else {
for (const agent of this.agents) {
await agent.waitForSync();
}
}
break;
case "disable-sync":
await this.getAgent(step.client).disableSync();
break;
case "enable-sync":
await this.getAgent(step.client).enableSync();
break;
case "pause-server":
this.serverControl.pause();
break;
case "resume-server":
this.serverControl.resume();
// Verify the server is actually responsive before proceeding.
// This replaces relying solely on hardcoded waits.
await this.serverControl.waitForReady();
break;
case "barrier":
await this.waitForConvergence();
break;
case "assert-content":
await this.getAgent(step.client).assertContent(
step.path,
step.content
);
break;
case "assert-exists":
await this.getAgent(step.client).assertExists(step.path);
break;
case "assert-not-exists":
await this.getAgent(step.client).assertNotExists(step.path);
break;
case "assert-consistent":
await this.assertConsistent(step.verify);
break;
default: {
const unknownStep = step as { type: string };
throw new Error(`Unknown step type: ${unknownStep.type}`);
}
}
}
/**
* Wait for all agents to reach a consistent state.
*
* Waiting for agents is done in two full rounds: the first round
* drains in-flight operations, but completing those operations can
* trigger new work on OTHER agents via server broadcasts. The second
* round waits for that cascading work to settle. Deeper cascades
* are handled by the outer retry loop.
*/
private async waitForConvergence(): Promise<void> {
this.logger.info("Barrier: waiting for convergence...");
const deadline = Date.now() + CONVERGENCE_TIMEOUT_MS;
let lastError: Error | undefined = undefined;
while (Date.now() < deadline) {
await this.waitAllAgentsSettled();
try {
await this.assertConsistent();
this.logger.info("Barrier complete: all clients converged");
return;
} catch (error) {
lastError =
error instanceof Error ? error : new Error(String(error));
this.logger.info("Barrier: not yet converged, retrying...");
await sleep(CONVERGENCE_RETRY_DELAY_MS);
}
}
// Final attempt — let the error propagate
await this.waitAllAgentsSettled();
try {
await this.assertConsistent();
this.logger.info("Barrier complete: all clients converged");
} catch (error) {
throw new Error(
`Convergence timed out after ${CONVERGENCE_TIMEOUT_MS}ms: ${error instanceof Error ? error.message : String(error)}`,
{ cause: lastError }
);
}
}
/**
* Wait for all agents to be simultaneously idle. Two full rounds are
* needed because completing work on agent A can trigger a server
* broadcast that enqueues new work on agent B, and vice versa.
*
* However, the 2nd sync may result in merges which can trigger another
* round of syncs, so this function should be called in a loop with a
* timeout to ensure true convergence rather than just waiting for the
* current round of syncs to complete.
*/
private async waitAllAgentsSettled(): Promise<void> {
for (let round = 0; round < 2; round++) {
for (const agent of this.agents) {
await agent.waitForSync();
}
}
}
private async assertConsistent(
verify?: (state: ClientState) => void
): Promise<void> {
this.logger.info("Asserting all clients are consistent...");
assert(this.agents.length >= 2, "Need at least 2 agents for consistency check");
const [referenceAgent] = this.agents;
const referenceFiles = (await referenceAgent.getFiles()).sort();
const referenceState: ClientState = { files: new Map() };
for (const file of referenceFiles) {
const content = await referenceAgent.getFileContent(file);
referenceState.files.set(file, content);
}
this.logger.info(
`Reference client has ${referenceFiles.length} files: ${referenceFiles.join(", ")}`
);
for (let i = 1; i < this.agents.length; i++) {
const agent = this.agents[i];
const files = (await agent.getFiles()).sort();
this.logger.info(
`Client ${i} has ${files.length} files: ${files.join(", ")}`
);
assert(
files.length === referenceFiles.length,
`File count mismatch: client 0 has ${referenceFiles.length} files, client ${i} has ${files.length} files`
);
for (let j = 0; j < files.length; j++) {
assert(
files[j] === referenceFiles[j],
`File list mismatch at index ${j}: client 0 has "${referenceFiles[j]}", client ${i} has "${files[j]}"`
);
}
for (const file of referenceFiles) {
const referenceContent = referenceState.files.get(file);
const agentContent = await agent.getFileContent(file);
assert(
referenceContent === agentContent,
`Content mismatch for ${file}:\nClient 0: "${referenceContent}"\nClient ${i}: "${agentContent}"`
);
}
}
this.logger.info("✓ All clients are consistent");
if (verify) {
this.logger.info("Running custom verification...");
try {
verify(referenceState);
} catch (error) {
const msg =
error instanceof Error ? error.message : String(error);
throw new Error(`Custom verification failed: ${msg}`);
}
this.logger.info("✓ Custom verification passed");
}
}
private async cleanup(): Promise<void> {
// Always resume the server in case a test paused it and then
// failed before reaching the resume step. Without this, all
// subsequent tests would hang because the server process is
// frozen (SIGSTOP) and can't respond to HTTP or WebSocket.
try {
this.serverControl.resume();
} catch {
// Server wasn't paused or isn't running — safe to ignore
}
this.logger.info("\nCleaning up agents...");
for (const agent of this.agents) {
try {
await agent.cleanup();
} catch (error) {
this.logger.warn(
`Agent cleanup error: ${error instanceof Error ? error.message : String(error)}`
);
}
}
this.agents = [];
this.logger.info("Cleanup complete");
}
}

67
frontend/deterministic-tests/src/tests/binary-pending-create-not-displaced.test.ts Normal file
View file

@ -0,0 +1,67 @@
import type { ClientState, TestDefinition } from "../test-definition";
import { assert } from "../utils/assert";
function verifyBothFilesExist(state: ClientState): void {
assert(
state.files.size === 2,
`Expected 2 files, got ${state.files.size}: ${[...state.files.keys()].join(", ")}`
);
assert(
state.files.has("data.bin"),
"Expected data.bin to exist"
);
assert(
state.files.has("data (1).bin"),
"Expected data (1).bin to exist"
);
const contents = new Set(state.files.values());
assert(
contents.has("binary data from client 0"),
`Expected one file to contain "binary data from client 0"`
);
assert(
contents.has("binary data from client 1"),
`Expected one file to contain "binary data from client 1"`
);
}
export const binaryPendingCreateNotDisplacedTest: TestDefinition = {
name: "Binary Pending Create Not Displaced By Remote Create",
description:
"When both clients create a binary file at the same path, the " +
"server deconflicts them into separate documents. Both files " +
"should exist on both clients after sync.",
clients: 2,
steps: [
{ type: "enable-sync", client: 0 },
{ type: "enable-sync", client: 1 },
// Both go offline
{ type: "disable-sync", client: 0 },
{ type: "disable-sync", client: 1 },
// Both create binary file at same path (use .bin extension)
{
type: "create",
client: 0,
path: "data.bin",
content: "binary data from client 0"
},
{
type: "create",
client: 1,
path: "data.bin",
content: "binary data from client 1"
},
// Both come online
{ type: "enable-sync", client: 0 },
{ type: "enable-sync", client: 1 },
{ type: "barrier" },
// Both files should exist (server deconflicted them)
{ type: "assert-consistent", verify: verifyBothFilesExist }
]
};

5
frontend/deterministic-tests/src/utils/assert.ts Normal file
View file

@ -0,0 +1,5 @@
export function assert(value: boolean, message: string): asserts value {
if (!value) {
throw new Error(message);
}
}

29
frontend/deterministic-tests/src/utils/find-free-port.ts Normal file
View file

@ -0,0 +1,29 @@
import * as net from "node:net";
export interface PortReservation {
port: number;
release: () => void;
}
/**
* Find a free port and keep it reserved until the caller explicitly releases it.
*/
export async function findFreePort(): Promise<PortReservation> {
return new Promise((resolve, reject) => {
const server = net.createServer();
server.listen(0, "127.0.0.1", () => {
const addr = server.address();
if (addr === null || typeof addr === "string") {
server.close();
reject(new Error("Failed to get port from server"));
return;
}
const { port } = addr;
resolve({
port,
release: () => server.close()
});
});
server.on("error", reject);
});
}

3
frontend/deterministic-tests/src/utils/sleep.ts Normal file
View file

@ -0,0 +1,3 @@
export async function sleep(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}

15
frontend/deterministic-tests/src/utils/with-timeout.ts Normal file
View file

@ -0,0 +1,15 @@
export async function withTimeout<T>(
promise: Promise<T>,
timeoutMs: number,
message: string
): Promise<T> {
let timeoutId: ReturnType<typeof setTimeout> | undefined = undefined;
const timeoutPromise = new Promise<never>((_resolve, reject) => {
timeoutId = setTimeout(() => {
reject(new Error(message));
}, timeoutMs);
});
return Promise.race([promise, timeoutPromise]).finally(() => {
clearTimeout(timeoutId);
});
}

12
frontend/deterministic-tests/tsconfig.json Normal file
View file

@ -0,0 +1,12 @@
{
"compilerOptions": {
"baseUrl": ".",
"strict": true,
"target": "ES2022",
"module": "CommonJS",
"esModuleInterop": true,
"lib": ["DOM", "ES2024"],
"moduleResolution": "node"
},
"exclude": ["./dist"]
}

30
frontend/deterministic-tests/webpack.config.js Normal file
View file

@ -0,0 +1,30 @@
const path = require("path");
const webpack = require("webpack");
module.exports = {
entry: "./src/cli.ts",
target: "node",
mode: "production",
optimization: {
minimize: false
},
module: {
rules: [
{
test: /\.ts$/,
use: "ts-loader"
}
]
},
resolve: {
extensions: [".ts", ".js"]
},
output: {
globalObject: "this",
filename: "cli.js",
path: path.resolve(__dirname, "dist")
},
plugins: [
new webpack.BannerPlugin({ banner: "#!/usr/bin/env node", raw: true })
]
};