.

frontend/test-client/package.json

@@ -0,0 +1,25 @@
+{
+    "name": "test-client",
+    "version": "0.8.0",
+    "private": true,
+    "bin": {
+        "test-client": "./dist/cli.js"
+    },
+    "scripts": {
+        "dev": "webpack watch --mode development",
+        "build": "webpack --mode production",
+        "test": "tsx --test src/**/*.test.ts"
+    },
+    "devDependencies": {
+        "@types/node": "^22.15.30",
+        "bufferutil": "^4.0.9",
+        "sync-client": "file:../sync-client",
+        "ts-loader": "^9.5.2",
+        "tslib": "2.8.1",
+        "tsx": "^4.20.5",
+        "typescript": "5.8.3",
+        "uuid": "^11.1.0",
+        "webpack": "^5.99.9",
+        "webpack-cli": "^6.0.1"
+    }
+}

frontend/test-client/src/agent/mock-agent.ts

@@ -0,0 +1,333 @@
+import { choose } from "../utils/choose";
+import { v4 as uuidv4 } from "uuid";
+import { assert } from "../utils/assert";
+import type { RelativePath, SyncSettings } from "sync-client";
+import { debugging, Logger, LogLevel } from "sync-client";
+import { MockClient } from "./mock-client";
+import { sleep } from "../utils/sleep";
+import type { LogLine } from "sync-client/dist/types/tracing/logger";
+
+export class MockAgent extends MockClient {
+	private readonly writtenContents: string[] = [];
+	private readonly pendingActions: Promise<unknown>[] = [];
+
+	// The renamed file finding algorithm isn't too smart so we can't both update and rename the same file
+	private doNotTouchWhileOffline: string[] = [];
+
+	public constructor(
+		initialSettings: Partial<SyncSettings>,
+		public readonly name: string,
+		private readonly doDeletes: boolean,
+		useSlowFileEvents: boolean,
+		private readonly jitterScaleInSeconds: number
+	) {
+		super(initialSettings, useSlowFileEvents);
+	}
+
+	public async init(): Promise<void> {
+		await super.init(
+			debugging.slowFetchFactory(this.jitterScaleInSeconds),
+			debugging.slowWebSocketFactory(
+				this.jitterScaleInSeconds,
+				new Logger() // this logger isn't wired anywhere, so messages to it will be ignored
+			)
+		);
+
+		assert(
+			(await this.client.checkConnection()).isSuccessful,
+			"Connection check failed"
+		);
+
+		this.client.logger.addOnMessageListener((logLine: LogLine) => {
+			const state = this.client.getSettings().isSyncEnabled
+				? "(online) "
+				: "(offline)";
+			const formatted = `[${this.name} ${state}] ${logLine.timestamp.toISOString()} ${logLine.level} ${logLine.message}`;
+
+			// HACK: we have to ensure the file has been synced if we want to change it offline without data loss
+			const historyEntry = /.*History entry: (.*.md).*/.exec(
+				logLine.message
+			);
+
+			if (historyEntry) {
+				this.doNotTouchWhileOffline =
+					this.doNotTouchWhileOffline.filter(
+						(file) => file !== historyEntry[1]
+					);
+			}
+			switch (logLine.level) {
+				case LogLevel.ERROR:
+					console.error(formatted);
+
+					if (!this.useSlowFileEvents) {
+						// Let's wait for the error to be caught if there was one
+						// eslint-disable-next-line @typescript-eslint/no-floating-promises
+						sleep(100).then(() => process.exit(1));
+					}
+
+					break;
+				case LogLevel.WARNING:
+					console.warn(formatted);
+					break;
+				case LogLevel.INFO:
+					console.info(formatted);
+					break;
+				case LogLevel.DEBUG:
+					console.debug(formatted);
+					break;
+			}
+		});
+
+		this.client.logger.info("Agent initialized");
+	}
+
+	public async act(): Promise<void> {
+		const options: (() => Promise<unknown>)[] = [
+			this.createFileAction.bind(this)
+		];
+
+		if (this.client.getSettings().isSyncEnabled) {
+			if (this.doNotTouchWhileOffline.length === 0) {
+				options.push(this.disableSyncAction.bind(this));
+			}
+		} else {
+			options.push(this.enableSyncAction.bind(this));
+		}
+
+		const files = await this.listAllFiles();
+
+		if (files.length > 0) {
+			options.push(
+				this.renameFileAction.bind(this, files),
+				this.updateFileAction.bind(this, files)
+			);
+
+			if (this.doDeletes) {
+				options.push(this.deleteFileAction.bind(this, files));
+			}
+		}
+
+		this.pendingActions.push(
+			(async (): Promise<unknown> => {
+				try {
+					return await choose(options)();
+				} catch (error) {
+					this.client.logger.error(
+						`Failed to perform an action: ${error}`
+					);
+					this.client.logger.info(JSON.stringify(this.data, null, 2));
+					this.client.logger.info(
+						JSON.stringify(this.localFiles, null, 2)
+					);
+					throw error;
+				}
+			})()
+		);
+	}
+
+	public async finish(): Promise<void> {
+		await this.client.setSetting("isSyncEnabled", true);
+		await Promise.all(this.pendingActions);
+		await this.client.waitAndStop();
+	}
+
+	public assertFileSystemsAreConsistent(otherAgent: MockAgent): void {
+		const globalFiles = Array.from(otherAgent.localFiles.keys());
+		const localFiles = Array.from(this.localFiles.keys());
+
+		const missingInOther = localFiles.filter(
+			(file) => !otherAgent.localFiles.has(file)
+		);
+		const missingInLocal = globalFiles.filter(
+			(file) => !this.localFiles.has(file)
+		);
+
+		try {
+			assert(
+				missingInOther.length === 0,
+				`Files from ${this.name} missing in ${otherAgent.name}: ${missingInOther.join(", ")}`
+			);
+			assert(
+				missingInLocal.length === 0,
+				`Files from ${otherAgent.name} missing in ${this.name}: ${missingInLocal.join(", ")}`
+			);
+
+			for (const file of globalFiles) {
+				const localContent = new TextDecoder().decode(
+					this.localFiles.get(file)
+				);
+				const otherContent = new TextDecoder().decode(
+					otherAgent.localFiles.get(file)
+				);
+				assert(
+					localContent === otherContent,
+					`Content mismatch for file ${file}:\n${localContent}\n${otherContent}`
+				);
+			}
+		} catch (e) {
+			this.client.logger.info(
+				"Local data: " + JSON.stringify(this.data, null, 2)
+			);
+			this.client.logger.info(
+				"Local files: " +
+					Array.from(otherAgent.localFiles.keys()).join(", ")
+			);
+			otherAgent.client.logger.info(
+				"Local data: " + JSON.stringify(otherAgent.data, null, 2)
+			);
+			otherAgent.client.logger.info(
+				"Local files: " +
+					Array.from(otherAgent.localFiles.keys()).join(", ")
+			);
+
+			throw e;
+		}
+	}
+
+	public assertAllContentIsPresentOnce(): void {
+		if (this.useSlowFileEvents) {
+			this.client.logger.info(
+				// We can't ensure that we have seen every single update
+				`Skipping content check for ${this.name} because slow file events are enabled`
+			);
+			return;
+		}
+
+		for (const content of this.writtenContents) {
+			const found = Array.from(this.localFiles.keys()).filter((key) => {
+				return new TextDecoder()
+					.decode(this.localFiles.get(key))
+					.includes(content);
+			});
+
+			if (this.doDeletes) {
+				assert(
+					found.length <= 1,
+					`[${this.name}] Content ${content} found in ${found.join(", ")}`
+				);
+			} else {
+				assert(
+					found.length >= 1,
+					`[${this.name}] Content ${content} not found in any files`
+				);
+
+				assert(
+					found.length <= 1,
+					`[${this.name}] Content ${content} found in multiple files: ${found.join(", ")}`
+				);
+
+				const [file] = found;
+				const fileContent = new TextDecoder().decode(
+					this.localFiles.get(file)
+				);
+				assert(
+					fileContent.split(content).length == 2,
+					`Content ${content} (of ${this.name}) found more than once in '${file}'. File content:\n${fileContent}`
+				);
+			}
+		}
+	}
+
+	private async createFileAction(): Promise<void> {
+		const file = this.getFileName();
+
+		if (
+			(!this.client.getSettings().isSyncEnabled &&
+				this.doNotTouchWhileOffline.includes(file)) ||
+			(await this.exists(file))
+		) {
+			return;
+		}
+
+		const content = this.getContent();
+		this.client.logger.info(
+			`Decided to create file ${file} with content ${content}`
+		);
+
+		return this.create(file, new TextEncoder().encode(` ${content} `));
+	}
+
+	private async disableSyncAction(): Promise<void> {
+		this.client.logger.info(`Decided to disable sync`);
+		await this.client.setSetting("isSyncEnabled", false);
+	}
+
+	private async enableSyncAction(): Promise<void> {
+		this.client.logger.info(`Decided to enable sync`);
+		await this.client.setSetting("isSyncEnabled", true);
+	}
+
+	private async renameFileAction(files: RelativePath[]): Promise<void> {
+		const file = choose(files);
+
+		// We can't edit files offline that have been updated while offline.
+		// Otherwise, the resolution logic couldn't handle it.
+		if (
+			!this.client.getSettings().isSyncEnabled &&
+			this.doNotTouchWhileOffline.includes(file)
+		) {
+			this.client.logger.info(
+				`Skipping file ${file} because it has been updated while offline`
+			);
+			return;
+		}
+
+		const newName = this.getFileName();
+
+		if (
+			(!this.client.getSettings().isSyncEnabled &&
+				this.doNotTouchWhileOffline.includes(newName)) ||
+			(await this.exists(newName))
+		) {
+			return;
+		}
+
+		this.client.logger.info(`Decided to rename file ${file} to ${newName}`);
+		this.doNotTouchWhileOffline.push(file, newName);
+
+		return this.rename(file, newName);
+	}
+
+	private async updateFileAction(files: RelativePath[]): Promise<void> {
+		const file = choose(files);
+
+		// We can't edit files offline that have been updated while offline.
+		// Otherwise, the resolution logic couldn't handle it.
+		if (
+			!this.client.getSettings().isSyncEnabled &&
+			this.doNotTouchWhileOffline.includes(file)
+		) {
+			this.client.logger.info(
+				`Skipping file ${file} because it has been updated while offline`
+			);
+			return;
+		}
+
+		const content = this.getContent();
+		this.client.logger.info(
+			`Decided to update file ${file} with ${content}`
+		);
+		this.doNotTouchWhileOffline.push(file);
+		await this.atomicUpdateText(file, (old) => ({
+			text: old.text + ` ${content} `,
+			cursors: []
+		}));
+	}
+
+	private async deleteFileAction(files: RelativePath[]): Promise<void> {
+		const file = choose(files);
+		this.client.logger.info(`Decided to delete file ${file}`);
+		return this.delete(file);
+	}
+
+	private getContent(): string {
+		const uuid = uuidv4();
+		this.writtenContents.push(uuid);
+		return uuid;
+	}
+
+	private getFileName(): string {
+		// Simulate name collisions between the clients
+		return `file-${Math.floor(Math.random() * 64)}.md`;
+	}
+}

frontend/test-client/src/agent/mock-client.ts

@@ -0,0 +1,195 @@
+import type { StoredDatabase } from "sync-client";
+import { assert } from "../utils/assert";
+import {
+	type RelativePath,
+	type FileSystemOperations,
+	type SyncSettings,
+	SyncClient
+} from "sync-client";
+import type { TextWithCursors } from "reconcile-text";
+export class MockClient implements FileSystemOperations {
+	protected readonly localFiles = new Map<string, Uint8Array>();
+	protected client!: SyncClient;
+
+	protected data: Partial<{
+		settings: Partial<SyncSettings>;
+		database: Partial<StoredDatabase>;
+	}> = {
+		database: {
+			// Assume all clients start at the same time so there's no need to fetch
+			// any shared state.
+			hasInitialSyncCompleted: true
+		}
+	};
+
+	public constructor(
+		initialSettings: Partial<SyncSettings>,
+		protected readonly useSlowFileEvents: boolean
+	) {
+		this.data.settings = initialSettings;
+	}
+
+	public async init(
+		fetchImplementation: typeof globalThis.fetch,
+		webSocketImplementation: typeof globalThis.WebSocket
+	): Promise<void> {
+		this.client = await SyncClient.create({
+			fs: this,
+			persistence: {
+				load: async () => this.data,
+				save: async (data) => void (this.data = data)
+			},
+			fetch: fetchImplementation,
+			webSocket: webSocketImplementation
+		});
+
+		await this.client.start();
+	}
+
+	public async listAllFiles(): Promise<RelativePath[]> {
+		return Array.from(this.localFiles.keys());
+	}
+
+	public async read(path: RelativePath): Promise<Uint8Array> {
+		const file = this.localFiles.get(path);
+		if (!file) {
+			throw new Error(`File ${path} does not exist`);
+		}
+		return file;
+	}
+
+	public async getFileSize(path: RelativePath): Promise<number> {
+		return (await this.read(path)).length;
+	}
+
+	public async exists(path: RelativePath): Promise<boolean> {
+		return this.localFiles.has(path);
+	}
+
+	public async create(
+		path: RelativePath,
+		newContent: Uint8Array
+	): Promise<void> {
+		if (this.localFiles.has(path)) {
+			throw new Error(`File ${path} already exists`);
+		}
+		this.client.logger.info(
+			`Creating file ${path} with content ${new TextDecoder().decode(newContent)}`
+		);
+		this.localFiles.set(path, newContent);
+
+		this.executeFileOperation(async () =>
+			this.client.syncLocallyCreatedFile(path)
+		);
+	}
+
+	public async createDirectory(_path: RelativePath): Promise<void> {
+		// This doesn't mean anything in our virtual FS representation
+	}
+
+	public async atomicUpdateText(
+		path: RelativePath,
+		updater: (currentContent: TextWithCursors) => TextWithCursors
+	): Promise<string> {
+		const file = this.localFiles.get(path);
+		if (!file) {
+			throw new Error(`File ${path} does not exist`);
+		}
+		const currentContent = new TextDecoder().decode(file);
+		const newContent = updater({ text: currentContent, cursors: [] }).text;
+		const newContentUint8Array = new TextEncoder().encode(newContent);
+		this.localFiles.set(path, newContentUint8Array);
+
+		if (!this.useSlowFileEvents) {
+			const existingParts = currentContent
+				.split(" ")
+				.map((part) => part.trim());
+			const newParts = newContent.split(" ").map((part) => part.trim());
+			existingParts.forEach((part) =>
+				// all changes should be additive
+				{
+					assert(
+						newParts.includes(part),
+						`Part ${part} not found in new content: ${newContent}`
+					);
+				}
+			);
+		}
+
+		this.client.logger.info(
+			`Updated file ${path} with:\n  current content: ${currentContent}\n  new content: ${newContent}`
+		);
+
+		this.executeFileOperation(async () =>
+			this.client.syncLocallyUpdatedFile({
+				relativePath: path
+			})
+		);
+
+		return newContent;
+	}
+
+	public async write(path: RelativePath, content: Uint8Array): Promise<void> {
+		const hasExisted = this.localFiles.has(path);
+		this.localFiles.set(path, content);
+
+		this.client.logger.info(
+			`Updated file ${path} with:\n  new content: ${new TextDecoder().decode(content)}`
+		);
+
+		this.executeFileOperation(async () => {
+			if (hasExisted) {
+				return this.client.syncLocallyUpdatedFile({
+					relativePath: path
+				});
+			} else {
+				return this.client.syncLocallyCreatedFile(path);
+			}
+		});
+	}
+
+	public async delete(path: RelativePath): Promise<void> {
+		this.client.logger.info(
+			`Deleting file: ${path} with:\n  content ${new TextDecoder().decode(this.localFiles.get(path))}`
+		);
+		this.localFiles.delete(path);
+
+		this.executeFileOperation(async () =>
+			this.client.syncLocallyDeletedFile(path)
+		);
+	}
+
+	public async rename(
+		oldPath: RelativePath,
+		newPath: RelativePath
+	): Promise<void> {
+		const file = this.localFiles.get(oldPath);
+		if (!file) {
+			throw new Error(`File ${oldPath} does not exist`);
+		}
+		this.localFiles.set(newPath, file);
+		if (oldPath !== newPath) {
+			this.localFiles.delete(oldPath);
+		}
+
+		this.client.logger.info(
+			`Renamed file: ${oldPath} -> ${newPath} with:\n  content ${new TextDecoder().decode(file)}`
+		);
+
+		this.executeFileOperation(async () =>
+			this.client.syncLocallyUpdatedFile({
+				oldPath,
+				relativePath: newPath
+			})
+		);
+	}
+
+	private executeFileOperation(callback: () => unknown): void {
+		if (this.useSlowFileEvents) {
+			// we aren't the best client and it takes some time to notice changes
+			setTimeout(callback, Math.random() * 100);
+		} else {
+			callback();
+		}
+	}
+}

frontend/test-client/src/cli.ts

@@ -0,0 +1,173 @@
+import type { SyncSettings } from "sync-client";
+import { MockAgent } from "./agent/mock-agent";
+import { sleep } from "./utils/sleep";
+import { v4 as uuidv4 } from "uuid";
+import { randomCasing } from "./utils/random-casing";
+
+const TEST_ITERATIONS = 5;
+
+// Simulate async file access by injecting waiting time before returning from file operations.
+let slowFileEvents = false;
+
+async function runTest({
+	agentCount,
+	concurrency,
+	iterations,
+	doDeletes,
+	useSlowFileEvents,
+	jitterScaleInSeconds
+}: {
+	agentCount: number;
+	concurrency: number;
+	iterations: number;
+	doDeletes: boolean;
+	useSlowFileEvents: boolean;
+	jitterScaleInSeconds: number;
+}): Promise<void> {
+	slowFileEvents = useSlowFileEvents;
+
+	const settings = `with ${agentCount} agents, concurrency ${concurrency}, iterations ${iterations}, doDeletes ${doDeletes}, jitterScaleInSeconds ${jitterScaleInSeconds}, useSlowFileEvents ${useSlowFileEvents}`;
+	console.info(`Running test ${settings}`);
+
+	const vaultName = uuidv4();
+	console.info(`Using vault name: ${vaultName}`);
+	const initialSettings: Partial<SyncSettings> = {
+		isSyncEnabled: true,
+		token: "   test-token-change-me     ", // same as in sync-server/config-e2e.yml with spaces
+		vaultName: randomCasing(vaultName) + (Math.random() > 0.5 ? "  " : ""), // extra spaces shouldn't matter
+		syncConcurrency: concurrency,
+		remoteUri: "http://localhost:3000"
+	};
+
+	const clients: MockAgent[] = [];
+	for (let i = 0; i < agentCount; i++) {
+		clients.push(
+			new MockAgent(
+				initialSettings,
+				`agent-${i}`,
+				doDeletes,
+				useSlowFileEvents,
+				jitterScaleInSeconds
+			)
+		);
+	}
+
+	try {
+		await Promise.all(clients.map(async (client) => client.init()));
+
+		for (let i = 0; i < iterations; i++) {
+			console.info(`Iteration ${i + 1}/${iterations}`);
+			await Promise.all(clients.map(async (client) => client.act()));
+			await sleep(100);
+		}
+
+		console.info("Stopping agents");
+
+		// Each agent can have unpushed changes which might conflict with eachother so each has to resolve the conflicts & push, and
+		for (const client of clients) {
+			try {
+				await client.finish();
+			} catch (err) {
+				if (!slowFileEvents) {
+					throw err;
+				}
+			}
+		}
+
+		// then we need a second pass to ensure that all agents pull the same state.
+		for (const client of clients) {
+			try {
+				await client.finish();
+			} catch (err) {
+				if (!slowFileEvents) {
+					throw err;
+				}
+			}
+		}
+
+		console.info("Agents finished successfully");
+
+		clients.slice(0, -1).forEach((client, i) => {
+			console.info(
+				`Checking consistency between ${client.name} and ${clients[i + 1].name}`
+			);
+			client.assertFileSystemsAreConsistent(clients[i]);
+			console.info(`Consistency check for ${client.name} passed`);
+		});
+
+		console.info("File systems found to be consistent");
+
+		clients.forEach((client) => {
+			console.info(`Checking content for ${client.name}`);
+			client.assertAllContentIsPresentOnce();
+			console.info(`Content check for ${client.name} passed`);
+		});
+
+		console.info(`Test passed ${settings}`);
+	} catch (err) {
+		console.error(`Test failed ${settings}`);
+		throw err;
+	}
+}
+
+async function runTests(): Promise<void> {
+	for (let i = 0; i < TEST_ITERATIONS; i++) {
+		for (const useSlowFileEvents of [false, true]) {
+			for (const concurrency of [
+				16,
+				1 // test with concurrency 1 to check for deadlocks
+			]) {
+				for (const doDeletes of [true, false]) {
+					await runTest({
+						agentCount: 2,
+						concurrency,
+						iterations: 100,
+						doDeletes,
+						useSlowFileEvents,
+						jitterScaleInSeconds: 0.75
+					});
+				}
+			}
+		}
+	}
+}
+
+process.on("uncaughtException", (error) => {
+	if (slowFileEvents) {
+		return;
+	}
+
+	if (
+		error instanceof Error &&
+		error.message.includes(
+			"WebSocket was closed before the connection was established"
+		)
+	) {
+		return;
+	}
+
+	console.error("Uncaught exception:", error);
+	process.exit(1);
+});
+
+process.on("unhandledRejection", (error, _promise) => {
+	if (error instanceof Error && error.message === "Sync was reset") {
+		return;
+	}
+
+	if (slowFileEvents) {
+		return;
+	}
+
+	console.error("Unhandled rejection:", error);
+	process.exit(1);
+});
+
+runTests()
+	.then(() => {
+		process.exit(0);
+	})
+	.catch((err: unknown) => {
+		console.error(err);
+		process.exit(1);
+	});

frontend/test-client/src/utils/assert.ts

@@ -0,0 +1,5 @@
+export function assert(value: boolean, message: string): asserts value {
+	if (!value) {
+		throw new Error(message);
+	}
+}

frontend/test-client/src/utils/choose.ts

@@ -0,0 +1,3 @@
+export function choose<T>(values: T[]): T {
+	return values[Math.floor(Math.random() * values.length)];
+}

frontend/test-client/src/utils/random-casing.test.ts

@@ -0,0 +1,13 @@
+import { describe, it } from "node:test";
+import assert from "node:assert";
+import { randomCasing } from "./random-casing";
+
+describe("randomCasing", () => {
+	it("simple test", () => {
+		const input =
+			"hello, this is a really long string with a lot of characters";
+		const result = randomCasing(input);
+		assert.strictEqual(result.toLowerCase(), input.toLowerCase());
+		assert.notStrictEqual(result, input);
+	});
+});

frontend/test-client/src/utils/random-casing.ts

@@ -0,0 +1,10 @@
+export function randomCasing(str: string): string {
+	const chars = str.split("");
+	const randomCasedChars = chars.map((char) => {
+		if (Math.random() < 0.5) {
+			return char.toUpperCase();
+		}
+		return char.toLowerCase();
+	});
+	return randomCasedChars.join("");
+}

frontend/test-client/src/utils/sleep.ts

@@ -0,0 +1,3 @@
+export async function sleep(ms: number): Promise<void> {
+	return new Promise((resolve) => setTimeout(resolve, ms));
+}

frontend/test-client/tsconfig.json

@@ -0,0 +1,17 @@
+{
+	"compilerOptions": {
+		"baseUrl": ".",
+		"strict": true,
+		"target": "ES2022",
+		"module": "CommonJS",
+		"esModuleInterop": true,
+		"lib": [
+			"DOM",
+			"ES2024",
+		],
+		"moduleResolution": "node"
+	},
+	"exclude": [
+		"./dist"
+	]
+}

frontend/test-client/webpack.config.js

@@ -0,0 +1,30 @@
+const path = require("path");
+const webpack = require("webpack");
+
+module.exports = {
+	entry: "./src/cli.ts",
+	target: "node",
+	mode: "production",
+	optimization: {
+		minimize: false
+	},
+	module: {
+		rules: [
+			{
+				test: /\.ts$/,
+				use: "ts-loader"
+			}
+		]
+	},
+	resolve: {
+		extensions: [".ts", ".js"]
+	},
+	output: {
+		globalObject: "this",
+		filename: "cli.js",
+		path: path.resolve(__dirname, "dist")
+	},
+	plugins: [
+		new webpack.BannerPlugin({ banner: "#!/usr/bin/env node", raw: true })
+	]
+};