Working for non-deletes

frontend/sync-client/src/file-operations/file-operations.test.ts

@@ -2,6 +2,7 @@ import type { FileSystemOperations } from "sync-client";
 import type {
 	Database,
 	DocumentMetadata,
+	DocumentRecord,
 	RelativePath
 } from "../persistence/database";
 import { FileOperations } from "./file-operations";
@@ -10,16 +11,16 @@ import { assertSetContainsExactly } from "../utils/assert-set-contains-exactly";
 
 describe("File operations", () => {
 	class MockDatabase {
-		public async move(
+		public move(
 			_oldRelativePath: RelativePath,
 			_newRelativePath: RelativePath
-		): Promise<void> {
+		): void {
 			// this is called but irrelevant for this mock
 		}
 
-		public getResolvedDocument(
+		public getDocumentByRelativePath(
-			_relativePath: RelativePath | undefined
+			_find: RelativePath
-		): DocumentMetadata | undefined {
+		): DocumentRecord | undefined {
 			return undefined;
 		}
 	}

frontend/sync-client/src/file-operations/file-operations.ts

@@ -71,27 +71,30 @@ export class FileOperations {
 				`Didn't expect ${path} to exist, deconflicting by moving it to '${deconflictedPath}'`
 			);
 
-			const existingMetadata = this.database.getResolvedDocument(path);
+			const document = this.database.getDocumentByRelativePath(path);
 			this.logger.debug(
-				`Existing metadata for ${path}: ${JSON.stringify(existingMetadata)}`
+				`Existing metadata for ${path}: ${JSON.stringify(document?.metadata)}`
 			);
+
+			this.logger.debug(
+				`We need to save what's at ${path} to ${deconflictedPath}`
+			);
+
 			if (
-				existingMetadata === undefined ||
+				document?.metadata !== undefined &&
-				existingMetadata.isDeleted ||
+				document.metadata.documentId === documentId
-				existingMetadata.documentId !== documentId ||
-				!documentId
 			) {
-				this.logger.debug(
-					`We need to save what's at ${path} to ${deconflictedPath}`
-				);
-				await this.move(path, deconflictedPath, documentId);
-				await this.database.move(path, deconflictedPath);
-			} else {
 				// This can happen if the document got moved both locally and remotely
 				// to the same file path. In this case, we shouldn't deconflict, however,
 				// we also can't overwrite otherwise we'd lose changes.
 				throw new FileNotFoundError(path);
 			}
+
+			this.logger.debug(
+				`We need to save what's at ${path} to ${deconflictedPath}`
+			);
+			await this.move(path, deconflictedPath, documentId);
+			// this.database.move(path, deconflictedPath);
 		} else {
 			await this.createParentDirectories(path);
 		}
@@ -135,7 +138,7 @@ export class FileOperations {
 				currentText = currentText.replace(/\r\n/g, "\n");
 				if (currentText !== expectedText) {
 					this.logger.debug(
-						`Performing a 3-way merge for ${path} with the expected content`
+						`Performing a 3-way merge for ${path} with the expected content:\n${expectedText}`
 					);
 
 					return mergeText(expectedText, currentText, newText);
@@ -174,21 +177,21 @@ export class FileOperations {
 			this.logger.debug(
 				`Conflict when moving '${oldPath}' to '${newPath}', the latter already exists, deconflicting by moving it to '${deconflictedPath}'`
 			);
-			const existingMetadata = this.database.getResolvedDocument(newPath);
+
+			const document = this.database.getDocumentByRelativePath(newPath);
+
 			if (
-				existingMetadata === undefined ||
+				document?.metadata !== undefined &&
-				existingMetadata.isDeleted ||
+				document.metadata.documentId === documentId
-				existingMetadata.documentId !== documentId ||
-				!documentId
 			) {
-				await this.move(newPath, deconflictedPath, documentId);
-				await this.database.move(oldPath, newPath);
-			} else {
 				// This can happen if the document got moved both locally and remotely
 				// to the same file path. In this case, we shouldn't deconflict, however,
 				// we also can't overwrite otherwise we'd lose changes.
 				throw new FileNotFoundError(newPath);
 			}
+
+			await this.move(newPath, deconflictedPath, documentId);
+			// this.database.move(oldPath, newPath);
 		} else {
 			await this.createParentDirectories(newPath);
 		}

frontend/sync-client/src/persistence/database.ts

@@ -1,3 +1,5 @@
+import type { Logger } from "../tracing/logger";
+
 export type VaultUpdateId = number;
 export type DocumentId = string;
 export type RelativePath = string;
@@ -8,20 +10,28 @@ export interface DocumentMetadata {
 	hash: string;
 	isDeleted: boolean;
 }
-
+export interface StoredDocumentMetadata {
-import type { Logger } from "src/tracing/logger";
+	relativePath: RelativePath;
+	parentVersionId: VaultUpdateId;
+	documentId: DocumentId;
+	hash: string;
+	isDeleted: boolean;
+}
 
 export interface StoredDatabase {
-	documents: Record<RelativePath, DocumentMetadata>;
+	documents: StoredDocumentMetadata[];
 	lastSeenUpdateId: VaultUpdateId | undefined;
 }
 
-export class Database {
+export interface DocumentRecord {
-	private documents = new Map<
+	identity: symbol;
-		RelativePath,
+	relativePath: RelativePath;
-		DocumentMetadata | Promise<DocumentMetadata | undefined>
+	metadata: DocumentMetadata | undefined;
-	>();
+	updates: Promise<void>[];
+}
 
+export class Database {
+	private documents: DocumentRecord[];
 	private lastSeenUpdateId: VaultUpdateId | undefined;
 
 	public constructor(
@@ -30,16 +40,17 @@ export class Database {
 		private readonly saveData: (data: StoredDatabase) => Promise<void>
 	) {
 		initialState ??= {};
-		if (initialState.documents) {
-			for (const [relativePath, metadata] of Object.entries(
-				initialState.documents
-			)) {
-				this.documents.set(relativePath, metadata);
-			}
-		}
-		this.ensureConsistency();
 
-		this.logger.debug(`Loaded ${this.documents.size} documents`);
+		this.documents =
+			initialState.documents?.map(({ relativePath, ...metadata }) => ({
+				relativePath,
+				identity: Symbol(),
+				metadata,
+				updates: []
+			})) ?? [];
+
+		this.ensureConsistency();
+		this.logger.debug(`Loaded ${this.documents.length} documents`);
 
 		this.lastSeenUpdateId = initialState.lastSeenUpdateId;
 		this.logger.debug(
@@ -48,62 +59,29 @@ export class Database {
 	}
 
 	public get length(): number {
-		return this.documents.size;
+		return this.documents.length;
 	}
 
-	public get resolvedDocuments(): [RelativePath, DocumentMetadata][] {
+	public get resolvedDocuments(): DocumentRecord[] {
-		// eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion
+		return this.documents.filter(({ metadata }) => metadata !== undefined);
-		return Array.from(this.documents.entries()).filter(
-			([_, metadata]) => !(metadata instanceof Promise)
-		) as [RelativePath, DocumentMetadata][];
 	}
 
 	public getLastSeenUpdateId(): VaultUpdateId | undefined {
 		return this.lastSeenUpdateId;
 	}
 
-	public async setLastSeenUpdateId(
+	public setLastSeenUpdateId(value: VaultUpdateId | undefined): void {
-		value: VaultUpdateId | undefined
-	): Promise<void> {
 		this.lastSeenUpdateId = value;
-		await this.save();
+		this.save();
 	}
 
-	public async resetSyncState(): Promise<void> {
+	public resetSyncState(): void {
-		this.documents = new Map();
+		this.documents = [];
 		this.lastSeenUpdateId = 0;
-		await this.save();
+		this.save();
 	}
 
-	public getDocumentByDocumentId(
+	public setDocument({
-		documentId: DocumentId
-	): [RelativePath, DocumentMetadata] | undefined {
-		return this.resolvedDocuments.find(
-			([_, metadata]) => metadata.documentId === documentId
-		);
-	}
-
-	public getDocumentByIdentity(
-		document:
-			| DocumentMetadata
-			| Promise<DocumentMetadata | undefined>
-			| undefined
-	):
-		| [
-				RelativePath,
-				DocumentMetadata | Promise<DocumentMetadata | undefined>
-		  ]
-		| undefined {
-		if (document === undefined) {
-			return undefined;
-		}
-
-		return Array.from(this.documents.entries()).find(
-			([_, metadata]) => metadata === document
-		);
-	}
-
-	public async setDocument({
 		documentId,
 		relativePath,
 		parentVersionId,
@@ -115,84 +93,142 @@ export class Database {
 		parentVersionId: VaultUpdateId;
 		hash: string;
 		isDeleted: boolean;
-	}): Promise<void> {
+	}): void {
-		this.documents.set(relativePath, {
+		const entry = this.getDocumentByRelativePath(relativePath);
-			documentId,
-			parentVersionId,
-			hash,
-			isDeleted
-		});
-		await this.save();
-	}
 
-	public async setDocumentPromise({
+		if (entry !== undefined) {
-		relativePath,
+			this.documents = this.documents.filter(
-		promise
+				({ identity }) => identity !== entry.identity
-	}: {
-		relativePath: RelativePath;
-		promise: Promise<DocumentMetadata | undefined>;
-	}): Promise<void> {
-		this.documents.set(relativePath, promise);
-		// No need to save as Promises don't get serialized
-		// and a crash would only result in the document being
-		// creatied again.
-	}
-
-	public getResolvedDocument(
-		relativePath: RelativePath | undefined
-	): DocumentMetadata | undefined {
-		if (relativePath == undefined) {
-			return undefined;
-		}
-
-		const metadata = this.documents.get(relativePath);
-		if (metadata instanceof Promise) {
-			return undefined;
-		}
-
-		return metadata;
-	}
-
-	public getDocument(
-		relativePath: RelativePath | undefined
-	): Promise<DocumentMetadata | undefined> | DocumentMetadata | undefined {
-		if (relativePath == undefined) {
-			return undefined;
-		}
-
-		return this.documents.get(relativePath);
-	}
-
-	public async move(
-		oldRelativePath: RelativePath,
-		newRelativePath: RelativePath
-	): Promise<void> {
-		const document = this.documents.get(oldRelativePath);
-		if (!document) {
-			return;
-		}
-
-		const resolvedDocument = this.getResolvedDocument(oldRelativePath);
-		if (
-			this.documents.has(newRelativePath) &&
-			resolvedDocument != undefined &&
-			resolvedDocument.isDeleted
-		) {
-			throw new Error(
-				`Cannot update physical path to path that is already in use: ${newRelativePath}`
 			);
 		}
 
-		this.documents.delete(oldRelativePath);
+		this.documents.push({
-		this.documents.set(newRelativePath, document);
+			// `entry` might be undefined if the document is new
+			identity: entry?.identity ?? Symbol(),
+			relativePath,
+			metadata: {
+				documentId,
+				parentVersionId,
+				hash,
+				isDeleted
+			},
+			updates: entry?.updates ?? []
+		});
 
-		await this.save();
+		this.save();
 	}
 
-	private async save(): Promise<void> {
+	public removeDocumentPromise(promise: Promise<void>): void {
+		const entry = this.getDocumentByUpdatePromise(promise);
+		entry.updates = entry.updates.filter((update) => update !== promise);
+		// No need to save as Promises don't get serialized
+	}
+
+	public getDocumentByRelativePath(
+		find: RelativePath
+	): DocumentRecord | undefined {
+		return this.documents.find(({ relativePath }) => relativePath === find);
+	}
+
+	public async getResolvedDocumentByRelativePath(
+		relativePath: RelativePath,
+		promise: Promise<void>
+	): Promise<DocumentRecord> {
+		let entry = this.getDocumentByRelativePath(relativePath);
+
+		if (entry === undefined) {
+			entry = {
+				relativePath,
+				identity: Symbol(),
+				metadata: undefined,
+				updates: []
+			};
+
+			this.documents.push(entry);
+		}
+
+		const currentPromises = entry.updates;
+		entry.updates = [...currentPromises, promise];
+		await Promise.all(currentPromises);
+
+		// Refetch the document as it might have been updated
+		return this.getDocumentByIdentity(entry.identity);
+	}
+
+	public getDocumentByUpdatePromise(promise: Promise<void>): DocumentRecord {
+		const result = this.documents.find(({ updates }) =>
+			updates.includes(promise)
+		);
+
+		if (result === undefined) {
+			throw new Error("Document not found by update promise");
+		}
+
+		return result;
+	}
+
+	public getDocumentByDocumentId(
+		documentId: DocumentId
+	): DocumentRecord | undefined {
+		return this.documents.find(
+			({ metadata }) => metadata?.documentId === documentId
+		);
+	}
+
+	public getDocumentByIdentity(find: symbol): DocumentRecord {
+		const result = this.documents.find(({ identity }) => identity === find);
+
+		if (result === undefined) {
+			throw new Error("Document not found by identity symbol");
+		}
+
+		return result;
+	}
+
+	public move(
+		oldRelativePath: RelativePath,
+		newRelativePath: RelativePath
+	): void {
+		const oldDocument = this.getDocumentByRelativePath(oldRelativePath);
+		if (oldDocument === undefined) {
+			throw new Error(
+				`Document to be moved not found: ${oldRelativePath}`
+			);
+		}
+
+		const newDocument = this.getDocumentByRelativePath(newRelativePath);
+		if (
+			newDocument !== undefined &&
+			newDocument.metadata?.isDeleted === false
+		) {
+			throw new Error(
+				`Cannot move document to existing path: ${newRelativePath}`
+			);
+		}
+
+		this.documents = this.documents.filter(
+			({ identity }) =>
+				identity !== oldDocument.identity &&
+				identity !== newDocument?.identity
+		);
+
+		this.documents.push({
+			...oldDocument,
+			relativePath: newRelativePath
+		});
+
+		this.save();
+	}
+
+	private save(): void {
 		this.ensureConsistency();
-		await this.saveData({
+		void this.saveData({
-			documents: Object.fromEntries(this.resolvedDocuments),
+			documents: this.resolvedDocuments.map(
+				({ relativePath, metadata }) => ({
+					relativePath,
+					...metadata
+				})
+			) as StoredDocumentMetadata[],
 			lastSeenUpdateId: this.lastSeenUpdateId
 		});
 	}
@@ -200,12 +236,16 @@ export class Database {
 	private ensureConsistency(): void {
 		const idToPath = new Map<string, string[]>();
 
-		this.resolvedDocuments.forEach(([name, metadata]) => {
+		this.resolvedDocuments
-			idToPath.set(metadata.documentId, [
+			.filter(({ metadata }) => metadata !== undefined)
-				...(idToPath.get(metadata.documentId) ?? []),
+			.forEach(({ metadata, relativePath }) => {
-				name
+				// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-			]);
+				idToPath.set(metadata!.documentId, [
-		});
+					// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+					...(idToPath.get(metadata!.documentId) ?? []),
+					relativePath
+				]);
+			});
 
 		const duplicates = Array.from(idToPath.entries())
 			.filter(([_, paths]) => paths.length > 1)

frontend/sync-client/src/services/sync-service.ts

@@ -109,6 +109,9 @@ export class SyncService {
 		contentBytes: Uint8Array;
 		createdDate: Date;
 	}): Promise<components["schemas"]["DocumentUpdateResponse"]> {
+		this.logger.debug(
+			`Updating document ${documentId} with parent version ${parentVersionId} & ${new TextDecoder().decode(contentBytes)} & ${relativePath}`
+		);
 		const formData = new FormData();
 		formData.append("parent_version_id", parentVersionId.toString());
 		formData.append("created_date", createdDate.toISOString());

frontend/sync-client/src/sync-client.ts

@@ -148,7 +148,7 @@ export class SyncClient {
 		this.stop();
 		await this._syncer.reset();
 		this._history.reset();
-		await this._database.resetSyncState();
+		this._database.resetSyncState();
 		this.logger.reset();
 	}
 

frontend/sync-client/src/sync-operations/syncer.ts

@@ -12,9 +12,10 @@ import { hash } from "src/utils/hash";
 import type { components } from "src/services/types";
 import type { Settings } from "src/persistence/settings";
 import type { FileOperations } from "src/file-operations/file-operations";
-import { findMatchingFileBasedOnHash } from "src/utils/find-matching-file-based-on-hash";
+import { findMatchingFile } from "src/utils/find-matching-file";
 import { UnrestrictedSyncer } from "./unrestricted-syncer";
 import { FileNotFoundError } from "src/file-operations/safe-filesystem-operations";
+import { createPromise } from "src/utils/create-promise";
 
 export class Syncer {
 	private readonly remainingOperationsListeners: ((
@@ -74,9 +75,10 @@ export class Syncer {
 				logger.debug(
 					`File has been deleted or moved before we had a chance to inspect it, skipping`
 				);
-			} else {
+				return undefined;
-				throw e;
 			}
+
+			throw e;
 		}
 	}
 
@@ -88,77 +90,95 @@ export class Syncer {
 
 	public async syncLocallyCreatedFile(
 		relativePath: RelativePath,
-		updateTime: Date
+		updateTime?: Date
 	): Promise<void> {
-		let resolve:
+		const [promise, resolve, reject] = createPromise();
-			| undefined
-			| ((metadata: DocumentMetadata | undefined) => void) = undefined;
 
-		const creationPromise = new Promise<DocumentMetadata | undefined>(
+		// Most likely, we're waiting for the previous delete to finish on the file at this path
-			(r) => (resolve = r)
+		const document = await this.database.getResolvedDocumentByRelativePath(
+			relativePath,
+			promise
 		);
 
-		await this.database.setDocumentPromise({
+		try {
-			relativePath,
+			await this.syncQueue.add(async () =>
-			promise: creationPromise
+				this.internalSyncer.unrestrictedSyncLocallyCreatedFile(
-		});
+					() =>
-
+						this.database.getDocumentByIdentity(document.identity),
-		await this.syncQueue.add(async () => {
-			// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-			resolve!(
-				await this.internalSyncer.unrestrictedSyncLocallyCreatedFile(
-					relativePath,
 					updateTime
 				)
 			);
-		});
+
+			resolve();
+		} catch (e) {
+			reject(e);
+		} finally {
+			this.database.removeDocumentPromise(promise);
+		}
 	}
 
 	public async syncLocallyDeletedFile(
 		relativePath: RelativePath
 	): Promise<void> {
-		let metadata = this.database.getDocument(relativePath);
+		const [promise, resolve, reject] = createPromise();
-		if (metadata !== undefined && !(metadata instanceof Promise)) {
-			metadata = Promise.resolve(metadata);
-		}
 
-		await this.syncQueue.add(async () =>
+		const document = await this.database.getResolvedDocumentByRelativePath(
-			this.internalSyncer.unrestrictedSyncLocallyDeletedFile(
+			relativePath,
-				relativePath,
+			promise
-				metadata
-			)
 		);
+
+		try {
+			await this.syncQueue.add(async () =>
+				this.internalSyncer.unrestrictedSyncLocallyDeletedFile(() =>
+					this.database.getDocumentByIdentity(document.identity)
+				)
+			);
+
+			resolve();
+		} catch (e) {
+			reject(e);
+		} finally {
+			this.database.removeDocumentPromise(promise);
+		}
 	}
 
 	public async syncLocallyUpdatedFile(args: {
 		oldPath?: RelativePath;
 		relativePath: RelativePath;
-		updateTime: Date;
+		updateTime?: Date;
 	}): Promise<void> {
-		if (args.oldPath === args.relativePath) {
-			throw new Error(
-				`Old path and new path are the same: ${args.oldPath}`
-			);
-		}
-
 		if (args.oldPath !== undefined) {
-			await this.database.move(args.oldPath, args.relativePath);
+			if (args.oldPath === args.relativePath) {
+				throw new Error(
+					`Old path and new path are the same: ${args.oldPath}`
+				);
+			}
+
+			this.database.move(args.oldPath, args.relativePath);
 		}
 
-		let metadata = this.database.getDocument(args.relativePath);
+		const [promise, resolve, reject] = createPromise();
-		if (metadata !== undefined && !(metadata instanceof Promise)) {
+
-			metadata = Promise.resolve(metadata);
+		const metadata = await this.database.getResolvedDocumentByRelativePath(
-		}
+			args.relativePath,
-		await this.syncQueue.add(async () =>
+			promise
-			this.internalSyncer.unrestrictedSyncLocallyUpdatedFile({
-				...args,
-				metadata
-			})
 		);
-	}
 
-	public async waitForSyncQueue(): Promise<void> {
+		try {
-		return this.syncQueue.onEmpty();
+			await this.syncQueue.add(async () =>
+				this.internalSyncer.unrestrictedSyncLocallyUpdatedFile({
+					...args,
+					getLatestDocument: () =>
+						this.database.getDocumentByIdentity(metadata.identity)
+				})
+			);
+
+			resolve();
+		} catch (e) {
+			reject(e);
+		} finally {
+			this.database.removeDocumentPromise(promise);
+		}
 	}
 
 	public async scheduleSyncForOfflineChanges(): Promise<void> {
@@ -217,6 +237,10 @@ export class Syncer {
 		}
 	}
 
+	public async waitForSyncQueue(): Promise<void> {
+		return this.syncQueue.onEmpty();
+	}
+
 	public async reset(): Promise<void> {
 		this.syncQueue.clear();
 		await this.syncQueue.onEmpty();
@@ -229,53 +253,67 @@ export class Syncer {
 	private async syncRemotelyUpdatedFile(
 		remoteVersion: components["schemas"]["DocumentVersionWithoutContent"]
 	): Promise<void> {
-		await this.syncQueue.add(async () =>
+		let document = this.database.getDocumentByDocumentId(
-			this.internalSyncer.unrestrictedSyncRemotelyUpdatedFile(
+			remoteVersion.documentId
-				remoteVersion
-			)
 		);
+
+		if (document === undefined) {
+			await this.syncQueue.add(async () =>
+				this.internalSyncer.unrestrictedSyncRemotelyUpdatedFile(
+					remoteVersion
+				)
+			);
+
+			return;
+		}
+
+		const [promise, resolve, reject] = createPromise();
+
+		document = await this.database.getResolvedDocumentByRelativePath(
+			document.relativePath,
+			promise
+		);
+
+		try {
+			await this.syncQueue.add(async () =>
+				this.internalSyncer.unrestrictedSyncRemotelyUpdatedFile(
+					remoteVersion,
+					() => this.database.getDocumentByIdentity(document.identity)
+				)
+			);
+
+			resolve();
+		} catch (e) {
+			reject(e);
+		} finally {
+			this.database.removeDocumentPromise(promise);
+		}
 	}
 
 	private async internalScheduleSyncForOfflineChanges(): Promise<void> {
 		const allLocalFiles = await this.operations.listAllFiles();
 
-		// This includes renamed files for now
 		let locallyPossiblyDeletedFiles = [
 			...this.database.resolvedDocuments
-		].filter(([path, _]) => !allLocalFiles.includes(path));
+		].filter(({ relativePath }) => !allLocalFiles.includes(relativePath));
 
 		const updates = Promise.all(
-			allLocalFiles.map(async (relativePath) =>
+			allLocalFiles.map(async (relativePath) => {
-				this.syncQueue.add(async () => {
+				if (
-					const metadata =
+					this.database.getDocumentByRelativePath(relativePath)
-						this.database.getResolvedDocument(relativePath);
+						?.metadata !== undefined
+				) {
+					this.logger.debug(
+						`Document ${relativePath} might have been updated locally, scheduling sync to validate and update it`
+					);
 
-					if (metadata) {
+					return this.syncLocallyUpdatedFile({
-						this.logger.debug(
+						relativePath
-							`Document ${relativePath} might have been updated locally, scheduling sync to validate and update it`
+					});
-						);
+				}
-						const updateTime =
-							await Syncer.forgivingFileNotFoundWrapper(
-								async () =>
-									this.operations.getModificationTime(
-										relativePath
-									),
-								this.logger
-							);
-						if (updateTime === undefined) {
-							return;
-						}
 
-						return this.internalSyncer.unrestrictedSyncLocallyUpdatedFile(
+				// Perhaps the file has been moved; let's check by looking at the deleted files
-							{
+				const contentHash = await this.syncQueue.add(async () => {
-								relativePath,
-								updateTime,
-								metadata: Promise.resolve(metadata)
-							}
-						);
-					}
-
-					// Perhaps the file has been moved. Let's check by looking at the deleted files
 					const contentBytes =
 						await Syncer.forgivingFileNotFoundWrapper(
 							async () => this.operations.read(relativePath),
@@ -284,90 +322,51 @@ export class Syncer {
 					if (contentBytes === undefined) {
 						return;
 					}
+					return hash(contentBytes);
+				});
 
-					const contentHash = hash(contentBytes);
+				if (contentHash == undefined) {
+					// The file was deleted before we had a chance to read it, no need to sync it here
+					return;
+				}
 
-					// todo: make this smarter so that offline files can be renamed & edited at the same time
+				const originalFile = findMatchingFile(
-					const originalFile = findMatchingFileBasedOnHash(
+					contentHash,
-						contentHash,
+					locallyPossiblyDeletedFiles
-						locallyPossiblyDeletedFiles
+				);
-					);
+				if (originalFile !== undefined) {
-					if (originalFile !== undefined) {
+					// `originalFile` hasn't been deleted but it got moved instead
-						// `originalFile` hasn't been deleted but it got moved instead
+					locallyPossiblyDeletedFiles =
-						locallyPossiblyDeletedFiles =
+						locallyPossiblyDeletedFiles.filter(
-							locallyPossiblyDeletedFiles.filter(
+							(item) =>
-								(item) => item[0] !== originalFile[0]
+								item.relativePath !== originalFile.relativePath
-							);
-
-						this.logger.debug(
-							`Document '${originalFile[0]}' was not found under its current path in the database but was found under a different path (${relativePath}), scheduling sync to move it`
 						);
 
-						const updateTime =
-							await Syncer.forgivingFileNotFoundWrapper(
-								async () =>
-									this.operations.getModificationTime(
-										relativePath
-									),
-								this.logger
-							);
-						if (updateTime === undefined) {
-							return;
-						}
-
-						return this.internalSyncer.unrestrictedSyncLocallyUpdatedFile(
-							{
-								oldPath: originalFile[0],
-								relativePath,
-								updateTime,
-								metadata: Promise.resolve(
-									this.database.getResolvedDocument(
-										relativePath
-									)
-								),
-								optimisations: {
-									contentBytes,
-									contentHash
-								}
-							}
-						);
-					}
-
 					this.logger.debug(
-						`Document ${relativePath} not found in database, scheduling sync to create it`
+						`Document '${originalFile.relativePath}' was not found under its current path in the database but was found under a different path (${relativePath}), scheduling sync to move it`
 					);
-					const updateTime =
+
-						await Syncer.forgivingFileNotFoundWrapper(
+					// We're outside of the pqueue, so we need to call the public wrapper
-							async () =>
+					return this.syncLocallyUpdatedFile({
-								this.operations.getModificationTime(
+						oldPath: originalFile.relativePath,
-									relativePath
+						relativePath
-								),
+					});
-							this.logger
+				}
-						);
+
-					if (updateTime === undefined) {
+				this.logger.debug(
-						return;
+					`Document ${relativePath} not found in database, scheduling sync to create it`
-					}
+				);
-					return this.internalSyncer.unrestrictedSyncLocallyCreatedFile(
+				// We're outside of the pqueue, so we need to call the public wrapper
-						relativePath,
+				return this.syncLocallyCreatedFile(relativePath);
-						updateTime
+			})
-					);
-				})
-			)
 		);
 
 		const deletes = Promise.all(
-			locallyPossiblyDeletedFiles.map(async ([relativePath, _]) => {
+			locallyPossiblyDeletedFiles.map(async ({ relativePath }) => {
 				this.logger.debug(
 					`Document ${relativePath} has been deleted locally, scheduling sync to delete it`
 				);
 
-				if (await this.operations.exists(relativePath)) {
-					this.logger.debug(
-						`Document ${relativePath} actually exists locally, skipping`
-					);
-					return Promise.resolve();
-				}
-
 				// We're outside of the pqueue, so we need to call the public wrapper
 				return this.syncLocallyDeletedFile(relativePath);
 			})
@@ -389,15 +388,7 @@ export class Syncer {
 		this.logger.info("Applying remote changes locally");
 
 		await Promise.all(
-			remote.latestDocuments
+			remote.latestDocuments.map(this.syncRemotelyUpdatedFile.bind(this))
-				.filter(
-					(remoteDocument) =>
-						remoteDocument.vaultUpdateId >
-						(this.database.getDocumentByDocumentId(
-							remoteDocument.documentId
-						)?.[1].parentVersionId ?? -1)
-				)
-				.map(this.syncRemotelyUpdatedFile.bind(this))
 		);
 
 		const lastSeenUpdateId = this.database.getLastSeenUpdateId();
@@ -405,7 +396,7 @@ export class Syncer {
 			lastSeenUpdateId === undefined ||
 			remote.lastUpdateId > lastSeenUpdateId
 		) {
-			await this.database.setLastSeenUpdateId(remote.lastUpdateId);
+			this.database.setLastSeenUpdateId(remote.lastUpdateId);
 		}
 	}
 

frontend/sync-client/src/sync-operations/unrestricted-syncer.ts

@@ -1,11 +1,12 @@
 import type {
 	Database,
 	DocumentMetadata,
+	DocumentRecord,
 	RelativePath
 } from "../persistence/database";
 
 import type { SyncService } from "src/services/sync-service";
-import type { Logger } from "src/tracing/logger";
+import { Logger } from "src/tracing/logger";
 import type { SyncHistory } from "src/tracing/sync-history";
 import { SyncSource, SyncStatus, SyncType } from "src/tracing/sync-history";
 import { EMPTY_HASH, hash } from "src/utils/hash";
@@ -31,37 +32,28 @@ export class UnrestrictedSyncer {
 	}
 
 	public async unrestrictedSyncLocallyCreatedFile(
-		relativePath: RelativePath,
+		getLatestDocument: () => DocumentRecord,
-		updateTime: Date,
+		updateTime?: Date
-		optimisations?: {
+	): Promise<void> {
-			contentBytes?: Uint8Array;
+		const { relativePath, metadata } = getLatestDocument();
-			contentHash?: string;
+
-		}
-	): Promise<DocumentMetadata | undefined> {
 		return this.executeSync(
 			[relativePath],
 			SyncType.CREATE,
 			SyncSource.PUSH,
 			async () => {
-				const localMetadata = this.database.getDocument(relativePath);
+				if (metadata !== undefined && !metadata.isDeleted) {
-
-				if (
-					!(localMetadata instanceof Promise) &&
-					localMetadata &&
-					!localMetadata.isDeleted
-				) {
 					this.logger.debug(
-						`Document metadata already exists for ${relativePath}, it must have been downloaded from the server`
+						`Document ${relativePath} already exists in the database, no need to create it again`
 					);
-
 					return;
 				}
 
-				const contentBytes =
+				const contentBytes = await this.operations.read(relativePath); // this can throw FileNotFoundError
-					optimisations?.contentBytes ??
+				const contentHash = hash(contentBytes);
-					(await this.operations.read(relativePath)); // this can throw FileNotFoundError
+
-				const contentHash =
+				updateTime ??=
-					optimisations?.contentHash ?? hash(contentBytes);
+					await this.operations.getModificationTime(relativePath); // this can throw FileNotFoundError
 
 				const response = await this.syncService.create({
 					relativePath,
@@ -69,95 +61,71 @@ export class UnrestrictedSyncer {
 					createdDate: updateTime
 				});
 
-				const currentMetadata =
+				const { relativePath: currentRelativePath } =
-					this.database.getDocumentByIdentity(localMetadata);
+					getLatestDocument();
-				if (!currentMetadata) {
-					throw new Error(
-						`Document metadata for ${relativePath} not found after creation`
-					);
-				}
 
 				this.history.addHistoryEntry({
 					status: SyncStatus.SUCCESS,
 					source: SyncSource.PUSH,
-					relativePath: currentMetadata[0],
+					relativePath,
 					message: `Successfully uploaded locally created file`,
 					type: SyncType.CREATE
 				});
 
 				const newMetadata = {
+					relativePath: currentRelativePath,
 					documentId: response.documentId,
 					parentVersionId: response.vaultUpdateId,
 					hash: contentHash,
 					isDeleted: false
 				};
 
-				await this.database.setDocument({
+				this.database.setDocument(newMetadata);
-					relativePath: currentMetadata[0],
-					...newMetadata
-				});
 
-				await this.tryIncrementVaultUpdateId(response.vaultUpdateId);
+				this.tryIncrementVaultUpdateId(response.vaultUpdateId);
-
-				return newMetadata;
 			}
 		);
 	}
 
 	public async unrestrictedSyncLocallyDeletedFile(
-		relativePath: RelativePath,
+		getLatestDocument: () => DocumentRecord
-		metadata: Promise<DocumentMetadata | undefined> | undefined
 	): Promise<void> {
+		let document = getLatestDocument();
 		await this.executeSync(
-			[relativePath],
+			[document.relativePath],
 			SyncType.DELETE,
 			SyncSource.PUSH,
 			async () => {
-				const localMetadata =
+				if (
-					metadata !== undefined
+					document.metadata === undefined ||
-						? await metadata
+					document.metadata.isDeleted
-						: this.database.getResolvedDocument(relativePath);
+				) {
-
+					this.logger.debug(
-				if (!localMetadata || localMetadata.isDeleted) {
+						`Document ${document.relativePath} has been already deleted, no need to delete it again`
-					this.logger.info(
-						`Locally deleted file hasn't been uploaded yet, so there's no need to delete it on the remote server`
 					);
-
 					return;
 				}
 
 				const response = await this.syncService.delete({
-					documentId: localMetadata.documentId,
+					documentId: document.metadata.documentId,
-					relativePath,
+					relativePath: document.relativePath,
-					createdDate: new Date() // We got the event now, so it must have been deleted just now
+					createdDate: new Date() // We've got the event now, so it must have been deleted just now
 				});
 
 				this.history.addHistoryEntry({
 					status: SyncStatus.SUCCESS,
 					source: SyncSource.PUSH,
-					relativePath,
+					relativePath: document.relativePath,
 					message: `Successfully deleted locally deleted file on the remote server`,
 					type: SyncType.DELETE
 				});
 
-				const currentMetadata = this.database.getDocumentByDocumentId(
+				document = getLatestDocument();
-					localMetadata.documentId
-				);
-
-				if (!currentMetadata || currentMetadata[1].isDeleted) {
-					this.logger.info(
-						`No metadata found for deleted file, '${relativePath}' must have been deleted by another operation`
-					);
-
-					return;
-				}
-
-				await this.operations.delete(currentMetadata[0]);
 
 				// We have to have a record of the delete in case there's an in-flight update for the same
 				// document which finishes after the delete has succeeded and would introduce a phantom metadata record.
-				await this.database.setDocument({
+				this.database.setDocument({
-					relativePath: currentMetadata[0],
+					relativePath: document.relativePath,
 					documentId: response.documentId,
 					parentVersionId: response.vaultUpdateId,
 					hash: EMPTY_HASH,
@@ -169,84 +137,64 @@ export class UnrestrictedSyncer {
 
 	public async unrestrictedSyncLocallyUpdatedFile({
 		oldPath,
-		relativePath,
+		getLatestDocument,
-		metadata,
+		updateTime
-		updateTime,
-		optimisations
 	}: {
 		oldPath?: RelativePath;
-		relativePath: RelativePath;
+		getLatestDocument: () => DocumentRecord;
-		metadata: Promise<DocumentMetadata | undefined> | undefined;
+		updateTime?: Date;
-		updateTime: Date;
-		optimisations?: {
-			contentBytes?: Uint8Array;
-			contentHash?: string;
-		};
 	}): Promise<void> {
+		let document = getLatestDocument();
+
 		await this.executeSync(
-			[oldPath, relativePath].filter((path) => path !== undefined),
+			[oldPath, document.relativePath].filter(
+				(path) => path !== undefined
+			),
 			SyncType.UPDATE,
 			SyncSource.PUSH,
 			async () => {
-				const localMetadata =
-					metadata !== undefined
-						? await metadata
-						: this.database.getResolvedDocument(relativePath);
-
-				if (!localMetadata || localMetadata.isDeleted) {
-					// It's fine, a subsequent sync operation must have dealt with this
-					return;
-				}
-
-				const contentBytes =
-					optimisations?.contentBytes ??
-					(await this.operations.read(relativePath)); // this can throw FileNotFoundError
-
-				let contentHash =
-					optimisations?.contentHash ?? hash(contentBytes);
-
 				if (
-					localMetadata.hash === contentHash &&
+					document.metadata === undefined ||
-					oldPath === undefined
+					document.metadata.isDeleted
 				) {
 					this.logger.debug(
-						`File hash of ${relativePath} matches with last synced version and the path hasn't changed; no need to sync`
+						`Document ${document.relativePath} has been already deleted, no need to update it, ${JSON.stringify(document)}, ${document.metadata?.isDeleted}`
 					);
 					return;
 				}
 
-				// Re-fetch based on the documentId instead of the relativePath because
+				const contentBytes = await this.operations.read(
-				// the relativePath might have changed since this operation was scheduled
+					document.relativePath
-				let latestMetadata = this.database.getDocumentByDocumentId(
+				); // this can throw FileNotFoundError
-					localMetadata.documentId
+				let contentHash = hash(contentBytes);
-				);
+
-				if (!latestMetadata || latestMetadata[1].isDeleted) {
+				if (
-					// It's fine, a subsequent sync operation must have dealt with this
+					document.metadata.hash === contentHash &&
+					oldPath === undefined
+				) {
+					this.logger.debug(
+						`File hash of ${document.relativePath} matches with last synced version and the path hasn't changed; no need to sync`
+					);
 					return;
 				}
 
+				updateTime ??= await this.operations.getModificationTime(
+					document.relativePath
+				); // this can throw FileNotFoundError;
+
 				const response = await this.syncService.put({
-					documentId: latestMetadata[1].documentId,
+					documentId: document.metadata.documentId,
-					parentVersionId: latestMetadata[1].parentVersionId,
+					parentVersionId: document.metadata.parentVersionId,
-					relativePath: latestMetadata[0],
+					relativePath: document.relativePath,
 					contentBytes,
 					createdDate: updateTime
 				});
 
-				latestMetadata = this.database.getDocumentByDocumentId(
-					response.documentId
-				);
-
-				if (!latestMetadata || latestMetadata[1].isDeleted) {
-					// The document has been deleted since this operation was scheduled
-					return;
-				}
-
 				if (
-					latestMetadata[1].parentVersionId >= response.vaultUpdateId
+					document.metadata.parentVersionId >= response.vaultUpdateId
 				) {
 					this.logger.debug(
-						`Document ${relativePath} is already more up to date than the fetched version`
+						`Document ${document.relativePath} is already more up to date than the fetched version`
 					);
 					return;
 				}
@@ -254,50 +202,42 @@ export class UnrestrictedSyncer {
 				this.history.addHistoryEntry({
 					status: SyncStatus.SUCCESS,
 					source: SyncSource.PUSH,
-					relativePath,
+					relativePath: document.relativePath,
 					message: `Successfully uploaded locally updated file to the remote server`,
 					type: SyncType.UPDATE
 				});
 
+				// Update relativePath which is the only property that can change while this is running (due to a move)
+				document = getLatestDocument();
+
 				if (response.isDeleted) {
-					await this.operations.delete(relativePath);
+					await this.operations.delete(document.relativePath);
 
 					this.history.addHistoryEntry({
 						status: SyncStatus.SUCCESS,
 						source: SyncSource.PULL,
-						relativePath,
+						relativePath: document.relativePath,
 						message:
 							"The file we tried to update had been deleted remotely, therefore, we have deleted it locally",
 						type: SyncType.DELETE
 					});
 
-					await this.database.setDocument({
+					this.database.setDocument({
 						documentId: response.documentId,
-						relativePath: latestMetadata[0],
+						relativePath: document.relativePath,
 						parentVersionId: response.vaultUpdateId,
 						hash: EMPTY_HASH,
 						isDeleted: true
 					});
 
-					await this.tryIncrementVaultUpdateId(
+					this.tryIncrementVaultUpdateId(response.vaultUpdateId);
-						response.vaultUpdateId
-					);
 
 					return;
 				}
 
-				if (
+				if (response.relativePath != document.relativePath) {
-					latestMetadata[1].parentVersionId >= response.vaultUpdateId
-				) {
-					this.logger.debug(
-						`Document ${relativePath} is already more up to date than the fetched version`
-					);
-					return;
-				}
-
-				if (response.relativePath != relativePath) {
 					await this.operations.move(
-						latestMetadata[0],
+						document.relativePath,
 						response.relativePath,
 						response.documentId
 					); // this can throw FileNotFoundError
@@ -316,155 +256,85 @@ export class UnrestrictedSyncer {
 					this.history.addHistoryEntry({
 						status: SyncStatus.SUCCESS,
 						source: SyncSource.PULL,
-						relativePath,
+						relativePath: document.relativePath,
 						message: `The file we updated had been updated remotely, so we downloaded the merged version`,
 						type: SyncType.UPDATE
 					});
 				}
 
-				await this.database.setDocument({
+				this.database.setDocument({
 					documentId: response.documentId,
 					relativePath:
-						response.relativePath != relativePath
+						response.relativePath != document.relativePath
 							? response.relativePath
-							: latestMetadata[0],
+							: document.relativePath,
 					parentVersionId: response.vaultUpdateId,
 					hash: contentHash,
 					isDeleted: response.isDeleted
 				});
 
-				await this.tryIncrementVaultUpdateId(response.vaultUpdateId);
+				this.tryIncrementVaultUpdateId(response.vaultUpdateId);
 			}
 		);
 	}
 
 	public async unrestrictedSyncRemotelyUpdatedFile(
-		remoteVersion: components["schemas"]["DocumentVersionWithoutContent"]
+		remoteVersion: components["schemas"]["DocumentVersionWithoutContent"],
+		getLatestDocument?: () => DocumentRecord
 	): Promise<void> {
 		await this.executeSync(
 			[remoteVersion.relativePath],
 			SyncType.UPDATE,
 			SyncSource.PULL,
 			async () => {
+				const localMetadata =
+					getLatestDocument?.() ??
+					this.database.getDocumentByDocumentId(
+						remoteVersion.documentId
+					);
+
+				if (
+					localMetadata?.metadata !== undefined &&
+					!localMetadata.metadata.isDeleted
+				) {
+					// If the file exists locally, let's pretend the user has updated it
+					// and deal with remote update/deletion within `unrestrictedSyncLocallyUpdatedFile`
+					if (
+						localMetadata.metadata.parentVersionId >=
+						remoteVersion.vaultUpdateId
+					) {
+						this.logger.debug(
+							`Document ${remoteVersion.relativePath} is already more up to date than the fetched version`
+						);
+						return;
+					}
+
+					return this.unrestrictedSyncLocallyUpdatedFile({
+						getLatestDocument: () =>
+							this.database.getDocumentByIdentity(
+								localMetadata.identity
+							)
+					});
+				}
+
 				const content = (
 					await this.syncService.get({
 						documentId: remoteVersion.documentId
 					})
 				).contentBase64;
 				const contentBytes = deserialize(content);
-				const contentHash = hash(contentBytes);
 
-				const localMetadata = this.database.getDocumentByDocumentId(
-					remoteVersion.documentId
-				);
-				if (
-					localMetadata?.[1].documentId ===
-						remoteVersion.documentId &&
-					localMetadata[1].parentVersionId >
-						remoteVersion.vaultUpdateId
-				) {
-					this.logger.info(
-						`Document ${remoteVersion.relativePath} is already up to date`
-					);
-					return;
-				}
-
-				const localBytes = await this.operations.read(
-					remoteVersion.relativePath
-				); // this can throw FileNotFoundError
-				const localHash = hash(localBytes);
-
-				if (localHash !== localMetadata?.[1].hash) {
-					this.logger.info(
-						`Document ${remoteVersion.relativePath} has pending local changes, so we shouldn't update it here`
-					);
-					return;
-				}
-
-				if (!localMetadata || localMetadata[1].isDeleted) {
-					if (remoteVersion.isDeleted) {
-						this.logger.info(
-							`Remotely deleted file hasn't been synced yet, so there's no need to delete it locally`
-						);
-						return;
-					}
-
-					await this.operations.create(
-						remoteVersion.relativePath,
-						contentBytes,
-						remoteVersion.documentId
-					);
-
-					await this.database.setDocument({
-						documentId: remoteVersion.documentId,
-						relativePath: remoteVersion.relativePath,
-						parentVersionId: remoteVersion.vaultUpdateId,
-						hash: hash(contentBytes),
-						isDeleted: remoteVersion.isDeleted
-					});
-
-					this.history.addHistoryEntry({
-						status: SyncStatus.SUCCESS,
-						source: SyncSource.PULL,
-						relativePath: remoteVersion.relativePath,
-						message: `Successfully downloaded remote file which hadn't existed locally`,
-						type: SyncType.CREATE
-					});
-					return;
-				}
-
-				const [relativePath, metadata] = localMetadata;
-				if (remoteVersion.vaultUpdateId <= metadata.parentVersionId) {
-					this.logger.debug(
-						`Document ${relativePath} is already up to date`
-					);
-					return;
-				}
-
-				if (remoteVersion.isDeleted) {
-					await this.operations.delete(relativePath);
-
-					this.history.addHistoryEntry({
-						status: SyncStatus.SUCCESS,
-						source: SyncSource.PULL,
-						relativePath: remoteVersion.relativePath,
-						message: `Successfully deleted remotely deleted file locally`,
-						type: SyncType.DELETE
-					});
-
-					await this.database.setDocument({
-						documentId: remoteVersion.documentId,
-						relativePath: relativePath,
-						parentVersionId: remoteVersion.vaultUpdateId,
-						hash: EMPTY_HASH,
-						isDeleted: true
-					});
-
-					return;
-				}
-
-				if (relativePath !== remoteVersion.relativePath) {
-					// TODO: this can fail, that's bad
-					await this.operations.move(
-						// this can throw FileNotFoundError
-						relativePath,
-						remoteVersion.relativePath,
-						remoteVersion.documentId
-					);
-				}
-
-				// todo: why
 				await this.operations.create(
 					remoteVersion.relativePath,
 					contentBytes,
 					remoteVersion.documentId
 				);
 
-				await this.database.setDocument({
+				this.database.setDocument({
 					documentId: remoteVersion.documentId,
 					relativePath: remoteVersion.relativePath,
 					parentVersionId: remoteVersion.vaultUpdateId,
-					hash: contentHash,
+					hash: hash(contentBytes),
 					isDeleted: remoteVersion.isDeleted
 				});
 
@@ -472,8 +342,8 @@ export class UnrestrictedSyncer {
 					status: SyncStatus.SUCCESS,
 					source: SyncSource.PULL,
 					relativePath: remoteVersion.relativePath,
-					message: `Successfully updated remotely updated file locally`,
+					message: `Successfully downloaded remote file which hadn't existed locally`,
-					type: SyncType.UPDATE
+					type: SyncType.CREATE
 				});
 			}
 		);
@@ -551,11 +421,9 @@ export class UnrestrictedSyncer {
 		this.locks.reset();
 	}
 
-	private async tryIncrementVaultUpdateId(
+	private tryIncrementVaultUpdateId(responseVaultUpdateId: number): void {
-		responseVaultUpdateId: number
-	): Promise<void> {
 		if (this.database.getLastSeenUpdateId() === responseVaultUpdateId - 1) {
-			await this.database.setLastSeenUpdateId(responseVaultUpdateId);
+			this.database.setLastSeenUpdateId(responseVaultUpdateId);
 		}
 	}
 }

frontend/sync-client/src/utils/create-promise.ts

@@ -0,0 +1,15 @@
+export function createPromise<T = void>(): [
+	Promise<T>,
+	(value: T) => void,
+	(error: unknown) => void
+] {
+	let resolve: undefined | ((resolved: T) => void) = undefined;
+	let reject: undefined | ((error: unknown) => void) = undefined;
+
+	const creationPromise = new Promise<T>(
+		(resolve_, reject_) => ((resolve = resolve_), (reject = reject_))
+	);
+
+	// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+	return [creationPromise, resolve!, reject!];
+}

frontend/sync-client/src/utils/find-matching-file.ts

@@ -1,14 +1,14 @@
-import type { DocumentMetadata, RelativePath } from "../persistence/database";
+import type { DocumentRecord } from "../persistence/database";
 import { EMPTY_HASH } from "./hash";
 
 // TODO: make this smarter so that offline files can be renamed & edited at the same time
 export function findMatchingFile(
 	contentHash: string,
-	candidates: [RelativePath, DocumentMetadata][]
+	candidates: DocumentRecord[]
-): [RelativePath, DocumentMetadata] | undefined {
+): DocumentRecord | undefined {
 	if (contentHash === EMPTY_HASH) {
 		return undefined;
 	}
 
-	return candidates.find(([_, metadata]) => metadata.hash === contentHash);
+	return candidates.find(({ metadata }) => metadata?.hash === contentHash);
 }

frontend/test-client/run.sh

@@ -1,21 +1,71 @@
 #!/bin/bash
 
 set -e
+set -o pipefail
+
+# Check if the argument is provided
+if [ $# -eq 0 ]; then
+    echo "Usage: $0 <number_of_processes>"
+    exit 1
+fi
+
+# Get the number of processes from the first argument
+process_count=$1
 
 npm run build
 
 pids=()
-for i in {1..10}; do
+for i in $(seq 1 $process_count); do
     node dist/cli.js 2>&1 | tee "log_${i}.log" &
     pids+=($!)
 done
 
-trap 'kill ${pids[@]} 2>/dev/null' SIGINT SIGTERM
+print_failed_log() {
+    for i in $(seq 1 $process_count); do
+        if [ -n "${pids[$i-1]}" ] && ! kill -0 ${pids[$i-1]} 2>/dev/null; then
+            # Get the exit code of the process
+            wait ${pids[$i-1]}
+            exit_code=$?
+            
+            # Only consider non-zero exit codes as failures
+            if [ $exit_code -ne 0 ]; then
+                echo "Process ${pids[$i-1]} failed with exit code $exit_code. Log file: $(pwd)/log_${i}.log"
+                return 0
+            else
+                echo "Process ${pids[$i-1]} completed successfully with exit code 0"
+                # Mark this PID as processed by setting it to empty
+                pids[$i-1]=""
+            fi
+        fi
+    done
+    return 1
+}
 
-for pid in ${pids[@]}; do
+# Monitor processes
-    if ! wait $pid; then
+while true; do
-        kill ${pids[@]} 2>/dev/null
+    if print_failed_log; then
-        echo "Process $pid failed, see log_$(echo ${pids[@]} | tr ' ' '\n' | grep -n "^$pid$" | cut -d: -f1).log"
+        # Kill remaining processes
+        for pid in "${pids[@]}"; do
+            if [ -n "$pid" ]; then
+                kill $pid 2>/dev/null || true
+            fi
+        done
         exit 1
     fi
+
+    # Check if all processes have completed
+    all_done=true
+    for pid in "${pids[@]}"; do
+        if [ -n "$pid" ] && kill -0 $pid 2>/dev/null; then
+            all_done=false
+            break
+        fi
+    done
+
+    if $all_done; then
+        echo "All processes completed successfully"
+        exit 0
+    fi
+
+    sleep 0.2
 done

frontend/test-client/src/agent/mock-agent.ts

@@ -64,7 +64,7 @@ export class MockAgent extends MockClient {
 
 					// Let's not ignore errors
 					// eslint-disable-next-line @typescript-eslint/no-floating-promises
-					sleep(1000).then(() => process.exit(1));
+					sleep(100).then(() => process.exit(1));
 
 					break;
 				case LogLevel.WARNING:

frontend/test-client/src/cli.ts

@@ -38,7 +38,9 @@ async function runTest({
 			)
 		);
 	}
+
 	// for debugging
+	// eslint-disable-next-line
 	(globalThis as any).clients = clients;
 
 	try {
@@ -88,11 +90,11 @@ async function runTest({
 }
 
 async function runTests(): Promise<void> {
-	const agentCounts = [2, 10];
+	const agentCounts = [2, 8];
-	const jitterScaleInSeconds = [0, 0.5, 3];
+	const jitterScaleInSeconds = [0.5, 0, 2];
-	const concurrencies = [1, 16];
+	const concurrencies = [1];
-	const iterations = [50, 300];
+	const iterations = [50, 200];
-	const doDeletes = [false];
+	const doDeletes = [true, false];
 
 	for (const agentCount of agentCounts) {
 		for (const concurrency of concurrencies) {
@@ -106,6 +108,7 @@ async function runTests(): Promise<void> {
 							doDeletes: deleteFiles,
 							jitterScaleInSeconds: jitter
 						});
+						return;
 					}
 				}
 			}
@@ -113,15 +116,13 @@ async function runTests(): Promise<void> {
 	}
 }
 
-process.on("uncaughtException", async (error) => {
+process.on("uncaughtException", (error) => {
 	console.error("Uncaught Exception:", error);
-	await sleep(1000);
 	process.exit(1);
 });
 
-process.on("unhandledRejection", async (reason, promise) => {
+process.on("unhandledRejection", (reason, _promise) => {
 	console.error("Unhandled Rejection:", reason);
-	await sleep(1000);
 	process.exit(1);
 });
 
@@ -131,6 +132,5 @@ runTests()
 	})
 	.catch(async (err: unknown) => {
 		console.error(err);
-		await sleep(1000);
 		process.exit(1);
 	});