Simplify syncing logic

frontend/sync-client/package.json

@@ -14,19 +14,17 @@
     },
     "devDependencies": {
         "byte-base64": "^1.1.0",
-        "minimatch": "^10.0.1",
+        "minimatch": "^10.1.1",
-        "p-queue": "^8.1.0",
+        "p-queue": "^9.0.1",
         "reconcile-text": "^0.8.0",
-        "uuid": "^13.0.0",
+        "@types/node": "^25.0.2",
-        "@types/node": "^24.8.1",
+        "ts-loader": "^9.5.4",
-        "ts-loader": "^9.5.2",
         "tslib": "2.8.1",
-        "tsx": "^4.20.6",
+        "tsx": "^4.21.0",
-        "typescript": "5.8.3",
+        "typescript": "5.9.3",
-        "webpack": "^5.99.9",
+        "webpack": "^5.103.0",
         "webpack-cli": "^6.0.1",
         "webpack-merge": "^6.0.1",
-        "@sentry/browser": "^10.8.0",
+        "@sentry/browser": "^10.30.0"
-        "ws": "^8.18.3"
     }
 }

frontend/sync-client/src/consts.ts

@@ -2,5 +2,6 @@ export const TIMEOUT_FOR_MERGING_HISTORY_ENTRIES_IN_SECONDS = 60;
 export const DIFF_CACHE_SIZE_MB = 2;
 export const MAX_LOG_MESSAGE_COUNT = 100000;
 export const MAX_HISTORY_ENTRY_COUNT = 5000;
-export const SUPPORTED_API_VERSION = 2;
+export const SUPPORTED_API_VERSION = 3;
-export const WEBSOCKET_DISCONNECT_TIMEOUT_IN_S = 10;
+export const WEBSOCKET_DISCONNECT_TIMEOUT_IN_SECONDS = 10;
+export const WEBSOCKET_CONNECTION_TIMEOUT_IN_SECONDS = 10;

frontend/sync-client/src/errors/authentication-error.ts

@@ -0,0 +1,6 @@
+export class AuthenticationError extends Error {
+    public constructor(message: string) {
+        super(message);
+        this.name = "AuthenticationError";
+    }
+}

frontend/sync-client/src/errors/file-not-found-error.ts

@@ -0,0 +1,9 @@
+export class FileNotFoundError extends Error {
+    public constructor(
+        message: string,
+        public readonly filePath: string
+    ) {
+        super(message);
+        this.name = "FileNotFoundError";
+    }
+}

frontend/sync-client/src/errors/server-version-mismatch-error.ts

@@ -0,0 +1,6 @@
+export class ServerVersionMismatchError extends Error {
+    public constructor(message: string) {
+        super(message);
+        this.name = "ServerVersionMismatchError";
+    }
+}

frontend/sync-client/src/errors/sync-reset-error.ts

@@ -0,0 +1,6 @@
+export class SyncResetError extends Error {
+    public constructor() {
+        super("SyncClient has been reset, cleaning up");
+        this.name = "SyncResetError";
+    }
+}

frontend/sync-client/src/file-operations/file-operations.test.ts

@@ -23,7 +23,7 @@ class MockServerConfig implements Pick<ServerConfig, "getConfig"> {
 
 class MockDatabase implements Partial<Database> {
     public getLatestDocumentByRelativePath(
-        _find: RelativePath
+        _target: RelativePath
     ): DocumentRecord | undefined {
         // no-op
         return undefined;

frontend/sync-client/src/file-operations/file-operations.ts

@@ -169,9 +169,9 @@ export class FileOperations {
         }
 
         await this.ensureClearPath(newPath);
-
         this.database.move(oldPath, newPath);
         await this.fs.rename(oldPath, newPath);
+
         await this.deletingEmptyParentDirectoriesOfDeletedFile(oldPath);
     }
 

frontend/sync-client/src/file-operations/safe-filesystem-operations.ts

@@ -2,7 +2,7 @@ import type { RelativePath } from "../persistence/database";
 import type { FileSystemOperations } from "./filesystem-operations";
 import type { Logger } from "../tracing/logger";
 import { Locks } from "../utils/data-structures/locks";
-import { FileNotFoundError } from "./file-not-found-error";
+import { FileNotFoundError } from "../errors/file-not-found-error";
 import type { TextWithCursors } from "reconcile-text";
 
 /**
@@ -17,7 +17,7 @@ export class SafeFileSystemOperations implements FileSystemOperations {
         private readonly fs: FileSystemOperations,
         private readonly logger: Logger
     ) {
-        this.locks = new Locks(logger);
+        this.locks = new Locks(SafeFileSystemOperations.name, logger);
     }
 
     public async listFilesRecursively(

frontend/sync-client/src/index.ts

@@ -2,6 +2,7 @@ import { awaitAll } from "./utils/await-all";
 import { logToConsole } from "./utils/debugging/log-to-console";
 import { slowFetchFactory } from "./utils/debugging/slow-fetch-factory";
 import { slowWebSocketFactory } from "./utils/debugging/slow-web-socket-factory";
+import { InMemoryFileSystem } from "./utils/debugging/in-memory-file-system";
 import { getRandomColor } from "./utils/get-random-color";
 import { lineAndColumnToPosition } from "./utils/line-and-column-to-position";
 import { positionToLineAndColumn } from "./utils/position-to-line-and-column";
@@ -27,8 +28,8 @@ export type { PersistenceProvider } from "./persistence/persistence";
 export type { CursorSpan } from "./services/types/CursorSpan";
 export type { ClientCursors } from "./services/types/ClientCursors";
 export type { NetworkConnectionStatus } from "./types/network-connection-status";
-export type { ServerVersionMismatchError } from "./services/server-version-mismatch-error";
+export type { ServerVersionMismatchError } from "./errors/server-version-mismatch-error";
-export type { AuthenticationError } from "./services/authentication-error";
+export type { AuthenticationError } from "./errors/authentication-error";
 export type { MaybeOutdatedClientCursors } from "./types/maybe-outdated-client-cursors";
 export { DocumentSyncStatus } from "./types/document-sync-status";
 export { SyncClient } from "./sync-client";
@@ -37,7 +38,8 @@ export type { TextWithCursors, CursorPosition } from "reconcile-text";
 export const debugging = {
     slowFetchFactory,
     slowWebSocketFactory,
-    logToConsole
+    logToConsole,
+    InMemoryFileSystem
 };
 
 export const utils = {

frontend/sync-client/src/persistence/database.ts

@@ -9,6 +9,7 @@ export type DocumentId = string;
 export type RelativePath = string;
 
 export interface DocumentMetadata {
+    documentId: DocumentId;
     parentVersionId: VaultUpdateId;
     hash: string;
     remoteRelativePath?: RelativePath;
@@ -25,7 +26,6 @@ export interface StoredDocumentMetadata {
 export interface StoredDatabase {
     documents: StoredDocumentMetadata[];
     lastSeenUpdateId: VaultUpdateId | undefined;
-    hasInitialSyncCompleted: boolean;
 }
 
 /**
@@ -36,17 +36,14 @@ export interface StoredDatabase {
  */
 export interface DocumentRecord {
     relativePath: RelativePath;
-    documentId: DocumentId;
     metadata: DocumentMetadata | undefined;
     isDeleted: boolean;
-    updates: Promise<unknown>[];
     parallelVersion: number;
 }
 
 export class Database {
     private documents: DocumentRecord[];
     private lastSeenUpdateIds: CoveredValues;
-    private hasInitialSyncCompleted: boolean;
 
     public constructor(
         private readonly logger: Logger,
@@ -56,16 +53,12 @@ export class Database {
         initialState ??= {};
 
         this.documents =
-            initialState.documents?.map(
+            initialState.documents?.map(({ relativePath, ...metadata }) => ({
-                ({ relativePath, documentId, ...metadata }) => ({
                 relativePath,
-                    documentId,
                 metadata,
                 isDeleted: false,
-                    updates: [],
                 parallelVersion: 0
-                })
+            })) ?? [];
-            ) ?? [];
 
         this.ensureConsistency();
         this.logger.debug(`Loaded ${this.documents.length} documents`);
@@ -79,12 +72,6 @@ export class Database {
         this.documents.forEach((doc) => {
             this.lastSeenUpdateIds.add(doc.metadata?.parentVersionId);
         });
-
-        this.hasInitialSyncCompleted =
-            initialState.hasInitialSyncCompleted ?? false;
-        this.logger.debug(
-            `Loaded hasInitialSyncCompleted: ${this.hasInitialSyncCompleted}`
-        );
     }
 
     public get length(): number {
@@ -127,91 +114,51 @@ export class Database {
 
     public updateDocumentMetadata(
         metadata: {
+            documentId: DocumentId;
             parentVersionId: VaultUpdateId;
             hash: string;
             remoteRelativePath: RelativePath;
         },
-        toUpdate: DocumentRecord
+        target: DocumentRecord
     ): void {
-        if (!this.documents.includes(toUpdate)) {
+        if (!this.documents.includes(target)) {
             throw new Error("Document not found in database");
         }
 
-        toUpdate.metadata = metadata;
+        this.logger.debug(
-
+            `Updating document metadata for ${target.relativePath} from ${JSON.stringify(
-        this.saveInTheBackground();
+                target.metadata,
-    }
+                null,
-
+                2
-    public removeDocumentPromise(promise: Promise<unknown>): void {
+            )} to ${JSON.stringify(metadata, null, 2)}`
-        const entry = this.documents.find(({ updates }) =>
-            updates.includes(promise)
         );
 
-        if (entry === undefined) {
+        target.metadata = metadata;
-            // This method should be idempotent and tolerant of
-            // stragglers calling it after the databse has been reset.
-            return;
-        }
 
-        removeFromArray(entry.updates, promise);
-        // No need to save as Promises don't get serialized
-    }
-
-    public removeDocument(find: DocumentRecord): void {
-        removeFromArray(this.documents, find);
         this.saveInTheBackground();
     }
 
     public getLatestDocumentByRelativePath(
-        find: RelativePath
+        target: RelativePath
     ): DocumentRecord | undefined {
         const candidates = this.documents.filter(
-            ({ relativePath }) => relativePath === find
+            ({ relativePath }) => relativePath === target
         );
         candidates.sort((a, b) => b.parallelVersion - a.parallelVersion); // descending
         return candidates[0];
     }
 
-    public async getResolvedDocumentByRelativePath(
-        relativePath: RelativePath,
-        promise: Promise<unknown>
-    ): Promise<DocumentRecord> {
-        const entry = this.getLatestDocumentByRelativePath(relativePath);
-
-        if (entry === undefined) {
-            throw new Error(
-                `Document not found by relative path: ${relativePath}, ${JSON.stringify(
-                    this.documents,
-                    null,
-                    2
-                )}`
-            );
-        }
-
-        const currentPromises = entry.updates;
-        entry.updates = [...currentPromises, promise];
-        await awaitAll(currentPromises);
-
-        return entry;
-    }
-
     public createNewPendingDocument(
-        documentId: DocumentId,
+        relativePath: RelativePath
-        relativePath: RelativePath,
-        promise: Promise<unknown>
     ): DocumentRecord {
-        this.logger.debug(
+        this.logger.debug(`Creating new pending document: ${relativePath}`);
-            `Creating new pending document: ${relativePath} (${documentId})`
-        );
         const previousEntry =
             this.getLatestDocumentByRelativePath(relativePath);
 
         const entry = {
             relativePath,
-            documentId,
             metadata: undefined,
             isDeleted: false,
-            updates: [promise],
             parallelVersion:
                 previousEntry?.parallelVersion === undefined
                     ? 0
@@ -219,39 +166,18 @@ export class Database {
         };
 
         this.documents.push(entry);
-        this.saveInTheBackground();
 
-        return entry;
+        // no need to save as we only save documents which have metadata
-    }
-
-    public createNewEmptyDocument(
-        documentId: DocumentId,
-        parentVersionId: VaultUpdateId,
-        relativePath: RelativePath
-    ): DocumentRecord {
-        const entry = {
-            relativePath,
-            documentId,
-            metadata: {
-                parentVersionId,
-                hash: EMPTY_HASH,
-                remoteRelativePath: relativePath
-            },
-            isDeleted: false,
-            updates: [],
-            parallelVersion: 0
-        };
-
-        this.documents.push(entry);
-        this.saveInTheBackground();
 
         return entry;
     }
 
     public getDocumentByDocumentId(
-        find: DocumentId
+        target: DocumentId
     ): DocumentRecord | undefined {
-        return this.documents.find(({ documentId }) => documentId === find);
+        return this.documents.find(
+            ({ metadata }) => metadata?.documentId === target
+        );
     }
 
     public move(
@@ -274,7 +200,7 @@ export class Database {
         }
 
         oldDocument.relativePath = newRelativePath;
-        // We're in a strange state where the target of the move has just got deleted,
+        // We might be in a strange state where the target of the move has just got deleted,
         // however, its metadata might already have a bunch of updates queued up for
         // the document at the new location. We need to keep these updates.
         oldDocument.parallelVersion =
@@ -286,19 +212,13 @@ export class Database {
     public delete(relativePath: RelativePath): void {
         const candidate = this.getLatestDocumentByRelativePath(relativePath);
         if (candidate === undefined) {
-            throw new Error(
+            return;
-                `Document not found by relative path: ${relativePath}`
-            );
         }
         candidate.isDeleted = true;
     }
 
-    public getHasInitialSyncCompleted(): boolean {
+    public removeDocument(target: DocumentRecord): void {
-        return this.hasInitialSyncCompleted;
+        removeFromArray(this.documents, target);
-    }
-
-    public setHasInitialSyncCompleted(value: boolean): void {
-        this.hasInitialSyncCompleted = value;
         this.saveInTheBackground();
     }
 
@@ -324,38 +244,45 @@ export class Database {
         this.lastSeenUpdateIds = new CoveredValues(
             0 // the first updateId will be 1 which is the first integer after -1
         );
-        this.hasInitialSyncCompleted = false;
         this.saveInTheBackground();
     }
 
     public async save(): Promise<void> {
         return this.saveData({
             documents: this.resolvedDocuments.map(
-                ({ relativePath, documentId, metadata }) => ({
+                ({ relativePath, metadata }) => ({
-                    documentId,
                     relativePath,
                     // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
                     ...metadata! // `resolvedDocuments` only returns docs with metadata set
                 })
             ),
-            lastSeenUpdateId: this.lastSeenUpdateIds.min,
+            lastSeenUpdateId: this.lastSeenUpdateIds.min
-            hasInitialSyncCompleted: this.hasInitialSyncCompleted
         });
     }
 
     private ensureConsistency(): void {
         const idToPath = new Map<string, string[]>();
 
-        this.resolvedDocuments.forEach(({ relativePath, documentId }) => {
+        this.resolvedDocuments.forEach(({ relativePath, metadata }) => {
-            idToPath.set(documentId, [
+            if (metadata === undefined) {
-                ...(idToPath.get(documentId) ?? []),
+                return;
+            }
+            idToPath.set(metadata.documentId, [
+                ...(idToPath.get(metadata.documentId) ?? []),
                 relativePath
             ]);
         });
 
         const duplicates = Array.from(idToPath.entries())
             .filter(([_, paths]) => paths.length > 1)
-            .map(([id, paths]) => `${id} (${paths.join(", ")})`);
+            .map(([id, paths]) => {
+                let details = "";
+                for (const path of paths) {
+                    const doc = this.getLatestDocumentByRelativePath(path);
+                    details += `\n- ${JSON.stringify(doc, null, 2)}`;
+                }
+                return `${id} (${paths.join(", ")}): ${details}`;
+            });
 
         if (duplicates.length > 0) {
             throw new Error(

frontend/sync-client/src/persistence/settings.ts

@@ -38,7 +38,7 @@ export class Settings {
     >();
 
     private settings: SyncSettings;
-    private readonly lock: Lock = new Lock();
+    private readonly lock: Lock;
 
     public constructor(
         private readonly logger: Logger,
@@ -50,6 +50,8 @@ export class Settings {
             ...(initialState ?? {})
         };
 
+        this.lock = new Lock(Settings.name, this.logger);
+
         this.logger.debug(
             `Loaded settings: ${JSON.stringify(this.settings, null, 2)}`
         );

frontend/sync-client/src/services/fetch-controller.test.ts

@@ -3,7 +3,7 @@ import { describe, it, mock, beforeEach, afterEach } from "node:test";
 import assert from "node:assert";
 import { FetchController } from "./fetch-controller";
 import { Logger } from "../tracing/logger";
-import { SyncResetError } from "./sync-reset-error";
+import { SyncResetError } from "../errors/sync-reset-error";
 import { sleep } from "../utils/sleep";
 
 describe("FetchController", () => {

frontend/sync-client/src/services/fetch-controller.ts

@@ -1,6 +1,6 @@
 import type { Logger } from "../tracing/logger";
 import { createPromise } from "../utils/create-promise";
-import { SyncResetError } from "./sync-reset-error";
+import { SyncResetError } from "../errors/sync-reset-error";
 
 /**
  * Offers a resettable fetch implementation that waits until syncing is enabled

frontend/sync-client/src/services/server-config.ts

@@ -1,6 +1,6 @@
 import { SUPPORTED_API_VERSION } from "../consts";
-import { AuthenticationError } from "./authentication-error";
+import { AuthenticationError } from "../errors/authentication-error";
-import { ServerVersionMismatchError } from "./server-version-mismatch-error";
+import { ServerVersionMismatchError } from "../errors/server-version-mismatch-error";
 import type { SyncService } from "./sync-service";
 import type { PingResponse } from "./types/PingResponse";
 
@@ -34,11 +34,6 @@ export class ServerConfig {
         }
     }
 
-    // warm the cache
-    public async initialize(): Promise<void> {
-        await this.getConfig();
-    }
-
     public async checkConnection(forceUpdate = false): Promise<{
         isSuccessful: boolean;
         message: string;

frontend/sync-client/src/services/sync-service.ts

@@ -8,7 +8,7 @@ import type { Logger } from "../tracing/logger";
 import type { Settings } from "../persistence/settings";
 import type { FetchController } from "./fetch-controller";
 import { sleep } from "../utils/sleep";
-import { SyncResetError } from "./sync-reset-error";
+import { SyncResetError } from "../errors/sync-reset-error";
 import type { SerializedError } from "./types/SerializedError";
 import type { DocumentVersionWithoutContent } from "./types/DocumentVersionWithoutContent";
 import type { DocumentUpdateResponse } from "./types/DocumentUpdateResponse";
@@ -66,19 +66,15 @@ export class SyncService {
     }
 
     public async create({
-        documentId,
         relativePath,
         contentBytes
     }: {
-        documentId?: DocumentId;
         relativePath: RelativePath;
         contentBytes: Uint8Array;
-    }): Promise<DocumentVersionWithoutContent> {
+    }): Promise<DocumentUpdateResponse> {
         return this.retryForever(async () => {
             const formData = new FormData();
-            if (documentId !== undefined) {
+
-                formData.append("document_id", documentId);
-            }
             formData.append("relative_path", relativePath);
             formData.append(
                 "content",
@@ -86,7 +82,7 @@ export class SyncService {
             );
 
             this.logger.debug(
-                `Creating document with id ${documentId} and relative path ${relativePath}`
+                `Creating document with relative path ${relativePath}`
             );
 
             const response = await this.client(this.getUrl("/documents"), {
@@ -103,8 +99,8 @@ export class SyncService {
                 );
             }
 
-            const result: DocumentVersionWithoutContent =
+            const result: DocumentUpdateResponse =
-                (await response.json()) as DocumentVersionWithoutContent; // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion
+                (await response.json()) as DocumentUpdateResponse; // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion
 
             this.logger.debug(`Created document ${JSON.stringify(result)}`);
 

frontend/sync-client/src/services/types/ClientCursors.ts

@@ -1,4 +1,8 @@
 // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
 import type { DocumentWithCursors } from "./DocumentWithCursors";
 
-export interface ClientCursors { userName: string, deviceId: string, documentsWithCursors: DocumentWithCursors[], }
+export interface ClientCursors {
+    userName: string;
+    deviceId: string;
+    documentsWithCursors: DocumentWithCursors[];
+}

frontend/sync-client/src/services/types/CreateDocumentVersion.ts

@@ -1,3 +1,6 @@
 // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
 
-export interface CreateDocumentVersion { relative_path: string, content: number[], }
+export interface CreateDocumentVersion {
+    relative_path: string;
+    content: number[];
+}

frontend/sync-client/src/services/types/CursorPositionFromClient.ts

@@ -1,4 +1,6 @@
 // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
 import type { DocumentWithCursors } from "./DocumentWithCursors";
 
-export interface CursorPositionFromClient { documentsWithCursors: DocumentWithCursors[], }
+export interface CursorPositionFromClient {
+    documentsWithCursors: DocumentWithCursors[];
+}

frontend/sync-client/src/services/types/CursorPositionFromServer.ts

@@ -1,4 +1,6 @@
 // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
 import type { ClientCursors } from "./ClientCursors";
 
-export interface CursorPositionFromServer { clients: ClientCursors[], }
+export interface CursorPositionFromServer {
+    clients: ClientCursors[];
+}

frontend/sync-client/src/services/types/CursorSpan.ts

@@ -1,3 +1,6 @@
 // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
 
-export interface CursorSpan { start: number, end: number, }
+export interface CursorSpan {
+    start: number;
+    end: number;
+}

frontend/sync-client/src/services/types/DeleteDocumentVersion.ts

@@ -1,3 +1,5 @@
 // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
 
-export type DeleteDocumentVersion = Record<string, never>;
+export interface DeleteDocumentVersion {
+    relativePath: string;
+}

frontend/sync-client/src/services/types/DocumentUpdateResponse.ts

@@ -5,4 +5,6 @@ import type { DocumentVersionWithoutContent } from "./DocumentVersionWithoutCont
 /**
  * Response to an update document request.
  */
-export type DocumentUpdateResponse = { "type": "FastForwardUpdate" } & DocumentVersionWithoutContent | { "type": "MergingUpdate" } & DocumentVersion;
+export type DocumentUpdateResponse =
+    | ({ type: "FastForwardUpdate" } & DocumentVersionWithoutContent)
+    | ({ type: "MergingUpdate" } & DocumentVersion);

frontend/sync-client/src/services/types/DocumentVersion.ts

@@ -1,3 +1,12 @@
 // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
 
-export interface DocumentVersion { vaultUpdateId: number, documentId: string, relativePath: string, updatedDate: string, contentBase64: string, isDeleted: boolean, userId: string, deviceId: string, }
+export interface DocumentVersion {
+    vaultUpdateId: number;
+    documentId: string;
+    relativePath: string;
+    updatedDate: string;
+    contentBase64: string;
+    isDeleted: boolean;
+    userId: string;
+    deviceId: string;
+}

frontend/sync-client/src/services/types/DocumentVersionWithoutContent.ts

@@ -1,3 +1,12 @@
 // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
 
-export interface DocumentVersionWithoutContent { vaultUpdateId: number, documentId: string, relativePath: string, updatedDate: string, isDeleted: boolean, userId: string, deviceId: string, contentSize: number, }
+export interface DocumentVersionWithoutContent {
+    vaultUpdateId: number;
+    documentId: string;
+    relativePath: string;
+    updatedDate: string;
+    isDeleted: boolean;
+    userId: string;
+    deviceId: string;
+    contentSize: number;
+}

frontend/sync-client/src/services/types/DocumentWithCursors.ts

@@ -1,4 +1,9 @@
 // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
 import type { CursorSpan } from "./CursorSpan";
 
-export interface DocumentWithCursors { vault_update_id: number | null, document_id: string, relative_path: string, cursors: CursorSpan[], }
+export interface DocumentWithCursors {
+    vault_update_id: number | null;
+    document_id: string;
+    relative_path: string;
+    cursors: CursorSpan[];
+}

frontend/sync-client/src/services/types/FetchLatestDocumentsResponse.ts

@@ -4,8 +4,10 @@ import type { DocumentVersionWithoutContent } from "./DocumentVersionWithoutCont
 /**
  * Response to a fetch latest documents request.
  */
-export interface FetchLatestDocumentsResponse { latestDocuments: DocumentVersionWithoutContent[], 
+export interface FetchLatestDocumentsResponse {
+    latestDocuments: DocumentVersionWithoutContent[];
     /**
      * The update ID of the latest document in the response.
      */
-lastUpdateId: bigint, }
+    lastUpdateId: bigint;
+}

frontend/sync-client/src/services/types/PingResponse.ts

@@ -7,18 +7,19 @@ export interface PingResponse {
     /**
      * Semantic version of the server.
      */
-serverVersion: string, 
+    serverVersion: string;
     /**
      * Whether the client is authenticated based on the sent Authorization
      * header.
      */
-isAuthenticated: boolean, 
+    isAuthenticated: boolean;
     /**
      * List of file extensions that are allowed to be merged.
      */
-mergeableFileExtensions: string[], 
+    mergeableFileExtensions: string[];
     /**
      * API version ensuring backwards & forwards compatibility between the client
      * and server.
      */
-supportedApiVersion: number, }
+    supportedApiVersion: number;
+}

frontend/sync-client/src/services/types/SerializedError.ts

@@ -1,3 +1,7 @@
 // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
 
-export interface SerializedError { errorType: string, message: string, causes: string[], }
+export interface SerializedError {
+    errorType: string;
+    message: string;
+    causes: string[];
+}

frontend/sync-client/src/services/types/UpdateTextDocumentVersion.ts

@@ -1,3 +1,7 @@
 // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
 
-export interface UpdateTextDocumentVersion { parentVersionId: number, relativePath: string, content: (number | string)[], }
+export interface UpdateTextDocumentVersion {
+    parentVersionId: number;
+    relativePath: string;
+    content: (number | string)[];
+}

frontend/sync-client/src/services/types/WebSocketClientMessage.ts

@@ -2,4 +2,6 @@
 import type { CursorPositionFromClient } from "./CursorPositionFromClient";
 import type { WebSocketHandshake } from "./WebSocketHandshake";
 
-export type WebSocketClientMessage = { "type": "handshake" } & WebSocketHandshake | { "type": "cursorPositions" } & CursorPositionFromClient;
+export type WebSocketClientMessage =
+    | ({ type: "handshake" } & WebSocketHandshake)
+    | ({ type: "cursorPositions" } & CursorPositionFromClient);

frontend/sync-client/src/services/types/WebSocketHandshake.ts

@@ -1,3 +1,7 @@
 // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
 
-export interface WebSocketHandshake { token: string, deviceId: string, lastSeenVaultUpdateId: number | null, }
+export interface WebSocketHandshake {
+    token: string;
+    deviceId: string;
+    lastSeenVaultUpdateId: number | null;
+}

frontend/sync-client/src/services/types/WebSocketServerMessage.ts

@@ -2,4 +2,6 @@
 import type { CursorPositionFromServer } from "./CursorPositionFromServer";
 import type { WebSocketVaultUpdate } from "./WebSocketVaultUpdate";
 
-export type WebSocketServerMessage = { "type": "vaultUpdate" } & WebSocketVaultUpdate | { "type": "cursorPositions" } & CursorPositionFromServer;
+export type WebSocketServerMessage =
+    | ({ type: "vaultUpdate" } & WebSocketVaultUpdate)
+    | ({ type: "cursorPositions" } & CursorPositionFromServer);

frontend/sync-client/src/services/types/WebSocketVaultUpdate.ts

@@ -1,4 +1,7 @@
 // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
 import type { DocumentVersionWithoutContent } from "./DocumentVersionWithoutContent";
 
-export interface WebSocketVaultUpdate { documents: DocumentVersionWithoutContent[], isInitialSync: boolean, }
+export interface WebSocketVaultUpdate {
+    documents: DocumentVersionWithoutContent[];
+    isInitialSync: boolean;
+}

frontend/sync-client/src/services/websocket-manager.test.ts

@@ -4,8 +4,6 @@ import assert from "node:assert";
 import { WebSocketManager } from "./websocket-manager";
 import type { Logger } from "../tracing/logger";
 import type { Settings } from "../persistence/settings";
-// eslint-disable-next-line @typescript-eslint/no-require-imports
-const WebSocket = require("ws") as typeof globalThis.WebSocket;
 
 class MockCloseEvent extends Event {
     public code: number;
@@ -91,10 +89,8 @@ function createMockFn<T extends (...args: unknown[]) => unknown>(
 describe("WebSocketManager", () => {
     let mockLogger: Logger = undefined as unknown as Logger;
     let mockSettings: Settings = undefined as unknown as Settings;
-    let deviceId = "test-device-123";
 
     beforeEach(() => {
-        deviceId = "test-device-123";
         const noop = (): void => {
             // Intentionally empty for mock
         };
@@ -116,7 +112,6 @@ describe("WebSocketManager", () => {
 
     it("cleans up promises after message handling", async () => {
         const manager = new WebSocketManager(
-            deviceId,
             mockLogger,
             mockSettings,
             MockWebSocket as unknown as typeof WebSocket
@@ -146,7 +141,6 @@ describe("WebSocketManager", () => {
 
     it("cleans up cursor position promises", async () => {
         const manager = new WebSocketManager(
-            deviceId,
             mockLogger,
             mockSettings,
             MockWebSocket as unknown as typeof WebSocket
@@ -176,7 +170,6 @@ describe("WebSocketManager", () => {
 
     it("logs handshake send errors", async () => {
         const manager = new WebSocketManager(
-            deviceId,
             mockLogger,
             mockSettings,
             MockWebSocket as unknown as typeof WebSocket
@@ -205,7 +198,6 @@ describe("WebSocketManager", () => {
 
     it("completes stop with timeout protection", async () => {
         const manager = new WebSocketManager(
-            deviceId,
             mockLogger,
             mockSettings,
             MockWebSocket as unknown as typeof WebSocket
@@ -220,7 +212,6 @@ describe("WebSocketManager", () => {
 
     it("clears old handlers on reconnection", async () => {
         const manager = new WebSocketManager(
-            deviceId,
             mockLogger,
             mockSettings,
             MockWebSocket as unknown as typeof WebSocket
@@ -257,7 +248,6 @@ describe("WebSocketManager", () => {
 
     it("tracks message handling promises", async () => {
         const manager = new WebSocketManager(
-            deviceId,
             mockLogger,
             mockSettings,
             MockWebSocket as unknown as typeof WebSocket

frontend/sync-client/src/services/websocket-manager.ts

@@ -6,7 +6,10 @@ import type { CursorPositionFromClient } from "./types/CursorPositionFromClient"
 import type { ClientCursors } from "./types/ClientCursors";
 import { createPromise } from "../utils/create-promise";
 import type { WebSocketVaultUpdate } from "./types/WebSocketVaultUpdate";
-import { WEBSOCKET_DISCONNECT_TIMEOUT_IN_S } from "../consts";
+import {
+    WEBSOCKET_DISCONNECT_TIMEOUT_IN_SECONDS,
+    WEBSOCKET_CONNECTION_TIMEOUT_IN_SECONDS
+} from "../consts";
 import { removeFromArray } from "../utils/remove-from-array";
 import { EventListeners } from "../utils/data-structures/event-listeners";
 import { awaitAll } from "../utils/await-all";
@@ -27,32 +30,17 @@ export class WebSocketManager {
     private isStopped = true;
     private resolveDisconnectingPromise: null | (() => unknown) = null;
     private reconnectTimeoutId: ReturnType<typeof setTimeout> | undefined;
+    private connectionTimeoutId: ReturnType<typeof setTimeout> | undefined;
 
     private readonly outstandingPromises: Promise<unknown>[] = [];
 
     private webSocket: WebSocket | undefined;
-    private readonly webSocketFactoryImplementation: typeof globalThis.WebSocket;
 
     public constructor(
-        private readonly deviceId: string,
         private readonly logger: Logger,
         private readonly settings: Settings,
-        webSocketImplementation?: typeof globalThis.WebSocket
+        private readonly webSocketFactoryImplementation: typeof globalThis.WebSocket = WebSocket
-    ) {
+    ) {}
-        if (webSocketImplementation) {
-            this.webSocketFactoryImplementation = webSocketImplementation;
-        } else {
-            if (
-                typeof globalThis !== "undefined" &&
-                typeof globalThis.WebSocket === "undefined"
-            ) {
-                // eslint-disable-next-line
-                this.webSocketFactoryImplementation = require("ws"); // polyfill for WebSocket in Node.js
-            } else {
-                this.webSocketFactoryImplementation = WebSocket;
-            }
-        }
-    }
 
     public get isWebSocketConnected(): boolean {
         return (
@@ -77,6 +65,11 @@ export class WebSocketManager {
             this.reconnectTimeoutId = undefined;
         }
 
+        if (this.connectionTimeoutId !== undefined) {
+            clearTimeout(this.connectionTimeoutId);
+            this.connectionTimeoutId = undefined;
+        }
+
         this.webSocket?.close(1000, "WebSocketManager has been stopped");
 
         // eslint-disable-next-line @typescript-eslint/init-declarations
@@ -85,10 +78,10 @@ export class WebSocketManager {
             timeoutId = setTimeout(() => {
                 reject(
                     new Error(
-                        `Timeout waiting for WebSocket to close after ${WEBSOCKET_DISCONNECT_TIMEOUT_IN_S} seconds`
+                        `Timeout waiting for WebSocket to close after ${WEBSOCKET_DISCONNECT_TIMEOUT_IN_SECONDS} seconds`
                     )
                 );
-            }, WEBSOCKET_DISCONNECT_TIMEOUT_IN_S * 1000);
+            }, WEBSOCKET_DISCONNECT_TIMEOUT_IN_SECONDS * 1000);
         });
 
         try {
@@ -171,7 +164,10 @@ export class WebSocketManager {
                 this.webSocket.onclose = null;
                 this.webSocket.onmessage = null;
                 this.webSocket.onerror = null;
-                this.webSocket.close();
+                this.webSocket.close(
+                    1000,
+                    "Closing previous WebSocket connection"
+                );
             } catch (e) {
                 this.logger.error(
                     `Failed to close previous WebSocket connection: ${e}`
@@ -187,7 +183,22 @@ export class WebSocketManager {
 
         this.webSocket = new this.webSocketFactoryImplementation(wsUri);
 
+        // Set connection timeout to handle cases where server is down and the WebSocket connection won't open
+        this.connectionTimeoutId = setTimeout(() => {
+            this.connectionTimeoutId = undefined;
+            this.logger.warn(
+                `WebSocket connection timeout after ${WEBSOCKET_CONNECTION_TIMEOUT_IN_SECONDS} seconds`
+            );
+            // Force close to trigger onclose handler which will schedule reconnection
+            this.webSocket?.close(1000, "Connection timeout");
+        }, WEBSOCKET_CONNECTION_TIMEOUT_IN_SECONDS * 1000);
+
         this.webSocket.onopen = (): void => {
+            if (this.connectionTimeoutId !== undefined) {
+                clearTimeout(this.connectionTimeoutId);
+                this.connectionTimeoutId = undefined;
+            }
+
             // Check if we've been stopped while connecting
             if (this.isStopped) {
                 this.webSocket?.close(
@@ -231,7 +242,18 @@ export class WebSocketManager {
             }
         };
 
+        this.webSocket.onerror = (error): void => {
+            this.logger.warn(
+                `WebSocket error occurred: ${error instanceof ErrorEvent ? error.message : "Unknown error"}`
+            );
+        };
+
         this.webSocket.onclose = (event): void => {
+            if (this.connectionTimeoutId !== undefined) {
+                clearTimeout(this.connectionTimeoutId);
+                this.connectionTimeoutId = undefined;
+            }
+
             this.logger.warn(
                 `WebSocket closed with code ${event.code} (${event.reason == "" ? "unknown reason" : event.reason})`
             );
@@ -241,10 +263,13 @@ export class WebSocketManager {
                 this.resolveDisconnectingPromise?.();
                 this.resolveDisconnectingPromise = null;
             } else {
+                const delay =
+                    this.settings.getSettings().webSocketRetryIntervalMs;
+                this.logger.info(`Reconnecting to WebSocket in ${delay}ms...`);
                 this.reconnectTimeoutId = setTimeout(() => {
                     this.reconnectTimeoutId = undefined;
                     this.initializeWebSocket();
-                }, this.settings.getSettings().webSocketRetryIntervalMs);
+                }, delay);
             }
         };
     }

frontend/sync-client/src/sync-client.ts

@@ -29,7 +29,6 @@ import { ServerConfig } from "./services/server-config";
 import type { EventListeners } from "./utils/data-structures/event-listeners";
 
 export class SyncClient {
-    private hasStartedOfflineSync = false;
     private hasFinishedOfflineSync = false;
     private hasStarted = false;
     private hasBeenDestroyed = false;
@@ -38,12 +37,12 @@ export class SyncClient {
     private readonly eventUnsubscribers: (() => void)[] = [];
 
     private constructor(
+        public readonly logger: Logger,
         private readonly history: SyncHistory,
         private readonly settings: Settings,
         private readonly database: Database,
         private readonly syncer: Syncer,
         private readonly webSocketManager: WebSocketManager,
-        public readonly logger: Logger,
         private readonly fetchController: FetchController,
         private readonly cursorTracker: CursorTracker,
         private readonly fileChangeNotifier: FileChangeNotifier,
@@ -195,7 +194,6 @@ export class SyncClient {
         );
 
         const webSocketManager = new WebSocketManager(
-            deviceId,
             logger,
             settings,
             webSocket
@@ -206,7 +204,6 @@ export class SyncClient {
             logger,
             database,
             settings,
-            syncService,
             webSocketManager,
             fileOperations,
             unrestrictedSyncer
@@ -214,18 +211,19 @@ export class SyncClient {
 
         const fileChangeNotifier = new FileChangeNotifier();
         const cursorTracker = new CursorTracker(
+            logger,
             database,
             webSocketManager,
             fileOperations,
             fileChangeNotifier
         );
         const client = new SyncClient(
+            logger,
             history,
             settings,
             database,
             syncer,
             webSocketManager,
-            logger,
             fetchController,
             cursorTracker,
             fileChangeNotifier,
@@ -337,12 +335,13 @@ export class SyncClient {
         this.database.reset();
         await this.database.save(); // ensure the new database reads as empty
         this.resetInMemoryState();
-        this.hasStartedOfflineSync = false;
         this.hasFinishedOfflineSync = false;
         this.serverConfig.reset();
 
+        if (this.settings.getSettings().isSyncEnabled) {
             await this.startSyncing();
         }
+    }
 
     public getSettings(): SyncSettings {
         return this.settings.getSettings();
@@ -410,12 +409,7 @@ export class SyncClient {
             return DocumentSyncStatus.SYNCING;
         }
 
-        const document =
+        return this.syncer.hasPendingOperationsForDocument(relativePath)
-            this.database.getLatestDocumentByRelativePath(relativePath);
-        if (document === undefined) {
-            return DocumentSyncStatus.SYNCING;
-        }
-        return document.updates.length > 0
             ? DocumentSyncStatus.SYNCING
             : DocumentSyncStatus.UP_TO_DATE;
     }
@@ -473,18 +467,17 @@ export class SyncClient {
         this.checkIfDestroyed("startSyncing");
         this.fetchController.finishReset();
 
-        await this.serverConfig.initialize();
+        // warm the cache
-        this.webSocketManager.start();
+        await this.serverConfig.getConfig();
 
-        if (!this.hasStartedOfflineSync) {
-            this.hasStartedOfflineSync = true;
         await this.syncer.scheduleSyncForOfflineChanges();
-        }
+        this.webSocketManager.start();
 
         this.hasFinishedOfflineSync = true;
     }
 
     private async pause(): Promise<void> {
+        this.hasFinishedOfflineSync = false;
         this.fetchController.startReset();
         await this.webSocketManager.stop();
         await this.waitUntilFinished();

frontend/sync-client/src/sync-operations/cursor-tracker.ts

@@ -10,6 +10,7 @@ import { hash } from "../utils/hash";
 import type { FileChangeNotifier } from "./file-change-notifier";
 import { Lock } from "../utils/data-structures/locks";
 import { EventListeners } from "../utils/data-structures/event-listeners";
+import { Logger } from "../tracing/logger";
 
 // Cursor positions are updated separately from documents. However, a given cursor position is only
 // valid within a certain version of the document it belongs to. This class tracks previous and the latest
@@ -22,7 +23,7 @@ export class CursorTracker {
         (cursors: MaybeOutdatedClientCursors[]) => unknown
     >();
 
-    private readonly updateLock = new Lock();
+    private readonly updateLock: Lock;
 
     private knownRemoteCursors: (ClientCursors & {
         upToDateness: DocumentUpToDateness;
@@ -33,11 +34,14 @@ export class CursorTracker {
         [];
 
     public constructor(
+        private readonly logger: Logger,
         private readonly database: Database,
         private readonly webSocketManager: WebSocketManager,
         private readonly fileOperations: FileOperations,
         private readonly fileChangeNotifier: FileChangeNotifier
     ) {
+        this.updateLock = new Lock(CursorTracker.name, logger);
+
         this.webSocketManager.onRemoteCursorsUpdateReceived.add(
             async (clientCursors) => {
                 await this.updateLock.withLock(async () => {
@@ -113,7 +117,7 @@ export class CursorTracker {
 
             documentsWithCursors.push({
                 relative_path: relativePath,
-                document_id: record.documentId,
+                document_id: record.metadata.documentId,
                 vault_update_id: record.metadata.parentVersionId,
                 cursors: cursors.map(({ start, end }) => ({
                     start: Math.min(start, end),

frontend/sync-client/src/sync-operations/syncer.ts

@@ -4,17 +4,14 @@ import type {
     DocumentRecord,
     RelativePath
 } from "../persistence/database";
-import type { SyncService } from "../services/sync-service";
 import type { Logger } from "../tracing/logger";
 import PQueue from "p-queue";
 import { hash } from "../utils/hash";
-import { v4 as uuidv4 } from "uuid";
 import type { Settings } from "../persistence/settings";
 import type { FileOperations } from "../file-operations/file-operations";
 import { findMatchingFile } from "../utils/find-matching-file";
 import type { UnrestrictedSyncer } from "./unrestricted-syncer";
-import { createPromise } from "../utils/create-promise";
+import { SyncResetError } from "../errors/sync-reset-error";
-import { SyncResetError } from "../services/sync-reset-error";
 import { Locks } from "../utils/data-structures/locks";
 import type { DocumentVersionWithoutContent } from "../services/types/DocumentVersionWithoutContent";
 import type { WebSocketVaultUpdate } from "../services/types/WebSocketVaultUpdate";
@@ -28,7 +25,7 @@ export class Syncer {
         (remainingOperations: number) => unknown
     >();
 
-    private readonly remoteDocumentsLock: Locks<DocumentId>;
+    public readonly updatedDocumentsByPathAndKeysLocks: Locks<string>; // can be DocumentId or RelativePath
 
     // FIFO to limit the number of concurrent sync operations
     private readonly syncQueue: PQueue;
@@ -42,16 +39,18 @@ export class Syncer {
         private readonly logger: Logger,
         private readonly database: Database,
         private readonly settings: Settings,
-        private readonly syncService: SyncService,
         private readonly webSocketManager: WebSocketManager,
         private readonly operations: FileOperations,
-        private readonly internalSyncer: UnrestrictedSyncer
+        private readonly unrestrictedSyncer: UnrestrictedSyncer
     ) {
         this.syncQueue = new PQueue({
             concurrency: settings.getSettings().syncConcurrency
         });
 
-        this.remoteDocumentsLock = new Locks<DocumentId>(this.logger);
+        this.updatedDocumentsByPathAndKeysLocks = new Locks<DocumentId>(
+            Syncer.name,
+            this.logger
+        );
 
         settings.onSettingsChanged.add((newSettings, oldSettings) => {
             if (newSettings.syncConcurrency !== oldSettings.syncConcurrency) {
@@ -83,52 +82,50 @@ export class Syncer {
         return this._isFirstSyncComplete;
     }
 
+    public hasPendingOperationsForDocument(relativePath: string): boolean {
+        return this.updatedDocumentsByPathAndKeysLocks.isLocked(relativePath);
+    }
+
     public async syncLocallyCreatedFile(
         relativePath: RelativePath
     ): Promise<void> {
+        // check whether someone else has already created the document in the database
         if (
             this.database.getLatestDocumentByRelativePath(relativePath)
                 ?.isDeleted === false
         ) {
+            // This is likely a consequence of us creating a file because of a remote update
+            // which triggered a local create, so we don't need to do anything here.
             this.logger.debug(
                 `Document ${relativePath} already exists in the database, skipping`
             );
             return;
         }
 
-        const [promise, resolve, reject] = createPromise();
+        const document = this.database.createNewPendingDocument(relativePath);
 
-        const id = uuidv4();
+        await this.enqueueSyncOperation(
-        const document = this.database.createNewPendingDocument(
+            async () =>
-            id,
+                this.unrestrictedSyncer.unrestrictedSyncLocallyCreatedOrUpdatedFile(
-            relativePath,
+                    {
-            promise
+                        document
-        );
-
-        try {
-            await this.syncQueue.add(async () =>
-                this.internalSyncer.unrestrictedSyncLocallyCreatedFile(document)
-            );
-
-            resolve();
-        } catch (e) {
-            reject(e);
-        } finally {
-            this.database.removeDocumentPromise(promise);
                     }
+                ),
+            [relativePath]
+        );
     }
 
     public async syncLocallyDeletedFile(
         relativePath: RelativePath
     ): Promise<void> {
-        if (
+        let document =
-            this.database.getLatestDocumentByRelativePath(relativePath)
+            this.database.getLatestDocumentByRelativePath(relativePath);
-                ?.isDeleted === true
+
-        ) {
+        if (document == null || document.isDeleted === true) {
             // This is must be a consequence of us deleting a file because of a remote update
             // which triggered a local delete, so we don't need to do anything here.
             this.logger.debug(
-                `Document ${relativePath} has already been markes as deleted, skipping`
+                `Document ${relativePath} has already been marked as deleted, skipping`
             );
             return;
         }
@@ -137,26 +134,13 @@ export class Syncer {
         // document which finishes after the delete has succeeded and would introduce a phantom metadata record.
         this.database.delete(relativePath);
 
-        const [promise, resolve, reject] = createPromise();
+        await this.enqueueSyncOperation(async () => {
-
+            await this.unrestrictedSyncer.unrestrictedSyncLocallyDeletedFile(
-        const document = await this.database.getResolvedDocumentByRelativePath(
+                document
-            relativePath,
-            promise
             );
 
-        try {
-            await this.syncQueue.add(async () =>
-                this.internalSyncer.unrestrictedSyncLocallyDeletedFile(document)
-            );
-
-            resolve();
-
             this.database.removeDocument(document);
-        } catch (e) {
+        }, [document?.metadata?.documentId, relativePath]);
-            reject(e);
-        } finally {
-            this.database.removeDocumentPromise(promise);
-        }
     }
 
     public async syncLocallyUpdatedFile({
@@ -166,38 +150,10 @@ export class Syncer {
         oldPath?: RelativePath;
         relativePath: RelativePath;
     }): Promise<void> {
-        if (oldPath !== undefined) {
+        const document =
-            // We might have moved the document in the database before calling this method,
+            this.database.getLatestDocumentByRelativePath(oldPath ?? relativePath);
-            // in that case, we mustn't move it again.
-            if (
-                this.database.getLatestDocumentByRelativePath(relativePath) ===
-                    undefined ||
-                this.database.getLatestDocumentByRelativePath(relativePath)
-                    ?.isDeleted === true
-            ) {
-                if (oldPath === relativePath) {
-                    throw new Error(
-                        `Old path and new path are the same: ${oldPath}`
-                    );
-                }
-
-                this.database.move(oldPath, relativePath);
-            }
-        }
-
-        let document =
-            this.database.getLatestDocumentByRelativePath(relativePath);
-
-        if (
-            oldPath !== undefined &&
-            document?.metadata?.remoteRelativePath === relativePath
-        ) {
-            this.logger.debug(
-                `Document ${relativePath} has been moved as a result of a remote update, skipping sync`
-            );
-            return;
-        }
 
+        // must have been removed after a successful delete
         if (document === undefined) {
             this.logger.debug(
                 `Cannot find document ${relativePath} in the database, skipping`
@@ -212,27 +168,47 @@ export class Syncer {
             return;
         }
 
-        const [promise, resolve, reject] = createPromise();
+        const documentAtNewPath =
+            this.database.getLatestDocumentByRelativePath(relativePath);
 
-        document = await this.database.getResolvedDocumentByRelativePath(
+        if (oldPath !== undefined) {
-            relativePath,
+            // We might have moved the document in the database before calling this method,
-            promise
+            // in that case, we mustn't move it again.
+            if (
+                documentAtNewPath === undefined ||
+                documentAtNewPath.isDeleted
+            ) {
+                if (oldPath === relativePath) {
+                    throw new Error(
+                        `Old path and new path are the same: ${oldPath}`
                     );
+                }
 
-        try {
+                this.database.move(oldPath, relativePath);
-            await this.syncQueue.add(async () =>
+            }
-                this.internalSyncer.unrestrictedSyncLocallyUpdatedFile({
+        }
+
+
+        if (
+            oldPath !== undefined &&
+            document?.metadata?.remoteRelativePath === relativePath
+        ) {
+            this.logger.debug(
+                `Document ${relativePath} has been moved as a result of a remote update, skipping sync`
+            );
+            return;
+        }
+
+        await this.enqueueSyncOperation(
+            async () =>
+                this.unrestrictedSyncer.unrestrictedSyncLocallyCreatedOrUpdatedFile(
+                    {
                         oldPath,
                         document
-                })
-            );
-
-            resolve();
-        } catch (e) {
-            reject(e);
-        } finally {
-            this.database.removeDocumentPromise(promise);
                     }
+                ),
+            [document.metadata?.documentId, relativePath, oldPath]
+        );
     }
 
     public async scheduleSyncForOfflineChanges(): Promise<void> {
@@ -257,8 +233,6 @@ export class Syncer {
                 `Not all local changes have been applied remotely: ${e}`
             );
             throw e;
-        } finally {
-            this.runningScheduleSyncForOfflineChanges = undefined;
         }
     }
 
@@ -271,6 +245,8 @@ export class Syncer {
         message: WebSocketVaultUpdate
     ): Promise<void> {
         try {
+            await this.scheduleSyncForOfflineChanges();
+
             const handlerPromise = awaitAll(
                 message.documents.map(async (document) =>
                     this.internalSyncRemotelyUpdatedFile(document)
@@ -296,7 +272,7 @@ export class Syncer {
     public reset(): void {
         this._isFirstSyncComplete = false;
         this.syncQueue.clear();
-        this.remoteDocumentsLock.reset();
+        this.updatedDocumentsByPathAndKeysLocks.reset();
         this.runningScheduleSyncForOfflineChanges = undefined;
     }
 
@@ -313,86 +289,26 @@ export class Syncer {
     private async internalSyncRemotelyUpdatedFile(
         remoteVersion: DocumentVersionWithoutContent
     ): Promise<void> {
-        let document = this.database.getDocumentByDocumentId(
+        const document = this.database.getDocumentByDocumentId(
             remoteVersion.documentId
         );
-
+        await this.enqueueSyncOperation(
-        if (document === undefined) {
+            async () =>
-            // Let's avoid the same documents getting created in parallel multiple times.
+                this.unrestrictedSyncer.unrestrictedSyncRemotelyUpdatedFile(
-            // There might be multiple tasks waiting for the lock
+                    remoteVersion,
-            return this.remoteDocumentsLock.withLock(
+                    document
-                remoteVersion.documentId,
+                ),
-                async () => {
+            [
-                    document = this.database.getDocumentByDocumentId(
+                document?.relativePath,
+                remoteVersion.relativePath,
                 remoteVersion.documentId
+            ]
         );
 
-                    // We're either the first one to get the lock, so we have to create the document in `unrestrictedSyncRemotelyUpdatedFile`
-                    if (document === undefined) {
-                        await this.syncQueue.add(async () =>
-                            this.internalSyncer.unrestrictedSyncRemotelyUpdatedFile(
-                                remoteVersion
-                            )
-                        );
-                    } else {
-                        const [promise, resolve, reject] = createPromise();
-
-                        document =
-                            await this.database.getResolvedDocumentByRelativePath(
-                                document.relativePath,
-                                promise
-                            );
-
-                        try {
-                            await this.syncQueue.add(async () =>
-                                this.internalSyncer.unrestrictedSyncRemotelyUpdatedFile(
-                                    remoteVersion,
-                                    document
-                                )
-                            );
-
-                            resolve();
-                        } catch (e) {
-                            reject(e);
-                        } finally {
-                            this.database.removeDocumentPromise(promise);
-                        }
-                    }
-
-                    this.database.addSeenUpdateId(remoteVersion.vaultUpdateId);
-                }
-            );
-        }
-
-        // We're either the first one to get the lock, so we have to create the document in `unrestrictedSyncRemotelyUpdatedFile`
-        const [promise, resolve, reject] = createPromise();
-
-        document = await this.database.getResolvedDocumentByRelativePath(
-            document.relativePath,
-            promise
-        );
-
-        try {
-            await this.syncQueue.add(async () =>
-                this.internalSyncer.unrestrictedSyncRemotelyUpdatedFile(
-                    remoteVersion,
-                    document
-                )
-            );
-
-            resolve();
-        } catch (e) {
-            reject(e);
-        } finally {
-            this.database.removeDocumentPromise(promise);
-        }
-
         this.database.addSeenUpdateId(remoteVersion.vaultUpdateId);
     }
 
     private async internalScheduleSyncForOfflineChanges(): Promise<void> {
-        await this.createFakeDocumentsFromRemoteState();
-
         const allLocalFiles = await this.operations.listFilesRecursively();
         this.logger.info(
             `Scheduling sync for ${allLocalFiles.length} local files`
@@ -409,7 +325,12 @@ export class Syncer {
             }
         }
 
-        await awaitAll(
+        interface Instruction {
+            type: "update" | "create";
+            relativePath: string;
+            oldPath?: string;
+        }
+        const instructions: (Instruction | undefined)[] = await awaitAll(
             allLocalFiles.map(async (relativePath) => {
                 if (
                     this.database.getLatestDocumentByRelativePath(relativePath)
@@ -419,16 +340,24 @@ export class Syncer {
                         `Document ${relativePath} might have been updated locally, scheduling sync to validate and update it`
                     );
 
-                    return this.syncLocallyUpdatedFile({
+                    return { type: "update", relativePath } as Instruction;
-                        relativePath
-                    });
                 }
 
                 // Perhaps the file has been moved; let's check by looking at the deleted files
                 const contentHash = await this.syncQueue.add(async () => {
+                    try {
                         const contentBytes =
                             await this.operations.read(relativePath); // this can throw FileNotFoundError
                         return hash(contentBytes);
+                    } catch (e) {
+                        if (
+                            e instanceof Error &&
+                            e.name === "FileNotFoundError"
+                        ) {
+                            return undefined;
+                        }
+                        throw e;
+                    }
                 });
 
                 if (contentHash == undefined) {
@@ -454,18 +383,21 @@ export class Syncer {
                         `Document '${originalFile.relativePath}' was not found under its current path in the database but was found under a different path (${relativePath}), scheduling sync to move it`
                     );
 
-                    // We're outside of the pqueue, so we need to call the public wrapper
+                    return {
-                    return this.syncLocallyUpdatedFile({
+                        type: "update",
                         oldPath: originalFile.relativePath,
                         relativePath
-                    });
+                    } as Instruction;
                 }
 
                 this.logger.debug(
                     `Document ${relativePath} not found in database, scheduling sync to create it`
                 );
-                // We're outside of the pqueue, so we need to call the public wrapper
+
-                return this.syncLocallyCreatedFile(relativePath);
+                return {
+                    type: "create",
+                    relativePath
+                } as Instruction;
             })
         );
 
@@ -481,42 +413,49 @@ export class Syncer {
                 return this.syncLocallyDeletedFile(relativePath);
             })
         );
-    }
 
-    /**
+        await awaitAll(
-    * Create fake documents in the database for all files that are present locally
+            instructions.map(async (instruction) => {
-    * and also exist remotely. This will stop the subequent syncs from duplicating
+                if (instruction === undefined) {
-    * the documents by creating the same documents from multiple clients.
-    */
-    private async createFakeDocumentsFromRemoteState(): Promise<void> {
-        if (this.database.getHasInitialSyncCompleted()) {
                     return;
                 }
 
-        const [allLocalFiles, remote] = await awaitAll([
+                if (instruction.type === "update") {
-            this.operations.listFilesRecursively(),
+                    // We're outside of the pqueue, so we need to call the public wrapper
-            this.syncQueue.add(async () => this.syncService.getAll())
+                    await this.syncLocallyUpdatedFile({
-        ]);
+                        oldPath: instruction.oldPath,
-
+                        relativePath: instruction.relativePath
-        if (remote !== undefined) {
-            remote.latestDocuments
-                .filter(
-                    (remoteDocument) =>
-                        allLocalFiles.includes(remoteDocument.relativePath) &&
-                        !remoteDocument.isDeleted &&
-                        this.database.getDocumentByDocumentId(
-                            remoteDocument.documentId
-                        ) === undefined
-                )
-                .forEach((remoteDocument) => {
-                    this.database.createNewEmptyDocument(
-                        remoteDocument.documentId,
-                        remoteDocument.vaultUpdateId,
-                        remoteDocument.relativePath
-                    );
                     });
+                    return;
+                }
+            })
+        );
+
+        // we have to ensure the deletes & updates have finished before starting creates,
+        // otherwise the server might return an existing document (that we're about to delete)
+        // instead of actually creating a new one
+        await awaitAll(
+            instructions.map(async (instruction) => {
+                if (instruction === undefined) {
+                    return;
                 }
 
-        this.database.setHasInitialSyncCompleted(true);
+                if (instruction.type === "create") {
+                    // We're outside of the pqueue, so we need to call the public wrapper
+                    await this.syncLocallyCreatedFile(instruction.relativePath);
+                    return;
+                }
+            })
+        );
+    }
+
+    private async enqueueSyncOperation<T>(
+        operation: () => Promise<T>,
+        keys: (string | undefined | null)[]
+    ): Promise<T> {
+        return this.updatedDocumentsByPathAndKeysLocks.withLock(
+            keys.filter((k) => k !== undefined && k !== null),
+            async () => this.syncQueue.add(operation)
+        );
     }
 }

frontend/sync-client/src/sync-operations/unrestricted-syncer.ts

@@ -3,7 +3,6 @@ import type {
     DocumentRecord,
     RelativePath
 } from "../persistence/database";
-
 import { diff } from "reconcile-text";
 import type { SyncService } from "../services/sync-service";
 import type { Logger } from "../tracing/logger";
@@ -18,13 +17,11 @@ import type {
 } from "../tracing/sync-history";
 import { SyncStatus, SyncType } from "../tracing/sync-history";
 import { EMPTY_HASH, hash } from "../utils/hash";
-
 import { base64ToBytes } from "byte-base64";
 import type { Settings } from "../persistence/settings";
 import type { FileOperations } from "../file-operations/file-operations";
-import { createPromise } from "../utils/create-promise";
+import { FileNotFoundError } from "../errors/file-not-found-error";
-import { FileNotFoundError } from "../file-operations/file-not-found-error";
+import { SyncResetError } from "../errors/sync-reset-error";
-import { SyncResetError } from "../services/sync-reset-error";
 import { globsToRegexes } from "../utils/globs-to-regexes";
 import type { DocumentVersion } from "../services/types/DocumentVersion";
 import type { DocumentUpdateResponse } from "../services/types/DocumentUpdateResponse";
@@ -60,115 +57,27 @@ export class UnrestrictedSyncer {
         });
     }
 
-    public async unrestrictedSyncLocallyCreatedFile(
+    public async unrestrictedSyncLocallyCreatedOrUpdatedFile({
-        document: DocumentRecord
-    ): Promise<void> {
-        const updateDetails: SyncCreateDetails = {
-            type: SyncType.CREATE,
-            relativePath: document.relativePath
-        };
-
-        return this.executeSync(updateDetails, async () => {
-            const originalRelativePath = document.relativePath;
-            if (document.isDeleted) {
-                this.logger.debug(
-                    `Document ${originalRelativePath} has been already deleted, no need to create it`
-                );
-                return;
-            }
-
-            const contentBytes =
-                await this.operations.read(originalRelativePath); // this can throw FileNotFoundError
-            const contentHash = hash(contentBytes);
-
-            const response = await this.syncService.create({
-                documentId: document.documentId,
-                relativePath: originalRelativePath,
-                contentBytes
-            });
-
-            // In case a document with the same name (but different ID) had existed remotely that we haven't known about
-            if (response.relativePath != originalRelativePath) {
-                this.logger.debug(
-                    `Document ${originalRelativePath} has been created remotely at a different path: ${response.relativePath}, moving it locally`
-                );
-                await this.operations.move(
-                    document.relativePath,
-                    response.relativePath
-                ); // this can throw FileNotFoundError
-            }
-
-            this.database.updateDocumentMetadata(
-                {
-                    parentVersionId: response.vaultUpdateId,
-                    hash: contentHash,
-                    remoteRelativePath: response.relativePath
-                },
-                document
-            );
-
-            this.database.addSeenUpdateId(response.vaultUpdateId);
-            await this.updateCache(
-                response.vaultUpdateId,
-                contentBytes,
-                response.relativePath
-            );
-
-            this.history.addHistoryEntry({
-                status: SyncStatus.SUCCESS,
-                details: updateDetails,
-                message: `Successfully uploaded locally created file`
-            });
-        });
-    }
-
-    public async unrestrictedSyncLocallyDeletedFile(
-        document: DocumentRecord
-    ): Promise<void> {
-        const updateDetails: SyncDeleteDetails = {
-            type: SyncType.DELETE,
-            relativePath: document.relativePath
-        };
-
-        await this.executeSync(updateDetails, async () => {
-            const response = await this.syncService.delete({
-                documentId: document.documentId,
-                relativePath: document.relativePath
-            });
-
-            this.database.updateDocumentMetadata(
-                {
-                    parentVersionId: response.vaultUpdateId,
-                    hash: EMPTY_HASH,
-                    remoteRelativePath: document.relativePath
-                },
-                document
-            );
-
-            this.database.addSeenUpdateId(response.vaultUpdateId);
-
-            this.history.addHistoryEntry({
-                status: SyncStatus.SUCCESS,
-                details: updateDetails,
-                message: `Successfully deleted locally deleted file on the server`,
-                author: response.userId
-            });
-        });
-    }
-
-    public async unrestrictedSyncLocallyUpdatedFile({
         oldPath,
-        document,
         // We use the same code path for both local and remote updates. We need to force the update
         // if there are no local changes but we know that the remote version is newer.
-        force = false
+        force = false,
+        document
     }: {
         oldPath?: RelativePath;
         force?: boolean;
         document: DocumentRecord;
     }): Promise<void> {
-        const updateDetails: SyncUpdateDetails | SyncMovedDetails =
+        const updateDetails:
-            oldPath !== undefined
+            | SyncCreateDetails
+            | SyncUpdateDetails
+            | SyncMovedDetails =
+            document.metadata === undefined
+                ? {
+                    type: SyncType.CREATE,
+                    relativePath: document.relativePath
+                }
+                : oldPath !== undefined
                     ? {
                         type: SyncType.MOVE,
                         relativePath: document.relativePath,
@@ -182,7 +91,7 @@ export class UnrestrictedSyncer {
         await this.executeSync(updateDetails, async () => {
             const originalRelativePath = document.relativePath;
 
-            if (document.isDeleted || document.metadata === undefined) {
+            if (document.isDeleted) {
                 this.logger.debug(
                     `Document ${document.relativePath} has been already deleted, no need to update it`
                 );
@@ -192,14 +101,28 @@ export class UnrestrictedSyncer {
             const contentBytes = await this.operations.read(
                 document.relativePath
             ); // this can throw FileNotFoundError
-            let contentHash = hash(contentBytes);
+            const contentHash = hash(contentBytes);
-
-            const areThereLocalChanges = !(
-                document.metadata.hash === contentHash && oldPath === undefined
-            );
 
             let response: DocumentVersion | DocumentUpdateResponse | undefined =
                 undefined;
+            if (document.metadata === undefined) {
+                response = await this.syncService.create({
+                    relativePath: originalRelativePath,
+                    contentBytes
+                });
+
+                await this.handleMaybeMergingResponse({
+                    document,
+                    response,
+                    contentHash,
+                    originalRelativePath,
+                    originalContentBytes: contentBytes,
+                    isCreate: true
+                });
+            } else {
+                const areThereLocalChanges =
+                    document.metadata.hash !== contentHash ||
+                    oldPath !== undefined;
 
                 if (areThereLocalChanges) {
                     const isText =
@@ -216,7 +139,7 @@ export class UnrestrictedSyncer {
                     response =
                         isText && cachedVersion !== undefined
                             ? await this.syncService.putText({
-                            documentId: document.documentId,
+                                documentId: document.metadata.documentId,
                                 parentVersionId:
                                     document.metadata.parentVersionId,
                                 relativePath: document.relativePath,
@@ -226,7 +149,7 @@ export class UnrestrictedSyncer {
                                 )
                             })
                             : await this.syncService.putBinary({
-                            documentId: document.documentId,
+                                documentId: document.metadata.documentId,
                                 parentVersionId:
                                     document.metadata.parentVersionId,
                                 relativePath: document.relativePath,
@@ -240,98 +163,32 @@ export class UnrestrictedSyncer {
                         return;
                     }
 
+                    // we use this code path (force == true) to sync remotely updated files which have no local changes
                     response = await this.syncService.get({
-                    documentId: document.documentId
+                        documentId: document.metadata.documentId
                     });
                 }
 
-            // `document` is mutable and reflects the latest state in the local database
+                await this.handleMaybeMergingResponse({
-            // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
+                    document,
-            if (document.isDeleted) {
+                    response,
-                this.logger.info(
+                    contentHash,
-                    `Document ${document.relativePath} has been deleted before we could finish updating it`
+                    originalRelativePath,
-                );
+                    originalContentBytes: contentBytes
-                this.database.addSeenUpdateId(response.vaultUpdateId);
+                });
-                return;
-            }
-
-            if (
-                // `Syncer` creates fake local document metadata for all remote docs with invalid hashes. The parent IDs will likely match
-                // the latest versions so we still need to update the local versions to turn the fakes into real metadata.
-                document.metadata.parentVersionId > response.vaultUpdateId
-            ) {
-                this.logger.debug(
-                    `Document ${document.relativePath} is already more up to date than the fetched version`
-                );
-                this.database.addSeenUpdateId(response.vaultUpdateId); // in case the previous `vaultUpdateId` update hasn't made it through
-                return;
-            }
-
-            if (response.isDeleted) {
-                return this.applyRemoteDeleteLocally(document, response);
-            }
-
-            let actualPath = document.relativePath;
-
-            if (response.relativePath != originalRelativePath) {
-                actualPath = response.relativePath;
-                // Make sure to update the remote relative path to avoid uploading
-                // the file as a result of this filesystem event.
-                document.metadata.remoteRelativePath = response.relativePath;
-                await this.operations.move(
-                    document.relativePath,
-                    response.relativePath
-                ); // this can throw FileNotFoundError
             }
 
             if (!("type" in response) || response.type === "MergingUpdate") {
-                const responseBytes = base64ToBytes(response.contentBase64);
-                contentHash = hash(responseBytes);
-
-                this.database.updateDocumentMetadata(
-                    {
-                        parentVersionId: response.vaultUpdateId,
-                        hash: contentHash,
-                        remoteRelativePath: response.relativePath
-                    },
-                    document
-                );
-                await this.operations.write(
-                    actualPath,
-                    contentBytes,
-                    responseBytes
-                );
-                await this.updateCache(
-                    response.vaultUpdateId,
-                    responseBytes,
-                    actualPath
-                );
-
                 if (!force) {
                     this.history.addHistoryEntry({
                         status: SyncStatus.SUCCESS,
                         details: updateDetails,
                         message: `The file we updated had been updated remotely, so we downloaded the merged version`
                     });
+                    return;
                 }
-            } else {
-                this.database.updateDocumentMetadata(
-                    {
-                        parentVersionId: response.vaultUpdateId,
-                        hash: contentHash,
-                        remoteRelativePath: response.relativePath
-                    },
-                    document
-                );
-                await this.updateCache(
-                    response.vaultUpdateId,
-                    contentBytes,
-                    actualPath
-                );
             }
 
-            this.database.addSeenUpdateId(response.vaultUpdateId);
-
             const actualUpdateDetails: SyncUpdateDetails | SyncMovedDetails =
                 oldPath !== undefined ||
                     response.relativePath != originalRelativePath
@@ -345,14 +202,7 @@ export class UnrestrictedSyncer {
                         relativePath: response.relativePath
                     };
 
-            if (areThereLocalChanges) {
+            if (!response.isDeleted) {
-                this.history.addHistoryEntry({
-                    status: SyncStatus.SUCCESS,
-                    details: actualUpdateDetails,
-                    message: `Successfully uploaded locally updated file to the server`,
-                    author: response.userId
-                });
-            } else {
                 this.history.addHistoryEntry({
                     status: SyncStatus.SUCCESS,
                     details: actualUpdateDetails,
@@ -360,10 +210,64 @@ export class UnrestrictedSyncer {
                     author: response.userId,
                     timestamp: new Date(response.updatedDate)
                 });
+            } else {
+                this.history.addHistoryEntry({
+                    status: SyncStatus.SUCCESS,
+                    details: {
+                        type: SyncType.DELETE,
+                        relativePath: document.relativePath
+                    },
+                    message:
+                        "Successfully deleted file which had been deleted remotely",
+                    author: response.userId,
+                    timestamp: new Date(response.updatedDate)
+                });
             }
         });
     }
 
+    public async unrestrictedSyncLocallyDeletedFile(
+        document: DocumentRecord
+    ): Promise<void> {
+        const updateDetails: SyncDeleteDetails = {
+            type: SyncType.DELETE,
+            relativePath: document.relativePath
+        };
+
+        await this.executeSync(updateDetails, async () => {
+            if (document.metadata === undefined) {
+                this.logger.debug(
+                    `Document ${document.relativePath} has never been synced, no need to delete it remotely`
+                );
+                return;
+            }
+
+            const response = await this.syncService.delete({
+                documentId: document.metadata.documentId,
+                relativePath: document.relativePath
+            });
+
+            this.database.updateDocumentMetadata(
+                {
+                    documentId: response.documentId,
+                    parentVersionId: response.vaultUpdateId,
+                    hash: EMPTY_HASH,
+                    remoteRelativePath: document.relativePath
+                },
+                document
+            );
+
+            this.database.addSeenUpdateId(response.vaultUpdateId);
+
+            this.history.addHistoryEntry({
+                status: SyncStatus.SUCCESS,
+                details: updateDetails,
+                message: `Successfully deleted locally deleted file on the server`,
+                author: response.userId
+            });
+        });
+    }
+
     public async unrestrictedSyncRemotelyUpdatedFile(
         remoteVersion: DocumentVersionWithoutContent,
         document?: DocumentRecord
@@ -382,13 +286,13 @@ export class UnrestrictedSyncer {
                     remoteVersion.vaultUpdateId
                 ) {
                     this.logger.debug(
-                        `Document ${remoteVersion.relativePath} is already at least as up to date as the fetched version`
+                        `Document ${document.relativePath} is already at least as up-to-date as the fetched version`
                     );
 
                     return;
                 }
 
-                return this.unrestrictedSyncLocallyUpdatedFile({
+                return this.unrestrictedSyncLocallyCreatedOrUpdatedFile({
                     document,
                     force: true
                 });
@@ -434,17 +338,15 @@ export class UnrestrictedSyncer {
 
             await this.operations.ensureClearPath(remoteVersion.relativePath);
 
-            const [promise, resolve] = createPromise();
             this.database.updateDocumentMetadata(
                 {
+                    documentId: remoteVersion.documentId,
                     parentVersionId: remoteVersion.vaultUpdateId,
                     hash: hash(contentBytes),
                     remoteRelativePath: remoteVersion.relativePath
                 },
                 this.database.createNewPendingDocument(
-                    remoteVersion.documentId,
+                    remoteVersion.relativePath
-                    remoteVersion.relativePath,
-                    promise
                 )
             );
 
@@ -458,9 +360,6 @@ export class UnrestrictedSyncer {
                 remoteVersion.relativePath
             );
 
-            resolve();
-            this.database.removeDocumentPromise(promise);
-
             this.history.addHistoryEntry({
                 status: SyncStatus.SUCCESS,
                 details: updateDetails,
@@ -471,10 +370,17 @@ export class UnrestrictedSyncer {
         });
     }
 
-    public async executeSync<T>(
+    private async executeSync<T>(
         details: SyncDetails,
         fn: () => Promise<T>
     ): Promise<T | undefined> {
+        if (!this.settings.getSettings().isSyncEnabled) {
+            this.logger.info(
+                `Skipping sync operation for file '${details.relativePath}' because sync is disabled`
+            );
+            return;
+        }
+
         for (const pattern of this.ignorePatterns) {
             if (pattern.test(details.relativePath)) {
                 this.logger.debug(
@@ -528,6 +434,127 @@ export class UnrestrictedSyncer {
         }
     }
 
+    private async handleMaybeMergingResponse({
+        document,
+        response,
+        contentHash,
+        originalRelativePath,
+        originalContentBytes,
+        isCreate
+    }: {
+        document: DocumentRecord;
+        response: DocumentVersion | DocumentUpdateResponse;
+        contentHash: string;
+        originalRelativePath: string;
+        originalContentBytes: Uint8Array;
+        isCreate?: boolean;
+    }): Promise<void> {
+        // `document` is mutable and reflects the latest state in the local database
+        if (document.isDeleted) {
+            this.logger.info(
+                `Document ${document.relativePath} has been deleted before we could finish updating it`
+            );
+            this.database.addSeenUpdateId(response.vaultUpdateId);
+            return;
+        }
+
+        if (
+            (document.metadata?.parentVersionId ?? 0) > response.vaultUpdateId
+        ) {
+            this.logger.debug(
+                `Document ${document.relativePath} is already more up to date than the fetched version`
+            );
+            this.database.addSeenUpdateId(response.vaultUpdateId); // in case the previous `vaultUpdateId` update hasn't made it through
+            return;
+        }
+
+        if (response.isDeleted) {
+            return this.applyRemoteDeleteLocally(document, response);
+        }
+
+        let actualPath = document.relativePath;
+
+        if (isCreate) {
+            // We have a file locally that got moved by another client to the same path as the one we're trying to create.
+            // The server returns a merging update for the document ID that already exists locally (but at another path).
+            // We have to merge these two documents by extending the provenance of the existing document and deleting
+            // the old document that the new document already contains the content for.
+            const existingDocument = this.database.getDocumentByDocumentId(
+                response.documentId
+            );
+            if (existingDocument !== undefined) {
+                this.logger.info(
+                    `Merging existing document ${existingDocument.relativePath} into ${document.relativePath
+                    } after concurrent move & creation`
+                );
+                if (!existingDocument.isDeleted) {
+                    this.database.delete(existingDocument.relativePath); // make sure syncLocallyDeletedFile doesn't actually schedule deleting the new file
+                    this.database.removeDocument(existingDocument);
+                    await this.operations.move(existingDocument.relativePath, document.relativePath);
+                } else {
+                    this.database.removeDocument(existingDocument);
+                }
+            }
+        }
+
+        // this can't happen on the creation path as we can only get a merging response if a document already exists remotely on the same path
+        if (response.relativePath != originalRelativePath) {
+            actualPath = response.relativePath;
+            // Make sure to update the remote relative path to avoid uploading
+            // the file as a result of this filesystem event.
+            if (document.metadata !== undefined) {
+                document.metadata.remoteRelativePath = response.relativePath;
+            }
+            await this.operations.move(
+                document.relativePath,
+                response.relativePath
+            ); // this can throw FileNotFoundError
+        }
+
+        if (!("type" in response) || response.type === "MergingUpdate") {
+            const responseBytes = base64ToBytes(response.contentBase64);
+            contentHash = hash(responseBytes);
+
+            this.database.updateDocumentMetadata(
+                {
+                    documentId: response.documentId,
+                    parentVersionId: response.vaultUpdateId,
+                    hash: contentHash,
+                    remoteRelativePath: response.relativePath
+                },
+                document
+            );
+
+            await this.operations.write(
+                actualPath,
+                originalContentBytes,
+                responseBytes
+            );
+            await this.updateCache(
+                response.vaultUpdateId,
+                responseBytes,
+                actualPath
+            );
+        } else {
+            this.database.updateDocumentMetadata(
+                {
+                    documentId: response.documentId,
+                    parentVersionId: response.vaultUpdateId,
+                    hash: contentHash,
+                    remoteRelativePath: response.relativePath
+                },
+                document
+            );
+            await this.updateCache(
+                response.vaultUpdateId,
+                originalContentBytes,
+                actualPath
+            );
+        }
+
+        this.database.addSeenUpdateId(response.vaultUpdateId);
+    }
+
     private getHistoryEntryForSkippedOversizedFile(
         sizeInBytes: number,
         relativePath: RelativePath
@@ -541,8 +568,7 @@ export class UnrestrictedSyncer {
                     type: SyncType.SKIPPED,
                     relativePath
                 },
-                message: `File size of ${sizeInMB} MB exceeds the maximum file size limit of ${
+                message: `File size of ${sizeInMB} MB exceeds the maximum file size limit of ${maxFileSizeMB
-                    maxFileSizeMB
                     } MB`
             };
         }
@@ -568,20 +594,10 @@ export class UnrestrictedSyncer {
         document: DocumentRecord,
         response: DocumentVersion | DocumentUpdateResponse
     ): Promise<void> {
-        this.history.addHistoryEntry({
-            status: SyncStatus.SUCCESS,
-            details: {
-                type: SyncType.DELETE,
-                relativePath: document.relativePath
-            },
-            message: "File has been deleted remotely, so we deleted it locally",
-            author: response.userId,
-            timestamp: new Date(response.updatedDate)
-        });
-
         this.database.delete(document.relativePath);
         this.database.updateDocumentMetadata(
             {
+                documentId: response.documentId,
                 parentVersionId: response.vaultUpdateId,
                 hash: EMPTY_HASH,
                 remoteRelativePath: response.relativePath

frontend/sync-client/src/utils/await-all.ts

@@ -9,7 +9,7 @@ type ResolvedTuple<T extends readonly unknown[]> = {
 export const awaitAll = async <T extends readonly unknown[]>(
     promises: PromiseTuple<T>
 ): Promise<ResolvedTuple<T>> => {
-    // eslint-disable-next-line no-restricted-properties
+    // eslint-disable-next-line no-restricted-properties, @typescript-eslint/await-thenable
     const result = await Promise.allSettled(promises);
     for (const res of result) {
         if (res.status === "rejected") {

frontend/sync-client/src/utils/create-client-id.ts

@@ -1,5 +1,3 @@
-import { v4 as uuidv4 } from "uuid";
-
 export function createClientId(): string {
     // @ts-expect-error, injected by webpack
     const packageVersion = __CURRENT_VERSION__; // eslint-disable-line
@@ -11,5 +9,5 @@ export function createClientId(): string {
               ? process.platform
               : "unknown";
 
-    return `vault-link/${packageVersion} (${uuidv4()}; ${platform})`;
+    return `vault-link/${packageVersion} (${Math.round(Math.random() * 1e10)}; ${platform})`;
 }

frontend/sync-client/src/utils/data-structures/locks.test.ts

@@ -5,18 +5,20 @@ import type { RelativePath } from "../../persistence/database";
 import { Locks } from "./locks";
 import { awaitAll } from "../await-all";
 import { sleep } from "../sleep";
-import { SyncResetError } from "../../services/sync-reset-error";
+import { SyncResetError } from "../../errors/sync-reset-error";
 
 describe("withLock", () => {
     const testPath: RelativePath = "test/document/path";
     const testPath2: RelativePath = "test/document/path2";
+    const testPath3: RelativePath = "test/document/path3";
+
     const logger = new Logger();
 
     // eslint-disable-next-line @typescript-eslint/init-declarations
     let locks: Locks<RelativePath>;
 
     beforeEach(() => {
-        locks = new Locks<RelativePath>(logger);
+        locks = new Locks<RelativePath>("locks-test", logger);
     });
 
     it("should execute function with single key lock", async () => {
@@ -56,22 +58,32 @@ describe("withLock", () => {
     it("should sort multiple keys to prevent deadlocks", async () => {
         const executionOrder: string[] = [];
 
-        // Start two concurrent operations with keys in different orders
+        await locks.waitForLock(testPath);
-        const promise1 = locks.withLock([testPath2, testPath], async () => {
+
+        const promise = awaitAll([
+            locks.withLock([testPath2, testPath3, testPath], async () => {
                 executionOrder.push("operation1-start");
-            await sleep(50);
                 executionOrder.push("operation1-end");
                 return "result1";
-        });
+            }),
 
-        const promise2 = locks.withLock([testPath, testPath2], async () => {
+            locks.withLock([testPath3, testPath, testPath2], async () => {
                 executionOrder.push("operation2-start");
-            await sleep(50);
                 executionOrder.push("operation2-end");
                 return "result2";
-        });
+            })
+        ]);
 
-        const [result1, result2] = await awaitAll([promise1, promise2]);
+        locks.unlock(testPath);
+
+        const [result1, result2] = await Promise.race([
+            promise,
+            new Promise<never>((_, reject) => {
+                setTimeout(() => {
+                    reject(new Error("Deadlock detected"));
+                }, 1000);
+            })
+        ]);
 
         assert.strictEqual(result1, "result1");
         assert.strictEqual(result2, "result2");
@@ -234,13 +246,14 @@ describe("withLock", () => {
 
 describe("reset", () => {
     const testPath: RelativePath = "test/document/path";
+    const testPath2: RelativePath = "test/document/path2";
     const logger = new Logger();
 
     // eslint-disable-next-line @typescript-eslint/init-declarations
     let locks: Locks<RelativePath>;
 
     beforeEach(() => {
-        locks = new Locks<RelativePath>(logger);
+        locks = new Locks<RelativePath>("locks-test", logger);
     });
 
     it("should reject pending waiters with SyncResetError while running operation completes", async () => {
@@ -289,4 +302,38 @@ describe("reset", () => {
         const result = await locks.withLock(testPath, () => "success");
         assert.strictEqual(result, "success");
     });
+
+    it("should release partially acquired locks when reset interrupts multi-key acquisition", async () => {
+        // Hold testPath2 so multi-key acquisition will block on it
+        await locks.waitForLock(testPath2);
+
+        // Start multi-key lock that will acquire testPath first, then block on testPath2
+        const multiKeyPromise = locks.withLock(
+            [testPath, testPath2],
+            async () => "multi"
+        );
+        void multiKeyPromise.catch(() => { }); // eslint-disable-line @typescript-eslint/no-empty-function
+
+        // Wait for the multi-key operation to acquire testPath and start waiting on testPath2
+        await sleep(10);
+
+        // Reset should reject the waiting operation
+        locks.reset();
+
+        await assert.rejects(multiKeyPromise, (err: Error) => {
+            assert.ok(err instanceof SyncResetError);
+            return true;
+        });
+
+        // The key that was already acquired (testPath) should now be released
+        // This would hang/timeout if the lock was leaked
+        const result = await Promise.race([
+            locks.withLock(testPath, () => "success"),
+            sleep(100).then(() => {
+                throw new Error("Lock was not released - deadlock detected");
+            })
+        ]);
+
+        assert.strictEqual(result, "success");
+    });
 });

frontend/sync-client/src/utils/data-structures/locks.ts

@@ -1,6 +1,5 @@
-import { SyncResetError } from "../../services/sync-reset-error";
+import { SyncResetError } from "../../errors/sync-reset-error";
 import type { Logger } from "../../tracing/logger";
-import { awaitAll } from "../await-all";
 
 /**
  * Manages exclusive locks on items to prevent concurrent modifications.
@@ -8,17 +7,20 @@ import { awaitAll } from "../await-all";
  *
  * @template T The type of the key used for locking
  */
+/** Waiter entry with callbacks */
+interface WaiterEntry<T> {
+    resolve: () => unknown;
+    reject: (err: unknown) => unknown;
+}
+
 export class Locks<T> {
     /** Currently locked keys */
     private readonly locked = new Set<T>();
 
-    /** Queue of resolve functions waiting for each key */
+    /** Queue of waiters for each key */
-    private readonly waiters = new Map<
+    private readonly waiters = new Map<T, WaiterEntry<T>[]>();
-        T,
-        [() => unknown, (err: unknown) => unknown][]
-    >();
 
-    public constructor(private readonly logger?: Logger) {}
+    public constructor(private readonly name: string, private readonly logger?: Logger) { }
 
     /**
      * Executes a function while holding exclusive locks on one or more keys.
@@ -59,12 +61,17 @@ export class Locks<T> {
         const uniqueKeys = Array.from(new Set(keys));
         uniqueKeys.sort((a, b) => String(a).localeCompare(String(b))); // Ensure consistent order to prevent deadlocks
 
-        await awaitAll(uniqueKeys.map(async (key) => this.waitForLock(key)));
+        const lockedKeys = [];
-
         try {
+            for (const key of uniqueKeys) {
+                // Must acquire locks in-order (not concurrently) to prevent deadlocks
+                await this.waitForLock(key);
+                lockedKeys.push(key);
+            }
+
             return await fn();
         } finally {
-            uniqueKeys.forEach((key) => {
+            lockedKeys.forEach((key) => {
                 this.unlock(key);
             });
         }
@@ -74,7 +81,7 @@ export class Locks<T> {
         // Resolve all waiting promises before clearing to prevent deadlock
         // Any operation waiting for a lock will be granted access immediately
         for (const waiting of this.waiters.values()) {
-            for (const [_, reject] of waiting) {
+            for (const { reject } of waiting) {
                 reject(new SyncResetError());
             }
         }
@@ -82,6 +89,10 @@ export class Locks<T> {
         this.waiters.clear();
     }
 
+    public isLocked(key: T): boolean {
+        return this.locked.has(key);
+    }
+
     /**
      * Attempts to acquire a lock immediately without waiting.
      * Must call `unlock()` if successful.
@@ -111,7 +122,7 @@ export class Locks<T> {
             return Promise.resolve();
         }
 
-        this.logger?.debug(`Waiting for lock on ${key}`);
+        this.logger?.debug(`Waiting for lock '${this.name}' on '${key}'`);
 
         return new Promise((resolve, reject) => {
             // DefaultDict behavior
@@ -121,7 +132,10 @@ export class Locks<T> {
                 this.waiters.set(key, waiting);
             }
 
-            waiting.push([resolve, reject]);
+            waiting.push({
+                resolve,
+                reject,
+            });
         });
     }
 
@@ -134,15 +148,20 @@ export class Locks<T> {
      */
     public unlock(key: T): void {
         if (!this.locked.has(key)) {
+            this.logger?.debug(
+                `Attempted to unlock '${this.name}' on '${key}' which is not locked`
+            );
             return;
         }
 
-        // Remove first waiter to ensure FIFO order
+        this.logger?.debug(`Releasing lock '${this.name}' on '${key}'`);
-        const [resolveNextWaiting, _] = this.waiters.get(key)?.shift() ?? [];
 
-        if (resolveNextWaiting) {
+        // Remove first waiter to ensure FIFO order
-            this.logger?.debug(`Granted lock on ${key}`);
+        const nextWaiter = this.waiters.get(key)?.shift();
-            resolveNextWaiting();
+
+        if (nextWaiter) {
+            this.logger?.debug(`Granted lock '${this.name}' on '${key}'`);
+            nextWaiter.resolve();
         } else {
             this.locked.delete(key);
         }
@@ -152,8 +171,8 @@ export class Locks<T> {
 export class Lock {
     private readonly locks: Locks<boolean>;
 
-    public constructor(logger?: Logger) {
+    public constructor(name: string, logger?: Logger) {
-        this.locks = new Locks(logger);
+        this.locks = new Locks(name, logger);
     }
 
     public async withLock<R>(fn: () => R | Promise<R>): Promise<R> {

frontend/sync-client/src/utils/debugging/in-memory-file-system.ts

@@ -0,0 +1,69 @@
+import type { RelativePath } from "../../persistence/database";
+import type { TextWithCursors } from "reconcile-text";
+import type { FileSystemOperations } from "../../file-operations/filesystem-operations";
+
+export class InMemoryFileSystem implements FileSystemOperations {
+    protected readonly files = new Map<string, Uint8Array>();
+
+    public async listFilesRecursively(
+        _root: RelativePath | undefined = undefined // we don't use multi-level paths during tests
+    ): Promise<RelativePath[]> {
+        return Array.from(this.files.keys());
+    }
+
+    public async read(path: RelativePath): Promise<Uint8Array> {
+        const file = this.files.get(path);
+        if (!file) {
+            throw new Error(`File ${path} does not exist`);
+        }
+        return file;
+    }
+
+    public async write(path: RelativePath, content: Uint8Array): Promise<void> {
+        this.files.set(path, content);
+    }
+
+    public async atomicUpdateText(
+        path: RelativePath,
+        updater: (current: TextWithCursors) => TextWithCursors
+    ): Promise<string> {
+        const file = this.files.get(path);
+        if (!file) {
+            throw new Error(`File ${path} does not exist`);
+        }
+        const currentContent = new TextDecoder().decode(file);
+        const newContent = updater({ text: currentContent, cursors: [] }).text;
+        this.files.set(path, new TextEncoder().encode(newContent));
+        return newContent;
+    }
+
+    public async getFileSize(path: RelativePath): Promise<number> {
+        return (await this.read(path)).length;
+    }
+
+    public async exists(path: RelativePath): Promise<boolean> {
+        return this.files.has(path);
+    }
+
+    public async createDirectory(_path: RelativePath): Promise<void> {
+        // This doesn't mean anything in our virtual FS representation
+    }
+
+    public async delete(path: RelativePath): Promise<void> {
+        this.files.delete(path);
+    }
+
+    public async rename(
+        oldPath: RelativePath,
+        newPath: RelativePath
+    ): Promise<void> {
+        const file = this.files.get(oldPath);
+        if (!file) {
+            throw new Error(`File ${oldPath} does not exist`);
+        }
+        this.files.set(newPath, file);
+        if (oldPath !== newPath) {
+            this.files.delete(oldPath);
+        }
+    }
+}

frontend/sync-client/src/utils/debugging/log-to-console.ts

@@ -1,10 +1,44 @@
-import type { SyncClient } from "../../sync-client";
+/* eslint-disable no-console */
-import type { LogLine } from "../../tracing/logger";
+import type { Logger, LogLine } from "../../tracing/logger";
 import { LogLevel } from "../../tracing/logger";
 
-export function logToConsole(client: SyncClient): void {
+const COLORS = {
-    client.logger.onLogEmitted.add((logLine: LogLine) => {
+    reset: "\x1b[0m",
-        const formatted = `${logLine.timestamp.toISOString()} ${logLine.level} ${logLine.message}`;
+    red: "\x1b[31m",
+    yellow: "\x1b[33m",
+    blue: "\x1b[34m",
+    gray: "\x1b[90m"
+};
+
+export function logToConsole(
+    logger: Logger,
+    { useColors = true }: { useColors?: boolean } = {}
+): void {
+    logger.onLogEmitted.add((logLine: LogLine) => {
+        const timestamp = logLine.timestamp.toISOString();
+        const message = logLine.message;
+
+        let color = "";
+        let reset = "";
+        if (useColors) {
+            reset = COLORS.reset;
+            switch (logLine.level) {
+                case LogLevel.ERROR:
+                    color = COLORS.red;
+                    break;
+                case LogLevel.WARNING:
+                    color = COLORS.yellow;
+                    break;
+                case LogLevel.INFO:
+                    color = COLORS.blue;
+                    break;
+                case LogLevel.DEBUG:
+                    color = COLORS.gray;
+                    break;
+            }
+        }
+
+        const formatted = `${timestamp} ${color}${logLine.level}${reset} ${message}`;
 
         switch (logLine.level) {
             case LogLevel.ERROR:

frontend/sync-client/src/utils/debugging/slow-web-socket-factory.ts

@@ -11,7 +11,7 @@ export function slowWebSocketFactory(
         private static readonly RECEIVE_KEY = "websocket-receive";
         private static readonly SEND_KEY = "websocket-send";
 
-        private readonly locks = new Locks(logger);
+        private readonly locks = new Locks(FlakyWebSocket.name, logger);
 
         public set onopen(callback: ((event: Event) => void) | null) {
             super.onopen = async (event: Event): Promise<void> => {

frontend/sync-client/webpack.config.js

@@ -49,11 +49,6 @@ module.exports = [
                 type: "umd"
             },
             globalObject: "this"
-        },
-        resolve: {
-            fallback: {
-                ws: false // Exclude `ws` from the browser bundle
-            }
         }
     }),
     merge(common, {
@@ -62,10 +57,6 @@ module.exports = [
             path: path.resolve(__dirname, "dist"),
             filename: "sync-client.node.js",
             libraryTarget: "commonjs2"
-        },
-        externals: {
-            bufferutil: "bufferutil",
-            "utf-8-validate": "utf-8-validate" // required for ws: https://github.com/websockets/ws/issues/2245#issuecomment-2250318733
         }
     })
 ];