Fix syncing when network latency is present (#4)

* WIP

* Add debug

* Dedupe inserts

* Add deterministic ordering

* Fix whitespaces

* Update insta

* Add integration test script

* Rename

* Add test

* Working for non-deletes

* omg it mostly works for deletes

* Isdeleted fix

* remove created dates

* update api

* Take document id

* No max attempt

* works

* Use string uuids

* .

* working!!!! (hopefully)

* Improve bundling

* Add module

* lint

* .

* lint

* Fix CI

* use toolchain

* clean up

* Add useSlowFileEvents

* Delete fuzz

* Fix CI

* use docker

* fix script

* clean up

* Clean up

* change node version

* Build docker image on every commit

* fix ci

* 1 db per vault

* Add scritps folder

* Bump versions

* Lint

* .

* Fix tests for real

* Style

* .

* try

* Consistent ordering

* Fix tests

* hmm

* .

* Clean up diff

* Fixes

* .

* Fix version bump

* .

* .

* .

frontend/sync-client/src/services/connected-state.ts

@@ -0,0 +1,51 @@
+import type { Settings } from "../persistence/settings";
+import type { Logger } from "../tracing/logger";
+import { createPromise } from "../utils/create-promise";
+import { retriedFetchFactory } from "../utils/retried-fetch";
+
+export class ConnectedState {
+	private resolveIsSyncEnabled: (() => void) | undefined;
+	private syncIsEnabled: Promise<void> | undefined;
+
+	public constructor(
+		settings: Settings,
+		private readonly logger: Logger
+	) {
+		settings.addOnSettingsChangeHandlers((newSettings, oldSettings) => {
+			if (!oldSettings.isSyncEnabled && newSettings.isSyncEnabled) {
+				this.handleComingOnline();
+			} else if (
+				oldSettings.isSyncEnabled &&
+				!newSettings.isSyncEnabled
+			) {
+				this.handleGoingOffline();
+			}
+		});
+	}
+
+	public getFetchImplementation(
+		fetch: typeof globalThis.fetch,
+		{ doRetries = true }: { doRetries: boolean } = { doRetries: true }
+	): typeof globalThis.fetch {
+		const retriedFetch = doRetries
+			? retriedFetchFactory(this.logger, fetch)
+			: fetch;
+
+		return async (input: RequestInfo | URL): Promise<Response> => {
+			if (this.syncIsEnabled !== undefined) {
+				await this.syncIsEnabled;
+			}
+			return retriedFetch(input);
+		};
+	}
+
+	private handleComingOnline(): void {
+		this.logger.debug("Sync is enabled");
+		this.resolveIsSyncEnabled?.();
+	}
+
+	private handleGoingOffline(): void {
+		this.logger.debug("Sync is disabled");
+		[this.syncIsEnabled, this.resolveIsSyncEnabled] = createPromise();
+	}
+}

frontend/sync-client/src/services/sync-service.ts

@@ -6,20 +6,22 @@ import type {
 	RelativePath,
 	VaultUpdateId
 } from "../persistence/database";
-import type { Logger } from "src/tracing/logger";
-import { retriedFetchFactory } from "src/utils/retried-fetch";
-import type { Settings } from "src/persistence/settings";
+import type { Logger } from "../tracing/logger";
+import type { Settings } from "../persistence/settings";
+import type { ConnectedState } from "./connected-state";
 
 export interface CheckConnectionResult {
 	isSuccessful: boolean;
 	message: string;
 }
+
 export class SyncService {
 	private client!: Client<paths>;
 	private clientWithoutRetries!: Client<paths>;
 	private _fetchImplementation: typeof globalThis.fetch = globalThis.fetch;
 
 	public constructor(
+		private readonly connectedState: ConnectedState,
 		private readonly settings: Settings,
 		private readonly logger: Logger
 	) {
@@ -52,17 +54,19 @@ export class SyncService {
 	}
 
 	public async create({
+		documentId,
 		relativePath,
-		contentBytes,
-		createdDate
+		contentBytes
 	}: {
+		documentId?: DocumentId;
 		relativePath: RelativePath;
 		contentBytes: Uint8Array;
-		createdDate: Date;
-	}): Promise<components["schemas"]["DocumentUpdateResponse"]> {
+	}): Promise<components["schemas"]["DocumentVersionWithoutContent"]> {
 		const formData = new FormData();
+		if (documentId !== undefined) {
+			formData.append("document_id", documentId);
+		}
 		formData.append("relative_path", relativePath);
-		formData.append("created_date", createdDate.toISOString());
 		formData.append("content", new Blob([contentBytes]));
 
 		const response = await this.client.POST(
@@ -100,18 +104,18 @@ export class SyncService {
 		parentVersionId,
 		documentId,
 		relativePath,
-		contentBytes,
-		createdDate
+		contentBytes
 	}: {
 		parentVersionId: VaultUpdateId;
 		documentId: DocumentId;
 		relativePath: RelativePath;
 		contentBytes: Uint8Array;
-		createdDate: Date;
 	}): Promise<components["schemas"]["DocumentUpdateResponse"]> {
+		this.logger.debug(
+			`Updating document ${documentId} with parent version ${parentVersionId} and relative path ${relativePath}`
+		);
 		const formData = new FormData();
 		formData.append("parent_version_id", parentVersionId.toString());
-		formData.append("created_date", createdDate.toISOString());
 		formData.append("relative_path", relativePath);
 		formData.append("content", new Blob([contentBytes]));
 
@@ -149,13 +153,11 @@ export class SyncService {
 
 	public async delete({
 		documentId,
-		relativePath,
-		createdDate
+		relativePath
 	}: {
 		documentId: DocumentId;
 		relativePath: RelativePath;
-		createdDate: Date;
-	}): Promise<void> {
+	}): Promise<components["schemas"]["DocumentVersionWithoutContent"]> {
 		const response = await this.client.DELETE(
 			"/vaults/{vault_id}/documents/{document_id}",
 			{
@@ -169,7 +171,6 @@ export class SyncService {
 					}
 				},
 				body: {
-					createdDate: createdDate.toISOString(),
 					relativePath
 				}
 			}
@@ -295,11 +296,17 @@ export class SyncService {
 	private createClient(remoteUri: string): void {
 		this.client = createClient<paths>({
 			baseUrl: remoteUri,
-			fetch: retriedFetchFactory(this.logger, this._fetchImplementation)
+			fetch: this.connectedState.getFetchImplementation(
+				this._fetchImplementation
+			)
 		});
 
 		this.clientWithoutRetries = createClient<paths>({
-			baseUrl: remoteUri
+			baseUrl: remoteUri,
+			fetch: this.connectedState.getFetchImplementation(
+				this._fetchImplementation,
+				{ doRetries: false }
+			)
 		});
 	}
 }

frontend/sync-client/src/services/types.ts

@@ -274,12 +274,13 @@ export interface paths {
 				};
 			};
 			responses: {
-				/** @description no content */
 				200: {
 					headers: {
 						[name: string]: unknown;
 					};
-					content?: never;
+					content: {
+						"application/json": components["schemas"]["DocumentVersionWithoutContent"];
+					};
 				};
 				default: {
 					headers: {
@@ -451,26 +452,25 @@ export interface components {
 		Array_of_uint8: number[];
 		CreateDocumentVersion: {
 			contentBase64: string;
-			/** Format: date-time */
-			createdDate: string;
+			/**
+			 * Format: uuid
+			 * @description The client can decide the document id (if it wishes to) in order to help with syncing. If the client does not provide a document id, the server will generate one. If the client provides a document id it must not already exist in the database.
+			 */
+			documentId?: string | null;
 			relativePath: string;
 		};
 		CreateDocumentVersionMultipart: {
 			content: components["schemas"]["Array_of_uint8"];
-			/** Format: date-time */
-			created_date: string;
+			/** Format: uuid */
+			document_id?: string | null;
 			relative_path: string;
 		};
 		DeleteDocumentVersion: {
-			/** Format: date-time */
-			createdDate: string;
 			relativePath: string;
 		};
-		/** @description Response to a update document request. */
+		/** @description Response to an update document request. */
 		DocumentUpdateResponse:
 			| {
-					/** Format: date-time */
-					createdDate: string;
 					/** Format: uuid */
 					documentId: string;
 					isDeleted: boolean;
@@ -479,14 +479,11 @@ export interface components {
 					type: "FastForwardUpdate";
 					/** Format: date-time */
 					updatedDate: string;
-					vaultId: string;
 					/** Format: int64 */
 					vaultUpdateId: number;
 			  }
 			| {
 					contentBase64: string;
-					/** Format: date-time */
-					createdDate: string;
 					/** Format: uuid */
 					documentId: string;
 					isDeleted: boolean;
@@ -495,34 +492,27 @@ export interface components {
 					type: "MergingUpdate";
 					/** Format: date-time */
 					updatedDate: string;
-					vaultId: string;
 					/** Format: int64 */
 					vaultUpdateId: number;
 			  };
 		DocumentVersion: {
 			contentBase64: string;
-			/** Format: date-time */
-			createdDate: string;
 			/** Format: uuid */
 			documentId: string;
 			isDeleted: boolean;
 			relativePath: string;
 			/** Format: date-time */
 			updatedDate: string;
-			vaultId: string;
 			/** Format: int64 */
 			vaultUpdateId: number;
 		};
 		DocumentVersionWithoutContent: {
-			/** Format: date-time */
-			createdDate: string;
 			/** Format: uuid */
 			documentId: string;
 			isDeleted: boolean;
 			relativePath: string;
 			/** Format: date-time */
 			updatedDate: string;
-			vaultId: string;
 			/** Format: int64 */
 			vaultUpdateId: number;
 		};
@@ -587,16 +577,12 @@ export interface components {
 		};
 		UpdateDocumentVersion: {
 			contentBase64: string;
-			/** Format: date-time */
-			createdDate: string;
 			/** Format: int64 */
 			parentVersionId: number;
 			relativePath: string;
 		};
 		UpdateDocumentVersionMultipart: {
 			content: components["schemas"]["Array_of_uint8"];
-			/** Format: date-time */
-			createdDate: string;
 			/** Format: int64 */
 			parentVersionId: number;
 			relativePath: string;