Stop text overflowing the history card

frontend/sync-client/package.json

@@ -16,7 +16,7 @@
         "byte-base64": "^1.1.0",
         "minimatch": "^10.0.1",
         "p-queue": "^8.1.0",
-        "reconcile-text": "^0.5.0",
+        "reconcile-text": "^0.7.1",
         "uuid": "^13.0.0"
     },
     "devDependencies": {

frontend/sync-client/src/file-operations/file-operations.ts

@@ -3,8 +3,9 @@ import type { FileSystemOperations } from "./filesystem-operations";
 import type { Database, RelativePath } from "../persistence/database";
 import { SafeFileSystemOperations } from "./safe-filesystem-operations";
 import type { TextWithCursors } from "reconcile-text";
-import { isBinary, reconcile } from "reconcile-text";
+import { reconcile } from "reconcile-text";
 import { isFileTypeMergable } from "../utils/is-file-type-mergable";
+import { isBinary } from "../utils/is-binary";
 
 export class FileOperations {
 	private static readonly PARENTHESES_REGEX = / \((\d+)\)$/;

frontend/sync-client/src/services/sync-service.ts

@@ -16,6 +16,7 @@ import type { DocumentVersion } from "./types/DocumentVersion";
 import type { FetchLatestDocumentsResponse } from "./types/FetchLatestDocumentsResponse";
 import type { PingResponse } from "./types/PingResponse";
 import type { DeleteDocumentVersion } from "./types/DeleteDocumentVersion";
+import type { UpdateTextDocumentVersion } from "./types/UpdateTextDocumentVersion";
 
 export interface CheckConnectionResult {
 	isSuccessful: boolean;
@@ -102,7 +103,64 @@ export class SyncService {
 		});
 	}
 
-	public async put({
+	public async putText({
+		parentVersionId,
+		documentId,
+		relativePath,
+		content
+	}: {
+		parentVersionId: VaultUpdateId;
+		documentId: DocumentId;
+		relativePath: RelativePath;
+		content: (number | string | bigint)[];
+	}): Promise<DocumentUpdateResponse> {
+		return this.withRetries(async () => {
+			this.logger.debug(
+				`Updating text document ${documentId} with parent version ${parentVersionId} and relative path ${relativePath}`
+			);
+
+			const request: UpdateTextDocumentVersion = {
+				parentVersionId,
+				relativePath,
+				content: content.map((c) => {
+					if (typeof c === "bigint") {
+						return Number(c);
+					}
+					return c;
+				})
+			};
+
+			const response = await this.client(
+				this.getUrl(`/documents/${documentId}/text`),
+				{
+					method: "PUT",
+					body: JSON.stringify(request),
+					headers: this.getDefaultHeaders({ type: "json" })
+				}
+			);
+
+			const result: SerializedError | DocumentUpdateResponse =
+				(await response.json()) as  // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion
+					| SerializedError
+					| DocumentUpdateResponse;
+
+			if ("errorType" in result) {
+				throw new Error(
+					`Failed to update document: ${SyncService.formatError(result)}`
+				);
+			}
+
+			this.logger.debug(
+				`Updated document ${JSON.stringify(result)} with id ${
+					result.documentId
+				}}`
+			);
+
+			return result;
+		});
+	}
+
+	public async putBinary({
 		parentVersionId,
 		documentId,
 		relativePath,
@@ -115,7 +173,7 @@ export class SyncService {
 	}): Promise<DocumentUpdateResponse> {
 		return this.withRetries(async () => {
 			this.logger.debug(
-				`Updating document ${documentId} with parent version ${parentVersionId} and relative path ${relativePath}`
+				`Updating binary document ${documentId} with parent version ${parentVersionId} and relative path ${relativePath}`
 			);
 			const formData = new FormData();
 			formData.append("parent_version_id", parentVersionId.toString());
@@ -126,7 +184,7 @@ export class SyncService {
 			);
 
 			const response = await this.client(
-				this.getUrl(`/documents/${documentId}`),
+				this.getUrl(`/documents/${documentId}/binary`),
 				{
 					method: "PUT",
 					body: formData,
@@ -171,10 +229,7 @@ export class SyncService {
 				{
 					method: "DELETE",
 					body: JSON.stringify(request),
-					headers: {
-						"Content-Type": "application/json",
-						...this.getDefaultHeaders()
-					}
+					headers: this.getDefaultHeaders({ type: "json" })
 				}
 			);
 
@@ -297,11 +352,21 @@ export class SyncService {
 		return `${safeRemoteUri}/vaults/${vaultName}${path}`;
 	}
 
-	private getDefaultHeaders(): Record<string, string> {
-		return {
+	private getDefaultHeaders(
+		{ type }: { type?: "json" | "form" } = { type: undefined }
+	): Record<string, string> {
+		const headers: Record<string, string> = {
 			"device-id": this.deviceId,
 			authorization: `Bearer ${this.settings.getSettings().token}`
 		};
+
+		if (type === "json") {
+			headers["Content-Type"] = "application/json";
+		} else if (type === "form") {
+			headers["Content-Type"] = "multipart/form-data";
+		}
+
+		return headers;
 	}
 
 	private async withRetries<T>(fn: () => Promise<T>): Promise<T> {

frontend/sync-client/src/services/types/UpdateBinaryDocumentVersion.ts

@@ -0,0 +1,7 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+
+export interface UpdateBinaryDocumentVersion {
+	parent_version_id: bigint;
+	relative_path: string;
+	content: number[];
+}

frontend/sync-client/src/services/types/UpdateTextDocumentVersion.ts

@@ -0,0 +1,7 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+
+export interface UpdateTextDocumentVersion {
+	parentVersionId: number;
+	relativePath: string;
+	content: (number | string)[];
+}

frontend/sync-client/src/sync-client.ts

@@ -21,13 +21,13 @@ import { CursorTracker } from "./sync-operations/cursor-tracker";
 import type { CursorSpan } from "./services/types/CursorSpan";
 import type { MaybeOutdatedClientCursors } from "./types/maybe-outdated-client-cursors";
 import { FileChangeNotifier } from "./sync-operations/file-change-notifier";
+import { FixedSizeDocumentCache } from "./utils/fix-sized-cache";
 
 export class SyncClient {
 	private static readonly MINIMUM_SAVE_INTERVAL_MS = 1000;
 	private hasStartedOfflineSync = false;
 	private hasFinishedOfflineSync = false;
 
-	// eslint-disable-next-line @typescript-eslint/max-params
 	private constructor(
 		private readonly history: SyncHistory,
 		private readonly settings: Settings,
@@ -135,13 +135,15 @@ export class SyncClient {
 			nativeLineEndings
 		);
 
+		const contentCache = new FixedSizeDocumentCache(1024 * 1024 * 2); // 2 MB cache
 		const unrestrictedSyncer = new UnrestrictedSyncer(
 			logger,
 			database,
 			settings,
 			syncService,
 			fileOperations,
-			history
+			history,
+			contentCache
 		);
 
 		const syncer = new Syncer(

frontend/sync-client/src/sync-operations/unrestricted-syncer.ts

@@ -4,6 +4,7 @@ import type {
 	RelativePath
 } from "../persistence/database";
 
+import { diff } from "reconcile-text";
 import type { SyncService } from "../services/sync-service";
 import type { Logger } from "../tracing/logger";
 import type {
@@ -27,6 +28,9 @@ import { globsToRegexes } from "../utils/globs-to-regexes";
 import type { DocumentVersion } from "../services/types/DocumentVersion";
 import type { DocumentUpdateResponse } from "../services/types/DocumentUpdateResponse";
 import type { DocumentVersionWithoutContent } from "../services/types/DocumentVersionWithoutContent";
+import type { FixedSizeDocumentCache } from "../utils/fix-sized-cache";
+import { isFileTypeMergable } from "../utils/is-file-type-mergable";
+import { isBinary } from "../utils/is-binary";
 
 export class UnrestrictedSyncer {
 	private ignorePatterns: RegExp[];
@@ -37,7 +41,8 @@ export class UnrestrictedSyncer {
 		private readonly settings: Settings,
 		private readonly syncService: SyncService,
 		private readonly operations: FileOperations,
-		private readonly history: SyncHistory
+		private readonly history: SyncHistory,
+		private readonly contentCache: FixedSizeDocumentCache
 	) {
 		this.ignorePatterns = globsToRegexes(
 			this.settings.getSettings().ignorePatterns,
@@ -87,8 +92,12 @@ export class UnrestrictedSyncer {
 				},
 				document
 			);
-
 			this.database.addSeenUpdateId(response.vaultUpdateId);
+			this.updateCache(
+				response.vaultUpdateId,
+				contentBytes,
+				response.relativePath
+			);
 
 			this.history.addHistoryEntry({
 				status: SyncStatus.SUCCESS,
@@ -178,12 +187,32 @@ export class UnrestrictedSyncer {
 				undefined;
 
 			if (areThereLocalChanges) {
-				response = await this.syncService.put({
-					documentId: document.documentId,
-					parentVersionId: document.metadata.parentVersionId,
-					relativePath: document.relativePath,
-					contentBytes
-				});
+				const isText =
+					!isBinary(contentBytes) &&
+					isFileTypeMergable(document.relativePath);
+				const cachedVersion = this.contentCache.get(
+					document.metadata.parentVersionId
+				);
+
+				response =
+					isText && cachedVersion !== undefined
+						? await this.syncService.putText({
+								documentId: document.documentId,
+								parentVersionId:
+									document.metadata.parentVersionId,
+								relativePath: document.relativePath,
+								content: diff(
+									new TextDecoder().decode(cachedVersion),
+									new TextDecoder().decode(contentBytes)
+								)
+							})
+						: await this.syncService.putBinary({
+								documentId: document.documentId,
+								parentVersionId:
+									document.metadata.parentVersionId,
+								relativePath: document.relativePath,
+								contentBytes
+							});
 			} else {
 				if (!force) {
 					this.logger.debug(
@@ -274,12 +303,16 @@ export class UnrestrictedSyncer {
 					},
 					document
 				);
-
 				await this.operations.write(
 					actualPath,
 					contentBytes,
 					responseBytes
 				);
+				this.updateCache(
+					response.vaultUpdateId,
+					contentBytes,
+					actualPath
+				);
 
 				if (!force) {
 					this.history.addHistoryEntry({
@@ -297,6 +330,11 @@ export class UnrestrictedSyncer {
 					},
 					document
 				);
+				this.updateCache(
+					response.vaultUpdateId,
+					contentBytes,
+					actualPath
+				);
 			}
 
 			this.database.addSeenUpdateId(response.vaultUpdateId);
@@ -423,6 +461,11 @@ export class UnrestrictedSyncer {
 				remoteVersion.relativePath,
 				contentBytes
 			);
+			this.updateCache(
+				remoteVersion.vaultUpdateId,
+				contentBytes,
+				remoteVersion.relativePath
+			);
 
 			resolve();
 			this.database.removeDocumentPromise(promise);
@@ -513,4 +556,14 @@ export class UnrestrictedSyncer {
 			};
 		}
 	}
+
+	private updateCache(
+		updateId: number,
+		contentBytes: Uint8Array,
+		filePath: RelativePath
+	): void {
+		if (isFileTypeMergable(filePath) && !isBinary(contentBytes)) {
+			this.contentCache.put(updateId, contentBytes);
+		}
+	}
 }

frontend/sync-client/src/utils/fix-sized-cache.test.ts

@@ -0,0 +1,62 @@
+import { describe, it } from "node:test";
+import assert from "node:assert";
+import { FixedSizeDocumentCache } from "./fix-sized-cache";
+
+describe("fixedSizeDocumentCache", () => {
+	it("happyPath", async () => {
+		const cache = new FixedSizeDocumentCache(4);
+		const doc1 = new Uint8Array([1, 2]);
+		const doc2 = new Uint8Array([3, 4]);
+		const doc3 = new Uint8Array([5, 6]);
+
+		cache.put(1, doc1);
+		assert.equal(cache.get(1), doc1);
+		cache.put(2, doc2);
+		assert.equal(cache.get(1), doc1);
+		assert.equal(cache.get(2), doc2);
+		cache.put(3, doc3);
+		assert.equal(cache.get(1), undefined);
+		assert.equal(cache.get(2), doc2);
+		assert.equal(cache.get(3), doc3);
+	});
+
+	it("updateExistingEntry", async () => {
+		const cache = new FixedSizeDocumentCache(4);
+		const doc1_v1 = new Uint8Array([1, 2]);
+		const doc1_v2 = new Uint8Array([3, 4]);
+		const doc2 = new Uint8Array([5, 6]);
+
+		cache.put(1, doc1_v1);
+		assert.equal(cache.get(1), doc1_v1);
+		cache.put(2, doc2);
+		assert.equal(cache.get(1), doc1_v1);
+		assert.equal(cache.get(2), doc2);
+		cache.put(1, doc1_v2); // Update doc1
+		assert.equal(cache.get(1), doc1_v2);
+		assert.equal(cache.get(2), doc2);
+	});
+
+	it("evictOldestEntry", async () => {
+		const cache = new FixedSizeDocumentCache(4);
+		const doc1 = new Uint8Array([1, 2]);
+		const doc2 = new Uint8Array([3, 4]);
+		const doc3 = new Uint8Array([5, 6]);
+
+		cache.put(1, doc1);
+		cache.put(2, doc2);
+		assert.equal(cache.get(2), doc2);
+		assert.equal(cache.get(1), doc1);
+		cache.put(3, doc3);
+		assert.equal(cache.get(1), doc1);
+		assert.equal(cache.get(2), undefined);
+		assert.equal(cache.get(3), doc3);
+	});
+
+	it("tooLargeEntry", async () => {
+		const cache = new FixedSizeDocumentCache(2);
+		const doc1 = new Uint8Array([1, 2, 3]);
+
+		cache.put(1, doc1);
+		assert.equal(cache.get(1), undefined);
+	});
+});

frontend/sync-client/src/utils/fix-sized-cache.ts

@@ -0,0 +1,57 @@
+// Implements an in-memory fixed-size cache for document contents,
+
+import type { VaultUpdateId } from "../persistence/database";
+
+// evicting the least recently used documents when the size limit is exceeded.
+export class FixedSizeDocumentCache {
+	private readonly maxSizeInBytes: number;
+	private currentSizeInBytes: number;
+	private readonly cache: Map<VaultUpdateId, Uint8Array>;
+	private usageOrder: VaultUpdateId[];
+
+	public constructor(maxSizeInBytes: number) {
+		this.maxSizeInBytes = maxSizeInBytes;
+		this.currentSizeInBytes = 0;
+		this.cache = new Map();
+		this.usageOrder = [];
+	}
+
+	public get(updateId: VaultUpdateId): Uint8Array | undefined {
+		const entry = this.cache.get(updateId);
+		if (entry) {
+			this.usageOrder = this.usageOrder.filter((id) => id !== updateId);
+			this.usageOrder.push(updateId);
+			return entry;
+		}
+		return undefined;
+	}
+
+	public put(updateId: VaultUpdateId, content: Uint8Array): void {
+		if (content.byteLength > this.maxSizeInBytes) {
+			// Document is too large to fit in the cache
+			return;
+		}
+
+		// If the document is already in the cache, update it
+		const existingEntry = this.cache.get(updateId);
+		if (existingEntry != null) {
+			this.currentSizeInBytes -= existingEntry.byteLength;
+			this.cache.delete(updateId);
+			this.usageOrder = this.usageOrder.filter((id) => id !== updateId);
+		}
+		this.cache.set(updateId, content);
+		this.usageOrder.push(updateId);
+		this.currentSizeInBytes += content.byteLength;
+
+		// Evict least recently used documents if over size limit
+		while (
+			this.currentSizeInBytes > this.maxSizeInBytes &&
+			this.usageOrder.length > 0
+		) {
+			const lruUpdateId = this.usageOrder.shift()!; // eslint-disable-line @typescript-eslint/no-non-null-assertion
+			const lruEntry = this.cache.get(lruUpdateId)!; // eslint-disable-line @typescript-eslint/no-non-null-assertion
+			this.cache.delete(lruUpdateId);
+			this.currentSizeInBytes -= lruEntry.byteLength;
+		}
+	}
+}

frontend/sync-client/src/utils/is-binary.ts

@@ -0,0 +1,16 @@
+// Text is unlikely to contain null bytes, so we can use that to distinguish binary files.
+export function isBinary(content: Uint8Array): boolean {
+	for (const byte of content) {
+		if (byte === 0) {
+			return true;
+		}
+	}
+
+	try {
+		new TextDecoder("utf-8", { fatal: true }).decode(content);
+	} catch {
+		return true;
+	}
+
+	return false;
+}