Extract reconcile (#85)

frontend/obsidian-plugin/jest.config.js

@@ -1,3 +1,3 @@
 module.exports = {
-	preset: "ts-jest/presets/js-with-babel-esm"
+	preset: "ts-jest"
 };

frontend/obsidian-plugin/src/obsidian-file-system.ts

@@ -1,13 +1,9 @@
 import type { Stat, Vault, Workspace } from "obsidian";
 import { MarkdownView, normalizePath } from "obsidian";
-import type {
-	FileSystemOperations,
-	RelativePath,
-	TextWithCursors
-} from "sync-client";
-import { lineAndColumnToPosition } from "./utils/line-and-column-to-position";
+import type { FileSystemOperations, RelativePath } from "sync-client";
 import { positionToLineAndColumn } from "./utils/position-to-line-and-column";
-import { getCursorsFromEditor } from "./views/cursors/get-cursors-from-editor";
+import { getSelectionsFromEditor } from "./views/cursors/get-selections-from-editor";
+import type { TextWithCursors, CursorPosition } from "reconcile-text";
 
 export class ObsidianFileSystemOperations implements FileSystemOperations {
 	public constructor(
@@ -80,18 +76,18 @@ export class ObsidianFileSystemOperations implements FileSystemOperations {
 		if (view?.file?.path === path) {
 			const text = view.editor.getValue();
 
-			const cursors = getCursorsFromEditor(view.editor).flatMap(
-				({ id, start: anchor, end: head }) => [
-					{
-						id: 2 * id,
-						characterPosition: anchor
-					},
-					{
-						id: 2 * id + 1,
-						characterPosition: head
-					}
-				]
-			);
+			const cursors: CursorPosition[] = getSelectionsFromEditor(
+				view.editor
+			).flatMap(({ id, start: anchor, end: head }) => [
+				{
+					id: 2 * id,
+					position: anchor
+				},
+				{
+					id: 2 * id + 1,
+					position: head
+				}
+			]);
 
 			const result = updater({
 				text,
@@ -109,13 +105,10 @@ export class ObsidianFileSystemOperations implements FileSystemOperations {
 				const from = result.cursors[2 * i];
 				const to = result.cursors[2 * i + 1];
 				const { line: fromLine, column: fromColumn } =
-					positionToLineAndColumn(
-						result.text,
-						from.characterPosition
-					);
+					positionToLineAndColumn(result.text, from.position);
 
 				const { line: toLine, column: toColumn } =
-					positionToLineAndColumn(result.text, to.characterPosition);
+					positionToLineAndColumn(result.text, to.position);
 
 				selections.push({
 					anchor: { line: fromLine, ch: fromColumn },

frontend/obsidian-plugin/src/vault-link-plugin.ts

@@ -1,9 +1,7 @@
 import type {
 	Editor,
-	EventRef,
 	MarkdownFileInfo,
 	TAbstractFile,
-	Workspace,
 	WorkspaceLeaf
 } from "obsidian";
 import type { MarkdownView } from "obsidian";
@@ -13,7 +11,6 @@ import { HistoryView } from "./views/history/history-view";
 import { StatusBar } from "./views/status-bar/status-bar";
 import { LogsView } from "./views/logs/logs-view";
 import { StatusDescription } from "./views/status-description/status-description";
-import type { CursorSpan, RelativePath } from "sync-client";
 import { SyncClient, rateLimit, DEFAULT_SETTINGS } from "sync-client";
 import { ObsidianFileSystemOperations } from "./obsidian-file-system";
 import { SyncSettingsTab } from "./views/settings/settings-tab";
@@ -24,7 +21,6 @@ import {
 	remoteCursorsPlugin,
 	setCursors
 } from "./views/cursors/remote-cursors-plugin";
-import { getCursorsFromEditor } from "./views/cursors/get-cursors-from-editor";
 import { LocalCursorUpdateListener } from "./views/cursors/local-cursor-update-listener";
 
 const MIN_WAIT_BETWEEN_UPDATES_IN_MS = 250;

frontend/obsidian-plugin/src/views/cursors/get-selections-from-editor.ts

@@ -1,13 +1,13 @@
 import type { Editor } from "obsidian";
 import { lineAndColumnToPosition } from "../../utils/line-and-column-to-position";
 
-export interface Cursor {
+export interface Selection {
 	id: number;
 	start: number;
 	end: number;
 }
 
-export function getCursorsFromEditor(editor: Editor): Cursor[] {
+export function getSelectionsFromEditor(editor: Editor): Selection[] {
 	const text = editor.getValue();
 	return editor.listSelections().map(({ anchor, head }, i) => ({
 		id: i,

frontend/obsidian-plugin/src/views/cursors/local-cursor-update-listener.ts

@@ -1,20 +1,20 @@
 import type { Workspace } from "obsidian";
-import { EventRef, Editor, MarkdownView, MarkdownFileInfo } from "obsidian";
-import type { Logger, SyncClient } from "sync-client";
-import type { Cursor } from "./get-cursors-from-editor";
-import { getCursorsFromEditor } from "./get-cursors-from-editor";
+import { MarkdownView } from "obsidian";
+import type { SyncClient } from "sync-client";
+import type { Selection } from "./get-selections-from-editor";
+import { getSelectionsFromEditor } from "./get-selections-from-editor";
 
 export class LocalCursorUpdateListener {
 	private static readonly UPDATE_INTERVAL_MS = 50;
 	private readonly eventHandle: NodeJS.Timeout;
-	private lastCursorState: Record<string, Cursor[]> = {};
+	private lastCursorState: Record<string, Selection[]> = {};
 
 	public constructor(
 		private readonly client: SyncClient,
 		private readonly workspace: Workspace
 	) {
 		this.eventHandle = setInterval(() => {
-			this.updateAllCursors();
+			this.updateAllSelections();
 		}, LocalCursorUpdateListener.UPDATE_INTERVAL_MS);
 	}
 
@@ -22,8 +22,8 @@ export class LocalCursorUpdateListener {
 		clearInterval(this.eventHandle);
 	}
 
-	private updateAllCursors(): void {
-		const currentCursors = this.getAllCursors();
+	private updateAllSelections(): void {
+		const currentCursors = this.getAllSelections();
 		if (
 			JSON.stringify(this.lastCursorState) ===
 			JSON.stringify(currentCursors)
@@ -40,8 +40,8 @@ export class LocalCursorUpdateListener {
 			});
 	}
 
-	private getAllCursors(): Record<string, Cursor[]> {
-		const cursors: Record<string, Cursor[]> = {};
+	private getAllSelections(): Record<string, Selection[]> {
+		const cursors: Record<string, Selection[]> = {};
 		this.workspace
 			.getLeavesOfType("markdown")
 			.map((leaf) => leaf.view)
@@ -51,7 +51,7 @@ export class LocalCursorUpdateListener {
 				if (!file) {
 					return;
 				}
-				cursors[file.path] = getCursorsFromEditor(view.editor);
+				cursors[file.path] = getSelectionsFromEditor(view.editor);
 			});
 		return cursors;
 	}

frontend/sync-client/jest.config.js

@@ -1,3 +1,3 @@
 module.exports = {
-	preset: "ts-jest/presets/js-with-babel-esm"
+	preset: "ts-jest"
 };

frontend/sync-client/package.json

@@ -10,19 +10,19 @@
     "scripts": {
         "dev": "webpack watch --mode development",
         "build": "webpack --mode production",
-        "test": "NODE_OPTIONS=\"$NODE_OPTIONS --experimental-vm-modules\" jest"
+        "test": "jest"
     },
     "dependencies": {
         "byte-base64": "^1.1.0",
         "minimatch": "^10.0.1",
         "p-queue": "^8.1.0",
-        "uuid": "^11.1.0"
+        "uuid": "^11.1.0",
+        "reconcile-text": "^0.5.0"
     },
     "devDependencies": {
         "@types/jest": "^29.5.14",
         "@types/node": "^22.15.30",
         "jest": "^29.7.0",
-        "sync_lib": "file:../../backend/sync_lib/pkg",
         "ts-jest": "^29.3.4",
         "ts-loader": "^9.5.2",
         "tslib": "2.8.1",
@@ -32,4 +32,4 @@
         "webpack-merge": "^6.0.1",
         "ws": "^8.18.2"
     }
-}
+}

frontend/sync-client/src/file-operations/file-operations.test.ts

@@ -6,12 +6,8 @@ import type {
 import { FileOperations } from "./file-operations";
 import { Logger } from "../tracing/logger";
 import { assertSetContainsExactly } from "../utils/assert-set-contains-exactly";
-import type {
-	FileSystemOperations,
-	TextWithCursors
-} from "./filesystem-operations";
-import init, { base64ToBytes } from "sync_lib";
-import fs from "fs";
+import type { FileSystemOperations } from "./filesystem-operations";
+import type { TextWithCursors } from "reconcile-text";
 
 class MockDatabase implements Partial<Database> {
 	public getLatestDocumentByRelativePath(
@@ -75,13 +71,6 @@ class FakeFileSystemOperations implements FileSystemOperations {
 }
 
 describe("File operations", () => {
-	beforeEach(async () => {
-		const wasmBin = fs.readFileSync(
-			"../../backend/sync_lib/pkg/sync_lib_bg.wasm"
-		);
-		await init({ module_or_path: wasmBin });
-	});
-
 	it("should deconflict renames", async () => {
 		const fileSystemOperations = new FakeFileSystemOperations();
 		const fileOperations = new FileOperations(

frontend/sync-client/src/file-operations/file-operations.ts

@@ -1,18 +1,10 @@
 import type { Logger } from "../tracing/logger";
-import type {
-	FileSystemOperations,
-	TextWithCursors
-} from "./filesystem-operations";
+import type { FileSystemOperations } from "./filesystem-operations";
 import type { Database, RelativePath } from "../persistence/database";
-import {
-	CursorPosition,
-	isBinary,
-	isFileTypeMergable,
-	mergeTextWithCursors,
-	TextWithCursors as RustTextWithCursors
-} from "sync_lib";
 import { SafeFileSystemOperations } from "./safe-filesystem-operations";
-
+import type { TextWithCursors } from "reconcile-text";
+import { isBinary, reconcile } from "reconcile-text";
+import { isFileTypeMergable } from "../utils/is-file-type-mergable";
 export class FileOperations {
 	private static readonly PARENTHESES_REGEX = / \((\d+)\)$/;
 	private readonly fs: SafeFileSystemOperations;
@@ -102,39 +94,25 @@ export class FileOperations {
 		await this.fs.atomicUpdateText(
 			path,
 			({ text, cursors }: TextWithCursors): TextWithCursors => {
-				text = text.replace(this.nativeLineEndings, "\n");
-
 				this.logger.debug(
 					`Performing a 3-way merge for ${path} with the expected content`
 				);
 
-				const left = new RustTextWithCursors(
-					text,
-					cursors.map(
-						(cursor) =>
-							new CursorPosition(
-								cursor.id,
-								cursor.characterPosition
-							)
-					)
+				text = text.replace(this.nativeLineEndings, "\n");
+				const merged = reconcile(
+					expectedText,
+					{ text, cursors },
+					newText
 				);
-				const right = new RustTextWithCursors(newText, []);
-				const merged = mergeTextWithCursors(expectedText, left, right);
 
-				const resultText = merged
-					.text()
-					.replace("\n", this.nativeLineEndings);
-
-				const resultCursors = merged.cursors().map((cursor) => ({
-					id: cursor.id(),
-					characterPosition: cursor.characterPosition()
-				}));
-
-				merged.free();
+				const resultText = merged.text.replace(
+					"\n",
+					this.nativeLineEndings
+				);
 
 				return {
 					text: resultText,
-					cursors: resultCursors
+					cursors: merged.cursors
 				};
 			}
 		);

frontend/sync-client/src/file-operations/filesystem-operations.ts

@@ -1,16 +1,6 @@
 import type { RelativePath } from "../persistence/database";
 
-export interface Cursor {
-	id: number;
-
-	/// The character position is the index of the character in the text where the text lines are separated by '\n' new line character even if the actual text uses different line endings.
-	characterPosition: number;
-}
-
-export interface TextWithCursors {
-	text: string;
-	cursors: Cursor[];
-}
+import type { TextWithCursors } from "reconcile-text";
 
 export interface FileSystemOperations {
 	// List all files that should be synced.

frontend/sync-client/src/file-operations/safe-filesystem-operations.ts

@@ -1,11 +1,9 @@
 import type { RelativePath } from "../persistence/database";
-import type {
-	FileSystemOperations,
-	TextWithCursors
-} from "./filesystem-operations";
+import type { FileSystemOperations } from "./filesystem-operations";
 import type { Logger } from "../tracing/logger";
 import { Locks } from "../utils/locks";
 import { FileNotFoundError } from "./file-not-found-error";
+import type { TextWithCursors } from "reconcile-text";
 
 /**
  * Decorates `FileSystemOperations` to replace errors with `FileNotFoundError`

frontend/sync-client/src/index.ts

@@ -13,11 +13,7 @@ export { Logger, LogLevel, LogLine } from "./tracing/logger";
 export { type SyncSettings, DEFAULT_SETTINGS } from "./persistence/settings";
 export { rateLimit } from "./utils/rate-limit";
 export type { RelativePath, StoredDatabase } from "./persistence/database";
-export type {
-	FileSystemOperations,
-	TextWithCursors,
-	Cursor
-} from "./file-operations/filesystem-operations";
+export type { FileSystemOperations } from "./file-operations/filesystem-operations";
 export type { PersistenceProvider } from "./persistence/persistence";
 export type { CursorSpan } from "./services/types/CursorSpan";
 export type { ClientCursors } from "./services/types/ClientCursors";

frontend/sync-client/src/sync-client.ts

@@ -1,5 +1,3 @@
-import initWasm from "sync_lib";
-import wasmBin from "../../../backend/sync_lib/pkg/sync_lib_bg.wasm";
 import type { PersistenceProvider } from "./persistence/persistence";
 import type { HistoryEntry, HistoryStats } from "./tracing/sync-history";
 import { SyncHistory } from "./tracing/sync-history";
@@ -79,11 +77,6 @@ export class SyncClient {
 
 		const history = new SyncHistory(logger);
 
-		await initWasm(
-			// eslint-disable-next-line
-			(wasmBin as any).default // it is loaded as a base64 string by webpack
-		);
-
 		let state = (await persistence.load()) ?? {
 			settings: undefined,
 			database: undefined

frontend/sync-client/src/utils/assert-set-contains-exactly.ts

@@ -1,4 +1,4 @@
-import assert from "assert";
+import * as assert from "assert";
 
 export function assertSetContainsExactly<T>(set: Set<T>, ...values: T[]): void {
 	assert(

frontend/sync-client/src/utils/deserialize.test.ts

@@ -1,18 +0,0 @@
-import init, { base64ToBytes } from "sync_lib";
-import fs from "fs";
-
-describe("deserialize", () => {
-	it("should serialize a Uint8Array to a base64 string", async () => {
-		const wasmBin = fs.readFileSync(
-			"../../backend/sync_lib/pkg/sync_lib_bg.wasm"
-		);
-		await init({ module_or_path: wasmBin });
-
-		const base64 = "SGVsbG8=";
-		const jsResult = base64ToBytes(base64);
-		const expected = new Uint8Array([72, 101, 108, 108, 111]);
-		expect(jsResult).toEqual(expected);
-		const rustResult = base64ToBytes(base64);
-		expect(jsResult).toEqual(rustResult);
-	});
-});

frontend/sync-client/src/utils/is-file-type-mergable.test.ts

@@ -0,0 +1,28 @@
+import { isFileTypeMergable } from "./is-file-type-mergable";
+
+describe("isFileTypeMergable", () => {
+	it("should return true for .md files", () => {
+		expect(isFileTypeMergable(".md")).toBe(true);
+		expect(isFileTypeMergable("hi.md")).toBe(true);
+		expect(isFileTypeMergable("my/path/to/my/document.md")).toBe(true);
+	});
+
+	it("should return true for .txt files", () => {
+		expect(isFileTypeMergable(".txt")).toBe(true);
+		expect(isFileTypeMergable("hi.txt")).toBe(true);
+		expect(isFileTypeMergable("my/path/to/my/document.txt")).toBe(true);
+	});
+
+	it("should be case insensitive", () => {
+		expect(isFileTypeMergable("hi.MD")).toBe(true);
+		expect(isFileTypeMergable("my/path/to/my/DOCUMENT.MD")).toBe(true);
+		expect(isFileTypeMergable("hi.TXT")).toBe(true);
+		expect(isFileTypeMergable("my/path/to/my/DOCUMENT.TXT")).toBe(true);
+	});
+
+	it("should return false for non-mergable file types", () => {
+		expect(isFileTypeMergable(".json")).toBe(false);
+		expect(isFileTypeMergable("HELLO.JSON")).toBe(false);
+		expect(isFileTypeMergable("my/config.yml")).toBe(false);
+	});
+});

frontend/sync-client/src/utils/is-file-type-mergable.ts

@@ -0,0 +1,6 @@
+export function isFileTypeMergable(pathOrFileName: string): boolean {
+	const parts = pathOrFileName.split(".");
+	const fileExtension = parts.at(-1) ?? "";
+
+	return ["md", "txt"].includes(fileExtension.toLowerCase());
+}

frontend/sync-client/src/utils/serialize.test.ts

@@ -1,18 +0,0 @@
-import { serialize } from "./serialize";
-import init, { bytesToBase64 } from "sync_lib";
-import fs from "fs";
-
-describe("serialize", () => {
-	it("should serialize a Uint8Array to a base64 string", async () => {
-		const wasmBin = fs.readFileSync(
-			"../../backend/sync_lib/pkg/sync_lib_bg.wasm"
-		);
-		await init({ module_or_path: wasmBin });
-
-		const data = new Uint8Array([72, 101, 108, 108, 111]);
-		const jsResult = serialize(data);
-		const rustResult = bytesToBase64(data);
-		expect(rustResult).toBe("SGVsbG8=");
-		expect(jsResult).toBe(rustResult);
-	});
-});

frontend/sync-client/src/utils/serialize.ts

@@ -1,5 +0,0 @@
-import { bytesToBase64 } from "byte-base64";
-
-export function serialize(data: Uint8Array): string {
-	return bytesToBase64(data);
-}

frontend/test-client/jest.config.js

@@ -1,3 +1,3 @@
 module.exports = {
-	preset: "ts-jest/presets/js-with-babel-esm"
+	preset: "ts-jest"
 };

frontend/test-client/src/agent/mock-client.ts

@@ -1,4 +1,4 @@
-import type { StoredDatabase, TextWithCursors } from "sync-client";
+import type { StoredDatabase } from "sync-client";
 import { assert } from "../utils/assert";
 import {
 	type RelativePath,
@@ -6,7 +6,7 @@ import {
 	type SyncSettings,
 	SyncClient
 } from "sync-client";
-
+import type { TextWithCursors } from "reconcile-text";
 export class MockClient implements FileSystemOperations {
 	protected readonly localFiles = new Map<string, Uint8Array>();
 	protected client!: SyncClient;

frontend/test-client/src/cli.ts

@@ -33,7 +33,7 @@ async function runTest({
 	console.info(`Using vault name: ${vaultName}`);
 	const initialSettings: Partial<SyncSettings> = {
 		isSyncEnabled: true,
-		token: "   test-token-change-me     ", // same as in backend/config-e2e.yml with spaces
+		token: "   test-token-change-me     ", // same as in sync-server/config-e2e.yml with spaces
 		vaultName: randomCasing(vaultName) + (Math.random() > 0.5 ? "  " : ""), // extra spaces shouldn't matter
 		syncConcurrency: concurrency,
 		remoteUri: "http://localhost:3000"