andras/reconcile
1
0
Fork 0
Code Issues Pull requests 5 Projects Releases Packages Wiki Activity Actions

Delete frontend

Browse source
This commit is contained in:
Andras Schmelczer 2025-06-15 10:31:03 +01:00
parent c6261d9fc1
commit cd308ad020
No known key found for this signature in database
GPG key ID: FC8F2C3D3D1A718C
111 changed files with 0 additions and 15368 deletions
Download patch file Download diff file Expand all files Collapse all files

55
frontend/eslint.config.mjs
View file

@ -1,55 +0,0 @@
import eslint from "@eslint/js";
import tseslint from "typescript-eslint";
import unusedImports from "eslint-plugin-unused-imports";
export default [
{
ignores: [
"sync-client/src/services/types.ts",
"**/dist/",
"**/*.mjs",
"**/*.js"
]
},
...tseslint.config({
plugins: {
"unused-imports": unusedImports
},
extends: [eslint.configs.recommended, tseslint.configs.all],
rules: {
"no-unused-vars": "off",
"@typescript-eslint/restrict-template-expressions": "off",
"@typescript-eslint/no-unused-vars": "off",
"@typescript-eslint/no-floating-promises": "error",
"@typescript-eslint/parameter-properties": "off",
"@typescript-eslint/require-await": "off",
"@typescript-eslint/class-methods-use-this": "off",
"@typescript-eslint/consistent-return": "off",
"@typescript-eslint/no-unsafe-argument": "off",
"@typescript-eslint/max-params": [
"error",
{
max: 6
}
],
"@typescript-eslint/no-magic-numbers": "off",
"@typescript-eslint/prefer-readonly-parameter-types": "off",
"@typescript-eslint/naming-convention": "off",
"unused-imports/no-unused-vars": [
"warn",
{
vars: "all",
varsIgnorePattern: "^_",
args: "after-used",
argsIgnorePattern: "^_"
}
]
},
languageOptions: {
parserOptions: {
projectService: true,
tsconfigRootDir: import.meta.dirname
}
}
})
];

0
frontend/obsidian-plugin/.hotreload
View file

92
frontend/obsidian-plugin/README.md
View file

@ -1,92 +0,0 @@
# Obsidian Sample Plugin
This is a sample plugin for Obsidian (https://obsidian.md).
This project uses TypeScript to provide type checking and documentation.
The repo depends on the latest plugin API (obsidian.d.ts) in TypeScript Definition format, which contains TSDoc comments describing what it does.
**Note:** The Obsidian API is still in early alpha and is subject to change at any time!
This sample plugin demonstrates some of the basic functionality the plugin API can do.
- Adds a ribbon icon, which shows a Notice when clicked.
- Adds a command "Open Sample Modal" which opens a Modal.
- Adds a plugin setting tab to the settings page.
- Registers a global click event and output 'click' to the console.
- Registers a global interval which logs 'setInterval' to the console.
## First time developing plugins?
Quick starting guide for new plugin devs:
- Check if [someone already developed a plugin for what you want](https://obsidian.md/plugins)! There might be an existing plugin similar enough that you can partner up with.
- Make a copy of this repo as a template with the "Use this template" button (login to GitHub if you don't see it).
- Clone your repo to a local development folder. For convenience, you can place this folder in your `.obsidian/plugins/your-plugin-name` folder.
- Install NodeJS, then run `npm i` in the command line under your repo folder.
- Run `npm run dev` to compile your plugin from `main.ts` to `main.js`.
- Make changes to `main.ts` (or create new `.ts` files). Those changes should be automatically compiled into `main.js`.
- Reload Obsidian to load the new version of your plugin.
- Enable plugin in settings window.
- For updates to the Obsidian API run `npm update` in the command line under your repo folder.
## Releasing new releases
- Update your `manifest.json` with your new version number, such as `1.0.1`, and the minimum Obsidian version required for your latest release.
- Update your `versions.json` file with `"new-plugin-version": "minimum-obsidian-version"` so older versions of Obsidian can download an older version of your plugin that's compatible.
- Create new GitHub release using your new version number as the "Tag version". Use the exact version number, don't include a prefix `v`. See here for an example: https://github.com/obsidianmd/obsidian-sample-plugin/releases
- Upload the files `manifest.json`, `main.js`, `styles.css` as binary attachments. Note: The manifest.json file must be in two places, first the root path of your repository and also in the release.
- Publish the release.
> You can simplify the version bump process by running `npm version patch`, `npm version minor` or `npm version major` after updating `minAppVersion` manually in `manifest.json`.
> The command will bump version in `manifest.json` and `package.json`, and add the entry for the new version to `versions.json`
## Adding your plugin to the community plugin list
- Check the [plugin guidelines](https://docs.obsidian.md/Plugins/Releasing/Plugin+guidelines).
- Publish an initial version.
- Make sure you have a `README.md` file in the root of your repo.
- Make a pull request at https://github.com/obsidianmd/obsidian-releases to add your plugin.
## How to use
- Clone this repo.
- Make sure your NodeJS is at least v16 (`node --version`).
- `npm i` or `yarn` to install dependencies.
- `npm run dev` to start compilation in watch mode.
## Manually installing the plugin
- Copy over `main.js`, `styles.css`, `manifest.json` to your vault `VaultFolder/.obsidian/plugins/your-plugin-id/`.
## Funding URL
You can include funding URLs where people who use your plugin can financially support it.
The simple way is to set the `fundingUrl` field to your link in your `manifest.json` file:
```json
{
"fundingUrl": "https://buymeacoffee.com"
}
```
If you have multiple URLs, you can also do:
```json
{
"fundingUrl": {
"Buy Me a Coffee": "https://buymeacoffee.com",
"GitHub Sponsor": "https://github.com/sponsors",
"Patreon": "https://www.patreon.com/"
}
}
```
## API Documentation
See https://github.com/obsidianmd/obsidian-api

3
frontend/obsidian-plugin/jest.config.js
View file

@ -1,3 +0,0 @@
module.exports = {
preset: "ts-jest/presets/js-with-babel-esm"
};

10
frontend/obsidian-plugin/manifest.json
View file

@ -1,10 +0,0 @@
{
"id": "vault-link",
"name": "VaultLink",
"version": "0.4.0",
"minAppVersion": "0.0.0",
"description": "Self-hosted synchronization and collaboration for your Vault.",
"author": "Andras Schmelczer",
"authorUrl": "https://schmelczer.dev",
"isDesktopOnly": false
}

39
frontend/obsidian-plugin/package.json
View file

@ -1,39 +0,0 @@
{
"name": "vault-link-obsidian-plugin",
"version": "0.4.0",
"description": "This is a sample plugin for Obsidian (https://obsidian.md)",
"main": "main.js",
"scripts": {
"dev": "webpack watch --mode development",
"build": "webpack --mode production",
"test": "jest",
"version": "node version-bump.mjs"
},
"keywords": [],
"author": "",
"license": "MIT",
"devDependencies": {
"@types/jest": "^29.5.14",
"@types/node": "^22.15.30",
"css-loader": "^7.1.2",
"date-fns": "^4.1.0",
"file-loader": "^6.2.0",
"fs-extra": "^11.3.0",
"jest": "^29.7.0",
"mini-css-extract-plugin": "^2.9.2",
"obsidian": "1.8.7",
"resolve-url-loader": "^5.0.0",
"sass": "^1.89.1",
"sass-loader": "^16.0.5",
"sync-client": "file:../sync-client",
"terser-webpack-plugin": "^5.3.14",
"ts-jest": "^29.3.4",
"ts-loader": "^9.5.2",
"tslib": "2.8.1",
"typescript": "5.8.3",
"url": "^0.11.4",
"virtual-scroller": "^1.13.1",
"webpack": "^5.99.9",
"webpack-cli": "^6.0.1"
}
}

178
frontend/obsidian-plugin/src/obsidian-file-system.ts
View file

@ -1,178 +0,0 @@
import type { Stat, Vault, Workspace } from "obsidian";
import { MarkdownView, normalizePath } from "obsidian";
import type {
FileSystemOperations,
RelativePath,
TextWithCursors
} from "sync-client";
import { lineAndColumnToPosition } from "./utils/line-and-column-to-position";
import { positionToLineAndColumn } from "./utils/position-to-line-and-column";
import { getCursorsFromEditor } from "./views/cursors/get-cursors-from-editor";
export class ObsidianFileSystemOperations implements FileSystemOperations {
public constructor(
private readonly vault: Vault,
private readonly workspace: Workspace
) {}
public async listAllFiles(): Promise<RelativePath[]> {
// Let's implement this by hand because vault.adapter.listAllFiles doesn't always return all files.
const allFiles = [];
const remainingFolders = [this.vault.getRoot().path];
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
while (true) {
const folder = remainingFolders.pop();
if (folder == undefined) {
break;
}
// This would be a very bad idea to sync as it would mess with
// the integrity of the sync database.
if (folder.endsWith(".obsidian/plugins/vault-link/data.json")) {
continue;
}
const files = await this.vault.adapter.list(normalizePath(folder));
allFiles.push(...files.files);
remainingFolders.push(...files.folders);
}
return allFiles;
}
public async read(path: RelativePath): Promise<Uint8Array> {
path = normalizePath(path);
const view = this.workspace.getActiveViewOfType(MarkdownView);
if (view?.file?.path === path) {
return new TextEncoder().encode(view.editor.getValue());
}
return new Uint8Array(await this.vault.adapter.readBinary(path));
}
public async write(path: RelativePath, content: Uint8Array): Promise<void> {
path = normalizePath(path);
const view = this.workspace.getActiveViewOfType(MarkdownView);
if (view?.file?.path === path) {
const position = view.editor.getCursor();
view.editor.setValue(new TextDecoder().decode(content));
view.editor.setCursor(position);
return;
}
return this.vault.adapter.writeBinary(
path,
// eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion
content.buffer as ArrayBuffer
);
}
public async atomicUpdateText(
path: RelativePath,
updater: (current: TextWithCursors) => TextWithCursors
): Promise<string> {
path = normalizePath(path);
const view = this.workspace.getActiveViewOfType(MarkdownView);
if (view?.file?.path === path) {
const text = view.editor.getValue();
const cursors = getCursorsFromEditor(view.editor).flatMap(
({ id, start: anchor, end: head }) => [
{
id: 2 * id,
characterPosition: anchor
},
{
id: 2 * id + 1,
characterPosition: head
}
]
);
const result = updater({
text,
cursors
});
if (result.text === text) {
return text;
}
view.editor.setValue(result.text);
const selections = [];
for (let i = 0; i < result.cursors.length / 2; i++) {
const from = result.cursors[2 * i];
const to = result.cursors[2 * i + 1];
const { line: fromLine, column: fromColumn } =
positionToLineAndColumn(
result.text,
from.characterPosition
);
const { line: toLine, column: toColumn } =
positionToLineAndColumn(result.text, to.characterPosition);
selections.push({
anchor: { line: fromLine, ch: fromColumn },
head: { line: toLine, ch: toColumn }
});
}
view.editor.setSelections(selections);
return result.text;
}
return this.vault.adapter.process(
path,
(text) =>
updater({
text,
cursors: []
}).text
);
}
public async getFileSize(path: RelativePath): Promise<number> {
return (await this.statFile(path)).size;
}
public async getModificationTime(path: RelativePath): Promise<Date> {
return new Date((await this.statFile(path)).mtime);
}
public async exists(path: RelativePath): Promise<boolean> {
return this.vault.adapter.exists(normalizePath(path));
}
public async createDirectory(path: RelativePath): Promise<void> {
return this.vault.adapter.mkdir(normalizePath(path));
}
public async delete(path: RelativePath): Promise<void> {
if (!(await this.vault.adapter.trashSystem(normalizePath(path)))) {
return this.vault.adapter.remove(normalizePath(path));
}
}
public async rename(
oldPath: RelativePath,
newPath: RelativePath
): Promise<void> {
return this.vault.adapter.rename(oldPath, newPath);
}
private async statFile(path: string): Promise<Stat> {
const file = await this.vault.adapter.stat(normalizePath(path));
if (!file) {
throw new Error(`File not found: ${path}`);
}
return file;
}
}

9
frontend/obsidian-plugin/src/utils/get-random-color.ts
View file

@ -1,9 +0,0 @@
export function getRandomColor(name: string): string {
let hash = 0;
for (let i = 0; i < name.length; i++) {
hash = (hash << 5) - hash + name.charCodeAt(i);
hash |= 0; // Convert to 32bit integer
}
const normalised = hash / 0x7fffffff;
return `hsl(${Math.abs(normalised * 360)}, 55%, 55%)`; // HSL color
}

42
frontend/obsidian-plugin/src/utils/line-and-column-to-position.test.ts
View file

@ -1,42 +0,0 @@
import { lineAndColumnToPosition } from "./line-and-column-to-position";
describe("lineAndColumnToPosition", () => {
it("should return the correct position for the first line", () => {
const text = "Hello\nWorld";
const position = lineAndColumnToPosition(text, 0, 3);
expect(position).toBe(3);
});
it("should return the correct position for the second line", () => {
const text = "Hello\nWorld";
const position = lineAndColumnToPosition(text, 1, 2);
expect(position).toBe(8);
});
it("should return the correct position for an empty string", () => {
const text = "";
const position = lineAndColumnToPosition(text, 0, 0);
expect(position).toBe(0);
});
it("with carrige return", () => {
expect(lineAndColumnToPosition("a\nb", 1, 1)).toBe(3);
expect(lineAndColumnToPosition("a\r\nb", 1, 1)).toBe(3);
});
it("should handle multi-line strings with varying lengths", () => {
const text = "Line1\nLongerLine2\nShort3";
const position = lineAndColumnToPosition(text, 2, 4);
expect(position).toBe(22);
});
it("should throw an error if the line number is out of range", () => {
const text = "Line1\nLine2";
expect(() => lineAndColumnToPosition(text, 3, 0)).toThrow();
});
it("should throw an error if the column number is out of range", () => {
const text = "Line1\nLine2";
expect(() => lineAndColumnToPosition(text, 1, 10)).toThrow();
});
});

34
frontend/obsidian-plugin/src/utils/line-and-column-to-position.ts
View file

@ -1,34 +0,0 @@
/**
* Converts line and column coordinates to an absolute character position in a text string.
*
* @param line - The zero-based line number
* @param column - The zero-based column number
* @param text - The text string to calculate position in
* @returns The absolute character position (zero-based index) in the text string
* @throws Error if line number is out of range
* @throws Error if column number is out of range
*/
export function lineAndColumnToPosition(
text: string,
line: number,
column: number
): number {
const lines = text.replace("\r", "").split("\n");
if (line >= lines.length) {
throw new Error(`Line number ${line} is out of range.`);
}
if (column > lines[line].length) {
throw new Error(`Column number ${column} is out of range.`);
}
let position = 0;
for (let i = 0; i < line; i++) {
position += lines[i].length + 1;
}
position += column;
return position;
}

61
frontend/obsidian-plugin/src/utils/position-to-line-and-column.test.ts
View file

@ -1,61 +0,0 @@
import { positionToLineAndColumn } from "./position-to-line-and-column";
describe("positionToLineAndColumn", () => {
test("converts position to line and column in multi-line text", () => {
const text = "ab\ncd\n";
expect(positionToLineAndColumn(text, 0)).toEqual({
line: 0,
column: 0
});
expect(positionToLineAndColumn(text, 1)).toEqual({
line: 0,
column: 1
});
expect(positionToLineAndColumn(text, 2)).toEqual({
line: 0,
column: 2
});
expect(positionToLineAndColumn(text, 3)).toEqual({
line: 1,
column: 0
});
expect(positionToLineAndColumn(text, 4)).toEqual({
line: 1,
column: 1
});
expect(positionToLineAndColumn(text, 6)).toEqual({
line: 2,
column: 0
});
});
test("with carrige returns", () => {
expect(positionToLineAndColumn("a\nb", 3)).toEqual({
line: 1,
column: 1
});
expect(positionToLineAndColumn("a\r\nb", 3)).toEqual({
line: 1,
column: 1
});
});
test("handles empty input", () => {
expect(positionToLineAndColumn("", 0)).toEqual({ line: 0, column: 0 });
});
test("handles positions at the end of text", () => {
const text = "End";
expect(positionToLineAndColumn(text, 3)).toEqual({
line: 0,
column: 3
});
});
test("throws error for position out of range", () => {
const text = "Short text";
expect(() => positionToLineAndColumn(text, 15)).toThrow();
expect(() => positionToLineAndColumn(text, -1)).toThrow();
});
});

36
frontend/obsidian-plugin/src/utils/position-to-line-and-column.ts
View file

@ -1,36 +0,0 @@
/**
* Converts a character position in text to line and column numbers.
*
* @param text The text content to analyze
* @param position The character position to convert
* @returns An object containing line and column numbers
* @throws Will throw an error if the position is negative or exceeds the text length
*/
export function positionToLineAndColumn(
text: string,
position: number
): { line: number; column: number } {
if (position < 0) {
throw new Error("Position cannot be negative");
}
text = text.replace("\r", "");
if (
position >
text.length + 1
// +1 to account for the cursor being after last character
) {
throw new Error(
`Position ${position} exceeds text length ${text.length}`
);
}
const textUpToPosition = text.substring(0, position);
const lines = textUpToPosition.split("\n");
const line = lines.length - 1;
const column = lines[lines.length - 1].length;
return { line, column };
}

23
frontend/obsidian-plugin/src/utils/register-console-for-logging.ts
View file

@ -1,23 +0,0 @@
import type { LogLine, SyncClient } from "sync-client";
import { LogLevel } from "sync-client";
export function registerConsoleForLogging(client: SyncClient): void {
client.logger.addOnMessageListener((logLine: LogLine) => {
const formatted = `${logLine.timestamp.toISOString()} ${logLine.level} ${logLine.message}`;
switch (logLine.level) {
case LogLevel.ERROR:
console.error(formatted);
break;
case LogLevel.WARNING:
console.warn(formatted);
break;
case LogLevel.INFO:
console.info(formatted);
break;
case LogLevel.DEBUG:
console.debug(formatted);
break;
}
});
}

220
frontend/obsidian-plugin/src/vault-link-plugin.ts
View file

@ -1,220 +0,0 @@
import type {
Editor,
EventRef,
MarkdownFileInfo,
TAbstractFile,
Workspace,
WorkspaceLeaf
} from "obsidian";
import type { MarkdownView } from "obsidian";
import { Platform, Plugin, TFile } from "obsidian";
import "../manifest.json";
import { HistoryView } from "./views/history/history-view";
import { StatusBar } from "./views/status-bar/status-bar";
import { LogsView } from "./views/logs/logs-view";
import { StatusDescription } from "./views/status-description/status-description";
import type { CursorSpan, RelativePath } from "sync-client";
import { SyncClient, rateLimit, DEFAULT_SETTINGS } from "sync-client";
import { ObsidianFileSystemOperations } from "./obsidian-file-system";
import { SyncSettingsTab } from "./views/settings/settings-tab";
import { registerConsoleForLogging } from "./utils/register-console-for-logging";
import { updateEditorStatusDisplay } from "./views/editor-sync-line/editor-sync-line";
import { remoteCursorsTheme } from "./views/cursors/remote-cursor-theme";
import {
remoteCursorsPlugin,
setCursors
} from "./views/cursors/remote-cursors-plugin";
import { getCursorsFromEditor } from "./views/cursors/get-cursors-from-editor";
import { LocalCursorUpdateListener } from "./views/cursors/local-cursor-update-listener";
const MIN_WAIT_BETWEEN_UPDATES_IN_MS = 250;
export default class VaultLinkPlugin extends Plugin {
private readonly disposables: (() => unknown)[] = [];
private settingsTab: SyncSettingsTab | undefined;
private client!: SyncClient;
private readonly rateLimitedUpdatesPerFile = new Map<
string,
() => Promise<unknown>
>();
public async onload(): Promise<void> {
DEFAULT_SETTINGS.ignorePatterns.push(
".obsidian/**",
".git/**",
".trash/**"
);
this.client = await SyncClient.create({
fs: new ObsidianFileSystemOperations(
this.app.vault,
this.app.workspace
),
persistence: {
load: this.loadData.bind(this),
save: this.saveData.bind(this)
},
nativeLineEndings: Platform.isWin ? "\r\n" : "\n"
});
registerConsoleForLogging(this.client);
const statusDescription = new StatusDescription(this.client);
this.settingsTab = new SyncSettingsTab({
app: this.app,
plugin: this,
syncClient: this.client,
statusDescription
});
this.addSettingTab(this.settingsTab);
new StatusBar(this, this.client);
this.registerView(
HistoryView.TYPE,
(leaf) => new HistoryView(this.client, leaf)
);
this.registerView(
LogsView.TYPE,
(leaf) => new LogsView(this.client, leaf)
);
this.registerEditorExtension([remoteCursorsTheme, remoteCursorsPlugin]);
this.client.addRemoteCursorsUpdateListener((cursors) => {
setCursors(cursors, this.app);
});
const cursorListener = new LocalCursorUpdateListener(
this.client,
this.app.workspace
);
this.disposables.push(() => {
cursorListener.dispose();
});
this.app.workspace.updateOptions();
this.addRibbonIcon(
HistoryView.ICON,
"Open VaultLink events",
async (_: MouseEvent) => this.activateView(HistoryView.TYPE)
);
this.addRibbonIcon(
LogsView.ICON,
"Open VaultLink logs",
async (_: MouseEvent) => this.activateView(LogsView.TYPE)
);
this.app.workspace.onLayoutReady(async () => {
this.registerEditorEvents();
void this.client.start();
const interval = setInterval(() => {
updateEditorStatusDisplay(this.app.workspace, this.client);
}, 200);
this.disposables.push(() => {
clearInterval(interval);
});
});
}
public onunload(): void {
this.client.stop();
this.disposables.forEach((disposable) => {
disposable();
});
}
public openSettings(): void {
// eslint-disable-next-line
(this.app as any).setting.open(); // this is undocumented
// eslint-disable-next-line
(this.app as any).setting.openTab(this.settingsTab); // this is undocumented
}
public closeSettings(): void {
// eslint-disable-next-line
(this.app as any).setting.close(); // this is undocumented
}
public async activateView(type: string): Promise<void> {
const { workspace } = this.app;
let leaf: WorkspaceLeaf | null = null;
const leaves = workspace.getLeavesOfType(type);
if (leaves.length > 0) {
[leaf] = leaves;
} else {
leaf = workspace.getRightLeaf(false);
await leaf?.setViewState({ type: type, active: true });
}
if (leaf) {
await workspace.revealLeaf(leaf);
}
}
private registerEditorEvents(): void {
[
this.app.workspace.on(
"editor-change",
async (
_editor: Editor,
info: MarkdownView | MarkdownFileInfo
) => {
const { file } = info;
if (file) {
await this.rateLimitedUpdate(file.path);
}
}
),
this.app.vault.on("create", async (file: TAbstractFile) => {
if (file instanceof TFile) {
await this.client.syncLocallyCreatedFile(file.path);
}
}),
this.app.vault.on("modify", async (file: TAbstractFile) => {
if (file instanceof TFile) {
await this.rateLimitedUpdate(file.path);
}
}),
this.app.vault.on("delete", async (file: TAbstractFile) => {
await this.client.syncLocallyDeletedFile(file.path);
}),
this.app.vault.on(
"rename",
async (file: TAbstractFile, oldPath: string) => {
if (file instanceof TFile) {
await this.client.syncLocallyUpdatedFile({
oldPath,
relativePath: file.path
});
}
}
)
].forEach((event) => {
this.registerEvent(event);
});
}
private async rateLimitedUpdate(path: string): Promise<void> {
if (!this.rateLimitedUpdatesPerFile.has(path)) {
this.rateLimitedUpdatesPerFile.set(
path,
rateLimit(
async () =>
this.client.syncLocallyUpdatedFile({
relativePath: path
}),
MIN_WAIT_BETWEEN_UPDATES_IN_MS
)
);
}
await this.rateLimitedUpdatesPerFile.get(path)?.();
}
}

17
frontend/obsidian-plugin/src/views/cursors/get-cursors-from-editor.ts
View file

@ -1,17 +0,0 @@
import type { Editor } from "obsidian";
import { lineAndColumnToPosition } from "../../utils/line-and-column-to-position";
export interface Cursor {
id: number;
start: number;
end: number;
}
export function getCursorsFromEditor(editor: Editor): Cursor[] {
const text = editor.getValue();
return editor.listSelections().map(({ anchor, head }, i) => ({
id: i,
start: lineAndColumnToPosition(text, anchor.line, anchor.ch),
end: lineAndColumnToPosition(text, head.line, head.ch)
}));
}

58
frontend/obsidian-plugin/src/views/cursors/local-cursor-update-listener.ts
View file

@ -1,58 +0,0 @@
import type { Workspace } from "obsidian";
import { EventRef, Editor, MarkdownView, MarkdownFileInfo } from "obsidian";
import type { Logger, SyncClient } from "sync-client";
import type { Cursor } from "./get-cursors-from-editor";
import { getCursorsFromEditor } from "./get-cursors-from-editor";
export class LocalCursorUpdateListener {
private static readonly UPDATE_INTERVAL_MS = 50;
private readonly eventHandle: NodeJS.Timeout;
private lastCursorState: Record<string, Cursor[]> = {};
public constructor(
private readonly client: SyncClient,
private readonly workspace: Workspace
) {
this.eventHandle = setInterval(() => {
this.updateAllCursors();
}, LocalCursorUpdateListener.UPDATE_INTERVAL_MS);
}
public dispose(): void {
clearInterval(this.eventHandle);
}
private updateAllCursors(): void {
const currentCursors = this.getAllCursors();
if (
JSON.stringify(this.lastCursorState) ===
JSON.stringify(currentCursors)
) {
return;
}
this.lastCursorState = currentCursors;
this.client
.updateLocalCursors(currentCursors)
.catch((error: unknown) => {
this.client.logger.error(
`Failed to update local cursors: ${error}`
);
});
}
private getAllCursors(): Record<string, Cursor[]> {
const cursors: Record<string, Cursor[]> = {};
this.workspace
.getLeavesOfType("markdown")
.map((leaf) => leaf.view)
.filter((view) => view instanceof MarkdownView)
.forEach((view) => {
const { file } = view;
if (!file) {
return;
}
cursors[file.path] = getCursorsFromEditor(view.editor);
});
return cursors;
}
}

63
frontend/obsidian-plugin/src/views/cursors/remote-cursor-theme.ts
View file

@ -1,63 +0,0 @@
import { EditorView } from "@codemirror/view";
const CARET_WIDTH = 2;
const DOT_RADIUS = 4;
export const remoteCursorsTheme = EditorView.baseTheme({
".selection-caret": {
position: "relative"
},
".selection-caret > *": {
position: "absolute",
backgroundColor: "inherit"
},
".selection-caret > .stick": {
left: 0,
top: 0,
transform: "translateX(-50%)",
width: `${CARET_WIDTH}px`,
height: "100%",
display: "block",
borderRadius: `${CARET_WIDTH / 2}px`,
animation: "blink-stick 1s steps(1) infinite"
},
"@keyframes blink-stick": {
"0%, 100%": { opacity: 1 },
"50%": { opacity: 0 }
},
".selection-caret > .dot": {
borderRadius: "50%",
width: `${DOT_RADIUS * 2}px`,
height: `${DOT_RADIUS * 2}px`,
top: `-${DOT_RADIUS}px`,
left: `-${DOT_RADIUS}px`,
transition: "transform .3s ease-in-out",
transformOrigin: "bottom center",
boxSizing: "border-box"
},
".selection-caret:hover > .dot": {
transform: "scale(0)"
},
".selection-caret > .info": {
top: "-1.3em",
left: `-${CARET_WIDTH / 2}px`,
fontSize: "0.9em",
userSelect: "none",
color: "white",
padding: "0 2px",
transition: "opacity .3s ease-in-out",
opacity: 0,
whiteSpace: "nowrap",
borderRadius: "3px 3px 3px 0"
},
".selection-caret:hover > .info": {
opacity: 1
}
});

46
frontend/obsidian-plugin/src/views/cursors/remote-cursor-widget.ts
View file

@ -1,46 +0,0 @@
import { AnnotationType, Annotation, RangeSet, Range } from "@codemirror/state";
import {
ViewUpdate,
ViewPlugin,
Decoration,
WidgetType
} from "@codemirror/view";
import type { PluginValue, DecorationSet, EditorView } from "@codemirror/view";
export class RemoteCursorWidget extends WidgetType {
public constructor(
private readonly color: string,
private readonly name: string
) {
super();
}
public toDOM(editor: EditorView): HTMLElement {
return editor.contentDOM.createEl(
"span",
{
cls: "selection-caret",
attr: {
style: `background-color: ${this.color}; border-color: ${this.color}`
}
},
(span) => {
span.createEl("div", {
cls: "stick"
});
span.createEl("div", {
cls: "dot"
});
span.createEl("div", {
cls: "info",
text: this.name
});
}
);
}
public eq(other: RemoteCursorWidget): boolean {
return other.color === this.color && other.name === this.name;
}
}

134
frontend/obsidian-plugin/src/views/cursors/remote-cursors-plugin.ts
View file

@ -1,134 +0,0 @@
import type { Range } from "@codemirror/state";
import { RangeSet, Annotation, AnnotationType } from "@codemirror/state";
import { ViewPlugin, Decoration, WidgetType } from "@codemirror/view";
import type {
PluginValue,
DecorationSet,
EditorView,
ViewUpdate
} from "@codemirror/view";
import { RemoteCursorWidget } from "./remote-cursor-widget";
import type { ClientCursors, CursorSpan } from "sync-client";
import type { App } from "obsidian";
import { MarkdownView } from "obsidian";
let cursors: {
name: string;
path: string;
span: CursorSpan;
}[] = [];
import { StateEffect } from "@codemirror/state";
import { getRandomColor } from "src/utils/get-random-color";
const forceUpdate = StateEffect.define();
export class RemoteCursorsPluginValue implements PluginValue {
public decorations: DecorationSet = RangeSet.of([]);
public update(update: ViewUpdate): void {
const decorations: Range<Decoration>[] = [];
cursors.forEach(({ name, span: { start, end } }) => {
const color = getRandomColor(name);
const startLine = update.view.state.doc.lineAt(start);
const endLine = update.view.state.doc.lineAt(end);
const attributes = {
style: `background-color: ${color};`
};
if (startLine.number === endLine.number) {
// selected content in a single line.
decorations.push({
from: start,
to: end,
value: Decoration.mark({
attributes
})
});
} else {
// selected content in multiple lines
// first, render text-selection in the first line
decorations.push({
from: start,
to: startLine.from + startLine.length,
value: Decoration.mark({
attributes
})
});
// render text-selection in the lines between the first and last line
for (let i = startLine.number + 1; i < endLine.number; i++) {
const currentLine = update.view.state.doc.line(i);
decorations.push({
from: currentLine.from,
to: currentLine.to,
value: Decoration.mark({
attributes
})
});
}
// render text-selection in the last line
decorations.push({
from: endLine.from,
to: end,
value: Decoration.mark({
attributes
})
});
}
decorations.push({
from: end,
to: end,
value: Decoration.widget({
side: end - start > 0 ? -1 : 1, // the local cursor should be rendered outside the remote selection
block: false,
widget: new RemoteCursorWidget(color, name)
})
});
});
this.decorations = Decoration.set(decorations, true);
}
}
export const remoteCursorsPlugin = ViewPlugin.fromClass(
RemoteCursorsPluginValue,
{
decorations: (v) => v.decorations
}
);
export function setCursors(clients: ClientCursors[], app: App): void {
cursors = clients.flatMap((client) => {
const clientCursors = client.cursors;
return Object.keys(clientCursors).flatMap((path) => {
const spans = clientCursors[path];
return spans
? spans.map((span) => ({
name: client.userName,
path,
span
}))
: [];
});
});
app.workspace
.getLeavesOfType("markdown")
.map((leaf) => leaf.view)
.filter((view) => view instanceof MarkdownView)
.forEach((view) => {
// @ts-expect-error, not typed
// eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion
const editor = view.editor.cm as EditorView;
editor.dispatch({
effects: [forceUpdate.of(null)]
});
});
}

43
frontend/obsidian-plugin/src/views/editor-sync-line/editor-sync-line.scss
View file

@ -1,43 +0,0 @@
.vault-link-sync-status {
position: absolute;
right: var(--size-4-4);
top: var(--size-4-2);
opacity: 0.7;
cursor: pointer;
> span {
opacity: 0;
position: absolute;
min-width: 200px;
text-align: right;
padding-right: var(--size-2-2);
top: 50%;
left: 0;
transform: translateY(-50%) translateX(-100%) translateY(-2px);
transition: opacity 200ms;
}
&:hover {
> span {
opacity: 1;
}
}
> .icon {
line-height: 0;
}
&.loading > .icon {
animation: spin 2s linear infinite;
@keyframes spin {
0% {
transform: rotate(0deg);
}
100% {
transform: rotate(360deg);
}
}
}
}

55
frontend/obsidian-plugin/src/views/editor-sync-line/editor-sync-line.ts
View file

@ -1,55 +0,0 @@
import type { Workspace } from "obsidian";
import { FileView, setIcon } from "obsidian";
import type { SyncClient } from "sync-client";
import { DocumentUpdateStatus } from "sync-client";
import "./editor-sync-line.scss";
export function updateEditorStatusDisplay(
workspace: Workspace,
client: SyncClient
): void {
workspace.iterateAllLeaves((leaf) => {
if (leaf.view instanceof FileView) {
const filePath = leaf.view.file?.path;
if (filePath == null) {
return;
}
const parent = leaf.view.contentEl.querySelector(".cm-editor");
if (parent == null) {
return;
}
const element =
parent.querySelector(".vault-link-sync-status") ??
parent.createDiv(
{
cls: "vault-link-sync-status"
},
(el) => {
el.createSpan({ text: "VaultLink sync state" });
el.createDiv({
cls: "icon"
});
}
);
const isLoading =
client.getDocumentSyncingStatus(filePath) ==
DocumentUpdateStatus.SYNCING;
if (isLoading) {
element.classList.add("loading");
} else {
element.classList.remove("loading");
}
const iconContainer = element.querySelector(".icon");
if (iconContainer != null) {
setIcon(
iconContainer as HTMLElement, // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion
isLoading ? "loader" : "circle-check"
);
}
}
});
}

61
frontend/obsidian-plugin/src/views/history/history-view.scss
View file

@ -1,61 +0,0 @@
.history-card {
padding: var(--size-4-4);
margin: var(--size-4-2);
background-color: var(--color-base-00);
border-radius: var(--radius-l);
container-type: inline-size;
&.clickable {
cursor: pointer;
}
&.success {
background-color: rgba(var(--color-green-rgb), 0.2);
}
&.error {
background-color: rgba(var(--color-red-rgb), 0.2);
}
&.skipped {
background-color: rgba(var(--color-green-rgb), 0.08);
}
.history-card-header {
display: flex;
justify-content: space-between;
align-items: center;
margin-bottom: var(--size-4-2);
gap: var(--size-4-2);
@container (max-width: 300px) {
flex-direction: column;
align-items: flex-start;
}
.history-card-title {
font: var(--font-monospace);
display: flex;
align-items: center;
gap: var(--size-4-2);
word-break: break-all;
margin: 0;
> span {
margin-bottom: var(--size-4-1);
}
}
.history-card-timestamp {
font-size: var(--font-ui-small);
font-style: italic;
color: var(--italic-color);
}
}
.history-card-message {
font-size: var(--font-ui-medium);
color: var(--color-base-70);
margin: 0;
}
}

221
frontend/obsidian-plugin/src/views/history/history-view.ts
View file

@ -1,221 +0,0 @@
import "./history-view.scss";
import type { IconName, WorkspaceLeaf } from "obsidian";
import { ItemView, setIcon } from "obsidian";
import { intlFormatDistance } from "date-fns";
import type { HistoryEntry, SyncClient } from "sync-client";
import { SyncType } from "sync-client";
export class HistoryView extends ItemView {
public static readonly TYPE = "history-view";
public static readonly ICON = "square-stack";
private timer: NodeJS.Timeout | null = null;
private historyContainer: HTMLElement | undefined;
private readonly historyEntryToElement = new Map<
HistoryEntry,
HTMLElement
>();
public constructor(
private readonly client: SyncClient,
leaf: WorkspaceLeaf
) {
super(leaf);
this.icon = HistoryView.ICON;
this.client.addSyncHistoryUpdateListener(
() =>
void this.updateView().catch((error: unknown) => {
this.client.logger.error(
`Failed to update history view: ${error}`
);
})
);
}
private static getSyncTypeIcon(type: SyncType | undefined): IconName {
switch (type) {
case SyncType.CREATE:
return "file-plus";
case SyncType.DELETE:
return "trash-2";
case SyncType.UPDATE:
return "file-pen-line";
case SyncType.MOVE:
return "move-right";
case SyncType.SKIPPED:
return "circle-slash";
case undefined:
default:
return "";
}
}
private static renderSyncItemTitle(
element: HTMLElement,
entry: HistoryEntry
): void {
const syncTypeIcon = HistoryView.getSyncTypeIcon(entry.details.type);
if (syncTypeIcon) {
setIcon(element.createDiv(), syncTypeIcon);
}
let fileName = entry.details.relativePath.split("/").pop() ?? "";
fileName = fileName.replace(/\.md$/, "");
element.createEl("span", {
text:
entry.details.type === SyncType.SKIPPED
? `Skipped: ${fileName}`
: fileName
});
}
private static updateTimeSince(
element: HTMLElement,
entry: HistoryEntry
): void {
const timestampElement = element.querySelector(
".history-card-timestamp"
);
if (timestampElement != null) {
timestampElement.textContent =
HistoryView.getTimestampAndAuthor(entry);
}
}
private static getTimestampAndAuthor(entry: HistoryEntry): string {
let content = intlFormatDistance(entry.timestamp, new Date());
if ("author" in entry && entry.author !== undefined) {
content += ` by ${entry.author}`;
}
return content;
}
public getViewType(): string {
return HistoryView.TYPE;
}
public getDisplayText(): string {
return "VaultLink history";
}
public async onOpen(): Promise<void> {
const container = this.containerEl.children[1];
container.createEl("h4", { text: "VaultLink history" });
this.historyContainer = container.createDiv({ cls: "logs-container" });
await this.updateView();
this.timer = setInterval(() => void this.updateView(), 1000);
}
public async onClose(): Promise<void> {
if (this.timer) {
clearInterval(this.timer);
}
}
private async updateView(): Promise<void> {
const container = this.historyContainer;
if (container === undefined) {
return;
}
// entries are newest first, but we prepend new ones
const entries = this.client.getHistoryEntries().toReversed();
if (this.historyEntryToElement.size === 0 && entries.length > 0) {
// Clear the "No update has happened yet" message
container.empty();
}
entries.forEach((entry) => {
const element = this.historyEntryToElement.get(entry);
if (element !== undefined) {
HistoryView.updateTimeSince(element, entry);
return;
}
const newElement = this.createHistoryCard(container, entry);
container.prepend(newElement);
this.historyEntryToElement.set(entry, newElement);
});
const newEntries = new Set(entries);
for (const [entry, element] of this.historyEntryToElement) {
if (!newEntries.has(entry)) {
element.remove();
this.historyEntryToElement.delete(entry);
}
}
if (entries.length === 0) {
container.empty();
container.createEl("p", {
text: "No update has happened yet."
});
}
}
private createHistoryCard(
container: HTMLElement,
entry: HistoryEntry
): HTMLElement {
return container.createDiv(
{
cls: ["history-card", entry.status.toLocaleLowerCase()]
},
(card) => {
if (
this.app.vault.getFileByPath(entry.details.relativePath) !=
null
) {
card.addEventListener("click", () => {
void this.app.workspace.openLinkText(
entry.details.relativePath,
entry.details.relativePath,
false
);
});
card.addClass("clickable");
}
card.createDiv(
{
cls: "history-card-header"
},
(header) => {
header.createEl(
"h5",
{
cls: "history-card-title"
},
(title) => {
HistoryView.renderSyncItemTitle(title, entry);
}
);
header.createSpan({
text: HistoryView.getTimestampAndAuthor(entry),
cls: "history-card-timestamp"
});
}
);
const body =
entry.details.type === SyncType.MOVE
? `${entry.message}. Moved from '${entry.details.movedFrom}' to '${entry.details.relativePath}'`
: `${entry.message}.`;
card.createEl("p", {
text: body,
cls: "history-card-message"
});
}
);
}
}

60
frontend/obsidian-plugin/src/views/logs/logs-view.scss
View file

@ -1,60 +0,0 @@
.logs-view {
display: flex;
flex-direction: column;
.verbosity-selector {
display: flex;
align-items: center;
justify-content: space-between;
font-weight: normal;
gap: var(--size-4-2);
margin: var(--size-4-4) var(--size-4-2);
h4 {
margin: 0;
}
select {
cursor: pointer;
}
}
.logs-container {
max-width: 100%;
overflow-y: auto;
.log-message {
font: var(--font-monospace);
margin-bottom: var(--size-2-1);
overflow-wrap: break-word;
white-space: pre-wrap;
user-select: all;
.timestamp {
padding: var(--size-2-1) var(--size-4-1);
border-radius: var(--radius-s);
background-color: var(--color-base-30);
font-size: var(--font-ui-small);
font-family: var(--font-monospace);
font-weight: var(--bold-weight);
margin-right: var(--size-4-1);
}
&.DEBUG {
color: var(--color-base-50);
}
&.INFO {
color: var(--color-base-100);
}
&.WARNING {
color: rgb(var(--color-yellow-rgb));
}
&.ERROR {
color: rgb(var(--color-red-rgb));
}
}
}
}

152
frontend/obsidian-plugin/src/views/logs/logs-view.ts
View file

@ -1,152 +0,0 @@
import "./logs-view.scss";
import type { WorkspaceLeaf } from "obsidian";
import { ItemView } from "obsidian";
import type { LogLine } from "sync-client";
import { LogLevel, type SyncClient } from "sync-client";
export class LogsView extends ItemView {
public static readonly TYPE = "logs-view";
public static readonly ICON = "logs";
private static readonly MAX_OFFSET_FROM_BOTTOM_WITH_AUTO_SCROLL_PX = 300;
private logsContainer: HTMLElement | undefined;
private readonly logLineToElement = new Map<LogLine, HTMLElement>();
private minLogLevel: LogLevel = LogLevel.INFO;
public constructor(
private readonly client: SyncClient,
leaf: WorkspaceLeaf
) {
super(leaf);
this.icon = LogsView.ICON;
this.client.logger.addOnMessageListener(() => {
this.updateView();
});
}
private static createLogLineElement(
container: HTMLElement,
logLine: LogLine
): HTMLElement {
return container.createDiv(
{
cls: ["log-message", logLine.level]
},
(messageContainer) => {
messageContainer.createEl("span", {
text: LogsView.formatTimestamp(logLine.timestamp),
cls: "timestamp"
});
messageContainer.createEl("span", {
text: logLine.message
});
}
);
}
private static formatTimestamp(timestamp: Date): string {
return timestamp.toTimeString().split(" ")[0];
}
public getViewType(): string {
return LogsView.TYPE;
}
public getDisplayText(): string {
return "VaultLink logs";
}
public async onOpen(): Promise<void> {
const container = this.containerEl.children[1];
container.addClass("logs-view");
const logLevels = [
{ label: "Debug", value: LogLevel.DEBUG },
{ label: "Info", value: LogLevel.INFO },
{ label: "Warn", value: LogLevel.WARNING },
{ label: "Error", value: LogLevel.ERROR }
];
container.createDiv(
{
cls: "verbosity-selector"
},
(verbositySection) => {
verbositySection.createEl("h4", {
text: "VaultLink logs"
});
verbositySection.createEl("select", {}, (dropdown) => {
logLevels.forEach(({ label, value }) =>
dropdown.createEl("option", { text: label, value })
);
dropdown.value = this.minLogLevel;
dropdown.addEventListener("change", () => {
// eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion
this.minLogLevel = dropdown.value as LogLevel;
this.logsContainer?.empty();
this.logLineToElement.clear();
this.updateView();
});
});
}
);
this.logsContainer = container.createDiv({ cls: "logs-container" });
this.updateView();
}
private updateView(): void {
const container = this.logsContainer;
if (container === undefined) {
return;
}
const logs = this.client.logger.getMessages(this.minLogLevel);
if (this.logLineToElement.size === 0 && logs.length > 0) {
// Clear the "No logs available yet" message
container.empty();
}
const shouldScroll =
container.scrollTop == 0 ||
container.scrollHeight -
container.clientHeight -
container.scrollTop <
LogsView.MAX_OFFSET_FROM_BOTTOM_WITH_AUTO_SCROLL_PX;
logs.forEach((message) => {
if (this.logLineToElement.has(message)) {
return;
}
const element = LogsView.createLogLineElement(container, message);
this.logLineToElement.set(message, element);
});
const newLines = new Set(logs);
for (const [logLine, element] of this.logLineToElement) {
if (!newLines.has(logLine)) {
element.remove();
this.logLineToElement.delete(logLine);
}
}
if (logs.length === 0) {
container.empty();
container.createEl("p", {
text: "No logs available yet."
});
} else if (shouldScroll) {
container.scrollTop = container.scrollHeight;
}
}
}

57
frontend/obsidian-plugin/src/views/settings/settings-tab.scss
View file

@ -1,57 +0,0 @@
@mixin number-card {
padding: var(--size-2-1) var(--size-4-1);
border-radius: var(--radius-s);
background-color: var(--color-base-30);
font-size: var(--font-ui-small);
&.good {
background-color: rgba(var(--color-green-rgb), 0.35);
}
&.bad {
background-color: rgba(var(--color-red-rgb), 0.35);
}
}
.vault-link-settings {
h2 {
display: flex;
align-items: center;
font-size: var(--h2-size);
.version {
@include number-card;
margin: var(--size-2-2) 0 0 var(--size-4-2);
background-color: var(--color-base-30);
color: var(--color-base-70);
font-size: var(--font-ui-smaller);
}
}
.button-container {
display: flex;
gap: var(--size-4-2);
}
h3 {
font-size: var(--font-ui-large);
margin-top: var(--heading-spacing);
}
button,
input[type="range"],
.checkbox-container,
.slider::-webkit-slider-thumb {
cursor: pointer;
}
input[type="text"],
textarea {
width: 250px;
}
textarea {
resize: none;
height: 75px;
}
}

383
frontend/obsidian-plugin/src/views/settings/settings-tab.ts
View file

@ -1,383 +0,0 @@
import "./settings-tab.scss";
import type { App } from "obsidian";
import { Notice, PluginSettingTab, Setting } from "obsidian";
import type VaultLinkPlugin from "src/vault-link-plugin";
import type { SyncClient, SyncSettings } from "sync-client";
import { HistoryView } from "../history/history-view";
import { LogsView } from "../logs/logs-view";
import type { StatusDescription } from "../status-description/status-description";
export class SyncSettingsTab extends PluginSettingTab {
private editedServerUri: string;
private editedToken: string;
private editedVaultName: string;
private readonly plugin: VaultLinkPlugin;
private readonly syncClient: SyncClient;
private readonly statusDescription: StatusDescription;
private statusDescriptionSubscription: (() => void) | undefined;
public constructor({
app,
plugin,
syncClient,
statusDescription
}: {
app: App;
plugin: VaultLinkPlugin;
syncClient: SyncClient;
statusDescription: StatusDescription;
}) {
super(app, plugin);
this.plugin = plugin;
this.syncClient = syncClient;
this.statusDescription = statusDescription;
this.editedServerUri = this.syncClient.getSettings().remoteUri;
this.editedToken = this.syncClient.getSettings().token;
this.editedVaultName = this.syncClient.getSettings().vaultName;
this.syncClient.addOnSettingsChangeListener(
(newSettings, oldSettings) => {
let hasChanged = false;
if (newSettings.remoteUri !== oldSettings.remoteUri) {
this.editedServerUri = newSettings.remoteUri;
hasChanged = true;
}
if (newSettings.token !== oldSettings.token) {
this.editedToken = newSettings.token;
hasChanged = true;
}
if (newSettings.vaultName !== oldSettings.vaultName) {
this.editedVaultName = newSettings.vaultName;
hasChanged = true;
}
if (hasChanged) {
this.display();
}
}
);
}
public display(): void {
const { containerEl } = this;
containerEl.empty();
containerEl.addClass("vault-link-settings");
this.renderSettingsHeader(containerEl);
this.renderConnectionSettings(containerEl);
this.renderSyncSettings(containerEl);
}
public hide(): void {
super.hide();
this.setStatusDescriptionSubscription();
}
private renderSettingsHeader(containerEl: HTMLElement): void {
containerEl.createEl("h2", { text: "VaultLink" }).createSpan({
text: this.plugin.manifest.version,
cls: "version"
});
containerEl.createDiv(
{
cls: "description"
},
(descriptionContainer) => {
this.setStatusDescriptionSubscription((): void => {
this.statusDescription.renderStatusDescription(
descriptionContainer
);
});
}
);
containerEl.createDiv(
{
cls: "button-container"
},
(buttonContainer) => {
buttonContainer.createEl(
"button",
{
text: "Show history"
},
(button) =>
(button.onclick = async (): Promise<void> => {
this.plugin.closeSettings();
await this.plugin.activateView(HistoryView.TYPE);
})
);
buttonContainer.createEl(
"button",
{
text: "Show logs"
},
(button) =>
(button.onclick = async (): Promise<void> => {
this.plugin.closeSettings();
await this.plugin.activateView(LogsView.TYPE);
})
);
}
);
}
private renderConnectionSettings(containerEl: HTMLElement): void {
containerEl.createEl("h3", { text: "Connection" });
const [title, updateTitle] = this.unsavedAwareSettingName(
"Server address",
"remoteUri"
);
new Setting(containerEl)
.setName(title)
.setDesc(
"Your VaultLink server's URL including the protocol and full path."
)
.setTooltip("This is the URL of the server you want to sync with.")
.addText((text) =>
text
.setPlaceholder("https://example.com:3000")
.setValue(this.editedServerUri.toLowerCase().trim())
.onChange((value) => {
this.editedServerUri = value.toLowerCase().trim();
updateTitle(value.toLowerCase().trim());
})
);
const [tokenTitle, updateTokenTitle] = this.unsavedAwareSettingName(
"Access token",
"token"
);
new Setting(containerEl)
.setName(tokenTitle)
.setClass("sync-settings-access-token")
.setDesc(
"Set the access token for the server that you can get from the server"
)
.setTooltip("todo, links to dcocs")
.addTextArea((text) =>
text
.setPlaceholder("ey...")
.setValue(this.editedToken.trim())
.onChange((value) => {
this.editedToken = value.trim();
updateTokenTitle(value.trim());
})
);
const [vaultNameTitle, updateVaultNameTitle] =
this.unsavedAwareSettingName("Vault name", "vaultName");
new Setting(containerEl)
.setName(vaultNameTitle)
.setDesc(
"Set the name of the remote vault that you want to sync with"
)
.setTooltip("todo, links to dcocs")
.addText((text) =>
text
.setPlaceholder("My Obsidian Vault")
.setValue(this.editedVaultName.toLowerCase().trim())
.onChange((value) => {
this.editedVaultName = value.toLowerCase().trim();
updateVaultNameTitle(value.toLowerCase().trim());
})
);
new Setting(containerEl)
.addButton((button) =>
button.setButtonText("Apply").onClick(async () => {
if (this.areThereUnsavedChanges()) {
await this.syncClient.setSettings({
vaultName: this.editedVaultName,
remoteUri: this.editedServerUri,
token: this.editedToken
});
new Notice(
"The changes have been applied successfully!"
);
await this.statusDescription.updateConnectionState();
} else {
new Notice("No changes to apply");
}
})
)
.addButton((button) =>
button.setButtonText("Test connection").onClick(async () => {
if (this.areThereUnsavedChanges()) {
new Notice(
"There are unsaved changes, testing with the currently saved settings"
);
}
new Notice(
(await this.syncClient.checkConnection()).serverMessage
);
await this.statusDescription.updateConnectionState();
})
);
}
private areThereUnsavedChanges(): boolean {
return (
this.editedServerUri !== this.syncClient.getSettings().remoteUri ||
this.editedToken !== this.syncClient.getSettings().token ||
this.editedVaultName !== this.syncClient.getSettings().vaultName
);
}
private renderSyncSettings(containerEl: HTMLElement): void {
containerEl.createEl("h3", { text: "Sync" });
new Setting(containerEl)
.setName("Enable sync")
.setDesc(
"Enable pulling and pushing changes to the remote server. The first time it's enabled, or after the sync state has been reset, all local files will be pushed to the server."
)
.setTooltip(
"Enable pulling and pushing changes to the remote server."
)
.addToggle((toggle) =>
toggle
.setValue(this.syncClient.getSettings().isSyncEnabled)
.onChange(async (value) =>
this.syncClient.setSetting("isSyncEnabled", value)
)
);
new Setting(containerEl)
.setName("Ignore patterns")
.setDesc(
"Patterns to ignore when syncing. Each line is a separate glob pattern. Patterns are matched against the relative path of the file. For example, to ignore all files in a folder named 'ignore', enter 'ignore/*'. To ignore all files with the extension '.log', enter '*.log'."
)
.addTextArea((text) =>
text
.setValue(
this.syncClient.getSettings().ignorePatterns.join("\n")
)
.setPlaceholder("Enter patterns to ignore, one per line")
.onChange(async (value) => {
const patterns = value
.split("\n")
.map((pattern) => pattern.trim())
.filter((pattern) => pattern.length > 0);
return this.syncClient.setSetting(
"ignorePatterns",
patterns
);
})
);
new Setting(containerEl)
.setName("Sync concurrency")
.setDesc(
"How many concurrent sync operations to run. Setting this value higher may increase the overall performance, however, it will require more memory as well. If you notice frequent crashes, especially on mobile, set this to 1."
)
.addSlider((text) =>
text
.setLimits(1, 16, 1)
.setDynamicTooltip()
.setInstant(false)
.setValue(this.syncClient.getSettings().syncConcurrency)
.onChange(async (value) =>
this.syncClient.setSetting("syncConcurrency", value)
)
);
new Setting(containerEl)
.setName("Maximum file size to be uploaded (MB)")
.setDesc(
"Set the maximum file size that can be uploaded to the server. Files larger than this size will be ignored."
)
.addText((input) =>
input
.setValue(
this.syncClient.getSettings().maxFileSizeMB.toString()
)
.onChange(async (value) => {
if (value === "") {
return;
}
let parsedValue = Number.parseFloat(value);
if (Number.isNaN(parsedValue) || parsedValue < 0) {
parsedValue =
this.syncClient.getSettings().maxFileSizeMB;
}
if (value !== parsedValue.toString()) {
input.setValue(parsedValue.toString());
}
return this.syncClient.setSetting(
"maxFileSizeMB",
parsedValue
);
})
);
new Setting(containerEl)
.setName("Danger zone")
.setDesc(
"Delete the local metadata database while leaving the local and remote files intact."
)
.addButton((button) =>
button.setButtonText("Reset sync state").onClick(async () => {
await this.syncClient.reset();
new Notice(
"Sync state has been reset, you will need to resync"
);
})
);
}
private setStatusDescriptionSubscription(
newSubscription?: () => void
): void {
if (this.statusDescriptionSubscription) {
this.statusDescription.removeStatusChangeListener(
this.statusDescriptionSubscription
);
}
this.statusDescriptionSubscription = newSubscription;
if (this.statusDescriptionSubscription) {
this.statusDescriptionSubscription();
this.statusDescription.addStatusChangeListener(
this.statusDescriptionSubscription
);
}
}
private unsavedAwareSettingName(
name: string,
settingName: keyof SyncSettings
): [
DocumentFragment,
(newValue: SyncSettings[keyof SyncSettings]) => void
] {
const titleContainer = document.createDocumentFragment();
const title = titleContainer.createEl("div", {
text: name,
cls: "setting-item-name"
});
const updateTitle = (
currentValue: SyncSettings[keyof SyncSettings]
): void => {
title.innerText = `${name}${
currentValue !== this.syncClient.getSettings()[settingName]
? " (unsaved)"
: ""
}`;
};
return [titleContainer, updateTitle];
}
}

14
frontend/obsidian-plugin/src/views/status-bar/status-bar.scss
View file

@ -1,14 +0,0 @@
.sync-status {
display: flex;
gap: var(--size-4-2);
* {
display: block;
}
.initialize-button {
padding: 0 var(--size-4-2);
background: rgba(var(--color-red-rgb), 0.4);
cursor: pointer;
}
}

77
frontend/obsidian-plugin/src/views/status-bar/status-bar.ts
View file

@ -1,77 +0,0 @@
import "./status-bar.scss";
import type { HistoryStats, SyncClient } from "sync-client";
import type VaultLinkPlugin from "../../vault-link-plugin";
export class StatusBar {
private readonly statusBarItem: HTMLElement;
private lastHistoryStats: HistoryStats | undefined;
private lastRemaining: number | undefined;
public constructor(
private readonly plugin: VaultLinkPlugin,
private readonly syncClient: SyncClient
) {
this.statusBarItem = plugin.addStatusBarItem();
this.syncClient.addSyncHistoryUpdateListener((status) => {
this.lastHistoryStats = status;
this.updateStatus();
});
this.syncClient.addRemainingSyncOperationsListener(
(remainingOperations) => {
this.lastRemaining = remainingOperations;
this.updateStatus();
}
);
this.syncClient.addOnSettingsChangeListener(() => {
this.updateStatus();
});
}
private updateStatus(): void {
this.statusBarItem.empty();
const container = this.statusBarItem.createDiv({
cls: ["sync-status"]
});
if (!this.syncClient.getSettings().isSyncEnabled) {
const button = container.createEl("button", {
text: "VaultLink is disabled, click to configure",
cls: "initialize-button"
});
button.onclick = (): void => {
this.plugin.openSettings();
};
return;
}
let hasShownMessage = false;
if ((this.lastRemaining ?? 0) > 0) {
hasShownMessage = true;
container.createSpan({ text: `${this.lastRemaining} ⏳` });
}
if ((this.lastHistoryStats?.success ?? 0) > 0) {
hasShownMessage = true;
container.createSpan({
text: `${this.lastHistoryStats?.success ?? 0} ✅`
});
}
if ((this.lastHistoryStats?.error ?? 0) > 0) {
hasShownMessage = true;
container.createSpan({
text: `${this.lastHistoryStats?.error ?? 0} ❌`
});
}
if (!hasShownMessage) {
container.createSpan({ text: "VaultLink is idle" });
}
}
}

32
frontend/obsidian-plugin/src/views/status-description/status-description.scss
View file

@ -1,32 +0,0 @@
@mixin number-card {
padding: var(--size-2-1) var(--size-4-1);
border-radius: var(--radius-s);
background-color: var(--color-base-30);
font-size: var(--font-ui-small);
&.good {
background-color: rgba(var(--color-green-rgb), 0.35);
}
&.bad {
background-color: rgba(var(--color-red-rgb), 0.35);
}
}
.status-description {
margin: var(--p-spacing) 0;
.number {
@include number-card;
font-family: var(--font-monospace);
font-weight: var(--bold-weight);
}
.error {
color: rgb(var(--color-red-rgb));
}
.warning {
color: rgb(var(--color-yellow-rgb));
}
}

148
frontend/obsidian-plugin/src/views/status-description/status-description.ts
View file

@ -1,148 +0,0 @@
import "./status-description.scss";
import type {
HistoryStats,
NetworkConnectionStatus,
SyncClient
} from "sync-client";
export class StatusDescription {
private lastHistoryStats: HistoryStats | undefined;
private lastRemaining: number | undefined;
private lastConnectionState: NetworkConnectionStatus | undefined;
private statusChangeListeners: (() => unknown)[] = [];
public constructor(private readonly syncClient: SyncClient) {
void this.updateConnectionState();
syncClient.addSyncHistoryUpdateListener((status) => {
this.lastHistoryStats = status;
this.updateDescription();
});
this.syncClient.addRemainingSyncOperationsListener(
(remainingOperations) => {
this.lastRemaining = remainingOperations;
this.updateDescription();
}
);
this.syncClient.addWebSocketStatusChangeListener(
() => void this.updateConnectionState()
);
this.syncClient.addOnSettingsChangeListener(
() => void this.updateConnectionState()
);
}
public async updateConnectionState(): Promise<void> {
this.lastConnectionState = await this.syncClient.checkConnection();
this.updateDescription();
}
public addStatusChangeListener(listener: () => void): void {
this.statusChangeListeners.push(listener);
}
public removeStatusChangeListener(listener: () => void): void {
this.statusChangeListeners = this.statusChangeListeners.filter(
(l) => l !== listener
);
}
public renderStatusDescription(container: HTMLElement): void {
container.empty();
container.addClass("status-description");
if (this.lastConnectionState == undefined) {
container.createSpan({
text: "VaultLink is starting up…",
cls: "warning"
});
return;
}
if (!this.lastConnectionState.isSuccessful) {
container.createSpan({
text: `VaultLink failed to connect to the remote server with error '${this.lastConnectionState.serverMessage}'`,
cls: "error"
});
return;
}
if (!this.lastConnectionState.isWebSocketConnected) {
container.createSpan({
text: `${this.lastConnectionState.serverMessage} but the WebSocket connection could not be established.`,
cls: "error"
});
return;
}
container.createSpan({ text: "VaultLink is connected to the server " });
container.createEl("a", {
text: this.syncClient.getSettings().remoteUri,
href: this.syncClient.getSettings().remoteUri
});
container.createSpan({
text: ` and has indexed approximately `
});
container.createSpan({
text: `${this.syncClient.documentCount}`,
cls: "number"
});
container.createSpan({
text: ` documents. `
});
if (
(this.lastRemaining ?? 0) === 0 &&
(this.lastHistoryStats?.success ?? 0) === 0 &&
(this.lastHistoryStats?.error ?? 0) === 0
) {
if (this.syncClient.getSettings().isSyncEnabled) {
container.createSpan({
text: "Syncing is enabled but VaultLink hasn't found anything to sync yet."
});
} else {
container.createSpan({
text: "However, syncing is disabled right now.",
cls: "warning"
});
}
return;
}
container.createSpan({
text: "The plugin has "
});
container.createSpan({
text: `${this.lastRemaining ?? 0}`,
cls: "number"
});
container.createSpan({
text: " outstanding operations while having succeeded "
});
container.createSpan({
text: `${this.lastHistoryStats?.success ?? 0}`,
cls: ["number", "good"]
});
container.createSpan({
text: " times and failed "
});
container.createSpan({
text: `${this.lastHistoryStats?.error ?? 0}`,
cls: ["number", "bad"]
});
container.createSpan({
text: " times."
});
}
private updateDescription(): void {
this.statusChangeListeners.forEach((listener) => {
listener();
});
}
}

17
frontend/obsidian-plugin/tsconfig.json
View file

@ -1,17 +0,0 @@
{
"compilerOptions": {
"baseUrl": ".",
"module": "ESNext",
"target": "ES2023",
"strict": true,
"moduleResolution": "bundler",
"allowSyntheticDefaultImports": true,
"lib": [
"DOM",
"ESNext"
]
},
"exclude": [
"./dist"
]
}

7
frontend/obsidian-plugin/version-bump.mjs
View file

@ -1,7 +0,0 @@
import { readFileSync, writeFileSync } from "fs";
const targetVersion = process.env.npm_package_version;
let manifest = JSON.parse(readFileSync("manifest.json", "utf8"));
manifest.version = targetVersion;
writeFileSync("manifest.json", JSON.stringify(manifest, null, "\t"));

117
frontend/obsidian-plugin/webpack.config.js
View file

@ -1,117 +0,0 @@
const path = require("path");
const TerserPlugin = require("terser-webpack-plugin");
const MiniCssExtractPlugin = require("mini-css-extract-plugin");
const fs = require("fs-extra");
module.exports = (env, argv) => ({
devtool: argv.mode === "development" ? "inline-source-map" : false,
entry: {
index: "./src/vault-link-plugin.ts"
},
watchOptions: {
ignored: "**/node_modules"
},
externals: {
obsidian: "commonjs obsidian",
electron: "commonjs electron",
"@codemirror/autocomplete": "commonjs @codemirror/autocomplete",
"@codemirror/collab": "commonjs @codemirror/collab",
"@codemirror/commands": "commonjs @codemirror/commands",
"@codemirror/language": "commonjs @codemirror/language",
"@codemirror/lint": "commonjs @codemirror/lint",
"@codemirror/search": "commonjs @codemirror/search",
"@codemirror/state": "commonjs @codemirror/state",
"@codemirror/view": "commonjs @codemirror/view"
},
optimization: {
minimizer: [
new TerserPlugin({
terserOptions: {
module: true
}
})
]
},
plugins: [
new MiniCssExtractPlugin({
filename: "styles.css"
}),
{
apply: (compiler) => {
if (argv.mode !== "development") {
return;
}
compiler.hooks.done.tap("Copy Files Plugin", (stats) => {
const source = path.resolve(__dirname, "dist");
const destinations = [
"/mnt/c/Users/Andras/Desktop/test/test/.obsidian/plugins/vault-link",
"/mnt/c/Users/Andras/Desktop/test/test2/.obsidian/plugins/vault-link",
"/home/andras/obsidian-test/.obsidian/plugins/vault-link"
];
destinations.forEach((destination) => {
fs.copy(source, destination)
.then(() =>
console.log(
"Files copied successfully after build!"
)
)
.catch((err) =>
console.error("Error copying files:", err)
);
fs.createFile(path.join(destination, ".hotreload"));
});
});
}
}
],
module: {
rules: [
{
test: /\.json$/i,
type: "asset/resource",
generator: {
filename: "[name][ext]"
}
},
{
test: /\.scss$/i,
use: [
MiniCssExtractPlugin.loader,
"css-loader",
"resolve-url-loader",
{
loader: "sass-loader",
options: {
sourceMap: true // required by resolve-url-loader
}
}
]
},
{
test: /\.ts$/,
use: ["ts-loader"]
}
]
},
resolve: {
extensions: [
".ts",
".js" // required for development
],
alias: {
root: __dirname,
src: path.resolve(__dirname, "src")
}
},
output: {
clean: true,
filename: "main.js",
library: {
type: "commonjs" // required for Obsidian
},
path: path.resolve(__dirname, "dist"),
publicPath: ""
}
});

7784
frontend/package-lock.json generated
View file

File diff suppressed because it is too large Load diff

30
frontend/package.json
View file

@ -1,30 +0,0 @@
{
"name": "my-workspace",
"private": true,
"workspaces": [
"sync-client",
"obsidian-plugin",
"test-client"
],
"prettier": {
"trailingComma": "none",
"tabWidth": 4,
"useTabs": true,
"endOfLine": "lf"
},
"scripts": {
"build": "npm run build --workspaces",
"dev": "concurrently --kill-others \"npm run dev -w sync-client\" \"npm run dev -w obsidian-plugin\"",
"test": "npm run test --workspaces",
"lint": "eslint --fix sync-client obsidian-plugin test-client && prettier --write \"**/*.ts\"",
"update": "ncu -u -ws"
},
"devDependencies": {
"concurrently": "^9.1.2",
"eslint": "9.28.0",
"eslint-plugin-unused-imports": "^4.1.4",
"npm-check-updates": "^18.0.1",
"prettier": "^3.5.3",
"typescript-eslint": "8.33.1"
}
}

3
frontend/sync-client/jest.config.js
View file

@ -1,3 +0,0 @@
module.exports = {
preset: "ts-jest/presets/js-with-babel-esm"
};

35
frontend/sync-client/package.json
View file

@ -1,35 +0,0 @@
{
"name": "sync-client",
"version": "0.4.0",
"main": "dist/sync-client.node.js",
"browser": "dist/sync-client.web.js",
"types": "dist/types/index.d.ts",
"files": [
"dist/**/*"
],
"scripts": {
"dev": "webpack watch --mode development",
"build": "webpack --mode production",
"test": "NODE_OPTIONS=\"$NODE_OPTIONS --experimental-vm-modules\" jest"
},
"dependencies": {
"byte-base64": "^1.1.0",
"minimatch": "^10.0.1",
"p-queue": "^8.1.0",
"uuid": "^11.1.0"
},
"devDependencies": {
"@types/jest": "^29.5.14",
"@types/node": "^22.15.30",
"jest": "^29.7.0",
"sync_lib": "file:../../backend/sync_lib/pkg",
"ts-jest": "^29.3.4",
"ts-loader": "^9.5.2",
"tslib": "2.8.1",
"typescript": "5.8.3",
"webpack": "^5.99.9",
"webpack-cli": "^6.0.1",
"webpack-merge": "^6.0.1",
"ws": "^8.18.2"
}
}

6
frontend/sync-client/src/file-operations/file-not-found-error.ts
View file

@ -1,6 +0,0 @@
export class FileNotFoundError extends Error {
public constructor(message: string) {
super(message);
this.name = "FileNotFoundError";
}
}

170
frontend/sync-client/src/file-operations/file-operations.test.ts
View file

@ -1,170 +0,0 @@
import type {
Database,
DocumentRecord,
RelativePath
} from "../persistence/database";
import { FileOperations } from "./file-operations";
import { Logger } from "../tracing/logger";
import { assertSetContainsExactly } from "../utils/assert-set-contains-exactly";
import type {
FileSystemOperations,
TextWithCursors
} from "./filesystem-operations";
import init, { base64ToBytes } from "sync_lib";
import fs from "fs";
class MockDatabase implements Partial<Database> {
public getLatestDocumentByRelativePath(
_find: RelativePath
): DocumentRecord | undefined {
// no-op
return undefined;
}
public move(
_oldRelativePath: RelativePath,
_newRelativePath: RelativePath
): void {
// no-op
}
}
class FakeFileSystemOperations implements FileSystemOperations {
public readonly names = new Set<string>();
public async listAllFiles(): Promise<RelativePath[]> {
throw new Error("Method not implemented.");
}
public async read(_path: RelativePath): Promise<Uint8Array> {
throw new Error("Method not implemented.");
}
public async write(
path: RelativePath,
_content: Uint8Array
): Promise<void> {
this.names.add(path);
}
public async atomicUpdateText(
_path: RelativePath,
_updater: (current: TextWithCursors) => TextWithCursors
): Promise<string> {
throw new Error("Method not implemented.");
}
public async getFileSize(_path: RelativePath): Promise<number> {
throw new Error("Method not implemented.");
}
public async getModificationTime(_path: RelativePath): Promise<Date> {
throw new Error("Method not implemented.");
}
public async exists(path: RelativePath): Promise<boolean> {
return this.names.has(path);
}
public async createDirectory(_path: RelativePath): Promise<void> {
// this is called but irrelevant for this mock
}
public async delete(_path: RelativePath): Promise<void> {
throw new Error("Method not implemented.");
}
public async rename(
oldPath: RelativePath,
newPath: RelativePath
): Promise<void> {
this.names.delete(oldPath);
this.names.add(newPath);
}
}
describe("File operations", () => {
beforeEach(async () => {
const wasmBin = fs.readFileSync(
"../../backend/sync_lib/pkg/sync_lib_bg.wasm"
);
await init({ module_or_path: wasmBin });
});
it("should deconflict renames", async () => {
const fileSystemOperations = new FakeFileSystemOperations();
const fileOperations = new FileOperations(
new Logger(),
new MockDatabase() as Database, // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion
fileSystemOperations
);
await fileOperations.create("a", new Uint8Array());
assertSetContainsExactly(fileSystemOperations.names, "a");
await fileOperations.move("a", "b");
assertSetContainsExactly(fileSystemOperations.names, "b");
await fileOperations.create("c", new Uint8Array());
assertSetContainsExactly(fileSystemOperations.names, "b", "c");
await fileOperations.move("c", "b");
assertSetContainsExactly(fileSystemOperations.names, "b", "b (1)");
await fileOperations.create("c", new Uint8Array());
await fileOperations.move("c", "b");
assertSetContainsExactly(
fileSystemOperations.names,
"b",
"b (1)",
"b (2)"
);
});
it("should deconflict renames with file extension", async () => {
const fileSystemOperations = new FakeFileSystemOperations();
const fileOperations = new FileOperations(
new Logger(),
new MockDatabase() as Database, // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion
fileSystemOperations
);
await fileOperations.create("b.md", new Uint8Array());
await fileOperations.create("c.md", new Uint8Array());
await fileOperations.move("c.md", "b.md");
assertSetContainsExactly(
fileSystemOperations.names,
"b.md",
"b (1).md"
);
await fileOperations.create("d.md", new Uint8Array());
await fileOperations.move("d.md", "b.md");
assertSetContainsExactly(
fileSystemOperations.names,
"b.md",
"b (1).md",
"b (2).md"
);
await fileOperations.create("file-23.md", new Uint8Array());
await fileOperations.create("file-23 (1).md", new Uint8Array());
await fileOperations.move("file-23.md", "file-23 (1).md");
assertSetContainsExactly(
fileSystemOperations.names,
"b.md",
"b (1).md",
"b (2).md",
"file-23 (1).md",
"file-23 (2).md"
);
});
it("should deconflict renames with paths", async () => {
const fileSystemOperations = new FakeFileSystemOperations();
const fileOperations = new FileOperations(
new Logger(),
new MockDatabase() as Database, // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion
fileSystemOperations
);
await fileOperations.create("a/b.c/d", new Uint8Array());
await fileOperations.create("a/b.c/e", new Uint8Array());
await fileOperations.move("a/b.c/d", "a/b.c/e");
assertSetContainsExactly(
fileSystemOperations.names,
"a/b.c/e",
"a/b.c/e (1)"
);
});
});

237
frontend/sync-client/src/file-operations/file-operations.ts
View file

@ -1,237 +0,0 @@
import type { Logger } from "../tracing/logger";
import type {
FileSystemOperations,
TextWithCursors
} from "./filesystem-operations";
import type { Database, RelativePath } from "../persistence/database";
import {
CursorPosition,
isBinary,
isFileTypeMergable,
mergeTextWithCursors,
TextWithCursors as RustTextWithCursors
} from "sync_lib";
import { SafeFileSystemOperations } from "./safe-filesystem-operations";
export class FileOperations {
private static readonly PARENTHESES_REGEX = / \((\d+)\)$/;
private readonly fs: SafeFileSystemOperations;
public constructor(
private readonly logger: Logger,
private readonly database: Database,
fs: FileSystemOperations,
private readonly nativeLineEndings = "\n"
) {
this.fs = new SafeFileSystemOperations(fs, logger);
}
public async listAllFiles(): Promise<RelativePath[]> {
return this.fs.listAllFiles();
}
public async read(path: RelativePath): Promise<Uint8Array> {
return this.fromNativeLineEndings(await this.fs.read(path));
}
/**
* Create a file at the specified path.
*
* If a file with the same name already exists, it is moved before creating the new one.
* Parent directories are created if necessary.
*/
public async create(
path: RelativePath,
newContent: Uint8Array
): Promise<void> {
await this.ensureClearPath(path);
return this.fs.write(path, this.toNativeLineEndings(newContent));
}
public async ensureClearPath(path: RelativePath): Promise<void> {
if (await this.fs.exists(path)) {
const deconflictedPath = await this.deconflictPath(path);
this.logger.debug(
`Didn't expect ${path} to exist, deconflicting by moving it to '${deconflictedPath}'`
);
this.database.move(path, deconflictedPath);
await this.fs.rename(path, deconflictedPath);
} else {
await this.createParentDirectories(path);
}
}
/**
* Update the file at the given path.
*
* Performs a 3-way merge before writing if the file's content differs from `expectedContent`.
* Does not recreate the file if it no longer exists, returning an empty array instead.
*/
public async write(
path: RelativePath,
expectedContent: Uint8Array,
newContent: Uint8Array
): Promise<void> {
if (!(await this.fs.exists(path))) {
this.logger.debug(
`The caller assumed ${path} exists, but it no longer, so we wont recreate it`
);
return;
}
if (
!isFileTypeMergable(path) ||
isBinary(expectedContent) ||
isBinary(newContent)
) {
this.logger.debug(
`The expected content is not mergable, so we won't perform a 3-way merge, just overwrite it`
);
await this.fs.write(
path,
// `newContent` might not be binary so we still have to ensure the line endings are correct
this.toNativeLineEndings(newContent)
);
return;
}
const expectedText = new TextDecoder().decode(expectedContent); // this comes from a previous read which must only have \n line endings
const newText = new TextDecoder().decode(newContent); // this comes from the server which stores text with \n line endings
await this.fs.atomicUpdateText(
path,
({ text, cursors }: TextWithCursors): TextWithCursors => {
text = text.replace(this.nativeLineEndings, "\n");
this.logger.debug(
`Performing a 3-way merge for ${path} with the expected content`
);
const left = new RustTextWithCursors(
text,
cursors.map(
(cursor) =>
new CursorPosition(
cursor.id,
cursor.characterPosition
)
)
);
const right = new RustTextWithCursors(newText, []);
const merged = mergeTextWithCursors(expectedText, left, right);
const resultText = merged
.text()
.replace("\n", this.nativeLineEndings);
const resultCursors = merged.cursors().map((cursor) => ({
id: cursor.id(),
characterPosition: cursor.characterPosition()
}));
merged.free();
return {
text: resultText,
cursors: resultCursors
};
}
);
}
public async delete(path: RelativePath): Promise<void> {
if (await this.exists(path)) {
return this.fs.delete(path);
} else {
this.logger.debug(`No need to delete '${path}', it doesn't exist`);
}
}
public async getFileSize(path: RelativePath): Promise<number> {
return this.fs.getFileSize(path);
}
public async exists(path: RelativePath): Promise<boolean> {
return this.fs.exists(path);
}
public async move(
oldPath: RelativePath,
newPath: RelativePath
): Promise<void> {
if (oldPath === newPath) {
return;
}
await this.ensureClearPath(newPath);
this.database.move(oldPath, newPath);
await this.fs.rename(oldPath, newPath);
}
private fromNativeLineEndings(content: Uint8Array): Uint8Array {
if (isBinary(content)) {
return content;
}
const decoder = new TextDecoder("utf-8");
let text = decoder.decode(content);
text = text.replace(this.nativeLineEndings, "\n");
return new TextEncoder().encode(text);
}
private toNativeLineEndings(content: Uint8Array): Uint8Array {
if (isBinary(content)) {
return content;
}
const decoder = new TextDecoder("utf-8");
let text = decoder.decode(content);
text = text.replace("\n", this.nativeLineEndings);
return new TextEncoder().encode(text);
}
private async createParentDirectories(path: string): Promise<void> {
const components = path.split("/");
if (components.length === 1) {
return;
}
for (let i = 1; i < components.length; i++) {
const parentDir = components.slice(0, i).join("/");
if (!(await this.fs.exists(parentDir))) {
await this.fs.createDirectory(parentDir);
}
}
}
private async deconflictPath(path: RelativePath): Promise<RelativePath> {
const pathParts = path.split("/");
const fileName = pathParts.pop();
if (fileName == "" || fileName == null) {
throw new Error(`Path '${path}' cannot be empty`);
}
let directory = pathParts.join("/");
if (directory) {
directory += "/";
}
const nameParts = fileName.split(".");
const extension =
nameParts.length > 1 ? "." + nameParts[nameParts.length - 1] : "";
let stem = extension ? nameParts.slice(0, -1).join(".") : fileName;
let currentCount = Number.parseInt(
FileOperations.PARENTHESES_REGEX.exec(stem)?.groups?.[0] ?? "0"
);
stem = stem.replace(FileOperations.PARENTHESES_REGEX, "");
let newName = path;
do {
currentCount++;
newName = `${directory}${stem} (${currentCount})${extension}`;
} while (await this.fs.exists(newName));
return newName;
}
}

45
frontend/sync-client/src/file-operations/filesystem-operations.ts
View file

@ -1,45 +0,0 @@
import type { RelativePath } from "../persistence/database";
export interface Cursor {
id: number;
/// The character position is the index of the character in the text where the text lines are separated by '\n' new line character even if the actual text uses different line endings.
characterPosition: number;
}
export interface TextWithCursors {
text: string;
cursors: Cursor[];
}
export interface FileSystemOperations {
// List all files that should be synced.
listAllFiles: () => Promise<RelativePath[]>;
// Read the content of a file.
read: (path: RelativePath) => Promise<Uint8Array>;
// Create or overwrite a file with the given content.
write: (path: RelativePath, content: Uint8Array) => Promise<void>;
// Atomically update the content of a text file.
atomicUpdateText: (
path: RelativePath,
updater: (current: TextWithCursors) => TextWithCursors
) => Promise<string>;
// Get the size of a file in bytes.
getFileSize: (path: RelativePath) => Promise<number>;
// Check if a file exists.
exists: (path: RelativePath) => Promise<boolean>;
// Create a directory at the specified path. All parent directories must already exist.
createDirectory: (path: RelativePath) => Promise<void>;
// Delete a file. It is expected that the path points to an existing file.
delete: (path: RelativePath) => Promise<void>;
// Rename a file. It is expected that the oldPath points to an existing file and the newPath does not exist.
rename: (oldPath: RelativePath, newPath: RelativePath) => Promise<void>;
}

169
frontend/sync-client/src/file-operations/safe-filesystem-operations.ts
View file

@ -1,169 +0,0 @@
import type { RelativePath } from "../persistence/database";
import type {
FileSystemOperations,
TextWithCursors
} from "./filesystem-operations";
import type { Logger } from "../tracing/logger";
import { Locks } from "../utils/locks";
import { FileNotFoundError } from "./file-not-found-error";
/**
* Decorates `FileSystemOperations` to replace errors with `FileNotFoundError`
* if the accessed file doesn't exist. It also ensures that there's at most a
* single request in-flight for any one file through the use of locks.
*/
export class SafeFileSystemOperations implements FileSystemOperations {
private readonly locks: Locks<RelativePath>;
public constructor(
private readonly fs: FileSystemOperations,
private readonly logger: Logger
) {
this.locks = new Locks(logger);
}
public async listAllFiles(): Promise<RelativePath[]> {
this.logger.debug("Listing all files");
const result = await this.fs.listAllFiles();
this.logger.debug(`Listed ${result.length} files`);
return result;
}
public async read(path: RelativePath): Promise<Uint8Array> {
this.logger.debug(`Reading file '${path}'`);
return this.safeOperation(
path,
this.decorateToHoldLock(path, async () => this.fs.read(path)),
"read"
);
}
public async write(path: RelativePath, content: Uint8Array): Promise<void> {
this.logger.debug(`Writing to file '${path}'`);
return this.decorateToHoldLock(path, async () =>
this.fs.write(path, content)
)();
}
public async atomicUpdateText(
path: RelativePath,
updater: (current: TextWithCursors) => TextWithCursors
): Promise<string> {
this.logger.debug(`Atomically updating file '${path}'`);
return this.safeOperation(
path,
this.decorateToHoldLock(path, async () =>
this.fs.atomicUpdateText(path, updater)
),
"atomicUpdateText"
);
}
public async getFileSize(path: RelativePath): Promise<number> {
// Logging this would be too noisy
return this.safeOperation(
path,
this.decorateToHoldLock(path, async () =>
this.fs.getFileSize(path)
),
"getFileSize"
);
}
public async exists(path: RelativePath): Promise<boolean> {
this.logger.debug(`Checking if file '${path}' exists`);
return this.decorateToHoldLock(path, async () =>
this.fs.exists(path)
)();
}
public async createDirectory(path: RelativePath): Promise<void> {
this.logger.debug(`Creating directory '${path}'`);
return this.decorateToHoldLock(path, async () =>
this.fs.createDirectory(path)
)();
}
public async delete(path: RelativePath): Promise<void> {
this.logger.debug(`Deleting file '${path}'`);
return this.decorateToHoldLock(path, async () =>
this.fs.delete(path)
)();
}
public async rename(
oldPath: RelativePath,
newPath: RelativePath
): Promise<void> {
this.logger.debug(`Renaming file '${oldPath}' to '${newPath}'`);
return this.safeOperation(
oldPath,
this.decorateToHoldLock([oldPath, newPath], async () =>
this.fs.rename(oldPath, newPath)
),
"rename"
);
}
/**
* Decorate an operation to ensure that the file is locked before running it
* and that the lock is released afterwards. This results in at-most one
* concurrent operation running per file.
*/
private decorateToHoldLock<T>(
pathOrPaths: RelativePath | RelativePath[],
operation: () => Promise<T>
): () => Promise<T> {
return async () => {
const paths = Array.isArray(pathOrPaths)
? pathOrPaths
: [pathOrPaths];
await Promise.all(
paths.map(async (path) => this.locks.waitForLock(path))
);
try {
return await operation();
} finally {
await Promise.all(
paths.map((path) => {
this.locks.unlock(path);
})
);
}
};
}
/**
* Decorate an operation to ensure that the file exists before running it.
* If the operation fails, it will check if the file still exists and throw
* a FileNotFoundError if it doesn't.
*/
private async safeOperation<T>(
path: RelativePath,
operation: () => Promise<T>,
operationName: string
): Promise<T> {
if (!(await this.fs.exists(path))) {
throw new FileNotFoundError(
`File '${path}' not found before trying to ${operationName}`
);
}
try {
return await operation();
} catch (error) {
// Without locking the file, this isn't atomic, however, it's good enough in practice.
// This will only break if the file exists, gets deleted and then immediately
// recreated while `operation` is running.
if (await this.fs.exists(path)) {
throw error;
} else {
throw new FileNotFoundError(
`File '${path}' not found when trying to ${operationName}`
);
}
}
}
}

26
frontend/sync-client/src/index.ts
View file

@ -1,26 +0,0 @@
export {
SyncType,
SyncStatus,
type HistoryStats,
type HistoryEntry,
type SyncDetails,
type SyncCreateDetails,
type SyncUpdateDetails,
type SyncMovedDetails,
type SyncDeleteDetails
} from "./tracing/sync-history";
export { Logger, LogLevel, LogLine } from "./tracing/logger";
export { type SyncSettings, DEFAULT_SETTINGS } from "./persistence/settings";
export { rateLimit } from "./utils/rate-limit";
export type { RelativePath, StoredDatabase } from "./persistence/database";
export type {
FileSystemOperations,
TextWithCursors,
Cursor
} from "./file-operations/filesystem-operations";
export type { PersistenceProvider } from "./persistence/persistence";
export type { CursorSpan } from "./services/types/CursorSpan";
export type { ClientCursors } from "./services/types/ClientCursors";
export type { NetworkConnectionStatus } from "./types/network-connection-status";
export { DocumentUpdateStatus } from "./types/document-update-status";
export { SyncClient } from "./sync-client";

358
frontend/sync-client/src/persistence/database.ts
View file

@ -1,358 +0,0 @@
import type { Logger } from "../tracing/logger";
import { EMPTY_HASH } from "../utils/hash";
import { CoveredValues } from "../utils/min-covered";
export type VaultUpdateId = number;
export type DocumentId = string;
export type RelativePath = string;
export interface DocumentMetadata {
parentVersionId: VaultUpdateId;
hash: string;
remoteRelativePath?: RelativePath;
}
export interface StoredDocumentMetadata {
relativePath: RelativePath;
documentId: DocumentId;
parentVersionId: VaultUpdateId;
remoteRelativePath?: RelativePath;
hash: string;
}
export interface StoredDatabase {
documents: StoredDocumentMetadata[];
lastSeenUpdateId: VaultUpdateId | undefined;
hasInitialSyncCompleted: boolean;
}
/**
* Represents a document in the database.
*
* It is mutable and its content should always represent the latest
* state of the document on disk based on the update events we have seen.
*/
export interface DocumentRecord {
relativePath: RelativePath;
documentId: DocumentId;
metadata: DocumentMetadata | undefined;
isDeleted: boolean;
updates: Promise<void>[];
parallelVersion: number;
}
export class Database {
private documents: DocumentRecord[];
private lastSeenUpdateIds: CoveredValues;
private hasInitialSyncCompleted: boolean;
public constructor(
private readonly logger: Logger,
initialState: Partial<StoredDatabase> | undefined,
private readonly saveData: (data: StoredDatabase) => Promise<void>
) {
initialState ??= {};
this.documents =
initialState.documents?.map(
({ relativePath, documentId, ...metadata }) => ({
relativePath,
documentId,
metadata,
isDeleted: false,
updates: [],
parallelVersion: 0
})
) ?? [];
this.ensureConsistency();
this.logger.debug(`Loaded ${this.documents.length} documents`);
const { lastSeenUpdateId } = initialState;
this.logger.debug(`Loaded last seen update id: ${lastSeenUpdateId}`);
this.lastSeenUpdateIds = new CoveredValues(
Math.max(0, lastSeenUpdateId ?? 0) // the first updateId will be 1 which is the first integer after -1
);
this.hasInitialSyncCompleted =
initialState.hasInitialSyncCompleted ?? false;
this.logger.debug(
`Loaded hasInitialSyncCompleted: ${this.hasInitialSyncCompleted}`
);
}
public get length(): number {
return this.documents.length;
}
public get resolvedDocuments(): DocumentRecord[] {
const paths = new Map<string, DocumentRecord[]>();
this.documents
.filter(({ metadata }) => metadata !== undefined)
.forEach((record) =>
paths.set(record.relativePath, [
record,
...(paths.get(record.relativePath) ?? [])
])
);
return Array.from(paths.values()).map((records) => {
records.sort(
(a, b) => b.parallelVersion - a.parallelVersion // descending
);
if (
records.length > 1 &&
records.some((current, i) =>
i === 0
? false
: records[i - 1].parallelVersion ===
current.parallelVersion
)
) {
throw new Error(
`Multiple documents with the same parallel version and path at ${records[0].relativePath}`
);
}
return records[0];
});
}
public updateDocumentMetadata(
metadata: {
parentVersionId: VaultUpdateId;
hash: string;
remoteRelativePath: RelativePath;
},
toUpdate: DocumentRecord
): void {
if (!this.documents.includes(toUpdate)) {
throw new Error("Document not found in database");
}
toUpdate.metadata = metadata;
this.save();
}
public removeDocumentPromise(promise: Promise<void>): void {
const entry = this.documents.find(({ updates }) =>
updates.includes(promise)
);
if (entry === undefined) {
// This method should be idempotent and tolerant of
// stragglers calling it after the databse has been reset.
return;
}
entry.updates = entry.updates.filter((update) => update !== promise);
// No need to save as Promises don't get serialized
}
public removeDocument(find: DocumentRecord): void {
this.documents = this.documents.filter((document) => document !== find);
this.save();
}
public getLatestDocumentByRelativePath(
find: RelativePath
): DocumentRecord | undefined {
const candidates = this.documents.filter(
({ relativePath }) => relativePath === find
);
candidates.sort((a, b) => b.parallelVersion - a.parallelVersion); // descending
return candidates[0];
}
public async getResolvedDocumentByRelativePath(
relativePath: RelativePath,
promise: Promise<void>
): Promise<DocumentRecord> {
const entry = this.getLatestDocumentByRelativePath(relativePath);
if (entry === undefined) {
throw new Error(
`Document not found by relative path: ${relativePath}, ${JSON.stringify(
this.documents,
null,
2
)}`
);
}
const currentPromises = entry.updates;
entry.updates = [...currentPromises, promise];
await Promise.all(currentPromises);
return entry;
}
public createNewPendingDocument(
documentId: DocumentId,
relativePath: RelativePath,
promise: Promise<void>
): DocumentRecord {
const previousEntry =
this.getLatestDocumentByRelativePath(relativePath);
const entry = {
relativePath,
documentId,
metadata: undefined,
isDeleted: false,
updates: [promise],
parallelVersion:
previousEntry?.parallelVersion === undefined
? 0
: previousEntry.parallelVersion + 1
};
this.documents.push(entry);
this.save();
return entry;
}
public createNewEmptyDocument(
documentId: DocumentId,
parentVersionId: VaultUpdateId,
relativePath: RelativePath
): DocumentRecord {
const entry = {
relativePath,
documentId,
metadata: {
parentVersionId,
hash: EMPTY_HASH,
remoteRelativePath: relativePath
},
isDeleted: false,
updates: [],
parallelVersion: 0
};
this.documents.push(entry);
this.save();
return entry;
}
public getDocumentByDocumentId(
find: DocumentId
): DocumentRecord | undefined {
return this.documents.find(({ documentId }) => documentId === find);
}
public move(
oldRelativePath: RelativePath,
newRelativePath: RelativePath
): void {
const oldDocument =
this.getLatestDocumentByRelativePath(oldRelativePath);
if (oldDocument === undefined) {
return;
}
const newDocument =
this.getLatestDocumentByRelativePath(newRelativePath);
if (newDocument?.isDeleted === false) {
throw new Error(
`Document already exists at new location: ${newRelativePath}`
);
}
oldDocument.relativePath = newRelativePath;
// We're in a strange state where the target of the move has just got deleted,
// however, its metadata might already have a bunch of updates queued up for
// the document at the new location. We need to keep these updates.
oldDocument.parallelVersion =
newDocument !== undefined ? newDocument.parallelVersion + 1 : 0;
this.save();
}
public delete(relativePath: RelativePath): void {
const candidate = this.getLatestDocumentByRelativePath(relativePath);
if (candidate === undefined) {
throw new Error(
`Document not found by relative path: ${relativePath}`
);
}
candidate.isDeleted = true;
}
public getHasInitialSyncCompleted(): boolean {
return this.hasInitialSyncCompleted;
}
public setHasInitialSyncCompleted(value: boolean): void {
this.hasInitialSyncCompleted = value;
this.save();
}
public getLastSeenUpdateId(): VaultUpdateId {
return this.lastSeenUpdateIds.min;
}
public addSeenUpdateId(value: number): void {
const previousMin = this.lastSeenUpdateIds.min;
this.lastSeenUpdateIds.add(value);
if (previousMin !== this.lastSeenUpdateIds.min) {
this.save();
}
}
public setLastSeenUpdateId(value: number): void {
this.lastSeenUpdateIds.min = value;
this.save();
}
public reset(): void {
this.documents = [];
this.lastSeenUpdateIds = new CoveredValues(
0 // the first updateId will be 1 which is the first integer after -1
);
this.hasInitialSyncCompleted = false;
this.save();
}
private save(): void {
this.ensureConsistency();
void this.saveData({
documents: this.resolvedDocuments.map(
({ relativePath, documentId, metadata }) => ({
documentId,
relativePath,
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
...metadata! // `resolvedDocuments` only returns docs with metadata set
})
),
lastSeenUpdateId: this.lastSeenUpdateIds.min,
hasInitialSyncCompleted: this.hasInitialSyncCompleted
});
}
private ensureConsistency(): void {
const idToPath = new Map<string, string[]>();
this.resolvedDocuments.forEach(({ relativePath, documentId }) => {
idToPath.set(documentId, [
...(idToPath.get(documentId) ?? []),
relativePath
]);
});
const duplicates = Array.from(idToPath.entries())
.filter(([_, paths]) => paths.length > 1)
.map(([id, paths]) => `${id} (${paths.join(", ")})`);
if (duplicates.length > 0) {
throw new Error(
"Document IDs are not unique, found duplicates: " +
duplicates.join("; ")
);
}
}
}

4
frontend/sync-client/src/persistence/persistence.ts
View file

@ -1,4 +0,0 @@
export interface PersistenceProvider<T> {
load: () => Promise<T | undefined>;
save: (data: T) => Promise<void>;
}

84
frontend/sync-client/src/persistence/settings.ts
View file

@ -1,84 +0,0 @@
import type { Logger } from "../tracing/logger";
export interface SyncSettings {
remoteUri: string;
token: string;
vaultName: string;
syncConcurrency: number;
isSyncEnabled: boolean;
maxFileSizeMB: number;
ignorePatterns: string[];
webSocketRetryIntervalMs: number;
}
export const DEFAULT_SETTINGS: SyncSettings = {
remoteUri: "",
token: "",
vaultName: "default",
syncConcurrency: 1,
isSyncEnabled: false,
maxFileSizeMB: 10,
ignorePatterns: [],
webSocketRetryIntervalMs: 3500
};
export class Settings {
private settings: SyncSettings;
private readonly onSettingsChangeHandlers: ((
newSettings: SyncSettings,
oldSettings: SyncSettings
) => void)[] = [];
public constructor(
private readonly logger: Logger,
initialState: Partial<SyncSettings> | undefined,
private readonly saveData: (data: SyncSettings) => Promise<void>
) {
this.settings = {
...DEFAULT_SETTINGS,
...(initialState ?? {})
};
this.logger.debug(
`Loaded settings: ${JSON.stringify(this.settings, null, 2)}`
);
}
public getSettings(): SyncSettings {
return this.settings;
}
public addOnSettingsChangeListener(
handler: (settings: SyncSettings, oldSettings: SyncSettings) => void
): void {
this.onSettingsChangeHandlers.push(handler);
}
public async setSetting<T extends keyof SyncSettings>(
key: T,
value: SyncSettings[T]
): Promise<void> {
this.logger.debug(`Setting '${key}' to '${value}'`);
await this.setSettings({
[key]: value
});
}
public async setSettings(value: Partial<SyncSettings>): Promise<void> {
const oldSettings = this.settings;
this.settings = {
...this.settings,
...value
};
this.onSettingsChangeHandlers.forEach((handler) => {
handler(this.settings, oldSettings);
});
await this.save();
}
private async save(): Promise<void> {
await this.saveData(this.settings);
}
}

98
frontend/sync-client/src/services/connection-status.ts
View file

@ -1,98 +0,0 @@
import type { Settings } from "../persistence/settings";
import type { Logger } from "../tracing/logger";
import { createPromise } from "../utils/create-promise";
import { SyncResetError } from "./sync-reset-error";
export class ConnectionStatus {
private static readonly UNTIL_RESOLUTION = Symbol();
private canFetch: boolean;
private until: Promise<symbol>;
private resolveUntil: (result: symbol) => void;
private rejectUntil: (reason: unknown) => void;
public constructor(
settings: Settings,
private readonly logger: Logger
) {
this.canFetch = settings.getSettings().isSyncEnabled;
[this.until, this.resolveUntil, this.rejectUntil] =
createPromise<symbol>();
settings.addOnSettingsChangeListener((newSettings, oldSettings) => {
if (oldSettings.isSyncEnabled != newSettings.isSyncEnabled) {
this.canFetch = newSettings.isSyncEnabled;
this.resolveUntil(ConnectionStatus.UNTIL_RESOLUTION);
[this.until, this.resolveUntil, this.rejectUntil] =
createPromise<symbol>();
}
});
}
private static getUrlFromInput(input: RequestInfo | URL): string {
if (input instanceof URL) {
return input.href;
}
if (typeof input === "string") {
return input;
}
return input.url;
}
public startReset(): void {
this.rejectUntil(new SyncResetError());
}
public finishReset(): void {
[this.until, this.resolveUntil, this.rejectUntil] = createPromise();
}
public getFetchImplementation(
logger: Logger,
fetch: typeof globalThis.fetch = globalThis.fetch
): typeof globalThis.fetch {
return async (
input: RequestInfo | URL,
init?: RequestInit
): Promise<Response> => {
while (!this.canFetch) {
await this.until;
}
try {
// https://github.com/jonbern/fetch-retry/blob/8684ef4e688375f623bd76f13add76dbc1d67cfb/index.js#L67C1-L70C21
const _input =
typeof Request !== "undefined" && input instanceof Request
? input.clone()
: input;
const fetchPromise = fetch(_input, init);
// We only want to catch rejections from `this.until`
let result: symbol | Response | undefined = undefined;
do {
result = await Promise.race([this.until, fetchPromise]);
} while (result === ConnectionStatus.UNTIL_RESOLUTION);
const fetchResult: Response = result as Response; // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion
if (!fetchResult.ok) {
this.logger.warn(
`Fetch for ${ConnectionStatus.getUrlFromInput(
input
)}, got status ${fetchResult.status}`
);
}
return fetchResult;
} catch (error) {
logger.warn(
`Fetch for ${ConnectionStatus.getUrlFromInput(
input
)}, got error: ${error}`
);
throw error;
}
};
}
}

6
frontend/sync-client/src/services/sync-reset-error.ts
View file

@ -1,6 +0,0 @@
export class SyncResetError extends Error {
public constructor() {
super("Sync was reset");
this.name = "SyncResetError";
}
}

319
frontend/sync-client/src/services/sync-service.ts
View file

@ -1,319 +0,0 @@
import type {
DocumentId,
RelativePath,
VaultUpdateId
} from "../persistence/database";
import type { Logger } from "../tracing/logger";
import type { Settings } from "../persistence/settings";
import type { ConnectionStatus } from "./connection-status";
import { sleep } from "../utils/sleep";
import { SyncResetError } from "./sync-reset-error";
import type { SerializedError } from "./types/SerializedError";
import type { DocumentVersionWithoutContent } from "./types/DocumentVersionWithoutContent";
import type { DocumentUpdateResponse } from "./types/DocumentUpdateResponse";
import type { DocumentVersion } from "./types/DocumentVersion";
import type { FetchLatestDocumentsResponse } from "./types/FetchLatestDocumentsResponse";
import type { PingResponse } from "./types/PingResponse";
import type { DeleteDocumentVersion } from "./types/DeleteDocumentVersion";
export interface CheckConnectionResult {
isSuccessful: boolean;
message: string;
}
export class SyncService {
private static readonly NETWORK_RETRY_INTERVAL_MS = 1000;
private readonly client: typeof globalThis.fetch;
private readonly pingClient: typeof globalThis.fetch;
public constructor(
private readonly deviceId: string,
private readonly connectionStatus: ConnectionStatus,
private readonly settings: Settings,
private readonly logger: Logger,
fetchImplementation: typeof globalThis.fetch = globalThis.fetch
) {
// ensure that if it's called a method, `this` won't be bound to the instance
const unboundFetch: typeof globalThis.fetch = async (...args) =>
fetchImplementation(...args);
this.client = this.connectionStatus.getFetchImplementation(
this.logger,
unboundFetch
);
this.pingClient = unboundFetch;
}
private static formatError(error: SerializedError): string {
let result = error.message;
if (error.causes.length > 0) {
const causes = error.causes.join(", ");
result += ` caused by: ${causes}`;
}
return result;
}
public async create({
documentId,
relativePath,
contentBytes
}: {
documentId?: DocumentId;
relativePath: RelativePath;
contentBytes: Uint8Array;
}): Promise<DocumentVersionWithoutContent> {
return this.withRetries(async () => {
const formData = new FormData();
if (documentId !== undefined) {
formData.append("document_id", documentId);
}
formData.append("relative_path", relativePath);
formData.append("content", new Blob([contentBytes]));
const response = await this.client(this.getUrl("/documents"), {
method: "POST",
body: formData,
headers: this.getDefaultHeaders()
});
const result: SerializedError | DocumentVersionWithoutContent =
(await response.json()) as // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion
| SerializedError
| DocumentVersionWithoutContent;
if ("errorType" in result) {
throw new Error(
`Failed to create document: ${SyncService.formatError(result)}`
);
}
this.logger.debug(
`Created document ${JSON.stringify(result)} with id ${
result.documentId
}`
);
return result;
});
}
public async put({
parentVersionId,
documentId,
relativePath,
contentBytes
}: {
parentVersionId: VaultUpdateId;
documentId: DocumentId;
relativePath: RelativePath;
contentBytes: Uint8Array;
}): Promise<DocumentUpdateResponse> {
return this.withRetries(async () => {
this.logger.debug(
`Updating document ${documentId} with parent version ${parentVersionId} and relative path ${relativePath}`
);
const formData = new FormData();
formData.append("parent_version_id", parentVersionId.toString());
formData.append("relative_path", relativePath);
formData.append("content", new Blob([contentBytes]));
const response = await this.client(
this.getUrl(`/documents/${documentId}`),
{
method: "PUT",
body: formData,
headers: this.getDefaultHeaders()
}
);
const result: SerializedError | DocumentUpdateResponse =
(await response.json()) as // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion
| SerializedError
| DocumentUpdateResponse;
if ("errorType" in result) {
throw new Error(
`Failed to update document: ${SyncService.formatError(result)}`
);
}
this.logger.debug(
`Updated document ${JSON.stringify(result)} with id ${
result.documentId
}}`
);
return result;
});
}
public async delete({
documentId,
relativePath
}: {
documentId: DocumentId;
relativePath: RelativePath;
}): Promise<DocumentVersionWithoutContent> {
return this.withRetries(async () => {
const request: DeleteDocumentVersion = {
relativePath
};
const response = await this.client(
this.getUrl(`/documents/${documentId}`),
{
method: "DELETE",
body: JSON.stringify(request),
headers: {
"Content-Type": "application/json",
...this.getDefaultHeaders()
}
}
);
const result: SerializedError | DocumentVersionWithoutContent =
(await response.json()) as // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion
| SerializedError
| DocumentVersionWithoutContent;
if ("errorType" in result) {
throw new Error(
`Failed to delete document: ${SyncService.formatError(result)}`
);
}
this.logger.debug(
`Deleted document ${relativePath} with id ${documentId}`
);
return result;
});
}
public async get({
documentId
}: {
documentId: DocumentId;
}): Promise<DocumentVersion> {
return this.withRetries(async () => {
const response = await this.client(
this.getUrl(`/documents/${documentId}`),
{
headers: this.getDefaultHeaders()
}
);
const result: SerializedError | DocumentVersion =
(await response.json()) as SerializedError | DocumentVersion; // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion
if ("errorType" in result) {
throw new Error(
`Failed to get document: ${SyncService.formatError(result)}`
);
}
this.logger.debug(
`Get document ${result.relativePath} with id ${result.documentId}`
);
return result;
});
}
public async getAll(
since?: VaultUpdateId
): Promise<FetchLatestDocumentsResponse> {
return this.withRetries(async () => {
const url = new URL(this.getUrl("/documents"));
if (since !== undefined) {
url.searchParams.append("since", since.toString());
}
const response = await this.client(url.toString(), {
headers: this.getDefaultHeaders()
});
const result: SerializedError | FetchLatestDocumentsResponse =
(await response.json()) as // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion
| SerializedError
| FetchLatestDocumentsResponse;
if ("errorType" in result) {
throw new Error(
`Failed to get documents: ${SyncService.formatError(result)}`
);
}
this.logger.debug(
`Got ${result.latestDocuments.length} document metadata`
);
return result;
});
}
public async checkConnection(): Promise<CheckConnectionResult> {
try {
const response = await this.pingClient(this.getUrl("/ping"), {
headers: this.getDefaultHeaders()
});
const result: PingResponse | SerializedError =
(await response.json()) as PingResponse | SerializedError; // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion
if ("errorType" in result) {
throw new Error(
`Failed to ping server: ${SyncService.formatError(result)}`
);
}
if (result.isAuthenticated) {
return {
isSuccessful: true,
message: `Successfully connected to server (version: ${result.serverVersion}) and authenticated`
};
}
return {
isSuccessful: false,
message: `Successfully connected to server (version: ${result.serverVersion}) but failed to authenticate`
};
} catch (e) {
return {
isSuccessful: false,
message: `Failed to connect to server: ${e}`
};
}
}
private getUrl(path: string): string {
const { vaultName, remoteUri } = this.settings.getSettings();
const safeRemoteUri = remoteUri.replace(/\/+$/, "");
return `${safeRemoteUri}/vaults/${vaultName}${path}`;
}
private getDefaultHeaders(): Record<string, string> {
return {
"device-id": this.deviceId,
authorization: `Bearer ${this.settings.getSettings().token}`
};
}
private async withRetries<T>(fn: () => Promise<T>): Promise<T> {
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
while (true) {
try {
return await fn();
} catch (e) {
// We must not retry errors coming from reset
if (e instanceof SyncResetError) {
throw e;
}
this.logger.error(
`Failed network call (${e}), retrying in ${SyncService.NETWORK_RETRY_INTERVAL_MS}ms`
);
await sleep(SyncService.NETWORK_RETRY_INTERVAL_MS);
}
}
}
}

8
frontend/sync-client/src/services/types/ClientCursors.ts
View file

@ -1,8 +0,0 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
import type { CursorSpan } from "./CursorSpan";
export interface ClientCursors {
userName: string;
deviceId: string;
cursors: Partial<Record<string, CursorSpan[]>>;
}

13
frontend/sync-client/src/services/types/CreateDocumentVersion.ts
View file

@ -1,13 +0,0 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
export interface CreateDocumentVersion {
/**
* The client can decide the document id (if it wishes to) in order
* to help with syncing. If the client does not provide a document id,
* the server will generate one. If the client provides a document id
* it must not already exist in the database.
*/
document_id: string | null;
relative_path: string;
content: number[];
}

6
frontend/sync-client/src/services/types/CursorPositionFromClient.ts
View file

@ -1,6 +0,0 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
import type { CursorSpan } from "./CursorSpan";
export interface CursorPositionFromClient {
documentToCursors: Partial<Record<string, CursorSpan[]>>;
}

6
frontend/sync-client/src/services/types/CursorPositionFromServer.ts
View file

@ -1,6 +0,0 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
import type { ClientCursors } from "./ClientCursors";
export interface CursorPositionFromServer {
clients: ClientCursors[];
}

6
frontend/sync-client/src/services/types/CursorSpan.ts
View file

@ -1,6 +0,0 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
export interface CursorSpan {
start: number;
end: number;
}

5
frontend/sync-client/src/services/types/DeleteDocumentVersion.ts
View file

@ -1,5 +0,0 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
export interface DeleteDocumentVersion {
relativePath: string;
}

10
frontend/sync-client/src/services/types/DocumentUpdateResponse.ts
View file

@ -1,10 +0,0 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
import type { DocumentVersion } from "./DocumentVersion";
import type { DocumentVersionWithoutContent } from "./DocumentVersionWithoutContent";
/**
* Response to an update document request.
*/
export type DocumentUpdateResponse =
| ({ type: "FastForwardUpdate" } & DocumentVersionWithoutContent)
| ({ type: "MergingUpdate" } & DocumentVersion);

12
frontend/sync-client/src/services/types/DocumentVersion.ts
View file

@ -1,12 +0,0 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
export interface DocumentVersion {
vaultUpdateId: number;
documentId: string;
relativePath: string;
updatedDate: string;
contentBase64: string;
isDeleted: boolean;
userId: string;
deviceId: string;
}

12
frontend/sync-client/src/services/types/DocumentVersionWithoutContent.ts
View file

@ -1,12 +0,0 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
export interface DocumentVersionWithoutContent {
vaultUpdateId: number;
documentId: string;
relativePath: string;
updatedDate: string;
isDeleted: boolean;
userId: string;
deviceId: string;
contentSize: number;
}

13
frontend/sync-client/src/services/types/FetchLatestDocumentsResponse.ts
View file

@ -1,13 +0,0 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
import type { DocumentVersionWithoutContent } from "./DocumentVersionWithoutContent";
/**
* Response to a fetch latest documents request.
*/
export interface FetchLatestDocumentsResponse {
latestDocuments: DocumentVersionWithoutContent[];
/**
* The update ID of the latest document in the response.
*/
lastUpdateId: bigint;
}

16
frontend/sync-client/src/services/types/PingResponse.ts
View file

@ -1,16 +0,0 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
/**
* Response to a ping request.
*/
export interface PingResponse {
/**
* Semantic version of the server.
*/
serverVersion: string;
/**
* Whether the client is authenticated based on the sent Authorization
* header.
*/
isAuthenticated: boolean;
}

7
frontend/sync-client/src/services/types/SerializedError.ts
View file

@ -1,7 +0,0 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
export interface SerializedError {
errorType: string;
message: string;
causes: string[];
}

7
frontend/sync-client/src/services/types/UpdateDocumentVersion.ts
View file

@ -1,7 +0,0 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
export interface UpdateDocumentVersion {
parent_version_id: bigint;
relative_path: string;
content: number[];
}

7
frontend/sync-client/src/services/types/WebSocketClientMessage.ts
View file

@ -1,7 +0,0 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
import type { CursorPositionFromClient } from "./CursorPositionFromClient";
import type { WebSocketHandshake } from "./WebSocketHandshake";
export type WebSocketClientMessage =
| ({ type: "handshake" } & WebSocketHandshake)
| ({ type: "cursorPositions" } & CursorPositionFromClient);

7
frontend/sync-client/src/services/types/WebSocketHandshake.ts
View file

@ -1,7 +0,0 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
export interface WebSocketHandshake {
token: string;
deviceId: string;
lastSeenVaultUpdateId: number | null;
}

7
frontend/sync-client/src/services/types/WebSocketServerMessage.ts
View file

@ -1,7 +0,0 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
import type { CursorPositionFromServer } from "./CursorPositionFromServer";
import type { WebSocketVaultUpdate } from "./WebSocketVaultUpdate";
export type WebSocketServerMessage =
| ({ type: "vaultUpdate" } & WebSocketVaultUpdate)
| ({ type: "cursorPositions" } & CursorPositionFromServer);

7
frontend/sync-client/src/services/types/WebSocketVaultUpdate.ts
View file

@ -1,7 +0,0 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
import type { DocumentVersionWithoutContent } from "./DocumentVersionWithoutContent";
export interface WebSocketVaultUpdate {
documents: DocumentVersionWithoutContent[];
isInitialSync: boolean;
}

209
frontend/sync-client/src/services/websocket-manager.ts
View file

@ -1,209 +0,0 @@
import type { Database } from "../persistence/database";
import type { Logger } from "../tracing/logger";
import type { Settings, SyncSettings } from "../persistence/settings";
import type { WebSocketServerMessage } from "./types/WebSocketServerMessage";
import type { Syncer } from "../sync-operations/syncer";
import type { WebSocketClientMessage } from "./types/WebSocketClientMessage";
import type { CursorPositionFromClient } from "./types/CursorPositionFromClient";
import type { ClientCursors } from "./types/ClientCursors";
export class WebSocketManager {
private readonly webSocketStatusChangeListeners: (() => unknown)[] = [];
private readonly remoteCursorsUpdateListeners: ((
cursors: ClientCursors[]
) => unknown)[] = [];
private refreshWebSocketInterval: NodeJS.Timeout | undefined;
private webSocket: WebSocket | undefined;
private readonly webSocketFactoryImplementation: typeof globalThis.WebSocket;
public constructor(
private readonly deviceId: string,
private readonly logger: Logger,
private readonly database: Database,
private readonly settings: Settings,
private readonly syncer: Syncer,
webSocketImplementation?: typeof globalThis.WebSocket
) {
if (webSocketImplementation) {
this.webSocketFactoryImplementation = webSocketImplementation;
} else {
if (
typeof globalThis !== "undefined" &&
typeof globalThis.WebSocket === "undefined"
) {
// eslint-disable-next-line
this.webSocketFactoryImplementation = require("ws"); // polyfill for WebSocket in Node.js
} else {
this.webSocketFactoryImplementation = WebSocket;
}
}
this.updateWebSocket(settings.getSettings());
settings.addOnSettingsChangeListener((newSettings, oldSettings) => {
if (
newSettings.remoteUri !== oldSettings.remoteUri ||
newSettings.vaultName !== oldSettings.vaultName ||
newSettings.token !== oldSettings.token ||
newSettings.isSyncEnabled !== oldSettings.isSyncEnabled
) {
this.updateWebSocket(newSettings);
}
});
this.setWebSocketRefreshInterval();
}
public get isWebSocketConnected(): boolean {
return (
this.webSocket?.readyState ===
this.webSocketFactoryImplementation.OPEN
);
}
public addWebSocketStatusChangeListener(listener: () => void): void {
this.webSocketStatusChangeListeners.push(listener);
}
public addRemoteCursorsUpdateListener(
listener: (cursors: ClientCursors[]) => void
): void {
this.remoteCursorsUpdateListeners.push(listener);
}
public async reset(): Promise<void> {
this.setWebSocketRefreshInterval();
this.updateWebSocket(this.settings.getSettings());
}
public stop(): void {
clearInterval(this.refreshWebSocketInterval);
try {
this.webSocket?.close();
} catch (e) {
this.logger.warn(`Failed to close WebSocket: ${e}`);
}
}
public updateLocalCursors(cursorPositions: CursorPositionFromClient): void {
if (!this.isWebSocketConnected) {
this.logger.warn(
"WebSocket is not connected, cannot send cursor positions"
);
return;
}
const message: WebSocketClientMessage = {
type: "cursorPositions",
...cursorPositions
};
this.webSocket?.send(JSON.stringify(message));
this.logger.info(
`Sent cursor positions: ${JSON.stringify(cursorPositions)}`
);
}
private updateWebSocket(settings: SyncSettings): void {
try {
this.webSocket?.close();
} catch (e) {
this.logger.warn(`Failed to close WebSocket: ${e}`);
}
if (!settings.isSyncEnabled) {
this.webSocket = undefined;
return;
}
const wsUri = new URL(settings.remoteUri);
wsUri.protocol = wsUri.protocol === "https" ? "wss" : "ws";
wsUri.pathname = `/vaults/${settings.vaultName}/ws`;
this.logger.info(`Connecting to WebSocket at ${wsUri.toString()}`);
this.webSocket = new this.webSocketFactoryImplementation(wsUri);
this.webSocket.onmessage = async (event): Promise<void> => {
// eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion
const message = JSON.parse(event.data) as WebSocketServerMessage;
if (message.type === "vaultUpdate") {
try {
await Promise.all(
message.documents.map(async (document) =>
this.syncer.syncRemotelyUpdatedFile(document)
)
);
if (message.isInitialSync && message.documents.length > 0) {
this.database.setLastSeenUpdateId(
message.documents
.map((document) => document.vaultUpdateId)
.reduce((a, b) => Math.max(a, b))
);
}
} catch (e) {
this.logger.error(
`Failed to sync remotely updated file: ${e}`
);
}
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
} else if (message.type === "cursorPositions") {
this.logger.info(
`Received cursor positions for ${JSON.stringify(message.clients)}`
);
this.remoteCursorsUpdateListeners.forEach((listener) => {
listener(
message.clients.filter(
(client) => client.deviceId !== this.deviceId
)
);
});
} else {
this.logger.warn(
`Received unknown message type: ${JSON.stringify(message)}`
);
}
};
// The JS WebSocket API doesn't support setting headers, so we have to send the token as a message
this.webSocket.onopen = (): void => {
this.logger.info("WebSocket connection opened");
this.webSocketStatusChangeListeners.forEach((listener) => {
listener();
});
const message: WebSocketClientMessage = {
type: "handshake",
deviceId: this.deviceId,
token: settings.token,
lastSeenVaultUpdateId: this.database.getLastSeenUpdateId()
};
this.webSocket?.send(JSON.stringify(message));
};
this.webSocket.onclose = (event): void => {
this.logger.warn(
`WebSocket closed with code ${event.code} (${event.reason == "" ? "unknown reason" : event.reason})`
);
this.webSocketStatusChangeListeners.forEach((listener) => {
listener();
});
};
}
private setWebSocketRefreshInterval(): void {
this.refreshWebSocketInterval = setInterval(() => {
if (
this.webSocket?.readyState ===
this.webSocketFactoryImplementation.CLOSED
) {
this.logger.info("WebSocket is closed, reconnecting...");
this.updateWebSocket(this.settings.getSettings());
}
}, this.settings.getSettings().webSocketRetryIntervalMs);
}
}

302
frontend/sync-client/src/sync-client.ts
View file

@ -1,302 +0,0 @@
import initWasm from "sync_lib";
import wasmBin from "../../../backend/sync_lib/pkg/sync_lib_bg.wasm";
import type { PersistenceProvider } from "./persistence/persistence";
import type { HistoryEntry, HistoryStats } from "./tracing/sync-history";
import { SyncHistory } from "./tracing/sync-history";
import { Logger } from "./tracing/logger";
import type { RelativePath, StoredDatabase } from "./persistence/database";
import { Database } from "./persistence/database";
import type { SyncSettings } from "./persistence/settings";
import { Settings } from "./persistence/settings";
import { SyncService } from "./services/sync-service";
import { Syncer } from "./sync-operations/syncer";
import type { FileSystemOperations } from "./file-operations/filesystem-operations";
import { FileOperations } from "./file-operations/file-operations";
import { ConnectionStatus } from "./services/connection-status";
import { UnrestrictedSyncer } from "./sync-operations/unrestricted-syncer";
import { rateLimit } from "./utils/rate-limit";
import type { NetworkConnectionStatus } from "./types/network-connection-status";
import { DocumentUpdateStatus } from "./types/document-update-status";
import { WebSocketManager } from "./services/websocket-manager";
import { createClientId } from "./utils/create-client-id";
import type { CursorSpan } from "./services/types/CursorSpan";
import type { ClientCursors } from "./services/types/ClientCursors";
export class SyncClient {
private static readonly MINIMUM_SAVE_INTERVAL_MS = 1000;
// eslint-disable-next-line @typescript-eslint/max-params
private constructor(
private readonly history: SyncHistory,
private readonly settings: Settings,
private readonly database: Database,
private readonly syncer: Syncer,
private readonly syncService: SyncService,
private readonly webSocketManager: WebSocketManager,
private readonly _logger: Logger,
private readonly connectionStatus: ConnectionStatus
) {
this.settings.addOnSettingsChangeListener(
(newSettings, oldSettings) => {
if (newSettings.vaultName !== oldSettings.vaultName) {
void this.reset();
}
}
);
}
public get logger(): Logger {
return this._logger;
}
public get documentCount(): number {
return this.database.length;
}
public static async create({
fs,
persistence,
fetch,
webSocket,
nativeLineEndings = "\n"
}: {
fs: FileSystemOperations;
persistence: PersistenceProvider<
Partial<{
settings: Partial<SyncSettings>;
database: Partial<StoredDatabase>;
}>
>;
fetch?: typeof globalThis.fetch;
webSocket?: typeof globalThis.WebSocket;
nativeLineEndings?: string;
}): Promise<SyncClient> {
const logger = new Logger();
const deviceId = createClientId();
logger.info(`Initialising SyncClient with client id ${deviceId}`);
const history = new SyncHistory(logger);
await initWasm(
// eslint-disable-next-line
(wasmBin as any).default // it is loaded as a base64 string by webpack
);
let state = (await persistence.load()) ?? {
settings: undefined,
database: undefined
};
const rateLimitedSave = rateLimit(
persistence.save,
SyncClient.MINIMUM_SAVE_INTERVAL_MS
);
const database = new Database(
logger,
state.database,
async (data): Promise<void> => {
state = { ...state, database: data };
await rateLimitedSave(state);
}
);
const settings = new Settings(
logger,
state.settings,
async (data): Promise<void> => {
state = { ...state, settings: data };
await rateLimitedSave(state);
}
);
const connectionStatus = new ConnectionStatus(settings, logger);
const syncService = new SyncService(
deviceId,
connectionStatus,
settings,
logger,
fetch
);
const fileOperations = new FileOperations(
logger,
database,
fs,
nativeLineEndings
);
const unrestrictedSyncer = new UnrestrictedSyncer(
logger,
database,
settings,
syncService,
fileOperations,
history
);
const syncer = new Syncer(
deviceId,
logger,
database,
settings,
syncService,
fileOperations,
unrestrictedSyncer
);
const webSocketManager = new WebSocketManager(
deviceId,
logger,
database,
settings,
syncer,
webSocket
);
const client = new SyncClient(
history,
settings,
database,
syncer,
syncService,
webSocketManager,
logger,
connectionStatus
);
logger.info("SyncClient initialised");
return client;
}
public async checkConnection(): Promise<NetworkConnectionStatus> {
const server = await this.syncService.checkConnection();
return {
isSuccessful: server.isSuccessful,
serverMessage: server.message,
isWebSocketConnected: this.webSocketManager.isWebSocketConnected
};
}
public getHistoryEntries(): readonly HistoryEntry[] {
return this.history.entries;
}
public addSyncHistoryUpdateListener(
listener: (stats: HistoryStats) => void
): void {
this.history.addSyncHistoryUpdateListener(listener);
}
public async start(): Promise<void> {
await this.syncer.scheduleSyncForOfflineChanges();
}
public stop(): void {
this.webSocketManager.stop();
}
public async waitAndStop(): Promise<void> {
this.stop();
await this.syncer.waitUntilFinished();
}
/// Wait for the in-flight operations to finish, reset all tracking,
/// and the local database but retain the settings.
/// The SyncClient can be used again after calling this method.
public async reset(): Promise<void> {
this.stop();
this.connectionStatus.startReset();
await this.syncer.reset();
await this.webSocketManager.reset();
this.history.reset();
this.database.reset();
this._logger.reset();
this.connectionStatus.finishReset();
void this.start();
}
public getSettings(): SyncSettings {
return this.settings.getSettings();
}
public async setSetting<T extends keyof SyncSettings>(
key: T,
value: SyncSettings[T]
): Promise<void> {
await this.settings.setSetting(key, value);
}
public async setSettings(value: Partial<SyncSettings>): Promise<void> {
await this.settings.setSettings(value);
}
public addOnSettingsChangeListener(
handler: (settings: SyncSettings, oldSettings: SyncSettings) => void
): void {
this.settings.addOnSettingsChangeListener(handler);
}
public addRemainingSyncOperationsListener(
listener: (remainingOperations: number) => void
): void {
this.syncer.addRemainingOperationsListener(listener);
}
public addWebSocketStatusChangeListener(listener: () => void): void {
this.webSocketManager.addWebSocketStatusChangeListener(listener);
}
public async syncLocallyCreatedFile(
relativePath: RelativePath
): Promise<void> {
return this.syncer.syncLocallyCreatedFile(relativePath);
}
public async syncLocallyDeletedFile(
relativePath: RelativePath
): Promise<void> {
return this.syncer.syncLocallyDeletedFile(relativePath);
}
public async syncLocallyUpdatedFile({
oldPath,
relativePath
}: {
oldPath?: RelativePath;
relativePath: RelativePath;
}): Promise<void> {
return this.syncer.syncLocallyUpdatedFile({
oldPath,
relativePath
});
}
public async updateLocalCursors(
documentToCursors: Record<RelativePath, CursorSpan[]>
): Promise<void> {
this.webSocketManager.updateLocalCursors({ documentToCursors });
}
public addRemoteCursorsUpdateListener(
listener: (cursors: ClientCursors[]) => void
): void {
this.webSocketManager.addRemoteCursorsUpdateListener(listener);
}
public getDocumentSyncingStatus(
relativePath: RelativePath
): DocumentUpdateStatus {
const document =
this.database.getLatestDocumentByRelativePath(relativePath);
if (document === undefined) {
return DocumentUpdateStatus.SYNCING;
}
return document.updates.length > 0
? DocumentUpdateStatus.SYNCING
: DocumentUpdateStatus.UP_TO_DATE;
}
}

442
frontend/sync-client/src/sync-operations/syncer.ts
View file

@ -1,442 +0,0 @@
import type {
Database,
DocumentId,
DocumentRecord,
RelativePath
} from "../persistence/database";
import type { SyncService } from "../services/sync-service";
import type { Logger } from "../tracing/logger";
import PQueue from "p-queue";
import { hash } from "../utils/hash";
import { v4 as uuidv4 } from "uuid";
import type { Settings, SyncSettings } from "../persistence/settings";
import type { FileOperations } from "../file-operations/file-operations";
import { findMatchingFile } from "../utils/find-matching-file";
import type { UnrestrictedSyncer } from "./unrestricted-syncer";
import { createPromise } from "../utils/create-promise";
import { SyncResetError } from "../services/sync-reset-error";
import { Locks } from "../utils/locks";
import type { DocumentVersionWithoutContent } from "../services/types/DocumentVersionWithoutContent";
export class Syncer {
private readonly remoteDocumentsLock: Locks<DocumentId>;
private readonly remainingOperationsListeners: ((
remainingOperations: number
) => void)[] = [];
private readonly syncQueue: PQueue;
private runningScheduleSyncForOfflineChanges: Promise<void> | undefined;
// eslint-disable-next-line @typescript-eslint/max-params
public constructor(
private readonly deviceId: string,
private readonly logger: Logger,
private readonly database: Database,
private readonly settings: Settings,
private readonly syncService: SyncService,
private readonly operations: FileOperations,
private readonly internalSyncer: UnrestrictedSyncer
) {
this.syncQueue = new PQueue({
concurrency: settings.getSettings().syncConcurrency
});
this.remoteDocumentsLock = new Locks<DocumentId>(this.logger);
settings.addOnSettingsChangeListener((newSettings, oldSettings) => {
if (newSettings.syncConcurrency !== oldSettings.syncConcurrency) {
this.syncQueue.concurrency = newSettings.syncConcurrency;
}
});
this.syncQueue.on("active", () => {
this.remainingOperationsListeners.forEach((listener) => {
listener(this.syncQueue.size);
});
});
}
public addRemainingOperationsListener(
listener: (remainingOperations: number) => void
): void {
this.remainingOperationsListeners.push(listener);
}
public async syncLocallyCreatedFile(
relativePath: RelativePath
): Promise<void> {
if (
this.database.getLatestDocumentByRelativePath(relativePath)
?.isDeleted === false
) {
this.logger.debug(
`Document ${relativePath} already exists in the database, skipping`
);
return;
}
const [promise, resolve, reject] = createPromise();
const id = uuidv4();
const document = this.database.createNewPendingDocument(
id,
relativePath,
promise
);
this.logger.debug(
`Creating new pending document ${relativePath} with id ${id}`
);
try {
await this.syncQueue.add(async () =>
this.internalSyncer.unrestrictedSyncLocallyCreatedFile(document)
);
resolve();
} catch (e) {
reject(e);
} finally {
this.database.removeDocumentPromise(promise);
}
}
public async syncLocallyDeletedFile(
relativePath: RelativePath
): Promise<void> {
if (
this.database.getLatestDocumentByRelativePath(relativePath)
?.isDeleted === true
) {
// This is must be a consequence of us deleting a file because of a remote update
// which triggered a local delete, so we don't need to do anything here.
this.logger.debug(
`Document ${relativePath} has already been markes as deleted, skipping`
);
return;
}
// We have to have a record of the delete in case there's an in-flight update for the same
// document which finishes after the delete has succeeded and would introduce a phantom metadata record.
this.database.delete(relativePath);
const [promise, resolve, reject] = createPromise();
const document = await this.database.getResolvedDocumentByRelativePath(
relativePath,
promise
);
try {
await this.syncQueue.add(async () =>
this.internalSyncer.unrestrictedSyncLocallyDeletedFile(document)
);
resolve();
this.database.removeDocument(document);
} catch (e) {
reject(e);
} finally {
this.database.removeDocumentPromise(promise);
}
}
public async syncLocallyUpdatedFile({
oldPath,
relativePath
}: {
oldPath?: RelativePath;
relativePath: RelativePath;
}): Promise<void> {
if (oldPath !== undefined) {
// We might have moved the document in the database before calling this method,
// in that case, we mustn't move it again.
if (
this.database.getLatestDocumentByRelativePath(relativePath) ===
undefined ||
this.database.getLatestDocumentByRelativePath(relativePath)
?.isDeleted === true
) {
if (oldPath === relativePath) {
throw new Error(
`Old path and new path are the same: ${oldPath}`
);
}
this.database.move(oldPath, relativePath);
}
}
let document =
this.database.getLatestDocumentByRelativePath(relativePath);
if (
oldPath !== undefined &&
document?.metadata?.remoteRelativePath === relativePath
) {
this.logger.debug(
`Document ${relativePath} has been moved as a result of a remote update, skipping sync`
);
return;
}
if (document === undefined) {
this.logger.debug(
`Cannot find document ${relativePath} in the database, skipping`
);
return;
}
if (document.isDeleted) {
this.logger.debug(
`Document ${relativePath} has been deleted locally, skipping`
);
return;
}
const [promise, resolve, reject] = createPromise();
document = await this.database.getResolvedDocumentByRelativePath(
relativePath,
promise
);
try {
await this.syncQueue.add(async () =>
this.internalSyncer.unrestrictedSyncLocallyUpdatedFile({
oldPath,
document
})
);
resolve();
} catch (e) {
reject(e);
} finally {
this.database.removeDocumentPromise(promise);
}
}
public async scheduleSyncForOfflineChanges(): Promise<void> {
if (this.runningScheduleSyncForOfflineChanges !== undefined) {
this.logger.debug("Uploading local changes is already in progress");
return this.runningScheduleSyncForOfflineChanges;
}
try {
this.runningScheduleSyncForOfflineChanges =
this.internalScheduleSyncForOfflineChanges();
await this.runningScheduleSyncForOfflineChanges;
this.logger.info(`All local changes have been applied remotely`);
} catch (e) {
if (e instanceof SyncResetError) {
this.logger.info(
"Failed to apply local changes remotely due to a reset"
);
return;
}
this.logger.error(
`Not all local changes have been applied remotely: ${e}`
);
throw e;
} finally {
this.runningScheduleSyncForOfflineChanges = undefined;
}
}
public async waitUntilFinished(): Promise<void> {
await this.runningScheduleSyncForOfflineChanges;
return this.syncQueue.onEmpty();
}
public async reset(): Promise<void> {
await this.waitUntilFinished();
}
public async syncRemotelyUpdatedFile(
remoteVersion: DocumentVersionWithoutContent
): Promise<void> {
let document = this.database.getDocumentByDocumentId(
remoteVersion.documentId
);
let hasLockToRelease = false;
if (document === undefined) {
// Let's avoid the same documents getting created in parallel multiple times.
// There might be multiple tasks waiting for the lock
await this.remoteDocumentsLock.waitForLock(
remoteVersion.documentId
);
hasLockToRelease = true;
document = this.database.getDocumentByDocumentId(
remoteVersion.documentId
);
}
try {
// We're either the first one to get the lock, so we have to create the document in `unrestrictedSyncRemotelyUpdatedFile`
if (document === undefined) {
await this.syncQueue.add(async () =>
this.internalSyncer.unrestrictedSyncRemotelyUpdatedFile(
remoteVersion
)
);
} else {
const [promise, resolve, reject] = createPromise();
document =
await this.database.getResolvedDocumentByRelativePath(
document.relativePath,
promise
);
try {
await this.syncQueue.add(async () =>
this.internalSyncer.unrestrictedSyncRemotelyUpdatedFile(
remoteVersion,
document
)
);
resolve();
} catch (e) {
reject(e);
} finally {
this.database.removeDocumentPromise(promise);
}
}
this.database.addSeenUpdateId(remoteVersion.vaultUpdateId);
} finally {
if (hasLockToRelease) {
this.remoteDocumentsLock.unlock(remoteVersion.documentId);
}
}
}
private async internalScheduleSyncForOfflineChanges(): Promise<void> {
await this.createFakeDocumentsFromRemoteState();
const allLocalFiles = await this.operations.listAllFiles();
let locallyPossiblyDeletedFiles: DocumentRecord[] = [];
for (const document of this.database.resolvedDocuments) {
if (
!document.isDeleted &&
!(await this.operations.exists(document.relativePath))
) {
locallyPossiblyDeletedFiles.push(document);
}
}
const updates = Promise.all(
allLocalFiles.map(async (relativePath) => {
if (
this.database.getLatestDocumentByRelativePath(relativePath)
?.metadata !== undefined
) {
this.logger.debug(
`Document ${relativePath} might have been updated locally, scheduling sync to validate and update it`
);
return this.syncLocallyUpdatedFile({
relativePath
});
}
// Perhaps the file has been moved; let's check by looking at the deleted files
const contentHash = await this.syncQueue.add(async () => {
const contentBytes =
await this.operations.read(relativePath); // this can throw FileNotFoundError
return hash(contentBytes);
});
if (contentHash == undefined) {
// The file was deleted before we had a chance to read it, no need to sync it here
return;
}
const originalFile = findMatchingFile(
contentHash,
locallyPossiblyDeletedFiles
);
if (originalFile !== undefined) {
// `originalFile` hasn't been deleted but it got moved instead
locallyPossiblyDeletedFiles =
locallyPossiblyDeletedFiles.filter(
(item) =>
item.relativePath !== originalFile.relativePath
);
this.logger.debug(
`Document '${originalFile.relativePath}' was not found under its current path in the database but was found under a different path (${relativePath}), scheduling sync to move it`
);
// We're outside of the pqueue, so we need to call the public wrapper
return this.syncLocallyUpdatedFile({
oldPath: originalFile.relativePath,
relativePath
});
}
this.logger.debug(
`Document ${relativePath} not found in database, scheduling sync to create it`
);
// We're outside of the pqueue, so we need to call the public wrapper
return this.syncLocallyCreatedFile(relativePath);
})
);
const deletes = Promise.all(
locallyPossiblyDeletedFiles.map(async ({ relativePath }) => {
this.logger.debug(
`Document ${relativePath} has been deleted locally, scheduling sync to delete it`
);
// We're outside of the pqueue, so we need to call the public wrapper
return this.syncLocallyDeletedFile(relativePath);
})
);
await Promise.all([updates, deletes]);
}
/**
* Create fake documents in the database for all files that are present locally
* and also exist remotely. This will stop the subequent syncs from duplicating
* the documents by creating the same documents from multiple clients.
*/
private async createFakeDocumentsFromRemoteState(): Promise<void> {
if (this.database.getHasInitialSyncCompleted()) {
return;
}
const [allLocalFiles, remote] = await Promise.all([
this.operations.listAllFiles(),
this.syncQueue.add(async () => this.syncService.getAll())
]);
if (remote !== undefined) {
remote.latestDocuments
.filter(
(remoteDocument) =>
allLocalFiles.includes(remoteDocument.relativePath) &&
!remoteDocument.isDeleted &&
this.database.getDocumentByDocumentId(
remoteDocument.documentId
) === undefined
)
.forEach((remoteDocument) => {
this.database.createNewEmptyDocument(
remoteDocument.documentId,
remoteDocument.vaultUpdateId,
remoteDocument.relativePath
);
});
}
this.database.setHasInitialSyncCompleted(true);
}
}

513
frontend/sync-client/src/sync-operations/unrestricted-syncer.ts
View file

@ -1,513 +0,0 @@
import type {
Database,
DocumentRecord,
RelativePath
} from "../persistence/database";
import type { SyncService } from "../services/sync-service";
import type { Logger } from "../tracing/logger";
import type {
CommonHistoryEntry,
SyncCreateDetails,
SyncDeleteDetails,
SyncDetails,
SyncHistory,
SyncMovedDetails,
SyncUpdateDetails
} from "../tracing/sync-history";
import { SyncStatus, SyncType } from "../tracing/sync-history";
import { EMPTY_HASH, hash } from "../utils/hash";
import { deserialize } from "../utils/deserialize";
import type { Settings } from "../persistence/settings";
import type { FileOperations } from "../file-operations/file-operations";
import { createPromise } from "../utils/create-promise";
import { FileNotFoundError } from "../file-operations/file-not-found-error";
import { SyncResetError } from "../services/sync-reset-error";
import { globsToRegexes } from "../utils/globs-to-regexes";
import type { DocumentVersion } from "../services/types/DocumentVersion";
import type { DocumentUpdateResponse } from "../services/types/DocumentUpdateResponse";
import type { DocumentVersionWithoutContent } from "../services/types/DocumentVersionWithoutContent";
export class UnrestrictedSyncer {
private ignorePatterns: RegExp[];
public constructor(
private readonly logger: Logger,
private readonly database: Database,
private readonly settings: Settings,
private readonly syncService: SyncService,
private readonly operations: FileOperations,
private readonly history: SyncHistory
) {
this.ignorePatterns = globsToRegexes(
this.settings.getSettings().ignorePatterns,
this.logger
);
this.settings.addOnSettingsChangeListener((newSettings) => {
this.ignorePatterns = globsToRegexes(
newSettings.ignorePatterns,
this.logger
);
});
}
public async unrestrictedSyncLocallyCreatedFile(
document: DocumentRecord
): Promise<void> {
const updateDetails: SyncCreateDetails = {
type: SyncType.CREATE,
relativePath: document.relativePath
};
return this.executeSync(updateDetails, async () => {
if (document.isDeleted) {
this.logger.debug(
`Document ${document.relativePath} has been already deleted, no need to create it`
);
return;
}
const contentBytes = await this.operations.read(
document.relativePath
); // this can throw FileNotFoundError
const contentHash = hash(contentBytes);
const response = await this.syncService.create({
documentId: document.documentId,
relativePath: document.relativePath,
contentBytes
});
this.database.updateDocumentMetadata(
{
parentVersionId: response.vaultUpdateId,
hash: contentHash,
remoteRelativePath: response.relativePath
},
document
);
this.database.addSeenUpdateId(response.vaultUpdateId);
this.history.addHistoryEntry({
status: SyncStatus.SUCCESS,
details: updateDetails,
message: `Successfully uploaded locally created file`
});
});
}
public async unrestrictedSyncLocallyDeletedFile(
document: DocumentRecord
): Promise<void> {
const updateDetails: SyncDeleteDetails = {
type: SyncType.DELETE,
relativePath: document.relativePath
};
await this.executeSync(updateDetails, async () => {
const response = await this.syncService.delete({
documentId: document.documentId,
relativePath: document.relativePath
});
this.database.updateDocumentMetadata(
{
parentVersionId: response.vaultUpdateId,
hash: EMPTY_HASH,
remoteRelativePath: document.relativePath
},
document
);
this.database.addSeenUpdateId(response.vaultUpdateId);
this.history.addHistoryEntry({
status: SyncStatus.SUCCESS,
details: updateDetails,
message: `Successfully deleted locally deleted file on the server`,
author: response.userId
});
});
}
public async unrestrictedSyncLocallyUpdatedFile({
oldPath,
document,
// We use the same code path for both local and remote updates. We need to force the update
// if there are no local changes but we know that the remote version is newer.
force = false
}: {
oldPath?: RelativePath;
force?: boolean;
document: DocumentRecord;
}): Promise<void> {
const updateDetails: SyncUpdateDetails | SyncMovedDetails =
oldPath !== undefined
? {
type: SyncType.MOVE,
relativePath: document.relativePath,
movedFrom: oldPath
}
: {
type: SyncType.UPDATE,
relativePath: document.relativePath
};
await this.executeSync(updateDetails, async () => {
const originalRelativePath = document.relativePath;
if (document.isDeleted || document.metadata === undefined) {
this.logger.debug(
`Document ${document.relativePath} has been already deleted, no need to update it`
);
return;
}
const contentBytes = await this.operations.read(
document.relativePath
); // this can throw FileNotFoundError
let contentHash = hash(contentBytes);
const areThereLocalChanges = !(
document.metadata.hash === contentHash && oldPath === undefined
);
let response: DocumentVersion | DocumentUpdateResponse | undefined =
undefined;
if (areThereLocalChanges) {
response = await this.syncService.put({
documentId: document.documentId,
parentVersionId: document.metadata.parentVersionId,
relativePath: document.relativePath,
contentBytes
});
} else {
if (!force) {
this.logger.debug(
`File hash of ${document.relativePath} matches with last synced version and the path hasn't changed; no need to sync`
);
return;
}
response = await this.syncService.get({
documentId: document.documentId
});
}
// `document` is mutable and reflects the latest state in the local database
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
if (document.isDeleted) {
this.logger.info(
`Document ${document.relativePath} has been deleted before we could finish updating it`
);
this.database.addSeenUpdateId(response.vaultUpdateId);
return;
}
if (
// `Syncer` creates fake local document metadata for all remote docs with invalid hashes. The parent IDs will likely match
// the latest versions so we still need to update the local versions to turn the fakes into real metadata.
document.metadata.parentVersionId > response.vaultUpdateId
) {
this.logger.debug(
`Document ${document.relativePath} is already more up to date than the fetched version`
);
this.database.addSeenUpdateId(response.vaultUpdateId); // in case the previous `vaultUpdateId` update hasn't made it through
return;
}
if (response.isDeleted) {
this.history.addHistoryEntry({
status: SyncStatus.SUCCESS,
details: {
type: SyncType.DELETE,
relativePath: document.relativePath
},
message:
"File has been deleted remotely, so we deleted it locally",
author: response.userId
});
this.database.delete(document.relativePath);
this.database.updateDocumentMetadata(
{
parentVersionId: response.vaultUpdateId,
hash: EMPTY_HASH,
remoteRelativePath: response.relativePath
},
document
);
await this.operations.delete(document.relativePath);
this.database.addSeenUpdateId(response.vaultUpdateId);
return;
}
let actualPath = document.relativePath;
if (response.relativePath != originalRelativePath) {
actualPath = response.relativePath;
// Make sure to update the remote relative path to avoid uploading
// the file as a result of this filesystem event.
document.metadata.remoteRelativePath = response.relativePath;
await this.operations.move(
document.relativePath,
response.relativePath
); // this can throw FileNotFoundError
}
if (!("type" in response) || response.type === "MergingUpdate") {
const responseBytes = deserialize(response.contentBase64);
contentHash = hash(responseBytes);
this.database.updateDocumentMetadata(
{
parentVersionId: response.vaultUpdateId,
hash: contentHash,
remoteRelativePath: response.relativePath
},
document
);
await this.operations.write(
actualPath,
contentBytes,
responseBytes
);
if (!force) {
this.history.addHistoryEntry({
status: SyncStatus.SUCCESS,
details: updateDetails,
message: `The file we updated had been updated remotely, so we downloaded the merged version`
});
}
} else {
this.database.updateDocumentMetadata(
{
parentVersionId: response.vaultUpdateId,
hash: contentHash,
remoteRelativePath: response.relativePath
},
document
);
}
this.database.addSeenUpdateId(response.vaultUpdateId);
const actualUpdateDetails: SyncUpdateDetails | SyncMovedDetails =
oldPath !== undefined ||
response.relativePath != originalRelativePath
? {
type: SyncType.MOVE,
relativePath: response.relativePath,
movedFrom: originalRelativePath
}
: {
type: SyncType.UPDATE,
relativePath: response.relativePath
};
if (areThereLocalChanges) {
this.history.addHistoryEntry({
status: SyncStatus.SUCCESS,
details: actualUpdateDetails,
message: `Successfully uploaded locally updated file to the server`,
author: response.userId
});
} else {
this.history.addHistoryEntry({
status: SyncStatus.SUCCESS,
details: actualUpdateDetails,
message: `Successfully downloaded remotely updated file from the server`,
author: response.userId
});
}
});
}
public async unrestrictedSyncRemotelyUpdatedFile(
remoteVersion: DocumentVersionWithoutContent,
document?: DocumentRecord
): Promise<void> {
const updateDetails: SyncCreateDetails = {
type: SyncType.CREATE,
relativePath: remoteVersion.relativePath
};
await this.executeSync(updateDetails, async () => {
if (document?.metadata !== undefined) {
// If the file exists locally, let's pretend the user has updated it
// and deal with remote update/deletion within `unrestrictedSyncLocallyUpdatedFile`
if (
document.metadata.parentVersionId >=
remoteVersion.vaultUpdateId
) {
this.logger.debug(
`Document ${remoteVersion.relativePath} is already at least as up to date as the fetched version`
);
return;
}
return this.unrestrictedSyncLocallyUpdatedFile({
document,
force: true
});
} else if (remoteVersion.isDeleted) {
// Either the document hasn't made it to us before and therefore we don't need to delete it,
// or we already have it, in which case the preceeding if would've dealt with it
this.logger.debug(
`Document ${remoteVersion.relativePath} has been deleted remotely, no need to sync`
);
return;
}
// Don't download oversized files
const historyEntryForSkippedOversizedFile =
this.getHistoryEntryForSkippedOversizedFile(
remoteVersion.contentSize,
remoteVersion.relativePath
);
if (historyEntryForSkippedOversizedFile !== undefined) {
this.history.addHistoryEntry(
historyEntryForSkippedOversizedFile
);
return;
}
const content = (
await this.syncService.get({
documentId: remoteVersion.documentId
})
).contentBase64;
// We're trying to create an entirely new document that didn't exist locally
document = this.database.getDocumentByDocumentId(
remoteVersion.documentId
);
// It can happen that a concurrent sync operation has already created the document, so we can bail here
if (document !== undefined) {
this.logger.debug(
`Document ${remoteVersion.relativePath} has already been created locally, no need to create it again`
);
return;
}
const contentBytes = deserialize(content);
await this.operations.ensureClearPath(remoteVersion.relativePath);
const [promise, resolve] = createPromise();
this.database.updateDocumentMetadata(
{
parentVersionId: remoteVersion.vaultUpdateId,
hash: hash(contentBytes),
remoteRelativePath: remoteVersion.relativePath
},
this.database.createNewPendingDocument(
remoteVersion.documentId,
remoteVersion.relativePath,
promise
)
);
await this.operations.create(
remoteVersion.relativePath,
contentBytes
);
resolve();
this.database.removeDocumentPromise(promise);
this.history.addHistoryEntry({
status: SyncStatus.SUCCESS,
details: updateDetails,
message: `Successfully downloaded remote file which hadn't existed locally`,
author: remoteVersion.userId
});
});
}
public async executeSync<T>(
details: SyncDetails,
fn: () => Promise<T>
): Promise<T | undefined> {
for (const pattern of this.ignorePatterns) {
if (pattern.test(details.relativePath)) {
this.logger.debug(
`File '${details.relativePath}' is ignored by the ignore pattern: ${pattern}`
);
return; // bail without SKIPPED status because we were told to ignore this file and we shouldn't clutter up the history
}
}
try {
// Only check the size of files which already exist locally.
if (await this.operations.exists(details.relativePath)) {
const sizeInBytes = await this.operations.getFileSize(
details.relativePath
);
const historyEntryForSkippedOversizedFile =
this.getHistoryEntryForSkippedOversizedFile(
sizeInBytes,
details.relativePath
);
if (historyEntryForSkippedOversizedFile !== undefined) {
this.history.addHistoryEntry(
historyEntryForSkippedOversizedFile
);
return;
}
}
return await fn();
} catch (e) {
if (e instanceof FileNotFoundError) {
// A subsequent sync operation must have been creating to deal with this
this.logger.info(
`Skiping file '${details.relativePath}' because it no longer exists when trying to ${details.type.toLocaleLowerCase()} it`
);
return;
}
if (e instanceof SyncResetError) {
this.logger.info(
`Interrupting sync operation because of a reset`
);
return;
} else {
this.history.addHistoryEntry({
status: SyncStatus.ERROR,
details,
message: `Failed to sync file '${details.relativePath}' because of ${e} when trying to ${details.type.toLocaleLowerCase()} it`
});
throw e;
}
}
}
private getHistoryEntryForSkippedOversizedFile(
sizeInBytes: number,
relativePath: RelativePath
): CommonHistoryEntry | undefined {
const sizeInMB = Math.round(sizeInBytes / 1024 / 1024);
const { maxFileSizeMB } = this.settings.getSettings();
if (sizeInMB > maxFileSizeMB) {
return {
status: SyncStatus.SKIPPED,
details: {
type: SyncType.SKIPPED,
relativePath
},
message: `File size of ${sizeInMB} MB exceeds the maximum file size limit of ${
maxFileSizeMB
} MB`
};
}
}
}

77
frontend/sync-client/src/tracing/logger.ts
View file

@ -1,77 +0,0 @@
export enum LogLevel {
DEBUG = "DEBUG",
INFO = "INFO",
WARNING = "WARNING",
ERROR = "ERROR"
}
const LOG_LEVEL_ORDER = {
[LogLevel.DEBUG]: 0,
[LogLevel.INFO]: 1,
[LogLevel.WARNING]: 2,
[LogLevel.ERROR]: 3
};
export class LogLine {
public timestamp = new Date();
public constructor(
public level: LogLevel,
public message: string
) {}
}
export class Logger {
private static readonly MAX_MESSAGES = 100000;
private readonly messages: LogLine[] = [];
private readonly onMessageListeners: ((message: LogLine) => void)[] = [];
public constructor(...onMessageListeners: ((message: LogLine) => void)[]) {
this.onMessageListeners = onMessageListeners;
}
public debug(message: string): void {
this.pushMessage(message, LogLevel.DEBUG);
}
public info(message: string): void {
this.pushMessage(message, LogLevel.INFO);
}
public warn(message: string): void {
this.pushMessage(message, LogLevel.WARNING);
}
public error(message: string): void {
this.pushMessage(message, LogLevel.ERROR);
}
public getMessages(mininumSeverity: LogLevel): LogLine[] {
return this.messages.filter(
(message) =>
LOG_LEVEL_ORDER[message.level] >=
LOG_LEVEL_ORDER[mininumSeverity]
);
}
public addOnMessageListener(listener: (message: LogLine) => void): void {
this.onMessageListeners.push(listener);
}
public reset(): void {
this.messages.length = 0;
this.debug("Logger has been reset");
}
private pushMessage(message: string, level: LogLevel): void {
const logLine = new LogLine(level, message);
this.messages.push(logLine);
while (this.messages.length > Logger.MAX_MESSAGES) {
this.messages.shift();
}
this.onMessageListeners.forEach((listener) => {
listener(logLine);
});
}
}

174
frontend/sync-client/src/tracing/sync-history.ts
View file

@ -1,174 +0,0 @@
import type { RelativePath } from "../persistence/database";
import type { Logger } from "./logger";
export interface SyncCreateDetails {
type: SyncType.CREATE;
relativePath: RelativePath;
}
export interface SyncUpdateDetails {
type: SyncType.UPDATE;
relativePath: RelativePath;
}
export interface SyncMovedDetails {
type: SyncType.MOVE;
relativePath: RelativePath;
movedFrom: RelativePath;
}
export interface SyncDeleteDetails {
type: SyncType.DELETE;
relativePath: RelativePath;
}
export interface SyncSkippedDetails {
type: SyncType.SKIPPED;
relativePath: RelativePath;
}
export type SyncDetails =
| SyncCreateDetails
| SyncUpdateDetails
| SyncDeleteDetails
| SyncMovedDetails
| SyncSkippedDetails;
export interface CommonHistoryEntry {
status: SyncStatus;
message: string;
details: SyncDetails;
author?: string;
}
export enum SyncType {
CREATE = "CREATE",
UPDATE = "UPDATE",
DELETE = "DELETE",
MOVE = "MOVE",
SKIPPED = "SKIPPED"
}
export enum SyncStatus {
SUCCESS = "SUCCESS",
ERROR = "ERROR",
SKIPPED = "SKIPPED"
}
export type HistoryEntry = CommonHistoryEntry & { timestamp: Date };
export interface HistoryStats {
success: number;
error: number;
}
export class SyncHistory {
private static readonly MAX_ENTRIES = 5000;
private static readonly TIMEOUT_FOR_MERGING_ENTRIES_IN_SECONDS = 60;
private _entries: HistoryEntry[] = [];
private readonly syncHistoryUpdateListeners: ((
status: HistoryStats
) => void)[] = [];
private status: HistoryStats = {
success: 0,
error: 0
};
public constructor(private readonly logger: Logger) {}
public get entries(): readonly HistoryEntry[] {
return this._entries;
}
/**
* Insert the entry at the beginning of the history list. If the entry
* already in the list, it will get moved to the beginning and updated.
*
* If the entry list is too long, the oldest entry will be removed.
*/
public addHistoryEntry(entry: CommonHistoryEntry): void {
const historyEntry = {
...entry,
timestamp: new Date()
};
const candidate = this.findSimilarRecentUpdateEntry(historyEntry);
if (candidate !== undefined) {
this._entries = this._entries.filter((e) => e !== candidate);
}
// Insert the entry at the beginning
this._entries.unshift(historyEntry);
if (this._entries.length > SyncHistory.MAX_ENTRIES) {
this._entries.pop();
}
this.updateSuccessCount(historyEntry);
}
public addSyncHistoryUpdateListener(
listener: (stats: HistoryStats) => void
): void {
this.syncHistoryUpdateListeners.push(listener);
listener({ ...this.status });
}
public reset(): void {
this._entries.length = 0;
this.status = {
success: 0,
error: 0
};
this.syncHistoryUpdateListeners.forEach((listener) => {
listener(this.status);
});
}
private findSimilarRecentUpdateEntry(
entry: HistoryEntry
): HistoryEntry | undefined {
if (entry.details.type !== SyncType.UPDATE) {
return;
}
const candidate = this._entries.find(
(e) =>
e.details.type === SyncType.UPDATE &&
e.details.relativePath === entry.details.relativePath
);
if (
candidate !== undefined &&
(this._entries[0] === candidate ||
candidate.timestamp.getTime() +
SyncHistory.TIMEOUT_FOR_MERGING_ENTRIES_IN_SECONDS * 1000 >
entry.timestamp.getTime())
) {
return candidate;
}
}
private updateSuccessCount(entry: HistoryEntry): void {
const message = `${entry.details.relativePath} - ${entry.message} (${entry.details.type.toLocaleLowerCase()})`;
switch (entry.status) {
case SyncStatus.SUCCESS:
this.status.success++;
this.logger.info(`History entry: ${message}`);
break;
case SyncStatus.ERROR:
this.status.error++;
this.logger.error(`Cannot sync file: ${message}`);
break;
case SyncStatus.SKIPPED:
this.logger.error(`Skipping file: ${message}`);
break;
}
this.syncHistoryUpdateListeners.forEach((listener) => {
listener(this.status);
});
}
}

4
frontend/sync-client/src/types/document-update-status.ts
View file

@ -1,4 +0,0 @@
export enum DocumentUpdateStatus {
UP_TO_DATE = "UP_TO_DATE",
SYNCING = "SYNCING"
}

5
frontend/sync-client/src/types/network-connection-status.ts
View file

@ -1,5 +0,0 @@
export interface NetworkConnectionStatus {
isSuccessful: boolean;
serverMessage: string;
isWebSocketConnected: boolean;
}

13
frontend/sync-client/src/utils/assert-set-contains-exactly.ts
View file

@ -1,13 +0,0 @@
import assert from "assert";
export function assertSetContainsExactly<T>(set: Set<T>, ...values: T[]): void {
assert(
set.size === values.length &&
Array.from(set).every((value) => values.includes(value)),
`Expected set to contain only ${values.map((v) => '"' + v + '"').join(", ")}, but it contained ${Array.from(
set
)
.map((v) => '"' + v + '"')
.join(", ")}`
);
}

15
frontend/sync-client/src/utils/create-client-id.ts
View file

@ -1,15 +0,0 @@
import { v4 as uuidv4 } from "uuid";
export function createClientId(): string {
// @ts-expect-error, injected by webpack
const packageVersion = __CURRENT_VERSION__; // eslint-disable-line
const platform =
typeof navigator !== "undefined"
? navigator.platform // eslint-disable-line @typescript-eslint/no-deprecated
: typeof process !== "undefined"
? process.platform
: "unknown";
return `vault-link/${packageVersion} (${uuidv4()}; ${platform})`;
}

19
frontend/sync-client/src/utils/create-promise.ts
View file

@ -1,19 +0,0 @@
/**
* A type-safe utility function to create a Promise with resolve and reject functions.
* @returns A tuple containing a Promise, a resolve function, and a reject function.
*/
export function createPromise<T = void>(): [
Promise<T>,
(value: T) => void,
(error: unknown) => void
] {
let resolve: undefined | ((resolved: T) => void) = undefined;
let reject: undefined | ((error: unknown) => void) = undefined;
const creationPromise = new Promise<T>(
(resolve_, reject_) => ((resolve = resolve_), (reject = reject_))
);
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
return [creationPromise, resolve!, reject!];
}

18
frontend/sync-client/src/utils/deserialize.test.ts
View file

@ -1,18 +0,0 @@
import init, { base64ToBytes } from "sync_lib";
import fs from "fs";
describe("deserialize", () => {
it("should serialize a Uint8Array to a base64 string", async () => {
const wasmBin = fs.readFileSync(
"../../backend/sync_lib/pkg/sync_lib_bg.wasm"
);
await init({ module_or_path: wasmBin });
const base64 = "SGVsbG8=";
const jsResult = base64ToBytes(base64);
const expected = new Uint8Array([72, 101, 108, 108, 111]);
expect(jsResult).toEqual(expected);
const rustResult = base64ToBytes(base64);
expect(jsResult).toEqual(rustResult);
});
});

5
frontend/sync-client/src/utils/deserialize.ts
View file

@ -1,5 +0,0 @@
import { base64ToBytes } from "byte-base64";
export function deserialize(data: string): Uint8Array {
return base64ToBytes(data);
}

14
frontend/sync-client/src/utils/find-matching-file.ts
View file

@ -1,14 +0,0 @@
import type { DocumentRecord } from "../persistence/database";
import { EMPTY_HASH } from "./hash";
// TODO: make this smarter so that offline files can be renamed & edited at the same time
export function findMatchingFile(
contentHash: string,
candidates: DocumentRecord[]
): DocumentRecord | undefined {
if (contentHash === EMPTY_HASH) {
return undefined;
}
return candidates.find(({ metadata }) => metadata?.hash === contentHash);
}

11
frontend/sync-client/src/utils/globs-to-regexes.test.ts
View file

@ -1,11 +0,0 @@
import { Logger } from "../tracing/logger";
import { globsToRegexes } from "./globs-to-regexes";
describe("globsToRegexes", () => {
it("basicExample", async () => {
const [regex] = globsToRegexes([".git/**"], new Logger());
expect(regex.test(".git/objects/object")).toBeTruthy();
expect(regex.test(".git/objects/.object")).toBeTruthy();
});
});

18
frontend/sync-client/src/utils/globs-to-regexes.ts
View file

@ -1,18 +0,0 @@
import { makeRe } from "minimatch";
import type { Logger } from "../tracing/logger";
export function globsToRegexes(globs: string[], logger: Logger): RegExp[] {
return globs
.map((pattern) => {
const result = makeRe(pattern, {
dot: true
});
if (result === false) {
logger.warn(
`Failed to parse ${pattern}' as a glob pattern, skipping it`
);
}
return result;
})
.filter((pattern) => pattern !== false);
}

12
frontend/sync-client/src/utils/hash.ts
View file

@ -1,12 +0,0 @@
// https://stackoverflow.com/questions/7616461/generate-a-hash-from-string-in-javascript
export function hash(content: Uint8Array): string {
let result = 0;
// eslint-disable-next-line @typescript-eslint/prefer-for-of
for (let i = 0; i < content.length; i++) {
result = (result << 5) - result + content[i];
result |= 0; // Convert to 32bit integer
}
return Math.abs(result).toString(16).padStart(8, "0");
}
export const EMPTY_HASH = hash(new Uint8Array(0));

27
frontend/sync-client/src/utils/is-equal-bytes.test.ts
View file

@ -1,27 +0,0 @@
import { isEqualBytes } from "./is-equal-bytes";
describe("isEqualBytes", () => {
it("should return true for equal byte arrays", () => {
const bytes1 = new Uint8Array([1, 2, 3, 4]);
const bytes2 = new Uint8Array([1, 2, 3, 4]);
expect(isEqualBytes(bytes1, bytes2)).toBe(true);
});
it("should return false for byte arrays of different lengths", () => {
const bytes1 = new Uint8Array([1, 2, 3, 4]);
const bytes2 = new Uint8Array([1, 2, 3]);
expect(isEqualBytes(bytes1, bytes2)).toBe(false);
});
it("should return true for empty byte arrays", () => {
const bytes1 = new Uint8Array([]);
const bytes2 = new Uint8Array([]);
expect(isEqualBytes(bytes1, bytes2)).toBe(true);
});
it("should return false for byte arrays with same length but different content", () => {
const bytes1 = new Uint8Array([1, 2, 3, 4]);
const bytes2 = new Uint8Array([4, 3, 2, 1]);
expect(isEqualBytes(bytes1, bytes2)).toBe(false);
});
});

13
frontend/sync-client/src/utils/is-equal-bytes.ts
View file

@ -1,13 +0,0 @@
export function isEqualBytes(bytes1: Uint8Array, bytes2: Uint8Array): boolean {
if (bytes1.length !== bytes2.length) {
return false;
}
for (let i = 0; i < bytes1.length; i++) {
if (bytes1[i] !== bytes2[i]) {
return false;
}
}
return true;
}

89
frontend/sync-client/src/utils/locks.test.ts
View file

@ -1,89 +0,0 @@
import { Logger } from "../tracing/logger";
import type { RelativePath } from "../persistence/database";
import { Locks } from "./locks";
describe("Document lock", () => {
const testPath: RelativePath = "test/document/path";
const logger = new Logger();
// eslint-disable-next-line @typescript-eslint/init-declarations
let locks: Locks<RelativePath>;
beforeEach(() => {
locks = new Locks<RelativePath>(logger);
});
test("should lock a document successfully", () => {
const result = locks.tryLock(testPath);
expect(result).toBe(true);
});
test("should not lock a document that is already locked", () => {
locks.tryLock(testPath);
const result = locks.tryLock(testPath);
expect(result).toBe(false);
});
test("should unlock a locked document", () => {
locks.tryLock(testPath);
locks.unlock(testPath);
const result = locks.tryLock(testPath);
expect(result).toBe(true);
locks.unlock(testPath);
});
test("should throw an error when unlocking a document that is not locked", () => {
expect(() => {
locks.unlock(testPath);
}).toThrow(`Document ${testPath} is not locked, cannot unlock`);
});
test("should wait for a document lock and resolve when unlocked", async () => {
locks.tryLock(testPath);
let resolved = false;
const waitPromise = locks.waitForLock(testPath).then(() => {
resolved = true;
});
locks.unlock(testPath);
await waitPromise;
expect(resolved).toBe(true);
});
test("should resolve multiple waiters in FIFO order", async () => {
locks.tryLock(testPath);
let firstResolved = false;
let secondResolved = false;
let thirdResolved = false;
const firstWaitPromise = locks.waitForLock(testPath).then(() => {
firstResolved = true;
});
const secondWaitPromise = locks.waitForLock(testPath).then(() => {
secondResolved = true;
});
const thirdWaitPromise = locks.waitForLock(testPath).then(() => {
thirdResolved = true;
});
locks.unlock(testPath);
await firstWaitPromise;
expect(firstResolved).toBe(true);
expect(secondResolved).toBe(false);
expect(thirdResolved).toBe(false);
locks.unlock(testPath);
await secondWaitPromise;
expect(secondResolved).toBe(true);
expect(thirdResolved).toBe(false);
locks.unlock(testPath);
await thirdWaitPromise;
expect(thirdResolved).toBe(true);
});
});

60
frontend/sync-client/src/utils/locks.ts
View file

@ -1,60 +0,0 @@
import type { Logger } from "../tracing/logger";
// Manages locks on T to prevent concurrent modifications
// allowing the client's FileOperations implementation to be simpler.
// Locks are granted in a first-in-first-out order.
export class Locks<T> {
private readonly locked = new Set<T>();
private readonly waiters = new Map<T, (() => unknown)[]>();
public constructor(private readonly logger: Logger) {}
public tryLock(key: T): boolean {
if (this.locked.has(key)) {
return false;
}
this.locked.add(key);
return true;
}
public async waitForLock(key: T): Promise<void> {
if (this.tryLock(key)) {
return Promise.resolve();
}
this.logger.debug(`Waiting for lock on ${key}`);
return new Promise((resolve) => {
let waiting = this.waiters.get(key);
if (!waiting) {
waiting = [];
this.waiters.set(key, waiting);
}
waiting.push(resolve);
});
}
public unlock(key: T): void {
if (!this.locked.has(key)) {
throw new Error(`Document ${key} is not locked, cannot unlock`);
}
// Remove the first element to ensure FIFO unblocking order
const nextWaiting = this.waiters.get(key)?.shift();
if (nextWaiting) {
this.logger.debug(`Granted lock on ${key}`);
nextWaiting();
} else {
this.locked.delete(key);
}
}
public reset(): void {
this.locked.clear();
this.waiters.clear();
}
}

60
frontend/sync-client/src/utils/min-covered.test.ts
View file

@ -1,60 +0,0 @@
import { CoveredValues } from "./min-covered";
describe("CoveredValues", () => {
test("should initialize with the given min value", () => {
const covered = new CoveredValues(5);
expect(covered.min).toBe(5);
});
test("should add values greater than min", () => {
const covered = new CoveredValues(0);
covered.add(3);
expect(covered.min).toBe(0);
covered.add(1);
expect(covered.min).toBe(1);
covered.add(4);
expect(covered.min).toBe(1);
covered.add(2);
expect(covered.min).toBe(4);
});
test("should ignore duplicate values", () => {
const covered = new CoveredValues(0);
covered.add(3);
covered.add(3);
covered.add(3);
expect(covered.min).toBe(0);
covered.add(1);
covered.add(2);
expect(covered.min).toBe(3);
});
test("should handle multiple consecutive values", () => {
const covered = new CoveredValues(132);
for (let i = 250; i > 132; i--) {
expect(covered.min).toBe(132);
covered.add(i);
}
expect(covered.min).toBe(250);
});
test("should handle adding values lower than current min", () => {
const covered = new CoveredValues(5);
covered.add(3);
expect(covered.min).toBe(5);
covered.add(6);
expect(covered.min).toBe(6);
});
test("should handle force setting min value", () => {
const covered = new CoveredValues(5);
covered.add(7);
covered.add(8);
covered.add(9);
expect(covered.min).toBe(5);
covered.min = 6;
expect(covered.min).toBe(6);
covered.add(10);
expect(covered.min).toBe(10);
});
});

56
frontend/sync-client/src/utils/min-covered.ts
View file

@ -1,56 +0,0 @@
/**
* A class that tracks the minimum covered value in a sequence of numbers.
* It keeps track of a minimum value based on the seen values.
*
* It expects integers slightly out of order and makes sure that the value of `min` is
* always the minimum of the seen values. This is done with bounded memory usage.
*
* @example
* ```typescript
* const covered = new CoveredValues(0);
* covered.add(2); // seenValues = [2], min = 0
* covered.add(1); // seenValues = [], min = 2
* covered.min; // returns 2
* ```
*/
export class CoveredValues {
private seenValues: number[] = [];
public constructor(private minValue: number) {}
public get min(): number {
return this.minValue;
}
public set min(value: number) {
this.minValue = Math.max(value, this.minValue);
this.seenValues = this.seenValues.filter((v) => v > value);
}
public add(value: number): void {
if (value < this.minValue) {
return;
}
let i = 0;
while (i < this.seenValues.length && this.seenValues[i] < value) {
i++;
}
if (i === this.seenValues.length) {
this.seenValues.push(value);
} else if (this.seenValues[i] === value) {
return;
} else {
this.seenValues.splice(i, 0, value);
}
while (
this.seenValues.length > 0 &&
this.seenValues[0] === this.minValue + 1
) {
this.seenValues.shift();
this.minValue++;
}
}
}

66
frontend/sync-client/src/utils/rate-limit.test.ts
View file

@ -1,66 +0,0 @@
import { rateLimit } from "./rate-limit";
import { jest } from "@jest/globals";
describe("rateLimit", () => {
beforeEach(() => {
jest.useFakeTimers();
});
afterEach(() => {
jest.useRealTimers();
});
it("should call the function immediately on first invocation", async () => {
const mockFn = jest
.fn<() => Promise<string>>()
.mockResolvedValue("result");
const rateLimited = rateLimit(mockFn, 100);
const promise = rateLimited();
expect(mockFn).toHaveBeenCalledTimes(1);
await promise;
});
it("should call the function again after the interval has passed", async () => {
const mockFn = jest
.fn<(value: number) => Promise<string>>()
.mockResolvedValue("result");
const rateLimited = rateLimit(mockFn, 100);
const promise1 = rateLimited(1);
await promise1;
jest.advanceTimersByTime(200);
const promise2 = rateLimited(2);
await promise2;
expect(mockFn).toHaveBeenCalledTimes(2);
expect(mockFn).toHaveBeenCalledWith(2);
});
it("should use the most recent arguments if multiple calls are made within interval", async () => {
const mockFn = jest
.fn<(value: string) => Promise<string>>()
.mockImplementation(async (val) => `${val}-result`);
const rateLimited = rateLimit(mockFn, 100);
const promise1 = rateLimited("first");
jest.advanceTimersByTime(10);
const promise2 = rateLimited("second");
jest.advanceTimersByTime(10);
const promise3 = rateLimited("third");
jest.advanceTimersByTime(1000);
expect(await promise1).toEqual("first-result");
expect(await promise2).toEqual("third-result");
expect(await promise3).toBeUndefined();
expect(mockFn).toHaveBeenCalledTimes(2);
expect(mockFn).toHaveBeenNthCalledWith(1, "first");
expect(mockFn).toHaveBeenNthCalledWith(2, "third");
});
});

58
frontend/sync-client/src/utils/rate-limit.ts
View file

@ -1,58 +0,0 @@
import { createPromise } from "./create-promise";
import { sleep } from "./sleep";
/**
* Creates a rate-limited version of a given asynchronous function.
* Ensures that the function is not called more frequently than specified by `minIntervalMs`.
* If the function is called while a previous call is still within the rate limit window,
* it will queue up the most recent arguments and execute them after the rate limit expires.
* Only the most recent call is preserved in the queue.
*
* @template T - Type of the function to be rate limited
* @param {T} fn - The asynchronous function to rate limit
* @param {number} minIntervalMs - The minimum interval in milliseconds between function calls
* @returns {(...args: Parameters<T>) => ReturnType<T> | Promise<undefined>} A decorated function that respects the rate limit.
* Returns the original function's return type when executed, or undefined if the call was superseded by a newer one.
*/
export function rateLimit<
R,
T extends (
...args: any // eslint-disable-line @typescript-eslint/no-explicit-any
) => Promise<R>
>(
fn: T,
minIntervalMs: number
): (...args: Parameters<T>) => Promise<R | undefined> {
let newArgs: Parameters<T> | undefined = undefined;
let running: Promise<unknown> | undefined = undefined;
const decoratedFn = async (
...args: Parameters<T>
): Promise<R | undefined> => {
if (running !== undefined) {
newArgs = args;
await running;
// args might have changed while we were waiting
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
if (newArgs === undefined) {
// we weren't the first one to wake up, that means a newer
// invocation is running now, we can just bail
return;
}
args = newArgs;
newArgs = undefined;
}
const [promise, resolve] = createPromise();
running = promise;
sleep(minIntervalMs)
.then(resolve)
.catch(() => {
// sleep cannot fail
});
return fn(...args);
};
return decoratedFn;
}

18
frontend/sync-client/src/utils/serialize.test.ts
View file

@ -1,18 +0,0 @@
import { serialize } from "./serialize";
import init, { bytesToBase64 } from "sync_lib";
import fs from "fs";
describe("serialize", () => {
it("should serialize a Uint8Array to a base64 string", async () => {
const wasmBin = fs.readFileSync(
"../../backend/sync_lib/pkg/sync_lib_bg.wasm"
);
await init({ module_or_path: wasmBin });
const data = new Uint8Array([72, 101, 108, 108, 111]);
const jsResult = serialize(data);
const rustResult = bytesToBase64(data);
expect(rustResult).toBe("SGVsbG8=");
expect(jsResult).toBe(rustResult);
});
});

5
frontend/sync-client/src/utils/serialize.ts
View file

@ -1,5 +0,0 @@
import { bytesToBase64 } from "byte-base64";
export function serialize(data: Uint8Array): string {
return bytesToBase64(data);
}

3
frontend/sync-client/src/utils/sleep.ts
View file

@ -1,3 +0,0 @@
export async function sleep(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}

17
frontend/sync-client/tsconfig.json
View file

@ -1,17 +0,0 @@
{
"compilerOptions": {
"module": "ESNext",
"target": "ESNext",
"strict": true,
"allowSyntheticDefaultImports": true,
"moduleResolution": "bundler",
"lib": [
"DOM" // to get `fetch` & `WebSocket`
],
"declaration": true,
"declarationDir": "./dist/types"
},
"exclude": [
"./dist"
]
}

71
frontend/sync-client/webpack.config.js
View file

@ -1,71 +0,0 @@
const path = require("path");
const { merge } = require("webpack-merge");
const webpack = require("webpack");
const packageJson = require("./package.json");
const common = {
entry: "./src/index.ts",
module: {
rules: [
{
test: /\.ts$/,
use: ["ts-loader"]
},
{
test: /\.wasm$/,
type: "asset/inline"
}
]
},
plugins: [
new webpack.DefinePlugin({
__CURRENT_VERSION__: JSON.stringify(packageJson.version)
})
],
optimization: {
// the consuming project should take care of minification
minimize: false
},
resolve: {
extensions: [".ts", ".js"],
alias: {
root: __dirname,
src: path.resolve(__dirname, "src")
}
},
performance: {
hints: false // it's a library, no need to warn about its size
}
};
module.exports = [
merge(common, {
target: "web",
output: {
path: path.resolve(__dirname, "dist"),
filename: "sync-client.web.js",
library: {
name: "SyncClient",
type: "umd"
},
globalObject: "this"
},
resolve: {
fallback: {
ws: false // Exclude `ws` from the browser bundle
}
}
}),
merge(common, {
target: "node",
output: {
path: path.resolve(__dirname, "dist"),
filename: "sync-client.node.js",
libraryTarget: "commonjs2"
},
externals: {
bufferutil: "bufferutil",
"utf-8-validate": "utf-8-validate" // required for ws: https://github.com/websockets/ws/issues/2245#issuecomment-2250318733
}
})
];

3
frontend/test-client/jest.config.js
View file

@ -1,3 +0,0 @@
module.exports = {
preset: "ts-jest/presets/js-with-babel-esm"
};

24
frontend/test-client/package.json
View file

@ -1,24 +0,0 @@
{
"name": "test-client",
"version": "0.4.0",
"private": true,
"bin": {
"test-client": "./dist/cli.js"
},
"scripts": {
"dev": "webpack watch --mode development",
"build": "webpack --mode production",
"test": "jest"
},
"devDependencies": {
"@types/node": "^22.15.30",
"sync-client": "file:../sync-client",
"ts-loader": "^9.5.2",
"tslib": "2.8.1",
"typescript": "5.8.3",
"uuid": "^11.1.0",
"webpack": "^5.99.9",
"webpack-cli": "^6.0.1",
"bufferutil": "^4.0.9"
}
}

332
frontend/test-client/src/agent/mock-agent.ts
View file

@ -1,332 +0,0 @@
import { choose } from "../utils/choose";
import { v4 as uuidv4 } from "uuid";
import { assert } from "../utils/assert";
import type { RelativePath, SyncSettings } from "sync-client";
import { LogLevel } from "sync-client";
import { MockClient } from "./mock-client";
import { sleep } from "../utils/sleep";
import type { LogLine } from "sync-client/dist/types/tracing/logger";
import { flakyFetchFactory } from "../utils/flaky-fetch";
import { flakyWebSocketFactory } from "../utils/flaky-websocket";
export class MockAgent extends MockClient {
private readonly writtenContents: string[] = [];
private readonly pendingActions: Promise<unknown>[] = [];
// The renamed file finding algorithm isn't too smart so we can't both update and rename the same file
private doNotTouchWhileOffline: string[] = [];
public constructor(
initialSettings: Partial<SyncSettings>,
public readonly name: string,
private readonly doDeletes: boolean,
useSlowFileEvents: boolean,
private readonly jitterScaleInSeconds: number
) {
super(initialSettings, useSlowFileEvents);
}
public async init(): Promise<void> {
await super.init(
flakyFetchFactory(this.jitterScaleInSeconds),
flakyWebSocketFactory(this.jitterScaleInSeconds)
);
assert(
(await this.client.checkConnection()).isSuccessful,
"Connection check failed"
);
this.client.logger.addOnMessageListener((logLine: LogLine) => {
const state = this.client.getSettings().isSyncEnabled
? "(online) "
: "(offline)";
const formatted = `[${this.name} ${state}] ${logLine.timestamp.toISOString()} ${logLine.level} ${logLine.message}`;
// HACK: we have to ensure the file has been synced if we want to change it offline without data loss
const historyEntry = /.*History entry: (.*.md).*/.exec(
logLine.message
);
if (historyEntry) {
this.doNotTouchWhileOffline =
this.doNotTouchWhileOffline.filter(
(file) => file !== historyEntry[1]
);
}
switch (logLine.level) {
case LogLevel.ERROR:
console.error(formatted);
if (!this.useSlowFileEvents) {
// Let's not ignore errors
// eslint-disable-next-line @typescript-eslint/no-floating-promises
sleep(100).then(() => process.exit(1));
}
break;
case LogLevel.WARNING:
console.warn(formatted);
break;
case LogLevel.INFO:
console.info(formatted);
break;
case LogLevel.DEBUG:
console.debug(formatted);
break;
}
});
this.client.logger.info("Agent initialized");
}
public async act(): Promise<void> {
const options: (() => Promise<unknown>)[] = [
this.createFileAction.bind(this)
];
if (this.client.getSettings().isSyncEnabled) {
if (this.doNotTouchWhileOffline.length === 0) {
options.push(this.disableSyncAction.bind(this));
}
} else {
options.push(this.enableSyncAction.bind(this));
}
const files = await this.listAllFiles();
if (files.length > 0) {
options.push(
this.renameFileAction.bind(this, files),
this.updateFileAction.bind(this, files)
);
if (this.doDeletes) {
options.push(this.deleteFileAction.bind(this, files));
}
}
this.pendingActions.push(
(async (): Promise<unknown> => {
try {
return await choose(options)();
} catch (error) {
this.client.logger.error(
`Failed to perform an action: ${error}`
);
this.client.logger.info(JSON.stringify(this.data, null, 2));
this.client.logger.info(
JSON.stringify(this.localFiles, null, 2)
);
throw error;
}
})()
);
}
public async finish(): Promise<void> {
await this.client.setSetting("isSyncEnabled", true);
await Promise.all(this.pendingActions);
await this.client.waitAndStop();
}
public assertFileSystemsAreConsistent(otherAgent: MockAgent): void {
const globalFiles = Array.from(otherAgent.localFiles.keys());
const localFiles = Array.from(this.localFiles.keys());
const missingInOther = localFiles.filter(
(file) => !otherAgent.localFiles.has(file)
);
const missingInLocal = globalFiles.filter(
(file) => !this.localFiles.has(file)
);
try {
assert(
missingInOther.length === 0,
`Files from ${this.name} missing in ${otherAgent.name}: ${missingInOther.join(", ")}`
);
assert(
missingInLocal.length === 0,
`Files from ${otherAgent.name} missing in ${this.name}: ${missingInLocal.join(", ")}`
);
for (const file of globalFiles) {
const localContent = new TextDecoder().decode(
this.localFiles.get(file)
);
const otherContent = new TextDecoder().decode(
otherAgent.localFiles.get(file)
);
assert(
localContent === otherContent,
`Content mismatch for file ${file}:\n${localContent}\n${otherContent}`
);
}
} catch (e) {
this.client.logger.info(
"Local data: " + JSON.stringify(this.data, null, 2)
);
this.client.logger.info(
"Local files: " +
Array.from(otherAgent.localFiles.keys()).join(", ")
);
otherAgent.client.logger.info(
"Local data: " + JSON.stringify(otherAgent.data, null, 2)
);
otherAgent.client.logger.info(
"Local files: " +
Array.from(otherAgent.localFiles.keys()).join(", ")
);
throw e;
}
}
public assertAllContentIsPresentOnce(): void {
if (this.useSlowFileEvents) {
this.client.logger.info(
// We can't ensure that we have seen every single update
`Skipping content check for ${this.name} because slow file events are enabled`
);
return;
}
for (const content of this.writtenContents) {
const found = Array.from(this.localFiles.keys()).filter((key) => {
return new TextDecoder()
.decode(this.localFiles.get(key))
.includes(content);
});
if (this.doDeletes) {
assert(
found.length <= 1,
`[${this.name}] Content ${content} found in ${found.join(", ")}`
);
} else {
assert(
found.length >= 1,
`[${this.name}] Content ${content} not found in any files`
);
assert(
found.length <= 1,
`[${this.name}] Content ${content} found in multiple files: ${found.join(", ")}`
);
const [file] = found;
const fileContent = new TextDecoder().decode(
this.localFiles.get(file)
);
assert(
fileContent.split(content).length == 2,
`Content ${content} (of ${this.name}) found more than once in '${file}'. File content:\n${fileContent}`
);
}
}
}
private async createFileAction(): Promise<void> {
const file = this.getFileName();
if (
(!this.client.getSettings().isSyncEnabled &&
this.doNotTouchWhileOffline.includes(file)) ||
(await this.exists(file))
) {
return;
}
const content = this.getContent();
this.client.logger.info(
`Decided to create file ${file} with content ${content}`
);
return this.create(file, new TextEncoder().encode(` ${content} `));
}
private async disableSyncAction(): Promise<void> {
this.client.logger.info(`Decided to disable sync`);
await this.client.setSetting("isSyncEnabled", false);
}
private async enableSyncAction(): Promise<void> {
this.client.logger.info(`Decided to enable sync`);
await this.client.setSetting("isSyncEnabled", true);
}
private async renameFileAction(files: RelativePath[]): Promise<void> {
const file = choose(files);
// We can't edit files offline that have been updated while offline.
// Otherwise, the resolution logic couldn't handle it.
if (
!this.client.getSettings().isSyncEnabled &&
this.doNotTouchWhileOffline.includes(file)
) {
this.client.logger.info(
`Skipping file ${file} because it has been updated while offline`
);
return;
}
const newName = this.getFileName();
if (
(!this.client.getSettings().isSyncEnabled &&
this.doNotTouchWhileOffline.includes(newName)) ||
(await this.exists(newName))
) {
return;
}
this.client.logger.info(`Decided to rename file ${file} to ${newName}`);
this.doNotTouchWhileOffline.push(file, newName);
return this.rename(file, newName);
}
private async updateFileAction(files: RelativePath[]): Promise<void> {
const file = choose(files);
// We can't edit files offline that have been updated while offline.
// Otherwise, the resolution logic couldn't handle it.
if (
!this.client.getSettings().isSyncEnabled &&
this.doNotTouchWhileOffline.includes(file)
) {
this.client.logger.info(
`Skipping file ${file} because it has been updated while offline`
);
return;
}
const content = this.getContent();
this.client.logger.info(
`Decided to update file ${file} with ${content}`
);
this.doNotTouchWhileOffline.push(file);
await this.atomicUpdateText(file, (old) => ({
text: old.text + ` ${content} `,
cursors: []
}));
}
private async deleteFileAction(files: RelativePath[]): Promise<void> {
const file = choose(files);
this.client.logger.info(`Decided to delete file ${file}`);
return this.delete(file);
}
private getContent(): string {
const uuid = uuidv4();
this.writtenContents.push(uuid);
return uuid;
}
private getFileName(): string {
// Simulate name collisions between the clients
return `file-${Math.floor(Math.random() * 64)}.md`;
}
}

Some files were not shown because too many files have changed in this diff Show more