andras/vault-link
1
0
Fork 0
Code Issues Pull requests 18 Projects Releases 55 Packages Wiki Activity Actions 1

Fix syncing when network latency is present (#4)

Browse source 
* WIP

* Add debug

* Dedupe inserts

* Add deterministic ordering

* Fix whitespaces

* Update insta

* Add integration test script

* Rename

* Add test

* Working for non-deletes

* omg it mostly works for deletes

* Isdeleted fix

* remove created dates

* update api

* Take document id

* No max attempt

* works

* Use string uuids

* .

* working!!!! (hopefully)

* Improve bundling

* Add module

* lint

* .

* lint

* Fix CI

* use toolchain

* clean up

* Add useSlowFileEvents

* Delete fuzz

* Fix CI

* use docker

* fix script

* clean up

* Clean up

* change node version

* Build docker image on every commit

* fix ci

* 1 db per vault

* Add scritps folder

* Bump versions

* Lint

* .

* Fix tests for real

* Style

* .

* try

* Consistent ordering

* Fix tests

* hmm

* .

* Clean up diff

* Fixes

* .

* Fix version bump

* .

* .

* .
This commit is contained in:
Andras Schmelczer 2025-03-16 20:13:49 +00:00 • committed by GitHub
parent bcf48c428d
commit 8b8f1d91d9
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
91 changed files with 2252 additions and 1586 deletions
Download patch file Download diff file Expand all files Collapse all files

35
frontend/sync-client/package.json
View file

@ -1,29 +1,36 @@
{
"name": "sync-client",
"version": "0.0.0",
"private": true,
"main": "dist/index.js",
"version": "0.0.30",
"main": "dist/sync-client.node.js",
"browser": "dist/sync-client.web.js",
"types": "dist/types/index.d.ts",
"files": [
"dist/**/*"
],
"scripts": {
"dev": "webpack watch --mode development",
"build": "webpack --mode production",
"test": "NODE_OPTIONS=\"$NODE_OPTIONS --experimental-vm-modules\" jest"
},
"dependencies": {
"byte-base64": "^1.1.0",
"fetch-retry": "^6.0.0",
"openapi-fetch": "0.13.5",
"openapi-typescript": "7.6.1",
"p-queue": "^8.1.0",
"uuid": "^11.1.0"
},
"devDependencies": {
"tslib": "2.8.1",
"typescript": "5.7.3",
"sync_lib": "file:../../backend/sync_lib/pkg",
"@types/jest": "^29.5.14",
"@types/node": "^22.13.5",
"@types/node": "^22.13.10",
"jest": "^29.7.0",
"ts-jest": "^29.2.6",
"p-queue": "^8.1.0",
"fetch-retry": "^6.0.0",
"byte-base64": "^1.1.0",
"openapi-fetch": "0.13.4",
"openapi-typescript": "7.6.1",
"ts-loader": "^9.5.2",
"tslib": "2.8.1",
"typescript": "5.8.2",
"webpack": "^5.98.0",
"webpack-cli": "^6.0.1"
"webpack-cli": "^6.0.1",
"webpack-merge": "^6.0.1",
"sync_lib": "file:../../backend/sync_lib/pkg"
}
}
}

3
frontend/sync-client/src/file-operations/document-locks.ts
View file

@ -1,6 +1,9 @@
import type { Logger } from "../tracing/logger";
import type { RelativePath } from "../persistence/database";
// Manages locks on documents to prevent concurrent modifications
// allowing the client's FileOperations implementation to be simpler.
// Locks are granted in a first-in-first-out order.
export class DocumentLocks {
private readonly locked = new Set<RelativePath>();
private readonly waiters = new Map<RelativePath, (() => void)[]>();

23
frontend/sync-client/src/file-operations/file-operations.test.ts
View file

@ -1,16 +1,27 @@
import type { FileSystemOperations } from "sync-client";
import type { Database, RelativePath } from "../persistence/database";
import type {
Database,
DocumentRecord,
RelativePath
} from "../persistence/database";
import { FileOperations } from "./file-operations";
import { Logger } from "../tracing/logger";
import { assertSetContainsExactly } from "../utils/assert-set-contains-exactly";
import type { FileSystemOperations } from "./filesystem-operations";
describe("File operations", () => {
class MockDatabase {
public async updatePath(
class MockDatabase implements Partial<Database> {
public getLatestDocumentByRelativePath(
_find: RelativePath
): DocumentRecord | undefined {
// no-op
return undefined;
}
public move(
_oldRelativePath: RelativePath,
_newRelativePath: RelativePath
): Promise<void> {
// this is called but irrelevant for this mock
): void {
// no-op
}
}

81
frontend/sync-client/src/file-operations/file-operations.ts
View file

@ -1,10 +1,6 @@
import type { Logger } from "src/tracing/logger";
import type { Logger } from "../tracing/logger";
import type { FileSystemOperations } from "./filesystem-operations";
import type {
Database,
DocumentId,
RelativePath
} from "src/persistence/database";
import type { Database, RelativePath } from "../persistence/database";
import { isBinary, isFileTypeMergable, mergeText } from "sync_lib";
import { SafeFileSystemOperations } from "./safe-filesystem-operations";
@ -17,7 +13,7 @@ export class FileOperations {
private readonly database: Database,
fs: FileSystemOperations
) {
this.fs = new SafeFileSystemOperations(fs);
this.fs = new SafeFileSystemOperations(fs, logger);
}
public async listAllFiles(): Promise<RelativePath[]> {
@ -35,7 +31,7 @@ export class FileOperations {
const decoder = new TextDecoder("utf-8");
// Normalize line endings to LF on Windows
// Normalize line-endings to LF on Windows
let text = decoder.decode(content);
text = text.replace(/\r\n/g, "\n");
@ -46,10 +42,6 @@ export class FileOperations {
return this.fs.getFileSize(path);
}
public async getModificationTime(path: RelativePath): Promise<Date> {
return this.fs.getModificationTime(path);
}
public async exists(path: RelativePath): Promise<boolean> {
return this.fs.exists(path);
}
@ -60,18 +52,23 @@ export class FileOperations {
path: RelativePath,
newContent: Uint8Array
): Promise<void> {
this.logger.debug(`Creating file: ${path}`);
await this.fs.write(path, newContent);
}
public async ensureClearPath(path: RelativePath): Promise<void> {
if (await this.fs.exists(path)) {
const deconflictedPath = await this.deconflictPath(path);
this.logger.debug(
`Didn't expect ${path} to exist, deconflicting by moving it to '${deconflictedPath}'`
);
await this.database.updatePath(path, deconflictedPath);
this.database.move(path, deconflictedPath);
await this.fs.rename(path, deconflictedPath);
} else {
await this.createParentDirectories(path);
}
await this.fs.write(path, newContent);
}
// Update the file at the given path.
@ -126,40 +123,25 @@ export class FileOperations {
return new TextEncoder().encode(resultText);
}
public async remove(path: RelativePath): Promise<void> {
this.logger.debug(`Deleting file: ${path}`);
return this.fs.delete(path);
public async delete(path: RelativePath): Promise<void> {
if (await this.exists(path)) {
this.logger.debug(`Deleting file: ${path}`);
return this.fs.delete(path);
} else {
this.logger.debug(`No need to delete '${path}', it doesn't exist`);
}
}
public async move(
oldPath: RelativePath,
newPath: RelativePath,
documentId?: DocumentId
newPath: RelativePath
): Promise<void> {
if (oldPath === newPath) {
return;
}
await this.ensureClearPath(newPath);
if (await this.fs.exists(newPath)) {
const deconflictedPath = await this.deconflictPath(newPath);
this.logger.debug(
`Conflict when moving '${oldPath}' to '${newPath}', the latter already exists, deconflicting by moving it to '${deconflictedPath}'`
);
const existingMetadata = this.database.getDocument(newPath);
if (
existingMetadata === undefined ||
existingMetadata.documentId !== documentId
) {
await this.database.updatePath(newPath, deconflictedPath);
await this.fs.rename(newPath, deconflictedPath);
} else {
await this.database.deleteDocument(newPath);
}
} else {
await this.createParentDirectories(newPath);
}
this.database.move(oldPath, newPath);
await this.fs.rename(oldPath, newPath);
}
@ -201,17 +183,12 @@ export class FileOperations {
);
stem = stem.replace(FileOperations.PARENTHESES_REGEX, "");
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
while (true) {
const newName =
currentCount === 0
? `${directory}${stem}${extension}`
: `${directory}${stem} (${currentCount})${extension}`;
if (await this.fs.exists(newName)) {
currentCount++;
} else {
return newName;
}
}
let newName = path;
do {
currentCount++;
newName = `${directory}${stem} (${currentCount})${extension}`;
} while (await this.fs.exists(newName));
return newName;
}
}

5
frontend/sync-client/src/file-operations/filesystem-operations.ts
View file

@ -1,4 +1,4 @@
import type { RelativePath } from "src/persistence/database";
import type { RelativePath } from "../persistence/database";
export interface FileSystemOperations {
listAllFiles: () => Promise<RelativePath[]>;
@ -9,11 +9,8 @@ export interface FileSystemOperations {
updater: (currentContent: string) => string
) => Promise<string>;
getFileSize: (path: RelativePath) => Promise<number>;
getModificationTime: (path: RelativePath) => Promise<Date>;
exists: (path: RelativePath) => Promise<boolean>;
createDirectory: (path: RelativePath) => Promise<void>;
delete: (path: RelativePath) => Promise<void>;
// Must be able to handle renaming to a file that already exists
rename: (oldPath: RelativePath, newPath: RelativePath) => Promise<void>;
}

109
frontend/sync-client/src/file-operations/safe-filesystem-operations.ts
View file

@ -1,5 +1,7 @@
import type { RelativePath } from "src/persistence/database";
import type { RelativePath } from "../persistence/database";
import type { FileSystemOperations } from "./filesystem-operations";
import type { Logger } from "../tracing/logger";
import { DocumentLocks } from "./document-locks";
export class FileNotFoundError extends Error {
public constructor(message: string) {
@ -9,71 +11,134 @@ export class FileNotFoundError extends Error {
}
// Decorate FileSystemOperations replacing errors with FileNotFoundError
// if the accessed file doesn't exist.
// if the accessed file doesn't exist. It also ensures that there's only
// ever a single request in-flight for any one file through the use of
// DocumentLocks.
export class SafeFileSystemOperations implements FileSystemOperations {
public constructor(private readonly fs: FileSystemOperations) {}
private readonly locks: DocumentLocks;
public constructor(
private readonly fs: FileSystemOperations,
private readonly logger: Logger
) {
this.locks = new DocumentLocks(logger);
}
public async listAllFiles(): Promise<RelativePath[]> {
return this.fs.listAllFiles();
}
public async read(path: RelativePath): Promise<Uint8Array> {
return this.safeOperation(path, async () => this.fs.read(path));
this.logger.debug(`Reading file: ${path}`);
return this.safeOperation(
path,
this.decorateToHoldLock(path, async () => this.fs.read(path)),
"read"
);
}
public async write(path: RelativePath, content: Uint8Array): Promise<void> {
return this.fs.write(path, content);
this.logger.debug(`Writing file: ${path}`);
return this.decorateToHoldLock(path, async () =>
this.fs.write(path, content)
)();
}
public async atomicUpdateText(
path: RelativePath,
updater: (currentContent: string) => string
): Promise<string> {
return this.safeOperation(path, async () =>
this.fs.atomicUpdateText(path, updater)
this.logger.debug(`Atomic update of file: ${path}`);
return this.safeOperation(
path,
this.decorateToHoldLock(path, async () =>
this.fs.atomicUpdateText(path, updater)
),
"atomicUpdateText"
);
}
public async getFileSize(path: RelativePath): Promise<number> {
return this.safeOperation(path, async () => this.fs.getFileSize(path));
}
public async getModificationTime(path: RelativePath): Promise<Date> {
return this.safeOperation(path, async () =>
this.fs.getModificationTime(path)
this.logger.debug(`Getting file size: ${path}`);
return this.safeOperation(
path,
this.decorateToHoldLock(path, async () =>
this.fs.getFileSize(path)
),
"getFileSize"
);
}
public async exists(path: RelativePath): Promise<boolean> {
return this.fs.exists(path);
this.logger.debug(`Checking if file exists: ${path}`);
return this.decorateToHoldLock(path, async () =>
this.fs.exists(path)
)();
}
public async createDirectory(path: RelativePath): Promise<void> {
return this.fs.createDirectory(path);
this.logger.debug(`Creating directory: ${path}`);
return this.decorateToHoldLock(path, async () =>
this.fs.createDirectory(path)
)();
}
public async delete(path: RelativePath): Promise<void> {
return this.fs.delete(path);
this.logger.debug(`Deleting file: ${path}`);
return this.decorateToHoldLock(path, async () =>
this.fs.delete(path)
)();
}
public async rename(
oldPath: RelativePath,
newPath: RelativePath
): Promise<void> {
return this.safeOperation(oldPath, async () =>
this.fs.rename(oldPath, newPath)
this.logger.debug(`Renaming file: ${oldPath} to ${newPath}`);
return this.safeOperation(
oldPath,
this.decorateToHoldLock([oldPath, newPath], async () =>
this.fs.rename(oldPath, newPath)
),
"rename"
);
}
private decorateToHoldLock<T>(
pathOrPaths: RelativePath | RelativePath[],
operation: () => Promise<T>
): () => Promise<T> {
return async () => {
const paths = Array.isArray(pathOrPaths)
? pathOrPaths
: [pathOrPaths];
await Promise.all(
paths.map(async (path) => this.locks.waitForDocumentLock(path))
);
try {
return await operation();
} finally {
await Promise.all(
paths.map((path) => {
this.locks.unlockDocument(path);
})
);
}
};
}
private async safeOperation<T>(
path: RelativePath,
operation: () => Promise<T>
operation: () => Promise<T>,
operationName: string
): Promise<T> {
// Without locking the file, this isn't atomic, however, it's good enough practicaly.
// This will only break if the file exists, gets deleted and then immediately
// recreated while `operation` is running.
if (!(await this.fs.exists(path))) {
throw new FileNotFoundError(path);
throw new FileNotFoundError(
`File not found: ${path} before trying to ${operationName}`
);
}
try {
return await operation();
@ -81,7 +146,9 @@ export class SafeFileSystemOperations implements FileSystemOperations {
if (await this.fs.exists(path)) {
throw error;
} else {
throw new FileNotFoundError(path);
throw new FileNotFoundError(
`File not found: ${path} when trying to ${operationName}`
);
}
}
}

301
frontend/sync-client/src/persistence/database.ts
View file

@ -1,23 +1,43 @@
import type { Logger } from "../tracing/logger";
export type VaultUpdateId = number;
export type DocumentId = string;
export type RelativePath = string;
export interface DocumentMetadata {
parentVersionId: VaultUpdateId;
documentId: DocumentId;
hash: string;
}
import type { Logger } from "src/tracing/logger";
export interface StoredDocumentMetadata {
relativePath: RelativePath;
documentId: DocumentId;
parentVersionId: VaultUpdateId;
hash: string;
}
export interface StoredDatabase {
documents: Record<RelativePath, DocumentMetadata>;
documents: StoredDocumentMetadata[];
lastSeenUpdateId: VaultUpdateId | undefined;
}
export class Database {
private documents = new Map<RelativePath, DocumentMetadata>();
/**
* Represents a document in the database.
*
* It is mutable and its content should always represent the latest
* state of the document on disk based on the update events we have seen.
*/
export interface DocumentRecord {
relativePath: RelativePath;
documentId: DocumentId;
metadata: DocumentMetadata | undefined;
isDeleted: boolean;
updates: Promise<void>[];
parallelVersion: number;
}
export class Database {
private documents: DocumentRecord[];
private lastSeenUpdateId: VaultUpdateId | undefined;
public constructor(
@ -26,16 +46,21 @@ export class Database {
private readonly saveData: (data: StoredDatabase) => Promise<void>
) {
initialState ??= {};
if (initialState.documents) {
for (const [relativePath, metadata] of Object.entries(
initialState.documents
)) {
this.documents.set(relativePath, metadata);
}
}
this.ensureConsistency();
this.logger.debug(`Loaded ${this.documents.size} documents`);
this.documents =
initialState.documents?.map(
({ relativePath, documentId, ...metadata }) => ({
relativePath,
documentId,
metadata,
isDeleted: false,
updates: [],
parallelVersion: 0
})
) ?? [];
this.ensureConsistency();
this.logger.debug(`Loaded ${this.documents.length} documents`);
this.lastSeenUpdateId = initialState.lastSeenUpdateId;
this.logger.debug(
@ -43,109 +68,213 @@ export class Database {
);
}
public getDocuments(): Map<RelativePath, DocumentMetadata> {
return this.documents;
public get length(): number {
return this.documents.length;
}
public get resolvedDocuments(): DocumentRecord[] {
const paths = new Map<string, DocumentRecord[]>();
this.documents
.filter(({ metadata }) => metadata !== undefined)
.forEach((record) =>
paths.set(record.relativePath, [
record,
...(paths.get(record.relativePath) ?? [])
])
);
return Array.from(paths.values()).map((records) => {
records.sort(
(a, b) => b.parallelVersion - a.parallelVersion // descending
);
if (
records.length > 1 &&
records.some((current, i) =>
i === 0
? false
: records[i - 1].parallelVersion ===
current.parallelVersion
)
) {
throw new Error(
`Multiple documents with the same parallel version and path at ${records[0].relativePath}`
);
}
return records[0];
});
}
public getLastSeenUpdateId(): VaultUpdateId | undefined {
return this.lastSeenUpdateId;
}
public async setLastSeenUpdateId(
value: VaultUpdateId | undefined
): Promise<void> {
public setLastSeenUpdateId(value: VaultUpdateId | undefined): void {
this.lastSeenUpdateId = value;
await this.save();
this.save();
}
public async resetSyncState(): Promise<void> {
this.documents = new Map();
public resetSyncState(): void {
this.documents = [];
this.lastSeenUpdateId = 0;
await this.save();
this.save();
}
public updateDocumentMetadata(
metadata: {
parentVersionId: VaultUpdateId;
hash: string;
},
toUpdate: DocumentRecord
): void {
if (!this.documents.includes(toUpdate)) {
throw new Error("Document not found in database");
}
toUpdate.metadata = metadata;
this.save();
}
public removeDocumentPromise(promise: Promise<void>): void {
const entry = this.documents.find(({ updates }) =>
updates.includes(promise)
);
if (entry === undefined) {
throw new Error("Document not found by update promise");
}
entry.updates = entry.updates.filter((update) => update !== promise);
// No need to save as Promises don't get serialized
}
public getLatestDocumentByRelativePath(
find: RelativePath
): DocumentRecord | undefined {
const candidates = this.documents.filter(
({ relativePath }) => relativePath === find
);
candidates.sort((a, b) => b.parallelVersion - a.parallelVersion); // descending
return candidates[0];
}
public async getResolvedDocumentByRelativePath(
relativePath: RelativePath,
promise: Promise<void>
): Promise<DocumentRecord> {
const entry = this.getLatestDocumentByRelativePath(relativePath);
if (entry === undefined) {
throw new Error(
`Document not found by relative path: ${relativePath}, ${JSON.stringify(
this.documents,
null,
2
)}`
);
}
const currentPromises = entry.updates;
entry.updates = [...currentPromises, promise];
await Promise.all(currentPromises);
return entry;
}
public createNewPendingDocument(
documentId: DocumentId,
relativePath: RelativePath,
promise: Promise<void>
): DocumentRecord {
const previousEntry =
this.getLatestDocumentByRelativePath(relativePath);
const entry = {
relativePath,
documentId,
metadata: undefined,
isDeleted: false,
updates: [promise],
parallelVersion:
previousEntry?.parallelVersion === undefined
? 0
: previousEntry.parallelVersion + 1
};
this.documents.push(entry);
this.save();
return entry;
}
public getDocumentByDocumentId(
documentId: DocumentId
): [RelativePath, DocumentMetadata] | undefined {
return [...this.documents.entries()].find(
([_, metadata]) => metadata.documentId === documentId
);
find: DocumentId
): DocumentRecord | undefined {
return this.documents.find(({ documentId }) => documentId === find);
}
public async setDocument({
documentId,
relativePath,
parentVersionId,
hash
}: {
documentId: DocumentId;
relativePath: RelativePath;
parentVersionId: VaultUpdateId;
hash: string;
}): Promise<void> {
this.documents.set(relativePath, {
documentId,
parentVersionId,
hash
});
await this.save();
}
public async removeDocument(relativePath: RelativePath): Promise<void> {
this.documents.delete(relativePath);
await this.save();
}
public getDocument(
relativePath: RelativePath
): DocumentMetadata | undefined {
return this.documents.get(relativePath);
}
public async deleteDocument(relativePath: RelativePath): Promise<void> {
this.documents.delete(relativePath);
await this.save();
}
public async updatePath(
public move(
oldRelativePath: RelativePath,
newRelativePath: RelativePath
): Promise<void> {
const document = this.documents.get(oldRelativePath);
if (!document) {
): void {
const oldDocument =
this.getLatestDocumentByRelativePath(oldRelativePath);
if (oldDocument === undefined) {
return;
}
const newDocument =
this.getLatestDocumentByRelativePath(newRelativePath);
if (newDocument?.isDeleted === false) {
throw new Error(
`Cannot update physical path for document that does not exist: ${oldRelativePath}`
`Document already exists at new location: ${newRelativePath}`
);
}
if (this.documents.has(newRelativePath)) {
throw new Error(
`Cannot update physical path to path that is already in use: ${newRelativePath}`
);
}
oldDocument.relativePath = newRelativePath;
// We're in a strange state where the target of the move has just got deleted,
// however, its metadata might already have a bunch of updates queued up for
// the document at the new location. We need to keep these updates.
oldDocument.parallelVersion =
newDocument !== undefined ? newDocument.parallelVersion + 1 : 0;
this.documents.delete(oldRelativePath);
this.documents.set(newRelativePath, document);
await this.save();
this.save();
}
private async save(): Promise<void> {
public delete(relativePath: RelativePath): void {
const candidate = this.getLatestDocumentByRelativePath(relativePath);
if (candidate === undefined) {
throw new Error(
`Document not found by relative path: ${relativePath}`
);
}
candidate.isDeleted = true;
}
private save(): void {
this.ensureConsistency();
await this.saveData({
documents: Object.fromEntries(this.documents.entries()),
void this.saveData({
documents: this.resolvedDocuments.map(
({ relativePath, documentId, metadata }) => ({
documentId,
relativePath,
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
...metadata! // resolvedDocuments only returns docs with metadata set
})
),
lastSeenUpdateId: this.lastSeenUpdateId
});
}
private ensureConsistency(): void {
const allMetadata = Array.from(this.documents.entries());
const idToPath = new Map<string, Array<string>>();
const idToPath = new Map<string, string[]>();
allMetadata.forEach(([name, metadata]) => {
idToPath.set(metadata.documentId, [
...(idToPath.get(metadata.documentId) ?? []),
name
this.resolvedDocuments.forEach(({ relativePath, documentId }) => {
idToPath.set(documentId, [
...(idToPath.get(documentId) ?? []),
relativePath
]);
});

4
frontend/sync-client/src/persistence/settings.ts
View file

@ -1,5 +1,5 @@
import type { Logger } from "src/tracing/logger";
import { LogLevel } from "src/tracing/logger";
import type { Logger } from "../tracing/logger";
import { LogLevel } from "../tracing/logger";
export interface SyncSettings {
remoteUri: string;

51
frontend/sync-client/src/services/connected-state.ts Normal file
View file

@ -0,0 +1,51 @@
import type { Settings } from "../persistence/settings";
import type { Logger } from "../tracing/logger";
import { createPromise } from "../utils/create-promise";
import { retriedFetchFactory } from "../utils/retried-fetch";
export class ConnectedState {
private resolveIsSyncEnabled: (() => void) | undefined;
private syncIsEnabled: Promise<void> | undefined;
public constructor(
settings: Settings,
private readonly logger: Logger
) {
settings.addOnSettingsChangeHandlers((newSettings, oldSettings) => {
if (!oldSettings.isSyncEnabled && newSettings.isSyncEnabled) {
this.handleComingOnline();
} else if (
oldSettings.isSyncEnabled &&
!newSettings.isSyncEnabled
) {
this.handleGoingOffline();
}
});
}
public getFetchImplementation(
fetch: typeof globalThis.fetch,
{ doRetries = true }: { doRetries: boolean } = { doRetries: true }
): typeof globalThis.fetch {
const retriedFetch = doRetries
? retriedFetchFactory(this.logger, fetch)
: fetch;
return async (input: RequestInfo | URL): Promise<Response> => {
if (this.syncIsEnabled !== undefined) {
await this.syncIsEnabled;
}
return retriedFetch(input);
};
}
private handleComingOnline(): void {
this.logger.debug("Sync is enabled");
this.resolveIsSyncEnabled?.();
}
private handleGoingOffline(): void {
this.logger.debug("Sync is disabled");
[this.syncIsEnabled, this.resolveIsSyncEnabled] = createPromise();
}
}

45
frontend/sync-client/src/services/sync-service.ts
View file

@ -6,20 +6,22 @@ import type {
RelativePath,
VaultUpdateId
} from "../persistence/database";
import type { Logger } from "src/tracing/logger";
import { retriedFetchFactory } from "src/utils/retried-fetch";
import type { Settings } from "src/persistence/settings";
import type { Logger } from "../tracing/logger";
import type { Settings } from "../persistence/settings";
import type { ConnectedState } from "./connected-state";
export interface CheckConnectionResult {
isSuccessful: boolean;
message: string;
}
export class SyncService {
private client!: Client<paths>;
private clientWithoutRetries!: Client<paths>;
private _fetchImplementation: typeof globalThis.fetch = globalThis.fetch;
public constructor(
private readonly connectedState: ConnectedState,
private readonly settings: Settings,
private readonly logger: Logger
) {
@ -52,17 +54,19 @@ export class SyncService {
}
public async create({
documentId,
relativePath,
contentBytes,
createdDate
contentBytes
}: {
documentId?: DocumentId;
relativePath: RelativePath;
contentBytes: Uint8Array;
createdDate: Date;
}): Promise<components["schemas"]["DocumentUpdateResponse"]> {
}): Promise<components["schemas"]["DocumentVersionWithoutContent"]> {
const formData = new FormData();
if (documentId !== undefined) {
formData.append("document_id", documentId);
}
formData.append("relative_path", relativePath);
formData.append("created_date", createdDate.toISOString());
formData.append("content", new Blob([contentBytes]));
const response = await this.client.POST(
@ -100,18 +104,18 @@ export class SyncService {
parentVersionId,
documentId,
relativePath,
contentBytes,
createdDate
contentBytes
}: {
parentVersionId: VaultUpdateId;
documentId: DocumentId;
relativePath: RelativePath;
contentBytes: Uint8Array;
createdDate: Date;
}): Promise<components["schemas"]["DocumentUpdateResponse"]> {
this.logger.debug(
`Updating document ${documentId} with parent version ${parentVersionId} and relative path ${relativePath}`
);
const formData = new FormData();
formData.append("parent_version_id", parentVersionId.toString());
formData.append("created_date", createdDate.toISOString());
formData.append("relative_path", relativePath);
formData.append("content", new Blob([contentBytes]));
@ -149,13 +153,11 @@ export class SyncService {
public async delete({
documentId,
relativePath,
createdDate
relativePath
}: {
documentId: DocumentId;
relativePath: RelativePath;
createdDate: Date;
}): Promise<void> {
}): Promise<components["schemas"]["DocumentVersionWithoutContent"]> {
const response = await this.client.DELETE(
"/vaults/{vault_id}/documents/{document_id}",
{
@ -169,7 +171,6 @@ export class SyncService {
}
},
body: {
createdDate: createdDate.toISOString(),
relativePath
}
}
@ -295,11 +296,17 @@ export class SyncService {
private createClient(remoteUri: string): void {
this.client = createClient<paths>({
baseUrl: remoteUri,
fetch: retriedFetchFactory(this.logger, this._fetchImplementation)
fetch: this.connectedState.getFetchImplementation(
this._fetchImplementation
)
});
this.clientWithoutRetries = createClient<paths>({
baseUrl: remoteUri
baseUrl: remoteUri,
fetch: this.connectedState.getFetchImplementation(
this._fetchImplementation,
{ doRetries: false }
)
});
}
}

36
frontend/sync-client/src/services/types.ts
View file

@ -274,12 +274,13 @@ export interface paths {
};
};
responses: {
/** @description no content */
200: {
headers: {
[name: string]: unknown;
};
content?: never;
content: {
"application/json": components["schemas"]["DocumentVersionWithoutContent"];
};
};
default: {
headers: {
@ -451,26 +452,25 @@ export interface components {
Array_of_uint8: number[];
CreateDocumentVersion: {
contentBase64: string;
/** Format: date-time */
createdDate: string;
/**
* Format: uuid
* @description The client can decide the document id (if it wishes to) in order to help with syncing. If the client does not provide a document id, the server will generate one. If the client provides a document id it must not already exist in the database.
*/
documentId?: string | null;
relativePath: string;
};
CreateDocumentVersionMultipart: {
content: components["schemas"]["Array_of_uint8"];
/** Format: date-time */
created_date: string;
/** Format: uuid */
document_id?: string | null;
relative_path: string;
};
DeleteDocumentVersion: {
/** Format: date-time */
createdDate: string;
relativePath: string;
};
/** @description Response to a update document request. */
/** @description Response to an update document request. */
DocumentUpdateResponse:
| {
/** Format: date-time */
createdDate: string;
/** Format: uuid */
documentId: string;
isDeleted: boolean;
@ -479,14 +479,11 @@ export interface components {
type: "FastForwardUpdate";
/** Format: date-time */
updatedDate: string;
vaultId: string;
/** Format: int64 */
vaultUpdateId: number;
}
| {
contentBase64: string;
/** Format: date-time */
createdDate: string;
/** Format: uuid */
documentId: string;
isDeleted: boolean;
@ -495,34 +492,27 @@ export interface components {
type: "MergingUpdate";
/** Format: date-time */
updatedDate: string;
vaultId: string;
/** Format: int64 */
vaultUpdateId: number;
};
DocumentVersion: {
contentBase64: string;
/** Format: date-time */
createdDate: string;
/** Format: uuid */
documentId: string;
isDeleted: boolean;
relativePath: string;
/** Format: date-time */
updatedDate: string;
vaultId: string;
/** Format: int64 */
vaultUpdateId: number;
};
DocumentVersionWithoutContent: {
/** Format: date-time */
createdDate: string;
/** Format: uuid */
documentId: string;
isDeleted: boolean;
relativePath: string;
/** Format: date-time */
updatedDate: string;
vaultId: string;
/** Format: int64 */
vaultUpdateId: number;
};
@ -587,16 +577,12 @@ export interface components {
};
UpdateDocumentVersion: {
contentBase64: string;
/** Format: date-time */
createdDate: string;
/** Format: int64 */
parentVersionId: number;
relativePath: string;
};
UpdateDocumentVersionMultipart: {
content: components["schemas"]["Array_of_uint8"];
/** Format: date-time */
createdDate: string;
/** Format: int64 */
parentVersionId: number;
relativePath: string;

28
frontend/sync-client/src/sync-client.ts
View file

@ -12,6 +12,7 @@ import { SyncService } from "./services/sync-service";
import { Syncer } from "./sync-operations/syncer";
import type { FileSystemOperations } from "./file-operations/filesystem-operations";
import { FileOperations } from "./file-operations/file-operations";
import { ConnectedState } from "./services/connected-state";
export class SyncClient {
private remoteListenerIntervalId: NodeJS.Timeout | null = null;
@ -42,7 +43,7 @@ export class SyncClient {
}
public get documentCount(): number {
return this._database.getDocuments().size;
return this._database.length;
}
public set fetchImplementation(fetch: typeof globalThis.fetch) {
@ -90,7 +91,9 @@ export class SyncClient {
}
);
const syncService = new SyncService(settings, logger);
const connectedState = new ConnectedState(settings, logger);
const syncService = new SyncService(connectedState, settings, logger);
const syncer = new Syncer(
logger,
@ -117,18 +120,13 @@ export class SyncClient {
);
settings.addOnSettingsChangeHandlers((newSettings, oldSettings) => {
client.registerRemoteEventListener(
newSettings.fetchChangesUpdateIntervalMs
);
if (!oldSettings.isSyncEnabled && newSettings.isSyncEnabled) {
syncer
.scheduleSyncForOfflineChanges()
.catch((_error: unknown) => {
logger.error(
"Failed to schedule sync for offline changes"
);
});
if (
newSettings.fetchChangesUpdateIntervalMs !==
oldSettings.fetchChangesUpdateIntervalMs
) {
client.registerRemoteEventListener(
newSettings.fetchChangesUpdateIntervalMs
);
}
});
@ -148,7 +146,7 @@ export class SyncClient {
this.stop();
await this._syncer.reset();
this._history.reset();
await this._database.resetSyncState();
this._database.resetSyncState();
this.logger.reset();
}

415
frontend/sync-client/src/sync-operations/syncer.ts
View file

@ -1,15 +1,17 @@
import type { Database, RelativePath } from "../persistence/database";
import type { SyncService } from "src/services/sync-service";
import type { Logger } from "src/tracing/logger";
import type { SyncHistory } from "src/tracing/sync-history";
import type { SyncService } from "../services/sync-service";
import type { Logger } from "../tracing/logger";
import type { SyncHistory } from "../tracing/sync-history";
import PQueue from "p-queue";
import { hash } from "src/utils/hash";
import type { components } from "src/services/types";
import type { Settings } from "src/persistence/settings";
import type { FileOperations } from "src/file-operations/file-operations";
import { findMatchingFileBasedOnHash } from "src/utils/find-matching-file-based-on-hash";
import { hash } from "../utils/hash";
import { v4 as uuidv4 } from "uuid";
import type { components } from "../services/types";
import type { Settings } from "../persistence/settings";
import type { FileOperations } from "../file-operations/file-operations";
import { findMatchingFile } from "../utils/find-matching-file";
import { UnrestrictedSyncer } from "./unrestricted-syncer";
import { FileNotFoundError } from "../file-operations/safe-filesystem-operations";
import { createPromise } from "../utils/create-promise";
export class Syncer {
private readonly remainingOperationsListeners: ((
@ -18,17 +20,15 @@ export class Syncer {
private readonly syncQueue: PQueue;
private runningScheduleSyncForOfflineChanges: Promise<void> | undefined =
undefined;
private runningApplyRemoteChangesLocally: Promise<void> | undefined =
undefined;
private runningScheduleSyncForOfflineChanges: Promise<void> | undefined;
private runningApplyRemoteChangesLocally: Promise<void> | undefined;
private readonly internalSyncer: UnrestrictedSyncer;
public constructor(
private readonly logger: Logger,
private readonly database: Database,
private readonly settings: Settings,
settings: Settings,
private readonly syncService: SyncService,
private readonly operations: FileOperations,
history: SyncHistory
@ -45,7 +45,9 @@ export class Syncer {
});
this.syncQueue.on("active", () => {
this.emitRemainingOperationsChange(this.syncQueue.size);
this.remainingOperationsListeners.forEach((listener) => {
listener(this.syncQueue.size);
});
});
this.internalSyncer = new UnrestrictedSyncer(
@ -65,48 +67,131 @@ export class Syncer {
}
public async syncLocallyCreatedFile(
relativePath: RelativePath,
updateTime: Date
relativePath: RelativePath
): Promise<void> {
await this.syncQueue.add(async () =>
this.internalSyncer.unrestrictedSyncLocallyCreatedFile(
relativePath,
updateTime
)
);
}
if (
this.database.getLatestDocumentByRelativePath(relativePath)
?.isDeleted === false
) {
this.logger.debug(
`Document ${relativePath} already exists in the database, skipping`
);
return;
}
public async syncLocallyUpdatedFile(args: {
oldPath?: RelativePath;
relativePath: RelativePath;
updateTime: Date;
}): Promise<void> {
await this.syncQueue.add(async () =>
this.internalSyncer.unrestrictedSyncLocallyUpdatedFile(args)
);
}
const [promise, resolve, reject] = createPromise();
public async waitForSyncQueue(): Promise<void> {
return this.syncQueue.onEmpty();
const document = this.database.createNewPendingDocument(
uuidv4(),
relativePath,
promise
);
try {
await this.syncQueue.add(async () =>
this.internalSyncer.unrestrictedSyncLocallyCreatedFile(document)
);
resolve();
} catch (e) {
reject(e);
} finally {
this.database.removeDocumentPromise(promise);
}
}
public async syncLocallyDeletedFile(
relativePath: RelativePath
): Promise<void> {
await this.syncQueue.add(async () =>
this.internalSyncer.unrestrictedSyncLocallyDeletedFile(relativePath)
// We have to have a record of the delete in case there's an in-flight update for the same
// document which finishes after the delete has succeeded and would introduce a phantom metadata record.
this.database.delete(relativePath);
const [promise, resolve, reject] = createPromise();
const document = await this.database.getResolvedDocumentByRelativePath(
relativePath,
promise
);
try {
await this.syncQueue.add(async () =>
this.internalSyncer.unrestrictedSyncLocallyDeletedFile(document)
);
resolve();
} catch (e) {
reject(e);
} finally {
this.database.removeDocumentPromise(promise);
}
}
public async scheduleSyncForOfflineChanges(): Promise<void> {
if (!this.settings.getSettings().isSyncEnabled) {
public async syncLocallyUpdatedFile({
oldPath,
relativePath
}: {
oldPath?: RelativePath;
relativePath: RelativePath;
}): Promise<void> {
if (
oldPath !== undefined &&
(this.database.getLatestDocumentByRelativePath(relativePath) ===
undefined ||
this.database.getLatestDocumentByRelativePath(relativePath)
?.isDeleted === true)
) {
if (oldPath === relativePath) {
throw new Error(
`Old path and new path are the same: ${oldPath}`
);
}
this.database.move(oldPath, relativePath);
}
let document =
this.database.getLatestDocumentByRelativePath(relativePath);
if (document === undefined) {
this.logger.debug(
`Syncing is disabled, not uploading local changes`
`Cannot find document ${relativePath} in the database, skipping`
);
return;
}
if (this.runningScheduleSyncForOfflineChanges != null) {
if (document.isDeleted) {
this.logger.debug(
`Document ${relativePath} has been deleted locally, skipping`
);
return;
}
const [promise, resolve, reject] = createPromise();
document = await this.database.getResolvedDocumentByRelativePath(
relativePath,
promise
);
try {
await this.syncQueue.add(async () =>
this.internalSyncer.unrestrictedSyncLocallyUpdatedFile({
oldPath,
document
})
);
resolve();
} catch (e) {
reject(e);
} finally {
this.database.removeDocumentPromise(promise);
}
}
public async scheduleSyncForOfflineChanges(): Promise<void> {
if (this.runningScheduleSyncForOfflineChanges !== undefined) {
this.logger.debug("Uploading local changes is already in progress");
return this.runningScheduleSyncForOfflineChanges;
}
@ -127,13 +212,6 @@ export class Syncer {
}
public async applyRemoteChangesLocally(): Promise<void> {
if (!this.settings.getSettings().isSyncEnabled) {
this.logger.debug(
`Syncing is disabled, not fetching remote changes`
);
return;
}
if (this.runningApplyRemoteChangesLocally != null) {
this.logger.debug(
"Applying remote changes locally is already in progress"
@ -154,6 +232,10 @@ export class Syncer {
}
}
public async waitForSyncQueue(): Promise<void> {
return this.syncQueue.onEmpty();
}
public async reset(): Promise<void> {
this.syncQueue.clear();
await this.syncQueue.onEmpty();
@ -163,115 +245,15 @@ export class Syncer {
this.internalSyncer.reset();
}
private async syncRemotelyUpdatedFile(
remoteVersion: components["schemas"]["DocumentVersionWithoutContent"]
): Promise<void> {
await this.syncQueue.add(async () =>
this.internalSyncer.unrestrictedSyncRemotelyUpdatedFile(
remoteVersion
)
);
}
private async internalScheduleSyncForOfflineChanges(): Promise<void> {
const allLocalFiles = await this.operations.listAllFiles();
// This includes renamed files for now
let locallyPossiblyDeletedFiles = [
...this.database.getDocuments().entries()
].filter(([path, _]) => !allLocalFiles.includes(path));
await Promise.all(
allLocalFiles.map(async (relativePath) =>
this.syncQueue.add(async () => {
const metadata = this.database.getDocument(relativePath);
if (metadata) {
this.logger.debug(
`Document ${relativePath} might have been updated locally, scheduling sync to validate and update it`
);
return this.internalSyncer.unrestrictedSyncLocallyUpdatedFile(
{
relativePath,
updateTime:
await this.operations.getModificationTime(
relativePath
)
}
);
}
// Perhaps the file has been moved. Let's check by looking at the deleted files
const contentBytes =
await this.operations.read(relativePath);
const contentHash = hash(contentBytes);
// todo: make this smarter so that offline files can be renamed & edited at the same time
const originalFile = findMatchingFileBasedOnHash(
contentHash,
locallyPossiblyDeletedFiles
);
if (originalFile !== undefined) {
// `originalFile` hasn't been deleted but it got moved instead
locallyPossiblyDeletedFiles =
locallyPossiblyDeletedFiles.filter(
(item) => item[0] !== originalFile[0]
);
this.logger.debug(
`Document '${originalFile[0]}' was not found under its current path in the database but was found under a different path (${relativePath}), scheduling sync to move it`
);
return this.internalSyncer.unrestrictedSyncLocallyUpdatedFile(
{
oldPath: originalFile[0],
relativePath: relativePath,
updateTime:
await this.operations.getModificationTime(
relativePath
),
optimisations: {
contentBytes,
contentHash
}
}
);
}
this.logger.debug(
`Document ${relativePath} not found in database, scheduling sync to create it`
);
return this.internalSyncer.unrestrictedSyncLocallyCreatedFile(
relativePath,
await this.operations.getModificationTime(relativePath)
);
})
)
);
await Promise.all(
locallyPossiblyDeletedFiles.map(async ([relativePath, _]) => {
this.logger.debug(
`Document ${relativePath} has been deleted locally, scheduling sync to delete it`
);
if (await this.operations.exists(relativePath)) {
this.logger.debug(
`Document ${relativePath} actually exists locally, skipping`
);
return Promise.resolve();
}
// We're outside of the pqueue, so we need to call the public wrapper
return this.syncLocallyDeletedFile(relativePath);
})
);
}
private async internalApplyRemoteChangesLocally(): Promise<void> {
const remote = await this.syncService.getAll(
this.database.getLastSeenUpdateId()
const remote = await this.syncQueue.add(async () =>
this.syncService.getAll(this.database.getLastSeenUpdateId())
);
if (!remote) {
throw new Error("Failed to fetch remote changes");
}
if (remote.latestDocuments.length === 0) {
this.logger.debug("No remote changes to apply");
return;
@ -280,9 +262,7 @@ export class Syncer {
this.logger.info("Applying remote changes locally");
await Promise.all(
remote.latestDocuments.map(async (remoteDocument) =>
this.syncRemotelyUpdatedFile(remoteDocument)
)
remote.latestDocuments.map(this.syncRemotelyUpdatedFile.bind(this))
);
const lastSeenUpdateId = this.database.getLastSeenUpdateId();
@ -290,13 +270,124 @@ export class Syncer {
lastSeenUpdateId === undefined ||
remote.lastUpdateId > lastSeenUpdateId
) {
await this.database.setLastSeenUpdateId(remote.lastUpdateId);
this.database.setLastSeenUpdateId(remote.lastUpdateId);
}
}
private emitRemainingOperationsChange(remainingOperations: number): void {
this.remainingOperationsListeners.forEach((listener) => {
listener(remainingOperations);
});
private async syncRemotelyUpdatedFile(
remoteVersion: components["schemas"]["DocumentVersionWithoutContent"]
): Promise<void> {
let document = this.database.getDocumentByDocumentId(
remoteVersion.documentId
);
const [promise, resolve, reject] = createPromise();
if (document === undefined) {
await this.syncQueue.add(async () =>
this.internalSyncer.unrestrictedSyncRemotelyUpdatedFile(
remoteVersion
)
);
} else {
document = await this.database.getResolvedDocumentByRelativePath(
document.relativePath,
promise
);
try {
await this.syncQueue.add(async () =>
this.internalSyncer.unrestrictedSyncRemotelyUpdatedFile(
remoteVersion,
document
)
);
resolve();
} catch (e) {
reject(e);
} finally {
this.database.removeDocumentPromise(promise);
}
}
}
private async internalScheduleSyncForOfflineChanges(): Promise<void> {
const allLocalFiles = await this.operations.listAllFiles();
let locallyPossiblyDeletedFiles = [
...this.database.resolvedDocuments
].filter(({ relativePath }) => !allLocalFiles.includes(relativePath));
const updates = Promise.all(
allLocalFiles.map(async (relativePath) => {
if (
this.database.getLatestDocumentByRelativePath(relativePath)
?.metadata !== undefined
) {
this.logger.debug(
`Document ${relativePath} might have been updated locally, scheduling sync to validate and update it`
);
return this.syncLocallyUpdatedFile({
relativePath
});
}
// Perhaps the file has been moved; let's check by looking at the deleted files
const contentHash = await this.syncQueue.add(async () => {
const contentBytes =
await this.operations.read(relativePath); // this can throw FileNotFoundError
return hash(contentBytes);
});
if (contentHash == undefined) {
// The file was deleted before we had a chance to read it, no need to sync it here
return;
}
const originalFile = findMatchingFile(
contentHash,
locallyPossiblyDeletedFiles
);
if (originalFile !== undefined) {
// `originalFile` hasn't been deleted but it got moved instead
locallyPossiblyDeletedFiles =
locallyPossiblyDeletedFiles.filter(
(item) =>
item.relativePath !== originalFile.relativePath
);
this.logger.debug(
`Document '${originalFile.relativePath}' was not found under its current path in the database but was found under a different path (${relativePath}), scheduling sync to move it`
);
// We're outside of the pqueue, so we need to call the public wrapper
return this.syncLocallyUpdatedFile({
oldPath: originalFile.relativePath,
relativePath
});
}
this.logger.debug(
`Document ${relativePath} not found in database, scheduling sync to create it`
);
// We're outside of the pqueue, so we need to call the public wrapper
return this.syncLocallyCreatedFile(relativePath);
})
);
const deletes = Promise.all(
locallyPossiblyDeletedFiles.map(async ({ relativePath }) => {
this.logger.debug(
`Document ${relativePath} has been deleted locally, scheduling sync to delete it`
);
// We're outside of the pqueue, so we need to call the public wrapper
return this.syncLocallyDeletedFile(relativePath);
})
);
await Promise.all([updates, deletes]);
}
}

687
frontend/sync-client/src/sync-operations/unrestricted-syncer.ts
View file

@ -1,19 +1,24 @@
import type { Database, RelativePath } from "../persistence/database";
import type {
Database,
DocumentRecord,
RelativePath
} from "../persistence/database";
import type { SyncService } from "src/services/sync-service";
import type { Logger } from "src/tracing/logger";
import type { SyncHistory } from "src/tracing/sync-history";
import { SyncSource, SyncStatus, SyncType } from "src/tracing/sync-history";
import { hash } from "src/utils/hash";
import type { components } from "src/services/types";
import { deserialize } from "src/utils/deserialize";
import type { Settings } from "src/persistence/settings";
import type { FileOperations } from "src/file-operations/file-operations";
import { FileNotFoundError } from "src/file-operations/safe-filesystem-operations";
import { DocumentLocks } from "./document-locks";
import type { SyncService } from "../services/sync-service";
import type { Logger } from "../tracing/logger";
import type { SyncHistory } from "../tracing/sync-history";
import { SyncSource, SyncStatus, SyncType } from "../tracing/sync-history";
import { EMPTY_HASH, hash } from "../utils/hash";
import type { components } from "../services/types";
import { deserialize } from "../utils/deserialize";
import type { Settings } from "../persistence/settings";
import type { FileOperations } from "../file-operations/file-operations";
import { FileNotFoundError } from "../file-operations/safe-filesystem-operations";
import { DocumentLocks } from "../file-operations/document-locks";
import { createPromise } from "../utils/create-promise";
export class UnrestrictedSyncer {
private readonly locks = new DocumentLocks();
private readonly locks: DocumentLocks;
public constructor(
private readonly logger: Logger,
@ -22,507 +27,375 @@ export class UnrestrictedSyncer {
private readonly syncService: SyncService,
private readonly operations: FileOperations,
private readonly history: SyncHistory
) {}
) {
this.locks = new DocumentLocks(logger);
}
public async unrestrictedSyncLocallyCreatedFile(
relativePath: RelativePath,
updateTime: Date,
optimisations?: {
contentBytes?: Uint8Array;
contentHash?: string;
}
document: DocumentRecord
): Promise<void> {
await this.executeWhileHoldingFileLock(
[relativePath],
return this.executeSync(
document.relativePath,
SyncType.CREATE,
SyncSource.PUSH,
async () => {
if (
(await this.operations.getFileSize(relativePath)) / // this can throw FileNotFoundError
1024 /
1024 >
this.settings.getSettings().maxFileSizeMB
) {
this.history.addHistoryEntry({
status: SyncStatus.ERROR,
relativePath,
message: `File size exceeds the maximum file size limit of ${
this.settings.getSettings().maxFileSizeMB
}MB`,
type: SyncType.CREATE
});
return;
}
const contentBytes =
optimisations?.contentBytes ??
(await this.operations.read(relativePath)); // this can throw FileNotFoundError
let contentHash =
optimisations?.contentHash ?? hash(contentBytes);
const localMetadata = this.database.getDocument(relativePath);
if (localMetadata) {
this.logger.debug(
`Document metadata already exists for ${relativePath}, it must have been downloaded from the server`
);
if (localMetadata.hash === contentHash) {
this.history.addHistoryEntry({
status: SyncStatus.NO_OP,
relativePath,
message: `File hash matches with last synced version, no need to sync`,
type: SyncType.UPDATE
});
return;
}
}
const contentBytes = await this.operations.read(
document.relativePath
); // this can throw FileNotFoundError
const contentHash = hash(contentBytes);
const response = await this.syncService.create({
relativePath,
contentBytes,
createdDate: updateTime
documentId: document.documentId,
relativePath: document.relativePath,
contentBytes
});
this.history.addHistoryEntry({
status: SyncStatus.SUCCESS,
source: SyncSource.PUSH,
relativePath,
relativePath: document.relativePath,
message: `Successfully uploaded locally created file`,
type: SyncType.CREATE
});
// The response can't have a different relative path than the one we sent
// because the relative path is the key when finding existing documents
// when a create request is sent.
this.database.updateDocumentMetadata(
{
parentVersionId: response.vaultUpdateId,
hash: contentHash
},
document
);
if (response.type === "MergingUpdate") {
const responseBytes = deserialize(response.contentBase64);
contentHash = hash(responseBytes);
this.tryIncrementVaultUpdateId(response.vaultUpdateId);
}
);
}
await this.operations.write(
relativePath,
contentBytes,
responseBytes
);
this.history.addHistoryEntry({
status: SyncStatus.SUCCESS,
source: SyncSource.PULL,
relativePath,
message: `The file we created locally has already existed remotely, so we have merged them`,
type: SyncType.UPDATE
});
}
await this.database.setDocument({
documentId: response.documentId,
relativePath: response.relativePath,
parentVersionId: response.vaultUpdateId,
hash: contentHash
public async unrestrictedSyncLocallyDeletedFile(
document: DocumentRecord
): Promise<void> {
await this.executeSync(
document.relativePath,
SyncType.DELETE,
SyncSource.PUSH,
async () => {
const response = await this.syncService.delete({
documentId: document.documentId,
relativePath: document.relativePath
});
await this.tryIncrementVaultUpdateId(response.vaultUpdateId);
this.history.addHistoryEntry({
status: SyncStatus.SUCCESS,
source: SyncSource.PUSH,
relativePath: document.relativePath,
message: `Successfully deleted locally deleted file on the remote server`,
type: SyncType.DELETE
});
this.database.updateDocumentMetadata(
{
parentVersionId: response.vaultUpdateId,
hash: EMPTY_HASH
},
document
);
}
);
}
public async unrestrictedSyncLocallyUpdatedFile({
oldPath,
relativePath,
updateTime,
optimisations
document,
force = false
}: {
oldPath?: RelativePath;
relativePath: RelativePath;
updateTime: Date;
optimisations?: {
contentBytes?: Uint8Array;
contentHash?: string;
};
force?: boolean;
document: DocumentRecord;
}): Promise<void> {
await this.executeWhileHoldingFileLock(
[oldPath, relativePath].filter((path) => path !== undefined),
await this.executeSync(
document.relativePath,
SyncType.UPDATE,
SyncSource.PUSH,
async () => {
// Check the new path first in case the metadata has been already moved
let localMetadata = this.database.getDocument(relativePath);
let metadataPath = relativePath;
const originalRelativePath = document.relativePath;
if (localMetadata === undefined && oldPath !== undefined) {
localMetadata = this.database.getDocument(oldPath);
metadataPath = oldPath;
}
if (!localMetadata) {
// It's fine, a subsequent sync operation must have dealt with this
if (document.metadata === undefined || document.isDeleted) {
this.logger.debug(
`Document ${document.relativePath} has been already deleted, no need to update it`
);
return;
}
if (
(await this.operations.getFileSize(relativePath)) / // this can throw FileNotFoundError
1024 /
1024 >
this.settings.getSettings().maxFileSizeMB
) {
this.history.addHistoryEntry({
status: SyncStatus.ERROR,
relativePath,
message: `File size exceeds the maximum file size limit of ${
this.settings.getSettings().maxFileSizeMB
}MB`,
type: SyncType.CREATE
});
return;
}
const contentBytes =
optimisations?.contentBytes ??
(await this.operations.read(relativePath)); // this can throw FileNotFoundError
let contentHash =
optimisations?.contentHash ?? hash(contentBytes);
const contentBytes = await this.operations.read(
document.relativePath
); // this can throw FileNotFoundError
let contentHash = hash(contentBytes);
if (
localMetadata.hash === contentHash &&
oldPath === undefined
document.metadata.hash === contentHash &&
oldPath === undefined &&
!force
) {
this.history.addHistoryEntry({
status: SyncStatus.NO_OP,
relativePath,
message: `File hash matches with last synced version, no need to sync`,
type: SyncType.UPDATE
});
this.logger.debug(
`File hash of ${document.relativePath} matches with last synced version and the path hasn't changed; no need to sync`
);
return;
}
const response = await this.syncService.put({
documentId: localMetadata.documentId,
parentVersionId: localMetadata.parentVersionId,
relativePath,
contentBytes,
createdDate: updateTime
documentId: document.documentId,
parentVersionId: document.metadata.parentVersionId,
relativePath: document.relativePath,
contentBytes
});
// `document` is mutable and reflects the latest state in the local database
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
if (document.isDeleted) {
this.logger.info(
`Document ${document.relativePath} has been deleted before we could finish updating it`
);
return;
}
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
if (document.metadata === undefined) {
throw new Error(
`Document ${document.relativePath} no longer has metadata after updating it, this cannot happen`
);
}
if (
document.metadata.parentVersionId >= response.vaultUpdateId
) {
this.logger.debug(
`Document ${document.relativePath} is already more up to date than the fetched version`
);
return;
}
this.history.addHistoryEntry({
status: SyncStatus.SUCCESS,
source: SyncSource.PUSH,
relativePath,
relativePath: document.relativePath,
message: `Successfully uploaded locally updated file to the remote server`,
type: SyncType.UPDATE
});
if (response.isDeleted) {
await this.operations.remove(oldPath ?? relativePath);
await this.database.removeDocument(oldPath ?? relativePath);
await this.tryIncrementVaultUpdateId(
response.vaultUpdateId
);
this.history.addHistoryEntry({
status: SyncStatus.SUCCESS,
source: SyncSource.PULL,
relativePath,
relativePath: document.relativePath,
message:
"The file we tried to update had been deleted remotely, therefore, we have deleted it locally",
type: SyncType.DELETE
});
this.database.delete(document.relativePath);
this.database.updateDocumentMetadata(
{
parentVersionId: response.vaultUpdateId,
hash: EMPTY_HASH
},
document
);
await this.operations.delete(document.relativePath);
this.tryIncrementVaultUpdateId(response.vaultUpdateId);
return;
}
if (
response.relativePath != relativePath &&
response.relativePath != oldPath
) {
await this.locks.waitForDocumentLock(response.relativePath);
let actualPath = document.relativePath;
if (response.relativePath != originalRelativePath) {
actualPath = response.relativePath;
await this.operations.move(
document.relativePath,
response.relativePath
); // this can throw FileNotFoundError
}
try {
if (response.relativePath != relativePath) {
// TODO: this can fail, that's bad
await this.operations.move(
// this can throw FileNotFoundError
relativePath,
response.relativePath,
response.documentId
);
}
if (response.type === "MergingUpdate") {
const responseBytes = deserialize(
response.contentBase64
);
contentHash = hash(responseBytes);
await this.operations.write(
response.relativePath,
contentBytes,
responseBytes
);
this.history.addHistoryEntry({
status: SyncStatus.SUCCESS,
source: SyncSource.PULL,
relativePath,
message: `The file we updated had been updated remotely, so we downloaded the merged version`,
type: SyncType.UPDATE
});
}
if (metadataPath !== response.relativePath) {
await this.database.updatePath(
metadataPath,
response.relativePath
);
}
await this.database.setDocument({
documentId: localMetadata.documentId,
relativePath: response.relativePath,
this.database.updateDocumentMetadata(
{
parentVersionId: response.vaultUpdateId,
hash: contentHash
});
},
document
);
await this.tryIncrementVaultUpdateId(
response.vaultUpdateId
if (response.type === "MergingUpdate") {
const responseBytes = deserialize(response.contentBase64);
contentHash = hash(responseBytes);
await this.operations.write(
actualPath,
contentBytes,
responseBytes
);
} finally {
if (
response.relativePath != relativePath &&
response.relativePath != oldPath
) {
this.locks.unlockDocument(response.relativePath);
}
}
}
);
}
public async unrestrictedSyncLocallyDeletedFile(
relativePath: RelativePath
): Promise<void> {
await this.executeWhileHoldingFileLock(
[relativePath],
SyncType.DELETE,
SyncSource.PUSH,
async () => {
const localMetadata = this.database.getDocument(relativePath);
if (!localMetadata) {
this.history.addHistoryEntry({
status: SyncStatus.NO_OP,
relativePath,
message: `Locally deleted file hasn't been uploaded yet, so there's no need to delete it on the remote server`,
type: SyncType.DELETE
status: SyncStatus.SUCCESS,
source: SyncSource.PULL,
relativePath: document.relativePath,
message: `The file we updated had been updated remotely, so we downloaded the merged version`,
type: SyncType.UPDATE
});
return;
}
await this.syncService.delete({
documentId: localMetadata.documentId,
relativePath,
createdDate: new Date() // We got the event now, so it must have been deleted just now
});
this.history.addHistoryEntry({
status: SyncStatus.SUCCESS,
source: SyncSource.PUSH,
relativePath,
message: `Successfully deleted locally deleted file on the remote server`,
type: SyncType.DELETE
});
await this.database.removeDocument(relativePath);
this.tryIncrementVaultUpdateId(response.vaultUpdateId);
}
);
}
public async unrestrictedSyncRemotelyUpdatedFile(
remoteVersion: components["schemas"]["DocumentVersionWithoutContent"]
remoteVersion: components["schemas"]["DocumentVersionWithoutContent"],
document?: DocumentRecord
): Promise<void> {
await this.executeWhileHoldingFileLock(
[remoteVersion.relativePath],
await this.executeSync(
remoteVersion.relativePath,
SyncType.UPDATE,
SyncSource.PULL,
async () => {
let localMetadata = this.database.getDocumentByDocumentId(
remoteVersion.documentId
);
if (
localMetadata &&
localMetadata[0] !== remoteVersion.relativePath
) {
await this.locks.waitForDocumentLock(localMetadata[0]);
}
// Waiting for the new lock might take a while so we need to fetch the database
// entry again in case it's changed.
localMetadata = this.database.getDocumentByDocumentId(
remoteVersion.documentId
);
if (!localMetadata) {
if (remoteVersion.isDeleted) {
this.history.addHistoryEntry({
status: SyncStatus.NO_OP,
source: SyncSource.PULL,
relativePath: remoteVersion.relativePath,
message: `Remotely deleted file hasn't been synced yet, so there's no need to delete it locally`,
type: SyncType.DELETE
});
if (document?.metadata !== undefined) {
// If the file exists locally, let's pretend the user has updated it
// and deal with remote update/deletion within `unrestrictedSyncLocallyUpdatedFile`
if (
document.metadata.parentVersionId >=
remoteVersion.vaultUpdateId
) {
this.logger.debug(
`Document ${remoteVersion.relativePath} is already more up to date than the fetched version`
);
return;
}
const content = (
await this.syncService.get({
documentId: remoteVersion.documentId
})
).contentBase64;
const contentBytes = deserialize(content);
await this.operations.create(
remoteVersion.relativePath,
contentBytes
return this.unrestrictedSyncLocallyUpdatedFile({
document,
force: true
});
} else if (remoteVersion.isDeleted) {
// Either the doc hasn't made it to us before and therefore we don't need to delete it,
// or we already have it, in which case the preceeding if will deal with it
this.logger.debug(
`Document ${remoteVersion.relativePath} has been deleted remotely, no need to sync`
);
await this.database.setDocument({
documentId: remoteVersion.documentId,
relativePath: remoteVersion.relativePath,
return;
}
const content = (
await this.syncService.get({
documentId: remoteVersion.documentId
})
).contentBase64;
document = this.database.getDocumentByDocumentId(
remoteVersion.documentId
);
if (document?.isDeleted === true) {
this.logger.info(
`Document ${remoteVersion.relativePath} has been deleted locally before we could finish updating it`
);
return;
}
if (
(document?.metadata?.parentVersionId ?? -1) >=
remoteVersion.vaultUpdateId
) {
this.logger.debug(
`Document ${remoteVersion.relativePath} is already more up to date than the fetched version`
);
return;
}
const contentBytes = deserialize(content);
await this.operations.ensureClearPath(
remoteVersion.relativePath
);
const [promise, resolve] = createPromise();
this.database.updateDocumentMetadata(
{
parentVersionId: remoteVersion.vaultUpdateId,
hash: hash(contentBytes)
});
this.history.addHistoryEntry({
status: SyncStatus.SUCCESS,
source: SyncSource.PULL,
relativePath: remoteVersion.relativePath,
message: `Successfully downloaded remote file which hadn't existed locally`,
type: SyncType.CREATE
});
return;
}
},
this.database.createNewPendingDocument(
remoteVersion.documentId,
remoteVersion.relativePath,
promise
)
);
const [relativePath, metadata] = localMetadata;
await this.operations.create(
remoteVersion.relativePath,
contentBytes
);
if (remoteVersion.vaultUpdateId <= metadata.parentVersionId) {
this.logger.debug(
`Document ${relativePath} is already up to date`
);
return;
}
resolve();
this.database.removeDocumentPromise(promise);
try {
if (remoteVersion.isDeleted) {
await this.operations.remove(relativePath);
await this.database.removeDocument(relativePath);
this.history.addHistoryEntry({
status: SyncStatus.SUCCESS,
source: SyncSource.PULL,
relativePath: remoteVersion.relativePath,
message: `Successfully deleted remotely deleted file locally`,
type: SyncType.DELETE
});
} else {
// TODO: this can fail, that's bad
const currentContent =
await this.operations.read(relativePath); // this can throw FileNotFoundError
const currentHash = hash(currentContent);
if (currentHash !== metadata.hash) {
this.logger.info(
`Document ${relativePath} has been updated both remotely and locally, letting the local file update event handle it`
);
return;
}
const content = (
await this.syncService.get({
documentId: remoteVersion.documentId
})
).contentBase64;
const contentBytes = deserialize(content);
const contentHash = hash(contentBytes);
if (relativePath !== remoteVersion.relativePath) {
// TODO: this can fail, that's bad
await this.operations.move(
// this can throw FileNotFoundError
relativePath,
remoteVersion.relativePath,
remoteVersion.documentId
);
await this.database.updatePath(
relativePath,
remoteVersion.relativePath
);
}
await this.operations.write(
remoteVersion.relativePath,
currentContent,
contentBytes
);
await this.database.setDocument({
documentId: remoteVersion.documentId,
relativePath: remoteVersion.relativePath,
parentVersionId: remoteVersion.vaultUpdateId,
hash: contentHash
});
this.history.addHistoryEntry({
status: SyncStatus.SUCCESS,
source: SyncSource.PULL,
relativePath: remoteVersion.relativePath,
message: `Successfully updated remotely updated file locally`,
type: SyncType.UPDATE
});
}
} finally {
if (relativePath !== remoteVersion.relativePath) {
this.locks.unlockDocument(relativePath);
}
}
this.history.addHistoryEntry({
status: SyncStatus.SUCCESS,
source: SyncSource.PULL,
relativePath: remoteVersion.relativePath,
message: `Successfully downloaded remote file which hadn't existed locally`,
type: SyncType.CREATE
});
}
);
}
public async executeWhileHoldingFileLock(
lockedPaths: RelativePath[],
public async executeSync<T>(
relativePath: RelativePath,
syncType: SyncType,
syncSource: SyncSource,
fn: () => Promise<void>
): Promise<void> {
const relativePath = lockedPaths[lockedPaths.length - 1];
if (!this.settings.getSettings().isSyncEnabled) {
this.logger.info(
`Syncing is disabled, not syncing ${relativePath}`
);
return;
}
fn: () => Promise<T>
): Promise<T | undefined> {
if (!this.operations.isFileEligibleForSync(relativePath)) {
this.logger.info(
`File ${relativePath} is not eligible for syncing`
);
this.history.addHistoryEntry({
status: SyncStatus.ERROR,
relativePath,
message: `File ${relativePath} is not eligible for syncing`,
type: syncType
});
return;
}
this.logger.debug(
`Syncing ${relativePath} (${syncSource} - ${syncType})`
);
await Promise.all(
lockedPaths.map(this.locks.waitForDocumentLock.bind(this.locks))
);
try {
await fn();
if (
(await this.operations.exists(relativePath)) &&
(await this.operations.getFileSize(relativePath)) / // this can throw FileNotFoundError
1024 /
1024 >
this.settings.getSettings().maxFileSizeMB
) {
this.history.addHistoryEntry({
status: SyncStatus.ERROR,
relativePath,
message: `File size exceeds the maximum file size limit of ${
this.settings.getSettings().maxFileSizeMB
}MB`,
type: syncType
});
return;
}
return await fn();
} catch (e) {
if (e instanceof FileNotFoundError) {
// A subsequent sync operation must have been creating to deal with this
this.history.addHistoryEntry({
status: SyncStatus.NO_OP,
relativePath,
message: `Skip ${syncSource.toLocaleLowerCase()} file because it no longer exists when trying to ${syncType.toLocaleLowerCase()} it`,
type: syncType,
source: syncSource
});
this.logger.info(
`Skip ${syncSource.toLocaleLowerCase()} file because it no longer exists when trying to ${syncType.toLocaleLowerCase()} it`
);
} else {
this.history.addHistoryEntry({
status: SyncStatus.ERROR,
@ -533,8 +406,6 @@ export class UnrestrictedSyncer {
});
throw e;
}
} finally {
lockedPaths.forEach(this.locks.unlockDocument.bind(this.locks));
}
}
@ -542,11 +413,9 @@ export class UnrestrictedSyncer {
this.locks.reset();
}
private async tryIncrementVaultUpdateId(
responseVaultUpdateId: number
): Promise<void> {
private tryIncrementVaultUpdateId(responseVaultUpdateId: number): void {
if (this.database.getLastSeenUpdateId() === responseVaultUpdateId - 1) {
await this.database.setLastSeenUpdateId(responseVaultUpdateId);
this.database.setLastSeenUpdateId(responseVaultUpdateId);
}
}
}

2
frontend/sync-client/src/tracing/sync-history.ts
View file

@ -1,4 +1,4 @@
import type { RelativePath } from "src/persistence/database";
import type { RelativePath } from "../persistence/database";
import type { Logger } from "./logger";
export interface CommonHistoryEntry {

15
frontend/sync-client/src/utils/create-promise.ts Normal file
View file

@ -0,0 +1,15 @@
export function createPromise<T = void>(): [
Promise<T>,
(value: T) => void,
(error: unknown) => void
] {
let resolve: undefined | ((resolved: T) => void) = undefined;
let reject: undefined | ((error: unknown) => void) = undefined;
const creationPromise = new Promise<T>(
(resolve_, reject_) => ((resolve = resolve_), (reject = reject_))
);
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
return [creationPromise, resolve!, reject!];
}

13
frontend/sync-client/src/utils/find-matching-file-based-on-hash.ts
View file

@ -1,13 +0,0 @@
import type { DocumentMetadata, RelativePath } from "src/persistence/database";
import { EMPTY_HASH } from "./hash";
export function findMatchingFileBasedOnHash(
contentHash: string,
candidates: [RelativePath, DocumentMetadata][]
): [RelativePath, DocumentMetadata] | undefined {
if (contentHash === EMPTY_HASH) {
return undefined;
}
return candidates.find(([_, metadata]) => metadata.hash === contentHash);
}

14
frontend/sync-client/src/utils/find-matching-file.ts Normal file
View file

@ -0,0 +1,14 @@
import type { DocumentRecord } from "../persistence/database";
import { EMPTY_HASH } from "./hash";
// TODO: make this smarter so that offline files can be renamed & edited at the same time
export function findMatchingFile(
contentHash: string,
candidates: DocumentRecord[]
): DocumentRecord | undefined {
if (contentHash === EMPTY_HASH) {
return undefined;
}
return candidates.find(({ metadata }) => metadata?.hash === contentHash);
}

2
frontend/sync-client/src/utils/hash.ts
View file

@ -6,7 +6,7 @@ export function hash(content: Uint8Array): string {
result = (result << 5) - result + content[i];
result |= 0; // Convert to 32bit integer
}
return Math.abs(result).toString(16);
return Math.abs(result).toString(16).padStart(8, "0");
}
export const EMPTY_HASH = hash(new Uint8Array(0));

3
frontend/sync-client/src/utils/retried-fetch.ts
View file

@ -1,6 +1,6 @@
import * as fetchRetryFactory from "fetch-retry";
import type { RequestInitRetryParams } from "fetch-retry";
import type { Logger } from "src/tracing/logger";
import type { Logger } from "../tracing/logger";
function getUrlFromInput(input: RequestInfo | URL): string {
if (input instanceof URL) {
@ -31,7 +31,6 @@ export function retriedFetchFactory(
}
return false;
},
retries: 6,
retryDelay: (attempt) => Math.pow(1.5, attempt) * 500,
...init
});

9
frontend/sync-client/tsconfig.json
View file

@ -1,12 +1,15 @@
{
"compilerOptions": {
"baseUrl": ".",
"module": "ESNext",
"target": "ESNext",
"strict": true,
"moduleResolution": "bundler",
"allowSyntheticDefaultImports": true,
"lib": ["DOM", "ESNext"]
"moduleResolution": "bundler",
"lib": [
"DOM" // to get "fetch"
],
"declaration": true,
"declarationDir": "./dist/types"
},
"exclude": ["./dist"]
}

56
frontend/sync-client/webpack.config.js
View file

@ -1,25 +1,13 @@
const path = require("path");
const { merge } = require("webpack-merge");
module.exports = (_env, _argv) => ({
const common = {
entry: "./src/index.ts",
devtool: "source-map",
target: "node",
module: {
rules: [
{
test: /\.ts$/,
use: [
{
loader: "ts-loader",
options: {
compilerOptions: {
declaration: true,
declarationDir: "./dist/types"
},
transpileOnly: false
}
}
]
use: ["ts-loader"]
},
{
test: /\.wasm$/,
@ -28,22 +16,40 @@ module.exports = (_env, _argv) => ({
]
},
optimization: {
// the consuming project should take care of minification
minimize: false
},
resolve: {
extensions: [".ts", ".js"],
extensions: [".ts"],
alias: {
root: __dirname,
src: path.resolve(__dirname, "src")
}
},
output: {
clean: true,
filename: "index.js",
library: {
name: "SyncClient",
type: "umd"
},
path: path.resolve(__dirname, "dist")
performance: {
hints: false // it's a library, no need to warn about its size
}
});
};
module.exports = [
merge(common, {
target: "web",
output: {
path: path.resolve(__dirname, "dist"),
filename: "sync-client.web.js",
library: {
name: "SyncClient",
type: "umd"
},
globalObject: "this"
}
}),
merge(common, {
target: "node",
output: {
path: path.resolve(__dirname, "dist"),
filename: "sync-client.node.js",
libraryTarget: "commonjs2"
}
})
];