andras/vault-link
1
0
Fork 0
Code Issues Pull requests 18 Projects Releases 55 Packages Wiki Activity Actions 1

Format & lint

Browse source
This commit is contained in:
Andras Schmelczer 2026-04-25 17:55:46 +01:00
parent fefac224b0
commit 7f62273e72
179 changed files with 2210 additions and 1319 deletions
Download patch file Download diff file Expand all files Collapse all files

15
frontend/sync-client/src/sync-operations/conflict-path.test.ts
View file

@ -31,10 +31,7 @@ describe("buildConflictFileName", () => {
0,
"stem length must be a whole number of families"
);
assert.ok(
!stem.endsWith("‍"),
"stem must not end with a dangling ZWJ"
);
assert.ok(!stem.endsWith("‍"), "stem must not end with a dangling ZWJ");
});
it("does not split a base character from its combining mark", () => {
@ -61,7 +58,10 @@ describe("buildConflictFileName", () => {
describe("CONFLICT_PATH_REGEX", () => {
it("does not misclassify user-authored names that start with `conflict-`", () => {
assert.strictEqual(CONFLICT_PATH_REGEX.test("conflict-resolution.md"), false);
assert.strictEqual(
CONFLICT_PATH_REGEX.test("conflict-resolution.md"),
false
);
});
it("only inspects the final path segment", () => {
@ -80,6 +80,9 @@ describe("CONFLICT_PATH_REGEX", () => {
});
it("round-trips with buildConflictFileName", () => {
assert.strictEqual(CONFLICT_PATH_REGEX.test(buildConflictFileName("note.md")), true);
assert.strictEqual(
CONFLICT_PATH_REGEX.test(buildConflictFileName("note.md")),
true
);
});
});

15
frontend/sync-client/src/sync-operations/conflict-path.ts
View file

@ -8,16 +8,10 @@
export const CONFLICT_PATH_REGEX =
/(?:^|\/)conflict-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}-[^/]*$/u;
const CONFLICT_PREFIX_LEN = "conflict-".length + 36 + 1;
const MAX_SEGMENT_BYTES = 255;
const MAX_ORIGINAL_BYTES = MAX_SEGMENT_BYTES - CONFLICT_PREFIX_LEN - 4;
export function buildConflictFileName(fileName: string): string {
const safeName = truncateFileNameToByteLimit(fileName, MAX_ORIGINAL_BYTES);
return `conflict-${crypto.randomUUID()}-${safeName}`;
}
function truncateFileNameToByteLimit(
fileName: string,
maxBytes: number
@ -34,7 +28,9 @@ function truncateFileNameToByteLimit(
const extensionBytes = encoder.encode(extension).byteLength;
const stemBudget = Math.max(0, maxBytes - extensionBytes);
const segmenter = new Intl.Segmenter(undefined, { granularity: "grapheme" });
const segmenter = new Intl.Segmenter(undefined, {
granularity: "grapheme"
});
let truncatedStem = "";
let usedBytes = 0;
for (const { segment } of segmenter.segment(stem)) {
@ -45,3 +41,8 @@ function truncateFileNameToByteLimit(
}
return truncatedStem + extension;
}
export function buildConflictFileName(fileName: string): string {
const safeName = truncateFileNameToByteLimit(fileName, MAX_ORIGINAL_BYTES);
return `conflict-${crypto.randomUUID()}-${safeName}`;
}

25
frontend/sync-client/src/sync-operations/cursor-tracker.ts
View file

@ -35,7 +35,7 @@ export class CursorTracker {
[];
public constructor(
private readonly logger: Logger,
logger: Logger,
private readonly queue: SyncEventQueue,
private readonly webSocketManager: WebSocketManager,
private readonly fileOperations: FileOperations,
@ -82,8 +82,7 @@ export class CursorTracker {
for (const clientCursor of this.knownRemoteCursors) {
if (
clientCursor.documentsWithCursors.some(
(document) =>
document.relativePath === relativePath
(document) => document.relativePath === relativePath
)
) {
clientCursor.upToDateness =
@ -135,7 +134,9 @@ export class CursorTracker {
const readContent = await this.fileOperations.read(
doc.relativePath
);
const record = this.queue.getSettledDocumentByPath(doc.relativePath);
const record = this.queue.getSettledDocumentByPath(
doc.relativePath
);
if (record?.remoteHash !== (await hash(readContent))) {
doc.vaultUpdateId = null;
}
@ -221,20 +222,18 @@ export class CursorTracker {
private async getDocumentUpToDateness(
document: DocumentWithCursors
): Promise<DocumentUpToDateness> {
const record = this.queue.getSettledDocumentByPath(document.relativePath);
const record = this.queue.getSettledDocumentByPath(
document.relativePath
);
if (!record) {
// the document of the cursor must be from the future
return DocumentUpToDateness.Later;
}
if (
record.parentVersionId < (document.vaultUpdateId ?? 0)
) {
if (record.parentVersionId < (document.vaultUpdateId ?? 0)) {
return DocumentUpToDateness.Later;
} else if (
(document.vaultUpdateId ?? 0) < record.parentVersionId
) {
} else if ((document.vaultUpdateId ?? 0) < record.parentVersionId) {
// the document of the cursor must be from the past
return DocumentUpToDateness.Prior;
}
@ -243,7 +242,9 @@ export class CursorTracker {
document.relativePath
);
const currentRecord = this.queue.getSettledDocumentByPath(document.relativePath);
const currentRecord = this.queue.getSettledDocumentByPath(
document.relativePath
);
return currentRecord?.remoteHash === (await hash(currentContent))
? DocumentUpToDateness.UpToDate
: DocumentUpToDateness.Prior;

34
frontend/sync-client/src/sync-operations/offline-change-detector.ts
View file

@ -8,8 +8,6 @@ import { FileNotFoundError } from "../errors/file-not-found-error";
import type { SyncEventQueue } from "./sync-event-queue";
import { removeFromArray } from "../utils/remove-from-array";
/**
* Scans the local filesystem and the document database to determine
* which files were created, updated, moved, or deleted while the
@ -20,8 +18,11 @@ export async function scheduleOfflineChanges(
operations: FileOperations,
queue: SyncEventQueue,
enqueueCreate: (path: RelativePath) => void,
enqueueUpdate: (args: { oldPath?: RelativePath; relativePath: RelativePath }) => void,
enqueueDelete: (path: RelativePath) => void,
enqueueUpdate: (args: {
oldPath?: RelativePath;
relativePath: RelativePath;
}) => void,
enqueueDelete: (path: RelativePath) => void
): Promise<void> {
const allLocalFiles = await operations.listFilesRecursively();
logger.info(`Scheduling sync for ${allLocalFiles.length} local files`);
@ -30,19 +31,14 @@ export async function scheduleOfflineChanges(
const locallyPossiblyDeletedFiles: DocumentWithPath[] = [];
for (const [path, record] of allDocuments.entries()) {
if (
record !== undefined
) {
locallyPossiblyDeletedFiles.push({ path, record });
}
locallyPossiblyDeletedFiles.push({ path, record });
}
const locallyPossibleCreatedFiles: RelativePath[] = [];
const syncedLocalFiles: RelativePath[] = [];
for (const localFile of allLocalFiles) {
if (allDocuments.has(localFile)
) {
if (allDocuments.has(localFile)) {
syncedLocalFiles.push(localFile);
} else {
locallyPossibleCreatedFiles.push(localFile);
@ -53,19 +49,27 @@ export async function scheduleOfflineChanges(
const content = await operations.read(path);
const contentHash = await hash(content);
const matchingDeletedFile = await findMatchingFile(contentHash, locallyPossiblyDeletedFiles);
const matchingDeletedFile = await findMatchingFile(
contentHash,
locallyPossiblyDeletedFiles
);
if (matchingDeletedFile !== undefined) {
logger.debug(
`File ${path} might have been moved from ${matchingDeletedFile.path} while offline, scheduling sync to move it`,
`File ${path} might have been moved from ${matchingDeletedFile.path} while offline, scheduling sync to move it`
);
enqueueUpdate({ oldPath: matchingDeletedFile.path, relativePath: path });
enqueueUpdate({
oldPath: matchingDeletedFile.path,
relativePath: path
});
removeFromArray(locallyPossiblyDeletedFiles, matchingDeletedFile);
removeFromArray(locallyPossibleCreatedFiles, path);
}
}
for (const path of locallyPossibleCreatedFiles) {
logger.debug(`File ${path} was created while offline, scheduling sync to create it`);
logger.debug(
`File ${path} was created while offline, scheduling sync to create it`
);
enqueueCreate(path);
}

124
frontend/sync-client/src/sync-operations/sync-event-queue.test.ts
View file

@ -9,8 +9,12 @@ import type { DocumentRecord, RelativePath } from "./types";
function createQueue(ignorePatterns: string[] = []): SyncEventQueue {
const logger = new Logger();
const settings = new Settings(logger, { ignorePatterns }, async () => { });
return new SyncEventQueue(settings, logger, undefined, async () => { });
const settings = new Settings(logger, { ignorePatterns }, async () => {
/* no-op */
});
return new SyncEventQueue(settings, logger, undefined, async () => {
/* no-op */
});
}
function fakeRemoteVersion(
@ -60,9 +64,7 @@ describe("SyncEventQueue", () => {
const third = await queue.next();
assert.strictEqual(third?.type, SyncEventType.LocalDelete);
if (third?.type === SyncEventType.LocalDelete) {
assert.strictEqual(third.documentId, "A");
}
assert.strictEqual(third.documentId, "A");
assert.strictEqual(await queue.next(), undefined);
});
@ -74,15 +76,11 @@ describe("SyncEventQueue", () => {
const first = await queue.next();
assert.strictEqual(first?.type, SyncEventType.LocalCreate);
if (first?.type === SyncEventType.LocalCreate) {
assert.strictEqual(first.path, "a.md");
}
assert.strictEqual(first.path, "a.md");
const second = await queue.next();
assert.strictEqual(second?.type, SyncEventType.LocalCreate);
if (second?.type === SyncEventType.LocalCreate) {
assert.strictEqual(second.path, "b.md");
}
assert.strictEqual(second.path, "b.md");
});
it("delete resolves documentId from path", async () => {
@ -93,14 +91,15 @@ describe("SyncEventQueue", () => {
const event = await queue.next();
assert.strictEqual(event?.type, SyncEventType.LocalDelete);
if (event?.type === SyncEventType.LocalDelete) {
assert.strictEqual(event.documentId, "A");
}
assert.strictEqual(event.documentId, "A");
});
it("delete for unknown path is silently ignored", async () => {
const queue = createQueue();
await queue.enqueue({ type: SyncEventType.LocalDelete, path: "unknown.md" });
await queue.enqueue({
type: SyncEventType.LocalDelete,
path: "unknown.md"
});
assert.strictEqual(queue.pendingUpdateCount, 0);
});
@ -112,11 +111,14 @@ describe("SyncEventQueue", () => {
await queue.setDocument("a.md", fakeRecord("A"));
assert.strictEqual(queue.syncedDocumentCount, 1);
assert.deepStrictEqual(queue.getSettledDocumentByPath("a.md"), fakeRecord("A"));
assert.deepStrictEqual(
queue.getSettledDocumentByPath("a.md"),
fakeRecord("A")
);
const found = queue.getDocumentByDocumentId("A");
assert.strictEqual(found?.path, "a.md");
assert.strictEqual(found?.record.documentId, "A");
assert.strictEqual(found.record.documentId, "A");
await queue.removeDocument("a.md");
assert.strictEqual(queue.syncedDocumentCount, 0);
@ -127,9 +129,16 @@ describe("SyncEventQueue", () => {
const queue = createQueue();
await queue.setDocument("a.md", fakeRecord("A"));
await queue.enqueue({ type: SyncEventType.LocalUpdate, path: "b.md", oldPath: "a.md" });
await queue.enqueue({
type: SyncEventType.LocalUpdate,
path: "b.md",
oldPath: "a.md"
});
assert.strictEqual(queue.getSettledDocumentByPath("a.md"), undefined);
assert.strictEqual(queue.getSettledDocumentByPath("b.md")?.documentId, "A");
assert.strictEqual(
queue.getSettledDocumentByPath("b.md")?.documentId,
"A"
);
});
it("create can be re-enqueued after being dequeued", async () => {
@ -144,11 +153,20 @@ describe("SyncEventQueue", () => {
it("silently ignores create events matching ignore patterns", async () => {
const queue = createQueue(["*.tmp", ".hidden/**"]);
await queue.enqueue({ type: SyncEventType.LocalCreate, path: "scratch.tmp" });
await queue.enqueue({ type: SyncEventType.LocalCreate, path: ".hidden/secret.md" });
await queue.enqueue({
type: SyncEventType.LocalCreate,
path: "scratch.tmp"
});
await queue.enqueue({
type: SyncEventType.LocalCreate,
path: ".hidden/secret.md"
});
assert.strictEqual(queue.pendingUpdateCount, 0);
await queue.enqueue({ type: SyncEventType.LocalCreate, path: "notes-new.md" });
await queue.enqueue({
type: SyncEventType.LocalCreate,
path: "notes-new.md"
});
assert.strictEqual(queue.pendingUpdateCount, 1);
await queue.enqueue({
@ -170,7 +188,10 @@ describe("SyncEventQueue", () => {
assert.strictEqual(queue.pendingUpdateCount, 0);
assert.strictEqual(queue.syncedDocumentCount, 1);
assert.strictEqual(queue.getSettledDocumentByPath("a.md")?.documentId, "A");
assert.strictEqual(
queue.getSettledDocumentByPath("a.md")?.documentId,
"A"
);
});
it("allSettledDocuments returns all tracked documents", async () => {
@ -186,24 +207,39 @@ describe("SyncEventQueue", () => {
it("loads initial state from persistence", () => {
const logger = new Logger();
const settings = new Settings(logger, {}, async () => { });
const queue = new SyncEventQueue(settings, logger, {
documents: [
{
relativePath: "a.md",
...fakeRecord("A", { parentVersionId: 5 })
},
{
relativePath: "b.md",
...fakeRecord("B", { parentVersionId: 3 })
}
],
lastSeenUpdateId: 4
}, async () => { });
const settings = new Settings(logger, {}, async () => {
/* no-op */
});
const queue = new SyncEventQueue(
settings,
logger,
{
documents: [
{
relativePath: "a.md",
...fakeRecord("A", { parentVersionId: 5 })
},
{
relativePath: "b.md",
...fakeRecord("B", { parentVersionId: 3 })
}
],
lastSeenUpdateId: 4
},
async () => {
/* no-op */
}
);
assert.strictEqual(queue.syncedDocumentCount, 2);
assert.strictEqual(queue.getSettledDocumentByPath("a.md")?.documentId, "A");
assert.strictEqual(queue.getSettledDocumentByPath("b.md")?.documentId, "B");
assert.strictEqual(
queue.getSettledDocumentByPath("a.md")?.documentId,
"A"
);
assert.strictEqual(
queue.getSettledDocumentByPath("b.md")?.documentId,
"B"
);
assert.strictEqual(queue.lastSeenUpdateId, 4);
});
@ -216,10 +252,16 @@ describe("SyncEventQueue", () => {
assert.ok(event?.type === SyncEventType.LocalCreate);
const createPromise = event.resolvers.promise;
await queue.resolveCreate(event, fakeRecord("DOC-1", { parentVersionId: 5 }));
await queue.resolveCreate(
event,
fakeRecord("DOC-1", { parentVersionId: 5 })
);
// Document is now settled
assert.strictEqual(queue.getSettledDocumentByPath("a.md")?.documentId, "DOC-1");
assert.strictEqual(
queue.getSettledDocumentByPath("a.md")?.documentId,
"DOC-1"
);
// Promise was resolved
assert.strictEqual(await createPromise, "DOC-1");

189
frontend/sync-client/src/sync-operations/sync-event-queue.ts
View file

@ -3,8 +3,8 @@ import type { Logger } from "../tracing/logger";
import { globsToRegexes } from "../utils/globs-to-regexes";
import { CONFLICT_PATH_REGEX } from "./conflict-path";
import { removeFromArray } from "../utils/remove-from-array";
import type { DocumentWithPath } from "./types";
import {
DocumentWithPath,
SyncEventType,
type DocumentId,
type DocumentRecord,
@ -12,27 +12,28 @@ import {
type RelativePath,
type StoredSyncState,
type SyncEvent,
type VaultUpdateId,
type VaultUpdateId
} from "./types";
import { MinCovered } from "../utils/data-structures/min-covered";
export class SyncEventQueue {
private _lastSeenUpdateId: MinCovered;
// Latest state of the filesystem as we know it, excluding
// unconfirmed creates but including pending deletes.
//
// It's always indexed by the latest path on disk.
//
//
// It maps a subset of the remote state onto the local filesystem.
private readonly documents = new Map<RelativePath, DocumentRecord>();
// All outstanding operations in order of occurrence,
// can include multiple generations of the same document,
// can include multiple generations of the same document,
// e.g.: a create, delete, create sequence for the same path.
//
// The paths within the events must always correspond to the latest
// path on disk, so the path of each event may be updated multiple
// times.
// times.
//
// It maps pending changes onto the local filesystem.
private readonly events: SyncEvent[] = [];
@ -40,8 +41,6 @@ export class SyncEventQueue {
// file creations for paths matching any of these patterns will be ignored
private ignorePatterns: RegExp[];
public _lastSeenUpdateId: MinCovered;
public constructor(
private readonly settings: Settings,
private readonly logger: Logger,
@ -70,17 +69,13 @@ export class SyncEventQueue {
this.documents.set(relativePath, record);
}
}
this._lastSeenUpdateId = new MinCovered(initialState.lastSeenUpdateId ?? 0);
this._lastSeenUpdateId = new MinCovered(
initialState.lastSeenUpdateId ?? 0
);
this.logger.debug(`Loaded ${this.documents.size} documents and lastSeenUpdateId=${this._lastSeenUpdateId} from storage`);
}
public get lastSeenUpdateId(): VaultUpdateId {
return this._lastSeenUpdateId.min;
}
public set lastSeenUpdateId(id: VaultUpdateId) {
this._lastSeenUpdateId.add(id);
this.logger.debug(
`Loaded ${this.documents.size} documents and lastSeenUpdateId=${this._lastSeenUpdateId.min} from storage`
);
}
public get pendingUpdateCount(): number {
@ -91,8 +86,19 @@ export class SyncEventQueue {
return this.documents.size;
}
public get lastSeenUpdateId(): VaultUpdateId {
return this._lastSeenUpdateId.min;
}
public set lastSeenUpdateId(id: VaultUpdateId) {
this._lastSeenUpdateId.add(id);
}
public async enqueue(input: FileSyncEvent): Promise<void> {
const path = (input.type === SyncEventType.RemoteChange) ? input.remoteVersion.relativePath : input.path;
const path =
input.type === SyncEventType.RemoteChange
? input.remoteVersion.relativePath
: input.path;
if (this.ignorePatterns.some((pattern) => pattern.test(path))) {
this.logger.info(
@ -106,22 +112,28 @@ export class SyncEventQueue {
return;
}
if (input.type === SyncEventType.LocalCreate) {
this.events.push({ type: SyncEventType.LocalCreate, path, originalPath: path, resolvers: Promise.withResolvers() });
this.events.push({
type: SyncEventType.LocalCreate,
path,
originalPath: path,
resolvers: Promise.withResolvers()
});
return;
}
const lookupPath = (input.type === SyncEventType.LocalUpdate && input.oldPath) ? input.oldPath : path;
const lookupPath =
input.type === SyncEventType.LocalUpdate &&
input.oldPath !== undefined
? input.oldPath
: path;
const record = this.documents.get(lookupPath);
// latest creation must take precedence as it's from the doc's latest generation
const pendingDocumentId: Promise<DocumentId> | undefined =
this.findLatestCreateForPath(lookupPath)?.resolvers.promise;
const documentId: DocumentId | undefined =
record?.documentId;
const documentId: DocumentId | undefined = record?.documentId;
if (pendingDocumentId === undefined && documentId === undefined) {
// we can get here when deleting a local document after a remote update
@ -129,7 +141,14 @@ export class SyncEventQueue {
}
if (input.type === SyncEventType.LocalDelete) {
this.events.push({ type: SyncEventType.LocalDelete, documentId: pendingDocumentId ?? documentId! });
const deleteId = pendingDocumentId ?? documentId;
if (deleteId === undefined) {
throw new Error("Unreachable: deleteId must be defined here");
}
this.events.push({
type: SyncEventType.LocalDelete,
documentId: deleteId
});
return;
}
@ -137,30 +156,43 @@ export class SyncEventQueue {
if (pendingDocumentId !== undefined) {
this.updatePendingCreatePath(input.oldPath, path);
} else {
if (record === undefined) {
throw new Error(
"Unreachable: record must be defined for non-pending update"
);
}
this.documents.delete(input.oldPath);
this.documents.set(path, record!);
this.documents.set(path, record);
for (const e of this.events) {
// It already has a docId, so there can't be a pending create event for it
if (e.type === SyncEventType.LocalUpdate && e.documentId === documentId) {
// It already has a docId, so there can't be a pending create event for it
if (
e.type === SyncEventType.LocalUpdate &&
e.documentId === documentId
) {
e.path = path;
}
}
await this.save();
}
return
return;
}
this.events.push({ type: SyncEventType.LocalUpdate, documentId: pendingDocumentId ?? documentId!, path, originalPath: path });
const updateId = pendingDocumentId ?? documentId;
if (updateId === undefined) {
throw new Error("Unreachable: updateId must be defined here");
}
this.events.push({
type: SyncEventType.LocalUpdate,
documentId: updateId,
path,
originalPath: path
});
}
public async next(): Promise<SyncEvent | undefined> {
return this.events.shift();
}
/**
* Call once a create has been acknowledged by the server.
*/
@ -170,19 +202,21 @@ export class SyncEventQueue {
): Promise<void> {
removeFromArray(this.events, event); // in case the create event is still pending
await this.setDocument(event.path, record);
event.resolvers?.resolve(record.documentId);
event.resolvers.resolve(record.documentId);
}
/**
* Update the settled document map and persist the new document version.
*/
public setDocument(path: RelativePath, record: DocumentRecord): Promise<void> {
public async setDocument(
path: RelativePath,
record: DocumentRecord
): Promise<void> {
this.documents.set(path, record);
return this.save();
}
public removeDocument(path: RelativePath): Promise<void> {
public async removeDocument(path: RelativePath): Promise<void> {
this.documents.delete(path);
return this.save();
}
@ -198,11 +232,7 @@ export class SyncEventQueue {
return undefined;
}
public getDocumentByDocumentIdOrFail(
target: DocumentId
): DocumentWithPath {
public getDocumentByDocumentIdOrFail(target: DocumentId): DocumentWithPath {
const result = this.getDocumentByDocumentId(target);
if (!result) {
throw new Error(`No document found with id ${target}`);
@ -210,10 +240,6 @@ export class SyncEventQueue {
return result;
}
public async save(): Promise<void> {
return this.saveData({
documents: Array.from(this.documents.entries()).map(
@ -227,16 +253,16 @@ export class SyncEventQueue {
}
// todo: let's remove
public getSettledDocumentByPath(path: RelativePath): DocumentRecord | undefined {
public getSettledDocumentByPath(
path: RelativePath
): DocumentRecord | undefined {
return this.documents.get(path);
}
public allSettledDocuments(): Map<RelativePath, DocumentRecord> {
return new Map(this.documents.entries());
}
public hasPendingEventsForPath(path: RelativePath): boolean {
const record = this.documents.get(path);
if (record === undefined) {
@ -252,7 +278,8 @@ export class SyncEventQueue {
e.documentId === docId) ||
(e.type === SyncEventType.RemoteChange &&
// we care about the local path not the remote
this.getDocumentByDocumentId(e.remoteVersion.documentId)?.path === path)
this.getDocumentByDocumentId(e.remoteVersion.documentId)
?.path === path)
);
}
@ -266,11 +293,10 @@ export class SyncEventQueue {
);
}
public async clearAllState(): Promise<void> {
this.clearPending();
this.documents.clear();
this._lastSeenUpdateId.reset()
this._lastSeenUpdateId.reset();
await this.save();
}
@ -279,29 +305,6 @@ export class SyncEventQueue {
this.events.length = 0;
}
private updatePendingCreatePath(
oldPath: RelativePath,
newPath: RelativePath
): void {
const createEvent = this.findLatestCreateForPath(oldPath);
if (createEvent === undefined) return;
const promise = createEvent.resolvers?.promise;
createEvent.path = newPath;
if (promise !== undefined) {
for (const e of this.events) {
if (
e.type === SyncEventType.LocalUpdate &&
e.documentId === promise
) {
e.path = newPath;
}
}
}
}
public findLatestCreateForPath(
path: RelativePath
): Extract<SyncEvent, { type: SyncEventType.LocalCreate }> | undefined {
@ -314,18 +317,34 @@ export class SyncEventQueue {
return undefined;
}
private updatePendingCreatePath(
oldPath: RelativePath,
newPath: RelativePath
): void {
const createEvent = this.findLatestCreateForPath(oldPath);
if (createEvent === undefined) return;
const { promise } = createEvent.resolvers;
createEvent.path = newPath;
private rejectAllPendingCreates(): void {
for (const event of this.events) {
if (event.type === SyncEventType.LocalCreate && event.resolvers !== undefined) {
event.resolvers.promise.catch(() => { /* suppressed — consumer may not be listening */ });
event.resolvers.reject(new Error("Create was cancelled"));
for (const e of this.events) {
if (
e.type === SyncEventType.LocalUpdate &&
e.documentId === promise
) {
e.path = newPath;
}
}
}
private rejectAllPendingCreates(): void {
for (const event of this.events) {
if (event.type === SyncEventType.LocalCreate) {
event.resolvers.promise.catch(() => {
/* suppressed — consumer may not be listening */
});
event.resolvers.reject(new Error("Create was cancelled"));
}
}
}
}

266
frontend/sync-client/src/sync-operations/syncer.ts
View file

@ -4,12 +4,15 @@ import {
type DocumentRecord,
type SyncEvent,
type RelativePath,
type VaultUpdateId,
type VaultUpdateId
} from "./types";
import type { Logger } from "../tracing/logger";
import { hash } from "../utils/hash";
import type { Settings } from "../persistence/settings";
import { MoveOnConflict, type FileOperations } from "../file-operations/file-operations";
import {
MoveOnConflict,
type FileOperations
} from "../file-operations/file-operations";
import { scheduleOfflineChanges } from "./offline-change-detector";
import { SyncResetError } from "../errors/sync-reset-error";
import type { DocumentVersionWithoutContent } from "../services/types/DocumentVersionWithoutContent";
@ -21,9 +24,7 @@ import type { SyncEventQueue } from "./sync-event-queue";
import type { SyncService } from "../services/sync-service";
import { FileNotFoundError } from "../errors/file-not-found-error";
import { HttpClientError } from "../errors/http-client-error";
import type {
SyncHistory
} from "../tracing/sync-history";
import type { SyncHistory } from "../tracing/sync-history";
import {
SyncStatus,
SyncType,
@ -79,7 +80,10 @@ export class Syncer {
}
public syncLocallyCreatedFile(relativePath: RelativePath): void {
void this.queue.enqueue({ type: SyncEventType.LocalCreate, path: relativePath });
void this.queue.enqueue({
type: SyncEventType.LocalCreate,
path: relativePath
});
this.ensureDraining();
}
@ -90,14 +94,18 @@ export class Syncer {
oldPath?: RelativePath;
relativePath: RelativePath;
}): void {
void this.queue.enqueue({ type: SyncEventType.LocalUpdate, path: relativePath, oldPath });
void this.queue.enqueue({
type: SyncEventType.LocalUpdate,
path: relativePath,
oldPath
});
this.ensureDraining();
}
public syncLocallyDeletedFile(relativePath: RelativePath): void {
void this.queue.enqueue({
type: SyncEventType.LocalDelete,
path: relativePath,
path: relativePath
});
this.ensureDraining();
}
@ -151,7 +159,6 @@ export class Syncer {
}
}
public reset(): void {
this._isFirstSyncStarted = false;
this.queue.clearPending();
@ -162,20 +169,14 @@ export class Syncer {
// fresh scan can only start once the prior one is done.
const current = this.runningScheduleSyncForOfflineChanges;
if (current !== undefined) {
current.finally(() => {
if (
this.runningScheduleSyncForOfflineChanges ===
current
) {
this.runningScheduleSyncForOfflineChanges =
undefined;
void current.finally(() => {
if (this.runningScheduleSyncForOfflineChanges === current) {
this.runningScheduleSyncForOfflineChanges = undefined;
}
});
}
}
private sendHandshakeMessage(): void {
const message: WebSocketClientMessage = {
type: "handshake",
@ -186,8 +187,6 @@ export class Syncer {
this.webSocketManager.sendHandshakeMessage(message);
}
private async internalScheduleSyncForOfflineChanges(): Promise<void> {
this.isScanning = true;
try {
@ -195,10 +194,18 @@ export class Syncer {
await this.drainPromise;
}
await scheduleOfflineChanges(
this.logger, this.operations, this.queue,
(path) => { this.syncLocallyCreatedFile(path); },
(args) => { this.syncLocallyUpdatedFile(args); },
(path) => { this.syncLocallyDeletedFile(path); },
this.logger,
this.operations,
this.queue,
(path) => {
this.syncLocallyCreatedFile(path);
},
(args) => {
this.syncLocallyUpdatedFile(args);
},
(path) => {
this.syncLocallyDeletedFile(path);
}
);
} finally {
this.isScanning = false;
@ -207,9 +214,6 @@ export class Syncer {
this.ensureDraining();
}
private ensureDraining(): void {
if (this.drainPromise !== undefined) return;
if (this.isScanning) return;
@ -218,7 +222,6 @@ export class Syncer {
});
}
private async drain(): Promise<void> {
let event = await this.queue.next();
while (event !== undefined) {
@ -271,8 +274,10 @@ export class Syncer {
`Skipping sync event '${event.type}' because the file no longer exists`
);
if (event.type === SyncEventType.LocalCreate) {
event.resolvers?.promise.catch(() => { });
event.resolvers?.reject(new Error("Create was cancelled"));
event.resolvers.promise.catch(() => {
/* suppressed */
});
event.resolvers.reject(new Error("Create was cancelled"));
}
return;
}
@ -285,10 +290,10 @@ export class Syncer {
// promise would otherwise hang forever, blocking any
// queued Delete / SyncLocal that `await`s it.
if (event.type === SyncEventType.LocalCreate) {
event.resolvers?.promise.catch(() => {
event.resolvers.promise.catch(() => {
/* suppressed */
});
event.resolvers?.reject(
event.resolvers.reject(
new Error(
`Create was cancelled — server rejected the request (${e.message})`
)
@ -300,10 +305,9 @@ export class Syncer {
}
}
private async skipIfOversized(event: SyncEvent): Promise<boolean> {
let sizeInBytes: number;
let relativePath: RelativePath;
let sizeInBytes = 0;
let relativePath: RelativePath = "";
switch (event.type) {
case SyncEventType.LocalDelete:
@ -316,7 +320,7 @@ export class Syncer {
case SyncEventType.RemoteChange:
if (event.remoteVersion.isDeleted) return false;
sizeInBytes = event.remoteVersion.contentSize;
relativePath = event.remoteVersion.relativePath;
({ relativePath } = event.remoteVersion);
break;
}
@ -329,8 +333,10 @@ export class Syncer {
this.history.addHistoryEntry(oversizedEntry);
if (event.type === SyncEventType.LocalCreate) {
event.resolvers?.promise.catch(() => { });
event.resolvers?.reject(new Error("Create was cancelled"));
event.resolvers.promise.catch(() => {
/* suppressed */
});
event.resolvers.reject(new Error("Create was cancelled"));
}
return true;
@ -354,9 +360,6 @@ export class Syncer {
}
}
private async processCreate(
event: Extract<SyncEvent, { type: SyncEventType.LocalCreate }>
): Promise<void> {
@ -378,13 +381,13 @@ export class Syncer {
createEvent: event
});
this.history.addHistoryEntry({
status: SyncStatus.SUCCESS,
details: { type: SyncType.CREATE, relativePath: effectivePath },
message: response.type === "MergingUpdate"
? "Created file and merged with existing remote version"
: "Successfully created file on the server",
message:
response.type === "MergingUpdate"
? "Created file and merged with existing remote version"
: "Successfully created file on the server",
author: response.userId,
timestamp: new Date(response.updatedDate)
});
@ -393,7 +396,7 @@ export class Syncer {
private async processDelete(
event: Extract<SyncEvent, { type: SyncEventType.LocalDelete }>
): Promise<void> {
let documentId = await event.documentId;
const documentId = await event.documentId;
const doc = this.queue.getDocumentByDocumentIdOrFail(documentId);
const relativePath = doc.path;
@ -406,7 +409,6 @@ export class Syncer {
await this.queue.removeDocument(doc.path);
this.queue.lastSeenUpdateId = response.vaultUpdateId;
this.history.addHistoryEntry({
status: SyncStatus.SUCCESS,
details: {
@ -421,16 +423,16 @@ export class Syncer {
private async processLocalUpdate(
event: Extract<SyncEvent, { type: SyncEventType.LocalUpdate }>
): Promise<void> {
let documentId = await event.documentId;
const documentId = await event.documentId;
const { path: diskPath, record } = this.queue.getDocumentByDocumentIdOrFail(documentId);
const { path: diskPath, record } =
this.queue.getDocumentByDocumentIdOrFail(documentId);
const contentBytes = await this.operations.read(diskPath);
const contentHash = await hash(contentBytes);
const hashChanged = contentHash !== record.remoteHash;
const pathChanged =
record.remoteRelativePath !== event.originalPath;
const pathChanged = record.remoteRelativePath !== event.originalPath;
if (!hashChanged && !pathChanged) {
this.logger.debug(
@ -443,12 +445,10 @@ export class Syncer {
record,
relativePath: event.originalPath,
contentBytes
}
);
});
this.queue.lastSeenUpdateId = response.vaultUpdateId;
await this.handleMaybeMergingResponse({
path: diskPath,
response,
@ -456,9 +456,7 @@ export class Syncer {
originalContentBytes: contentBytes
});
const isMerge =
"type" in response && response.type === "MergingUpdate";
const isMerge = "type" in response && response.type === "MergingUpdate";
this.history.addHistoryEntry({
status: SyncStatus.SUCCESS,
details: {
@ -489,12 +487,12 @@ export class Syncer {
// response)
createEvent?: Extract<SyncEvent, { type: SyncEventType.LocalCreate }>;
}): Promise<void> {
let record = {
const record = {
documentId: response.documentId,
parentVersionId: response.vaultUpdateId,
remoteRelativePath: response.relativePath
};
let remoteHash: string;
let remoteHash = "";
if ("type" in response && response.type === "MergingUpdate") {
const responseBytes = base64ToBytes(response.contentBase64);
@ -506,11 +504,7 @@ export class Syncer {
remoteHash = await hash(responseBytes);
await this.updateCache(
response.vaultUpdateId,
responseBytes,
path
);
await this.updateCache(response.vaultUpdateId, responseBytes, path);
} else {
// Fast-forward update: no merge needed
remoteHash = contentHash;
@ -524,13 +518,16 @@ export class Syncer {
if (createEvent === undefined) {
// a http response will always be more up-to-date than any queued remote update
this.operations.move(path, response.relativePath, MoveOnConflict.EXISTING);
await this.operations.move(
path,
response.relativePath,
MoveOnConflict.EXISTING
);
await this.queue.setDocument(response.relativePath, {
...record,
remoteHash
});
} else {
// The response to a create must contain the path from the create request
await this.queue.resolveCreate(createEvent, {
@ -542,7 +539,6 @@ export class Syncer {
this.queue.lastSeenUpdateId = response.vaultUpdateId;
}
private async processRemoteChange(
event: Extract<SyncEvent, { type: SyncEventType.RemoteChange }>
): Promise<void> {
@ -556,10 +552,16 @@ export class Syncer {
// trying to delete a document we've already scheduled for deletion locally
return;
}
return this.processRemoteDelete(documentWithPath.path, remoteVersion);
return this.processRemoteDelete(
documentWithPath.path,
remoteVersion
);
}
if (documentWithPath?.record.parentVersionId ?? 0 >= remoteVersion.vaultUpdateId) {
if (
(documentWithPath?.record.parentVersionId ?? 0) >=
remoteVersion.vaultUpdateId
) {
this.queue.lastSeenUpdateId = remoteVersion.vaultUpdateId;
this.logger.debug(
`Document ${remoteVersion.relativePath} is already up-to-date or has newer local changes; skipping remote update`
@ -569,26 +571,36 @@ export class Syncer {
if (documentWithPath !== undefined) {
// must be the update to an existing doc
return this.processRemoteUpdate(documentWithPath.path, documentWithPath.record, remoteVersion);
return this.processRemoteUpdate(
documentWithPath.path,
documentWithPath.record,
remoteVersion
);
}
const pendingCreate = this.queue.findLatestCreateForPath(remoteVersion.relativePath);
const pendingCreate = this.queue.findLatestCreateForPath(
remoteVersion.relativePath
);
if (pendingCreate === undefined) {
return this.processRemoteCreateForNewDocument(remoteVersion);
} else {
return this.processRemoteCreateForPendingDocument(remoteVersion, pendingCreate);
return this.processRemoteCreateForPendingDocument(
remoteVersion,
pendingCreate
);
}
}
private async processRemoteDelete(path: RelativePath, remoteVersion: DocumentVersionWithoutContent): Promise<void> {
private async processRemoteDelete(
path: RelativePath,
remoteVersion: DocumentVersionWithoutContent
): Promise<void> {
await this.operations.delete(path);
await this.queue.removeDocument(path);
this.queue.lastSeenUpdateId = remoteVersion.vaultUpdateId;
this.history.addHistoryEntry({
status: SyncStatus.SUCCESS,
details: {
@ -602,22 +614,29 @@ export class Syncer {
});
}
private async processRemoteUpdate(path: RelativePath, record: DocumentRecord, remoteVersion: DocumentVersionWithoutContent): Promise<void> {
if (
record.parentVersionId >=
remoteVersion.vaultUpdateId
) {
this.logger.debug(
`Document ${path} is already up-to-date`
);
private async processRemoteUpdate(
path: RelativePath,
record: DocumentRecord,
remoteVersion: DocumentVersionWithoutContent
): Promise<void> {
if (record.parentVersionId >= remoteVersion.vaultUpdateId) {
this.logger.debug(`Document ${path} is already up-to-date`);
return;
}
if (!this.queue.hasPendingLocalEventsForDocumentId(remoteVersion.documentId)) {
if (
!this.queue.hasPendingLocalEventsForDocumentId(
remoteVersion.documentId
)
) {
// no local changes
const currentContent = await this.operations.read(path);
const remoteContent = await this.syncService.getDocumentVersionContent({ documentId: remoteVersion.documentId, vaultUpdateId: remoteVersion.vaultUpdateId });
this.operations.write(path, currentContent, remoteContent);
const remoteContent =
await this.syncService.getDocumentVersionContent({
documentId: remoteVersion.documentId,
vaultUpdateId: remoteVersion.vaultUpdateId
});
await this.operations.write(path, currentContent, remoteContent);
await this.updateCache(
remoteVersion.vaultUpdateId,
@ -625,20 +644,26 @@ export class Syncer {
path
);
this.queue.lastSeenUpdateId = remoteVersion.vaultUpdateId;
} // else we don't need to update the content, a subsequent local update will do that
} // else we don't need to update the content, a subsequent local update will do that
this.syncRemotelyUpdatedFile({ // schedule it so that the lastSeenUpdateId remains consistent
document:
remoteVersion
})
void this.syncRemotelyUpdatedFile({
// schedule it so that the lastSeenUpdateId remains consistent
document: remoteVersion
});
// wait for a local edit to do the actual updating here, so we can't even update the lastSeenUpdateId here
const conflictingDoc = this.queue.getSettledDocumentByPath(remoteVersion.relativePath);
const actualRelativePath = await this.operations.move(path, remoteVersion.relativePath, conflictingDoc?.parentVersionId ?? 0 < remoteVersion.vaultUpdateId ? MoveOnConflict.EXISTING : MoveOnConflict.NEW);
const conflictingDoc = this.queue.getSettledDocumentByPath(
remoteVersion.relativePath
);
const actualRelativePath = await this.operations.move(
path,
remoteVersion.relativePath,
(conflictingDoc?.parentVersionId ?? 0) < remoteVersion.vaultUpdateId
? MoveOnConflict.EXISTING
: MoveOnConflict.NEW
);
this.queue.setDocument(actualRelativePath, {
await this.queue.setDocument(actualRelativePath, {
...record,
remoteRelativePath: actualRelativePath
});
@ -651,22 +676,28 @@ export class Syncer {
movedFrom: path
},
// todo: eh
message: `File was renamed remotely from ${path} to ${actualRelativePath}`,
message: `File was renamed remotely from ${path} to ${actualRelativePath}`
});
}
private async processRemoteCreateForNewDocument(remoteVersion: DocumentVersionWithoutContent): Promise<void> {
private async processRemoteCreateForNewDocument(
remoteVersion: DocumentVersionWithoutContent
): Promise<void> {
const remoteContent = await this.syncService.getDocumentVersionContent({
documentId: remoteVersion.documentId,
vaultUpdateId: remoteVersion.vaultUpdateId
});
const conflictingDoc = this.queue.getSettledDocumentByPath(remoteVersion.relativePath);
const conflictingDoc = this.queue.getSettledDocumentByPath(
remoteVersion.relativePath
);
const actualPath = await this.operations.create(
remoteVersion.relativePath,
remoteContent,
conflictingDoc?.parentVersionId ?? 0 < remoteVersion.vaultUpdateId ? MoveOnConflict.EXISTING : MoveOnConflict.NEW
(conflictingDoc?.parentVersionId ?? 0) < remoteVersion.vaultUpdateId
? MoveOnConflict.EXISTING
: MoveOnConflict.NEW
);
await this.updateCache(
@ -703,7 +734,10 @@ export class Syncer {
// We must avoid duplicating files.
private async processRemoteCreateForPendingDocument(
remoteVersion: DocumentVersionWithoutContent,
pendingCreateEvent: Extract<SyncEvent, { type: SyncEventType.LocalCreate }>
pendingCreateEvent: Extract<
SyncEvent,
{ type: SyncEventType.LocalCreate }
>
): Promise<void> {
const remoteContent = await this.syncService.getDocumentVersionContent({
documentId: remoteVersion.documentId,
@ -712,7 +746,9 @@ export class Syncer {
const remoteHash = await hash(remoteContent);
const path = remoteVersion.relativePath;
const currentContent = await this.operations.read(pendingCreateEvent.path);
const currentContent = await this.operations.read(
pendingCreateEvent.path
);
await this.operations.write(path, currentContent, remoteContent);
await this.updateCache(
@ -735,25 +771,21 @@ export class Syncer {
type: SyncType.UPDATE,
relativePath: path
},
message:
`Adopted remote create at ${path}`,
message: `Adopted remote create at ${path}`,
author: remoteVersion.userId,
timestamp: new Date(remoteVersion.updatedDate)
});
}
private async sendUpdate(
{ record, relativePath, contentBytes }: {
record: DocumentRecord,
relativePath: RelativePath,
contentBytes: Uint8Array
}
): Promise<DocumentUpdateResponse> {
private async sendUpdate({
record,
relativePath,
contentBytes
}: {
record: DocumentRecord;
relativePath: RelativePath;
contentBytes: Uint8Array;
}): Promise<DocumentUpdateResponse> {
const isText =
!isBinary(contentBytes) &&
isFileTypeMergable(
@ -783,8 +815,6 @@ export class Syncer {
});
}
private async updateCache(
updateId: VaultUpdateId,
contentBytes: Uint8Array,

47
frontend/sync-client/src/sync-operations/types.ts
View file

@ -29,36 +29,41 @@ export enum SyncEventType {
LocalCreate = "local-create",
LocalUpdate = "local-update", // includes both content and path changes
LocalDelete = "local-delete",
RemoteChange = "remote-change", // includes every type of create/update/delete coming from the server
RemoteChange = "remote-change" // includes every type of create/update/delete coming from the server
}
export type FileSyncEvent =
| { type: SyncEventType.LocalCreate; path: RelativePath }
| {
type: SyncEventType.LocalUpdate; path: RelativePath; oldPath?: RelativePath // oldPath is undefined for content changes
}
type: SyncEventType.LocalUpdate;
path: RelativePath;
oldPath?: RelativePath; // oldPath is undefined for content changes
}
| { type: SyncEventType.LocalDelete; path: RelativePath }
| { type: SyncEventType.RemoteChange; remoteVersion: DocumentVersionWithoutContent };
| {
type: SyncEventType.RemoteChange;
remoteVersion: DocumentVersionWithoutContent;
};
export type SyncEvent =
| {
type: SyncEventType.LocalCreate;
path: RelativePath; // current path on disk
originalPath: RelativePath; // original path on disk when the event was queued
resolvers: PromiseWithResolvers<DocumentId>
}
type: SyncEventType.LocalCreate;
path: RelativePath; // current path on disk
originalPath: RelativePath; // original path on disk when the event was queued
resolvers: PromiseWithResolvers<DocumentId>;
}
| {
type: SyncEventType.LocalUpdate;
documentId: DocumentId | Promise<DocumentId>; // if it's a promise, the promise is fulfilled once the document's create event is processed
path: RelativePath; // current path on disk
originalPath: RelativePath; // original path on disk when the event was queued
// no need to store the old path in case of a rename; the server will figure it out from the parent's path
}
type: SyncEventType.LocalUpdate;
documentId: DocumentId | Promise<DocumentId>; // if it's a promise, the promise is fulfilled once the document's create event is processed
path: RelativePath; // current path on disk
originalPath: RelativePath; // original path on disk when the event was queued
// no need to store the old path in case of a rename; the server will figure it out from the parent's path
}
| {
type: SyncEventType.LocalDelete;
documentId: DocumentId | Promise<DocumentId>; // if it's a promise, the promise is fulfilled once the document's create event is processed
}
type: SyncEventType.LocalDelete;
documentId: DocumentId | Promise<DocumentId>; // if it's a promise, the promise is fulfilled once the document's create event is processed
}
| {
type: SyncEventType.RemoteChange;
remoteVersion: DocumentVersionWithoutContent;
};
type: SyncEventType.RemoteChange;
remoteVersion: DocumentVersionWithoutContent;
};