diff --git a/frontend/sync-client/src/sync-operations/sync-event-queue.test.ts b/frontend/sync-client/src/sync-operations/sync-event-queue.test.ts index d2e32268..8c4d68ab 100644 --- a/frontend/sync-client/src/sync-operations/sync-event-queue.test.ts +++ b/frontend/sync-client/src/sync-operations/sync-event-queue.test.ts @@ -8,8 +8,8 @@ import { SyncEventType } from "./types"; function createQueue(ignorePatterns: string[] = []): SyncEventQueue { const logger = new Logger(); - const settings = new Settings(logger, { ignorePatterns }, async () => {}); - return new SyncEventQueue(settings, logger, undefined, async () => {}); + const settings = new Settings(logger, { ignorePatterns }, async () => { }); + return new SyncEventQueue(settings, logger, undefined, async () => { }); } function fakeRemoteVersion( @@ -30,48 +30,47 @@ function fakeRemoteVersion( } describe("SyncEventQueue", () => { - it("sync-local followed by delete for the same document returns only the delete", () => { + it("sync-local followed by delete for the same document returns only the delete", async () => { const queue = createQueue(); queue.setDocument("a.md", { documentId: "A", parentVersionId: 1, - hash: "hash-a" + remoteHash: "hash-a" }); - queue.enqueue({ type: SyncEventType.SyncLocal, documentId: "A" }); - queue.enqueue({ type: SyncEventType.SyncLocal, documentId: "A" }); + queue.enqueue({ type: SyncEventType.SyncLocal, documentId: "A", path: "a.md", originalPath: "a.md" }); + queue.enqueue({ type: SyncEventType.SyncLocal, documentId: "A", path: "a.md", originalPath: "a.md" }); queue.enqueue({ type: SyncEventType.Delete, documentId: "A", - path: "a.md", }); - const event = queue.next(); + const event = await queue.next(); assert.strictEqual(event?.type, SyncEventType.Delete); if (event?.type === SyncEventType.Delete) { assert.strictEqual(event.documentId, "A"); } - assert.strictEqual(queue.next(), undefined); + assert.strictEqual(await queue.next(), undefined); }); - it("sync-local events for the same document coalesce to one", () => { + it("sync-local events for the same document coalesce to one", async () => { const queue = createQueue(); queue.setDocument("a.md", { documentId: "A", parentVersionId: 1, - hash: "hash-a" + remoteHash: "hash-a" }); - queue.enqueue({ type: SyncEventType.SyncLocal, documentId: "A" }); - queue.enqueue({ type: SyncEventType.SyncLocal, documentId: "A" }); - queue.enqueue({ type: SyncEventType.SyncLocal, documentId: "A" }); + queue.enqueue({ type: SyncEventType.SyncLocal, documentId: "A", path: "a.md", originalPath: "a.md" }); + queue.enqueue({ type: SyncEventType.SyncLocal, documentId: "A", path: "a.md", originalPath: "a.md" }); + queue.enqueue({ type: SyncEventType.SyncLocal, documentId: "A", path: "a.md", originalPath: "a.md" }); - const event = queue.next(); + const event = await queue.next(); assert.strictEqual(event?.type, SyncEventType.SyncLocal); - assert.strictEqual(queue.next(), undefined); + assert.strictEqual(await queue.next(), undefined); }); - it("sync-remote events for the same documentId coalesce to the last one", () => { + it("sync-remote events for the same documentId coalesce to the last one", async () => { const queue = createQueue(); queue.enqueue({ @@ -87,116 +86,63 @@ describe("SyncEventQueue", () => { remoteVersion: fakeRemoteVersion("A", { vaultUpdateId: 3 }) }); - const event = queue.next(); + const event = await queue.next(); assert.strictEqual(event?.type, SyncEventType.SyncRemote); if (event?.type === SyncEventType.SyncRemote) { assert.strictEqual(event.remoteVersion.vaultUpdateId, 3); } - assert.strictEqual(queue.next(), undefined); + assert.strictEqual(await queue.next(), undefined); }); - it("create events are returned FIFO", () => { + it("create events are returned FIFO", async () => { const queue = createQueue(); - queue.enqueue({ type: SyncEventType.Create, path: "a.md" }); - queue.enqueue({ type: SyncEventType.Create, path: "b.md" }); + queue.enqueue({ type: SyncEventType.Create, path: "a.md", originalPath: "a.md" }); + queue.enqueue({ type: SyncEventType.Create, path: "b.md", originalPath: "b.md" }); - const first = queue.next(); + const first = await queue.next(); assert.strictEqual(first?.type, SyncEventType.Create); if (first?.type === SyncEventType.Create) { assert.strictEqual(first.path, "a.md"); } - const second = queue.next(); + const second = await queue.next(); assert.strictEqual(second?.type, SyncEventType.Create); if (second?.type === SyncEventType.Create) { assert.strictEqual(second.path, "b.md"); } }); - it("duplicate creates for the same path are skipped", () => { - const queue = createQueue(); - queue.enqueue({ type: SyncEventType.Create, path: "a.md" }); - queue.enqueue({ type: SyncEventType.Create, path: "a.md" }); - assert.strictEqual(queue.size, 1); - }); - - it("create is skipped if the path already has a tracked document", () => { - const queue = createQueue(); - queue.setDocument("a.md", { - documentId: "A", - parentVersionId: 1, - hash: "hash-a" - }); - - queue.enqueue({ type: SyncEventType.Create, path: "a.md" }); - assert.strictEqual(queue.size, 0); - }); - - it("delete uses the provided documentId", () => { + it("delete uses the provided documentId", async () => { const queue = createQueue(); queue.enqueue({ type: SyncEventType.Delete, documentId: "A", - path: "a.md", }); - const event = queue.next(); + const event = await queue.next(); assert.strictEqual(event?.type, SyncEventType.Delete); if (event?.type === SyncEventType.Delete) { assert.strictEqual(event.documentId, "A"); } }); - it("updateCreatePath updates the path of a create event in the queue", () => { - const queue = createQueue(); - queue.enqueue({ type: SyncEventType.Create, path: "old.md" }); - - const updated = queue.updateCreatePath("old.md", "new.md"); - assert.strictEqual(updated, true); - assert.strictEqual(queue.hasCreateEvent("old.md"), false); - assert.strictEqual(queue.hasCreateEvent("new.md"), true); - - const event = queue.next(); - assert.strictEqual(event?.type, SyncEventType.Create); - if (event?.type === SyncEventType.Create) { - assert.strictEqual(event.path, "new.md"); - } - }); - - it("updateCreatePath returns false when no create event exists", () => { - const queue = createQueue(); - const updated = queue.updateCreatePath("old.md", "new.md"); - assert.strictEqual(updated, false); - }); - - it("hasCreateEvent detects pending creates", () => { - const queue = createQueue(); - assert.strictEqual(queue.hasCreateEvent("a.md"), false); - - queue.enqueue({ type: SyncEventType.Create, path: "a.md" }); - assert.strictEqual(queue.hasCreateEvent("a.md"), true); - - queue.next(); - assert.strictEqual(queue.hasCreateEvent("a.md"), false); - }); - it("document store CRUD operations work correctly", () => { const queue = createQueue(); - assert.strictEqual(queue.getDocument("a.md"), undefined); + assert.strictEqual(queue.getSettledDocumentByPath("a.md"), undefined); assert.strictEqual(queue.documentCount, 0); queue.setDocument("a.md", { documentId: "A", parentVersionId: 1, - hash: "hash-a" + remoteHash: "hash-a" }); assert.strictEqual(queue.documentCount, 1); - assert.deepStrictEqual(queue.getDocument("a.md"), { + assert.deepStrictEqual(queue.getSettledDocumentByPath("a.md"), { documentId: "A", parentVersionId: 1, - hash: "hash-a" + remoteHash: "hash-a" }); const found = queue.getDocumentByDocumentId("A"); @@ -205,7 +151,7 @@ describe("SyncEventQueue", () => { queue.removeDocument("a.md"); assert.strictEqual(queue.documentCount, 0); - assert.strictEqual(queue.getDocument("a.md"), undefined); + assert.strictEqual(queue.getSettledDocumentByPath("a.md"), undefined); }); it("moveDocument moves a document and returns displaced documentId", () => { @@ -213,18 +159,18 @@ describe("SyncEventQueue", () => { queue.setDocument("a.md", { documentId: "A", parentVersionId: 1, - hash: "hash-a" + remoteHash: "hash-a" }); queue.setDocument("b.md", { documentId: "B", parentVersionId: 2, - hash: "hash-b" + remoteHash: "hash-b" }); const displacedId = queue.moveDocument("a.md", "b.md"); assert.strictEqual(displacedId, "B"); - assert.strictEqual(queue.getDocument("a.md"), undefined); - assert.strictEqual(queue.getDocument("b.md")?.documentId, "A"); + assert.strictEqual(queue.getSettledDocumentByPath("a.md"), undefined); + assert.strictEqual(queue.getSettledDocumentByPath("b.md")?.documentId, "A"); assert.strictEqual(queue.documentCount, 1); }); @@ -233,140 +179,193 @@ describe("SyncEventQueue", () => { queue.setDocument("a.md", { documentId: "A", parentVersionId: 1, - hash: "hash-a" + remoteHash: "hash-a" }); const displacedId = queue.moveDocument("a.md", "b.md"); assert.strictEqual(displacedId, undefined); - assert.strictEqual(queue.getDocument("b.md")?.documentId, "A"); + assert.strictEqual(queue.getSettledDocumentByPath("b.md")?.documentId, "A"); }); - it("interleaved events for different documents are not confused", () => { + it("interleaved events for different documents are not confused", async () => { const queue = createQueue(); queue.setDocument("a.md", { documentId: "A", parentVersionId: 1, - hash: "hash-a" + remoteHash: "hash-a" }); queue.setDocument("b.md", { documentId: "B", parentVersionId: 2, - hash: "hash-b" + remoteHash: "hash-b" }); - queue.enqueue({ type: SyncEventType.SyncLocal, documentId: "A" }); - queue.enqueue({ type: SyncEventType.SyncLocal, documentId: "B" }); + queue.enqueue({ type: SyncEventType.SyncLocal, documentId: "A", path: "a.md", originalPath: "a.md" }); + queue.enqueue({ type: SyncEventType.SyncLocal, documentId: "B", path: "b.md", originalPath: "b.md" }); queue.enqueue({ type: SyncEventType.Delete, documentId: "A", - path: "a.md", }); - queue.enqueue({ type: SyncEventType.SyncLocal, documentId: "B" }); + queue.enqueue({ type: SyncEventType.SyncLocal, documentId: "B", path: "b.md", originalPath: "b.md" }); // First next() should see the delete for A (coalescing sync-local + delete) - const first = queue.next(); + const first = await queue.next(); assert.strictEqual(first?.type, SyncEventType.Delete); if (first?.type === SyncEventType.Delete) { assert.strictEqual(first.documentId, "A"); } // Remaining should be the coalesced sync-local for B - const second = queue.next(); + const second = await queue.next(); assert.strictEqual(second?.type, SyncEventType.SyncLocal); if (second?.type === SyncEventType.SyncLocal) { assert.strictEqual(second.documentId, "B"); } - assert.strictEqual(queue.next(), undefined); + assert.strictEqual(await queue.next(), undefined); }); - it("delete discards subsequent sync-remote events for the same document", () => { + it("delete discards subsequent sync-remote events for the same document", async () => { const queue = createQueue(); queue.enqueue({ type: SyncEventType.Delete, documentId: "A", - path: "a.md", }); queue.enqueue({ type: SyncEventType.SyncRemote, remoteVersion: fakeRemoteVersion("A", { vaultUpdateId: 5 }) }); - const event = queue.next(); + const event = await queue.next(); assert.strictEqual(event?.type, SyncEventType.Delete); - assert.strictEqual(queue.next(), undefined); + assert.strictEqual(await queue.next(), undefined); }); - it("delete discards subsequent sync-local and sync-remote for the same document", () => { + it("delete discards subsequent sync-local and sync-remote for the same document", async () => { const queue = createQueue(); queue.setDocument("a.md", { documentId: "A", parentVersionId: 1, - hash: "hash-a" + remoteHash: "hash-a" }); queue.enqueue({ type: SyncEventType.Delete, documentId: "A", - path: "a.md", }); - queue.enqueue({ type: SyncEventType.SyncLocal, documentId: "A" }); - queue.enqueue({ type: SyncEventType.Create, path: "b.md" }); + queue.enqueue({ type: SyncEventType.SyncLocal, documentId: "A", path: "a.md", originalPath: "a.md" }); + queue.enqueue({ type: SyncEventType.Create, path: "b.md", originalPath: "b.md" }); queue.enqueue({ type: SyncEventType.SyncRemote, remoteVersion: fakeRemoteVersion("A", { vaultUpdateId: 5 }) }); - const first = queue.next(); + const first = await queue.next(); assert.strictEqual(first?.type, SyncEventType.Delete); // Only the unrelated create should remain - const second = queue.next(); + const second = await queue.next(); assert.strictEqual(second?.type, SyncEventType.Create); - assert.strictEqual(queue.next(), undefined); + assert.strictEqual(await queue.next(), undefined); }); - it("delete with empty documentId does not discard other events", () => { + it("delete with promise documentId does not discard other events", async () => { const queue = createQueue(); queue.setDocument("a.md", { documentId: "A", parentVersionId: 1, - hash: "hash-a" + remoteHash: "hash-a" }); + queue.enqueue({ type: SyncEventType.Create, path: "unknown.md", originalPath: "unknown.md" }); + const createPromise = queue.getCreatePromise("unknown.md"); + assert.ok(createPromise !== undefined); + const event = await queue.next(); // dequeue the create + assert.ok(event?.type === SyncEventType.Create); + // Resolve so the delete's await doesn't hang + event.resolvers!.resolve("NEW"); + queue.enqueue({ type: SyncEventType.Delete, - documentId: "", - path: "unknown.md", + documentId: createPromise, }); - queue.enqueue({ type: SyncEventType.SyncLocal, documentId: "A" }); + queue.enqueue({ type: SyncEventType.SyncLocal, documentId: "A", path: "a.md", originalPath: "a.md" }); - queue.next(); - const second = queue.next(); + await queue.next(); // delete + const second = await queue.next(); assert.strictEqual(second?.type, SyncEventType.SyncLocal); }); - it("create can be re-enqueued after being dequeued", () => { + it("getCreatePromise returns a promise resolved by the event's resolvers", async () => { const queue = createQueue(); - queue.enqueue({ type: SyncEventType.Create, path: "a.md" }); - queue.next(); + queue.enqueue({ type: SyncEventType.Create, path: "a.md", originalPath: "a.md" }); - queue.enqueue({ type: SyncEventType.Create, path: "a.md" }); + const promise = queue.getCreatePromise("a.md"); + assert.ok(promise !== undefined); + + // The syncer resolves via event.resolvers after dequeuing + const event = await queue.next(); + assert.ok(event?.type === SyncEventType.Create); + assert.ok(event.resolvers !== undefined); + event.resolvers.resolve("resolved-id"); + + assert.strictEqual(await promise, "resolved-id"); + }); + + it("rejecting the event's resolvers rejects the create promise", async () => { + const queue = createQueue(); + queue.enqueue({ type: SyncEventType.Create, path: "a.md", originalPath: "a.md" }); + + const promise = queue.getCreatePromise("a.md"); + assert.ok(promise !== undefined); + + const event = await queue.next(); + assert.ok(event?.type === SyncEventType.Create); + assert.ok(event.resolvers !== undefined); + event.resolvers.promise.catch(() => { }); + event.resolvers.reject(new Error("cancelled")); + + await assert.rejects(promise); + }); + + it("clear rejects all pending create promises", async () => { + const queue = createQueue(); + queue.enqueue({ type: SyncEventType.Create, path: "a.md", originalPath: "a.md" }); + queue.enqueue({ type: SyncEventType.Create, path: "b.md", originalPath: "b.md" }); + + const promiseA = queue.getCreatePromise("a.md"); + const promiseB = queue.getCreatePromise("b.md"); + assert.ok(promiseA !== undefined); + assert.ok(promiseB !== undefined); + + queue.clear(); + + await assert.rejects(promiseA); + await assert.rejects(promiseB); + }); + + it("create can be re-enqueued after being dequeued", async () => { + const queue = createQueue(); + queue.enqueue({ type: SyncEventType.Create, path: "a.md", originalPath: "a.md" }); + await queue.next(); + + queue.enqueue({ type: SyncEventType.Create, path: "a.md", originalPath: "a.md" }); assert.strictEqual(queue.size, 1); }); it("silently ignores create events matching ignore patterns", () => { const queue = createQueue(["*.tmp", ".hidden/**"]); - queue.enqueue({ type: SyncEventType.Create, path: "scratch.tmp" }); + queue.enqueue({ type: SyncEventType.Create, path: "scratch.tmp", originalPath: "scratch.tmp" }); queue.enqueue({ type: SyncEventType.Create, path: ".hidden/secret.md", + originalPath: ".hidden/secret.md", }); assert.strictEqual(queue.size, 0); - queue.enqueue({ type: SyncEventType.Create, path: "notes-new.md" }); + queue.enqueue({ type: SyncEventType.Create, path: "notes-new.md", originalPath: "notes-new.md" }); assert.strictEqual(queue.size, 1); queue.enqueue({ @@ -381,10 +380,10 @@ describe("SyncEventQueue", () => { queue.setDocument("a.md", { documentId: "A", parentVersionId: 1, - hash: "hash-a" + remoteHash: "hash-a" }); - queue.enqueue({ type: SyncEventType.Create, path: "b.md" }); - queue.enqueue({ type: SyncEventType.SyncLocal, documentId: "A" }); + queue.enqueue({ type: SyncEventType.Create, path: "b.md", originalPath: "b.md" }); + queue.enqueue({ type: SyncEventType.SyncLocal, documentId: "A", path: "a.md", originalPath: "a.md" }); assert.strictEqual(queue.size, 2); @@ -392,7 +391,7 @@ describe("SyncEventQueue", () => { assert.strictEqual(queue.size, 0); assert.strictEqual(queue.documentCount, 1); - assert.strictEqual(queue.getDocument("a.md")?.documentId, "A"); + assert.strictEqual(queue.getSettledDocumentByPath("a.md")?.documentId, "A"); }); it("allDocuments returns all tracked documents", () => { @@ -400,15 +399,15 @@ describe("SyncEventQueue", () => { queue.setDocument("a.md", { documentId: "A", parentVersionId: 1, - hash: "hash-a" + remoteHash: "hash-a" }); queue.setDocument("b.md", { documentId: "B", parentVersionId: 2, - hash: "hash-b" + remoteHash: "hash-b" }); - const docs = queue.allDocuments(); + const docs = queue.allSettledDocuments(); assert.strictEqual(docs.length, 2); const paths = docs.map(([p]) => p).sort(); assert.deepStrictEqual(paths, ["a.md", "b.md"]); @@ -416,28 +415,157 @@ describe("SyncEventQueue", () => { it("loads initial state from persistence", () => { const logger = new Logger(); - const settings = new Settings(logger, {}, async () => {}); + const settings = new Settings(logger, {}, async () => { }); const queue = new SyncEventQueue(settings, logger, { documents: [ { relativePath: "a.md", documentId: "A", parentVersionId: 5, - hash: "hash-a" + remoteHash: "hash-a" }, { relativePath: "b.md", documentId: "B", parentVersionId: 3, - hash: "hash-b" + remoteHash: "hash-b" } ], lastSeenUpdateId: 4 - }, async () => {}); + }, async () => { }); assert.strictEqual(queue.documentCount, 2); - assert.strictEqual(queue.getDocument("a.md")?.documentId, "A"); - assert.strictEqual(queue.getDocument("b.md")?.documentId, "B"); - assert.strictEqual(queue.getLastSeenUpdateId(), 5); + assert.strictEqual(queue.getSettledDocumentByPath("a.md")?.documentId, "A"); + assert.strictEqual(queue.getSettledDocumentByPath("b.md")?.documentId, "B"); + assert.strictEqual(queue.lastSeenUpdateId, 5); + }); + + it("trackedPaths combines documents and pending events", () => { + const queue = createQueue(); + queue.setDocument("a.md", { + documentId: "A", + parentVersionId: 1, + remoteHash: "hash-a" + }); + queue.setDocument("b.md", { + documentId: "B", + parentVersionId: 2, + remoteHash: "hash-b" + }); + + // Pending create adds a path + queue.enqueue({ type: SyncEventType.Create, path: "c.md", originalPath: "c.md" }); + // Pending delete removes a path + queue.enqueue({ + type: SyncEventType.Delete, + documentId: "A", + }); + + const paths = queue.trackedPaths(); + assert.deepStrictEqual( + [...paths].sort(), + ["b.md", "c.md"] + ); + }); + + it("trackedPaths handles create-delete-create for the same path", () => { + const queue = createQueue(); + + queue.enqueue({ type: SyncEventType.Create, path: "a.md", originalPath: "a.md" }); + queue.enqueue({ + type: SyncEventType.Delete, + documentId: Promise.resolve("X"), + }); + queue.enqueue({ type: SyncEventType.Create, path: "a.md", originalPath: "a.md" }); + + const paths = queue.trackedPaths(); + assert.ok(paths.has("a.md")); + }); + + it("trackedPaths applies moves for promise-based SyncLocal events", () => { + const queue = createQueue(); + + queue.enqueue({ type: SyncEventType.Create, path: "a.md", originalPath: "a.md" }); + const createPromise = queue.getCreatePromise("a.md")!; + + // File was renamed from a.md to b.md + queue.enqueue({ + type: SyncEventType.SyncLocal, + documentId: createPromise, + path: "b.md", + originalPath: "a.md", + }); + + const paths = queue.trackedPaths(); + assert.ok(!paths.has("a.md")); + assert.ok(paths.has("b.md")); + }); + + it("trackedPaths tracks multiple moves for the same pending create", () => { + const queue = createQueue(); + + queue.enqueue({ type: SyncEventType.Create, path: "a.md", originalPath: "a.md" }); + const createPromise = queue.getCreatePromise("a.md")!; + + queue.enqueue({ + type: SyncEventType.SyncLocal, + documentId: createPromise, + path: "b.md", + originalPath: "a.md", + }); + queue.enqueue({ + type: SyncEventType.SyncLocal, + documentId: createPromise, + path: "c.md", + originalPath: "a.md", + }); + + const paths = queue.trackedPaths(); + assert.ok(!paths.has("a.md")); + assert.ok(!paths.has("b.md")); + assert.ok(paths.has("c.md")); + }); + + it("resolveCreate settles the document and replaces promise documentIds in the queue", async () => { + const queue = createQueue(); + + queue.enqueue({ type: SyncEventType.Create, path: "a.md", originalPath: "a.md" }); + const createPromise = queue.getCreatePromise("a.md")!; + + // Dependent events enqueued while create is in flight + queue.enqueue({ + type: SyncEventType.SyncLocal, + documentId: createPromise, + path: "a.md", + originalPath: "a.md", + }); + queue.enqueue({ + type: SyncEventType.Delete, + documentId: createPromise, + }); + + const event = await queue.next(); // dequeue the create + assert.ok(event?.type === SyncEventType.Create); + + queue.resolveCreate(event, { + documentId: "DOC-1", + parentVersionId: 5, + remoteHash: "hash-1", + }); + + // Document is now settled + assert.strictEqual(queue.getSettledDocumentByPath("a.md")?.documentId, "DOC-1"); + + // Promise was resolved + assert.strictEqual(await createPromise, "DOC-1"); + + // Remaining events have string documentIds instead of promises. + // The SyncLocal + Delete for "DOC-1" coalesce: sync-local is + // discarded and the delete is returned (standard coalescing). + const deleteEvt = await queue.next(); + assert.ok(deleteEvt?.type === SyncEventType.Delete); + assert.strictEqual(deleteEvt.documentId, "DOC-1"); + + assert.strictEqual(await queue.next(), undefined); }); }); diff --git a/frontend/sync-client/src/sync-operations/sync-event-queue.ts b/frontend/sync-client/src/sync-operations/sync-event-queue.ts index 362c35dc..dc79c0db 100644 --- a/frontend/sync-client/src/sync-operations/sync-event-queue.ts +++ b/frontend/sync-client/src/sync-operations/sync-event-queue.ts @@ -14,9 +14,19 @@ import { } from "./types"; export class SyncEventQueue { - private readonly events: SyncEvent[] = []; + // latest state of the filesystem as we know it, excluding + // unconfirmed creates but including pending deletes, + // it's always indexed by the latest path on disk private readonly documents = new Map(); - private readonly recentlyDeletedDocumentIds = new Set(); + + // all outstanding operations in order of occurrence, + // can include multiple generations of the same document, + // e.g.: a create, delete, create sequence for the same path. + // The paths for the events must always correspond to the latest + // path on disk, so the path of each event may be updated multiple + // times. + private readonly events: SyncEvent[] = []; + private lastSeenUpdateIds: CoveredValues; private ignorePatterns: RegExp[]; @@ -55,7 +65,7 @@ export class SyncEventQueue { this.lastSeenUpdateIds.add(record.parentVersionId); } - this.logger.debug(`Loaded ${this.documents.size} documents`); + this.logger.debug(`Loaded ${this.documents.size} documents and lastSeenUpdateId=${this.lastSeenUpdateIds.min}`); } public get size(): number { @@ -66,10 +76,15 @@ export class SyncEventQueue { return this.documents.size; } - public getLastSeenUpdateId(): VaultUpdateId { + public get lastSeenUpdateId(): VaultUpdateId { return this.lastSeenUpdateIds.min; } + public set lastSeenUpdateId(value: number) { + this.lastSeenUpdateIds.min = value; + this.saveInTheBackground(); + } + public addSeenUpdateId(value: number): void { const previousMin = this.lastSeenUpdateIds.min; this.lastSeenUpdateIds.add(value); @@ -78,12 +93,8 @@ export class SyncEventQueue { } } - public setLastSeenUpdateId(value: number): void { - this.lastSeenUpdateIds.min = value; - this.saveInTheBackground(); - } - public getDocument(path: RelativePath): DocumentRecord | undefined { + public getSettledDocumentByPath(path: RelativePath): DocumentRecord | undefined { return this.documents.get(path); } @@ -104,86 +115,96 @@ export class SyncEventQueue { } public removeDocument(path: RelativePath): void { - const record = this.documents.get(path); - if (record !== undefined) { - this.recentlyDeletedDocumentIds.add(record.documentId); - } this.documents.delete(path); this.saveInTheBackground(); } /** - * Move a document from oldPath to newPath. - * If the target path is occupied by a different document, it is removed - * and its documentId is returned so the caller can handle the displacement. + * Settle a Create event: add the document to the settled map, + * resolve the create promise, and replace promise-based documentId + * references in the event queue with the actual string documentId. */ - public moveDocument( - oldPath: RelativePath, - newPath: RelativePath - ): DocumentId | undefined { - const record = this.documents.get(oldPath); - if (record === undefined) return undefined; + public resolveCreate( + event: Extract, + record: DocumentRecord + ): void { + const promise = event.resolvers?.promise; - let displacedDocumentId: DocumentId | undefined = undefined; - const existingAtTarget = this.documents.get(newPath); - if ( - existingAtTarget !== undefined && - existingAtTarget.documentId !== record.documentId - ) { - displacedDocumentId = existingAtTarget.documentId; - this.recentlyDeletedDocumentIds.add(displacedDocumentId); - this.documents.delete(newPath); + this.documents.set(event.path, record); + event.resolvers?.resolve(record.documentId); + + if (promise !== undefined) { + for (const e of this.events) { + if ( + (e.type === SyncEventType.SyncLocal || e.type === SyncEventType.Delete) && + e.documentId === promise + ) { + (e as { documentId: DocumentId | Promise }).documentId = record.documentId; + } + } } - this.documents.delete(oldPath); - this.documents.set(newPath, record); this.saveInTheBackground(); - return displacedDocumentId; } - public wasRecentlyDeleted(documentId: DocumentId): boolean { - return this.recentlyDeletedDocumentIds.has(documentId); + public getCreatePromise(path: RelativePath): Promise | undefined { + const event = this.findLastCreate(path); + if (event === undefined) return undefined; + event.resolvers ??= Promise.withResolvers(); + return event.resolvers.promise; } - public unmarkRecentlyDeleted(documentId: DocumentId): void { - this.recentlyDeletedDocumentIds.delete(documentId); - } - - - public allDocuments(): [RelativePath, DocumentRecord][] { + public allSettledDocuments(): [RelativePath, DocumentRecord][] { return Array.from(this.documents.entries()); } - public hasCreateEvent(path: RelativePath): boolean { - return this.events.some( - (e) => e.type === SyncEventType.Create && e.path === path - ); - } + /** + * Returns the set of paths we expect to exist on disk by replaying + * the event queue on top of the settled documents map. + */ + public trackedPaths(): Set { + const paths = new Set(this.documents.keys()); + // Track current path for each pending create so moves can be applied + const pendingPaths = new Map, RelativePath>(); - public updateCreatePath( - oldPath: RelativePath, - newPath: RelativePath - ): boolean { for (const event of this.events) { - if (event.type === SyncEventType.Create && event.path === oldPath) { - event.path = newPath; - return true; - } + if (event.type === SyncEventType.Create) { + paths.add(event.path); + if (event.resolvers !== undefined) { + pendingPaths.set(event.resolvers.promise, event.path); + } + } else if (event.type === SyncEventType.Delete) { + if (typeof event.documentId === "string") { + const path = this.getDocumentByDocumentId(event.documentId)?.path; + if (path) { + paths.delete(path); + } else { + throw new Error(`Delete event for unknown documentId ${event.documentId}`); + } + } else { + const path = pendingPaths.get(event.documentId); + if (!path) { + throw new Error(`Delete event with unresolved documentId promise`); + } + paths.delete(path); + } + } // no need to handle SyncLocal as path updates are applied to this.documents immediately when the event is enqueued } - return false; + return paths; } public hasPendingEventsForPath(path: RelativePath): boolean { const record = this.documents.get(path); - const docId = record?.documentId; + if (!record) { + return true; // if we don't know about this path, it must be pending creation + } + const docId = record.documentId; return this.events.some( (e) => (e.type === SyncEventType.Create && e.path === path) || (e.type === SyncEventType.SyncLocal && - docId !== undefined && e.documentId === docId) || (e.type === SyncEventType.Delete && - docId !== undefined && e.documentId === docId) || (e.type === SyncEventType.SyncRemote && e.remoteVersion.relativePath === path) @@ -203,31 +224,26 @@ export class SyncEventQueue { } public resetState(): void { + this.rejectAllPendingCreates(); this.documents.clear(); - this.recentlyDeletedDocumentIds.clear(); this.lastSeenUpdateIds = new CoveredValues(0); this.saveInTheBackground(); } public clear(): void { + this.rejectAllPendingCreates(); this.events.length = 0; - this.recentlyDeletedDocumentIds.clear(); } public enqueue(event: SyncEvent): void { if (this.isIgnored(event)) return; - if (event.type === SyncEventType.Create) { - if (this.documents.has(event.path)) return; - if (this.hasCreateEvent(event.path)) return; - } - this.events.push(event); } - public next(): SyncEvent | undefined { + public async next(): Promise { if (this.events.length === 0) return undefined; const [first] = this.events; @@ -244,9 +260,7 @@ export class SyncEventQueue { if (first.type === SyncEventType.Delete) { this.events.shift(); const { documentId } = first; - if (documentId !== "") { - this.removeAllEventsForDocumentId(documentId); - } + this.removeAllEventsForDocumentId(await documentId); return first; } @@ -261,16 +275,18 @@ export class SyncEventQueue { e.documentId === documentId ); if (deleteEvent !== undefined) { - this.removeAllSyncLocalsForDocumentId(documentId); + this.removeAllSyncLocalsForDocumentId(await documentId); removeFromArray(this.events, deleteEvent); return deleteEvent; } - // Coalesce multiple sync-locals for the same documentId to the last one + // Coalesce multiple sync-locals for the same documentId and + // original path to the last one const matching = this.events.filter( (e) => e.type === SyncEventType.SyncLocal && - e.documentId === documentId + e.documentId === documentId && + e.originalPath === first.originalPath ); const result = matching[matching.length - 1]; for (const item of matching) { @@ -328,6 +344,49 @@ export class SyncEventQueue { } } + public updatePendingCreatePath( + oldPath: RelativePath, + newPath: RelativePath + ): void { + const createEvent = this.findLastCreate(oldPath); + if (createEvent === undefined) return; + + const promise = createEvent.resolvers?.promise; + createEvent.path = newPath; + + if (promise !== undefined) { + for (const e of this.events) { + if ( + e.type === SyncEventType.SyncLocal && + e.documentId === promise + ) { + e.path = newPath; + } + } + } + } + + private findLastCreate( + path: RelativePath + ): Extract | undefined { + for (let i = this.events.length - 1; i >= 0; i--) { + const e = this.events[i]; + if (e.type === SyncEventType.Create && e.path === path) { + return e; + } + } + return undefined; + } + + private rejectAllPendingCreates(): void { + for (const event of this.events) { + if (event.type === SyncEventType.Create && event.resolvers !== undefined) { + event.resolvers.promise.catch(() => { /* suppressed — consumer may not be listening */ }); + event.resolvers.reject(new Error("Create was cancelled")); + } + } + } + private saveInTheBackground(): void { void this.save().catch((error: unknown) => { this.logger.error(`Error saving sync state: ${error}`); diff --git a/frontend/sync-client/src/sync-operations/types.ts b/frontend/sync-client/src/sync-operations/types.ts index f722aa8a..1bf99b8a 100644 --- a/frontend/sync-client/src/sync-operations/types.ts +++ b/frontend/sync-client/src/sync-operations/types.ts @@ -7,7 +7,7 @@ export type RelativePath = string; export interface DocumentRecord { documentId: DocumentId; parentVersionId: VaultUpdateId; - hash: string; + remoteHash: string; remoteRelativePath?: RelativePath; } @@ -23,18 +23,26 @@ export interface StoredSyncState { export enum SyncEventType { Create = "create", SyncLocal = "sync-local", - SyncRemote = "sync-remote", Delete = "delete", + SyncRemote = "sync-remote", } export type SyncEvent = - | { type: SyncEventType.Create; path: RelativePath } - | { type: SyncEventType.SyncLocal; documentId: DocumentId } + | { + type: SyncEventType.Create; + path: RelativePath; // current path on disk + originalPath: RelativePath; // original path on disk when the event was created + resolvers?: PromiseWithResolvers + } + | { + type: SyncEventType.SyncLocal; + documentId: DocumentId | Promise; // if it's a promise, the promise is fulfilled once the document's create event is processed + path: RelativePath; // current path on disk + originalPath: RelativePath; // original path on disk when the event was created + } | { type: SyncEventType.Delete; - documentId: DocumentId; - path: RelativePath; - displacedAtVersion?: VaultUpdateId; + documentId: DocumentId | Promise; // if it's a promise, the promise is fulfilled once the document's create event is processed } | { type: SyncEventType.SyncRemote;